<commit_before>
extern crate quickcheck;
extern crate bnf;

#[cfg(test)]
mod tests {
    use quickcheck::{QuickCheck, TestResult, Arbitrary, Gen};
    use bnf::Grammar;

    #[derive(PartialEq, Debug, Clone)]
    struct Meta {
        bnf: String,
    }

    // Modified version of BNF for BNF from
    // https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_form#Further_examples
    const BNF_FOR_BNF: &str =
            "<syntax>        ::= <rule> | <rule> <syntax>
            <rule>           ::= <opt-whitespace> \"<\" <rule-name> \">\"
                                <opt-whitespace> \"::=\" <opt-whitespace>
                                <expression> <line-end>
            <opt-whitespace> ::= \" \" <opt-whitespace> | \"\"
            <expression>     ::= <list> | <list> <opt-whitespace> \"|\"
                                <opt-whitespace> <expression>
            <line-end>       ::= <opt-whitespace> <EOL> | <line-end> <line-end>
            <list>           ::= <term> | <term> <opt-whitespace> <list>
            <term>           ::= <literal> | \"<\" <rule-name> \">\"
            <literal>        ::= '\"' <text1> '\"' | \"'\" <text2> \"'\"
            <text1>          ::= \"\" | <character1> <text1>
            <text2>          ::= \"\" | <character2> <text2>
            <character>      ::= <letter> | <digit> | <symbol>
            <letter>         ::= \"A\" | \"B\" | \"C\" | \"D\" | \"E\" | \"F\"
                                | \"G\" | \"H\" | \"I\" | \"J\" | \"K\" | \"L\"
                                | \"M\" | \"N\" | \"O\" | \"P\" | \"Q\" | \"R\"
                                | \"S\" | \"T\" | \"U\" | \"V\" | \"W\" | \"X\"
                                | \"Y\" | \"Z\" | \"a\" | \"b\" | \"c\" | \"d\"
                                | \"e\" | \"f\" | \"g\" | \"h\" | \"i\" | \"j\"
                                | \"k\" | \"l\" | \"m\" | \"n\" | \"o\" | \"p\"
                                | \"q\" | \"r\" | \"s\" | \"t\" | \"u\" | \"v\"
                                | \"w\" | \"x\" | \"y\" | \"z\"

            <digit>          ::= \"0\" | \"1\" | \"2\" | \"3\" | \"4\" | \"5\"
                                | \"6\" | \"7\" | \"8\" | \"9\"

            <symbol>         ::=  \"|\" | \" \" | \"-\" | \"!\" | \"#\" | \"$\"
                                | \"%\" | \"&\" | \"(\" | \")\" | \"*\" | \"+\"
                                | \",\" | \"-\" | \".\" | \"/\" | \":\" | \";\"
                                |\">\" | \"=\" | \"<\" | \"?\" | \"@\" | \"[\"
                                | \"\\\" | \"]\" | \"^\" | \"_\" | \"`\"
                                | \"{{\" | \"}}\" | \"~\"

            <character1>     ::= <character> | \"'\"
            <character2>     ::= <character> | '\"'
            <rule-name>      ::= <letter> | <rule-name> <rule-char>
            <rule-char>      ::= <letter> | <digit> | \"-\"
            <EOL>            ::= \"\n\"";

    impl Arbitrary for Meta {
        fn arbitrary<G: Gen>(_: &mut G) -> Meta {
            // Generate Grammar object from grammar for BNF grammars
            let grammar = Grammar::from_str(BNF_FOR_BNF);
            assert!(grammar.is_ok(), "{:?} should be Ok", grammar);

            // generate a random valid grammar from the above
            let sentence = grammar.unwrap().generate();
            assert!(sentence.is_ok(), "{:?} should be Ok", sentence);

            Meta { bnf: sentence.unwrap() }
        }
    }

    fn prop_grammar_from_str(meta: Meta) -> TestResult {
        // parse a randomly generated grammar to a Grammar object
        let meta_grammar = Grammar::from_str(&meta.bnf);
        TestResult::from_bool(meta_grammar.is_ok())
    }

    #[test]
    fn test_generated_grammars() {
        QuickCheck::new().quickcheck(prop_grammar_from_str as fn(Meta) -> TestResult)
    }
}
<commit_msg>looks like there was a non-terminating rule in the bnf for bnf used for tests<commit_after>
extern crate quickcheck;
extern crate bnf;

#[cfg(test)]
mod tests {
    use quickcheck::{QuickCheck, TestResult, Arbitrary, Gen};
    use bnf::Grammar;

    #[derive(PartialEq, Debug, Clone)]
    struct Meta {
        bnf: String,
    }

    // Modified version of BNF for BNF from
    // https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_form#Further_examples
    const BNF_FOR_BNF: &str =
            "<syntax>        ::= <rule> | <rule> <syntax>
            <rule>           ::= <opt-whitespace> \"<\" <rule-name> \">\"
                                <opt-whitespace> \"::=\" <opt-whitespace>
                                <expression> <line-end>
            <opt-whitespace> ::=  \"\" | \" \" <opt-whitespace>
            <expression>     ::= <list> | <list> <opt-whitespace> \"|\"
                                <opt-whitespace> <expression>
            <line-end>       ::= <opt-whitespace> <EOL> | <line-end> <line-end>
            <list>           ::= <term> | <term> <opt-whitespace> <list>
            <term>           ::= <literal> | \"<\" <rule-name> \">\"
            <literal>        ::= '\"' <text1> '\"' | \"'\" <text2> \"'\"
            <text1>          ::= \"\" | <character1> <text1>
            <text2>          ::= \"\" | <character2> <text2>
            <character>      ::= <letter> | <digit> | <symbol>
            <letter>         ::= \"A\" | \"B\" | \"C\" | \"D\" | \"E\" | \"F\"
                                | \"G\" | \"H\" | \"I\" | \"J\" | \"K\" | \"L\"
                                | \"M\" | \"N\" | \"O\" | \"P\" | \"Q\" | \"R\"
                                | \"S\" | \"T\" | \"U\" | \"V\" | \"W\" | \"X\"
                                | \"Y\" | \"Z\" | \"a\" | \"b\" | \"c\" | \"d\"
                                | \"e\" | \"f\" | \"g\" | \"h\" | \"i\" | \"j\"
                                | \"k\" | \"l\" | \"m\" | \"n\" | \"o\" | \"p\"
                                | \"q\" | \"r\" | \"s\" | \"t\" | \"u\" | \"v\"
                                | \"w\" | \"x\" | \"y\" | \"z\"

            <digit>          ::= \"0\" | \"1\" | \"2\" | \"3\" | \"4\" | \"5\"
                                | \"6\" | \"7\" | \"8\" | \"9\"

            <symbol>         ::=  \"|\" | \" \" | \"-\" | \"!\" | \"#\" | \"$\"
                                | \"%\" | \"&\" | \"(\" | \")\" | \"*\" | \"+\"
                                | \",\" | \"-\" | \".\" | \"/\" | \":\" | \";\"
                                |\">\" | \"=\" | \"<\" | \"?\" | \"@\" | \"[\"
                                | \"\\\" | \"]\" | \"^\" | \"_\" | \"`\"
                                | \"{{\" | \"}}\" | \"~\"

            <character1>     ::= <character> | \"'\"
            <character2>     ::= <character> | '\"'
            <rule-name>      ::= <letter> | <rule-name> <rule-char>
            <rule-char>      ::= <letter> | <digit> | \"-\"
            <EOL>            ::= \"\n\"";

    impl Arbitrary for Meta {
        fn arbitrary<G: Gen>(_: &mut G) -> Meta {
            // Generate Grammar object from grammar for BNF grammars
            let grammar = Grammar::from_str(BNF_FOR_BNF);
            assert!(grammar.is_ok(), "{:?} should be Ok", grammar);

            // generate a random valid grammar from the above
            let sentence = grammar.unwrap().generate();
            assert!(sentence.is_ok(), "{:?} should be Ok", sentence);

            Meta { bnf: sentence.unwrap() }
        }
    }

    fn prop_grammar_from_str(meta: Meta) -> TestResult {
        // parse a randomly generated grammar to a Grammar object
        let meta_grammar = Grammar::from_str(&meta.bnf);
        TestResult::from_bool(meta_grammar.is_ok())
    }

    #[test]
    fn test_generated_grammars() {
        QuickCheck::new().quickcheck(prop_grammar_from_str as fn(Meta) -> TestResult)
    }
}
<|endoftext|>


<commit_before>use std::env;
use std::fmt;
use std::fs::{self, PathExt};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process;
use std::str::FromStr;
use std::sync::atomic;

use csv;

use Csv;

static XSV_INTEGRATION_TEST_DIR: &'static str = "xit";

static NEXT_ID: atomic::AtomicUsize = atomic::ATOMIC_USIZE_INIT;

pub struct Workdir {
    root: PathBuf,
    dir: PathBuf,
    flexible: bool,
}

impl Workdir {
    pub fn new(name: &str) -> Workdir {
        let id = NEXT_ID.fetch_add(1, atomic::Ordering::SeqCst);
        let root = env::current_exe().unwrap()
                       .parent()
                       .expect("executable's directory")
                       .to_path_buf();
        let dir = root.join(XSV_INTEGRATION_TEST_DIR)
                      .join(name)
                      .join(&format!("test-{}", id));

        // I don't get why this is necessary, but Travis seems to need it?
        let md = fs::metadata(&dir);
        if fs::metadata(&dir).map(|md| md.is_dir()).unwrap_or(false) {
            if let Err(err) = fs::remove_dir_all(&dir) {
                panic!("Could not remove directory '{:?}': {}", dir, err);
            }
        }
        if fs::metadata(&dir).map(|md| md.is_file()).unwrap_or(false) {
            if let Err(err) = fs::remove_file(&dir) {
                panic!("Could not remove file '{:?}': {}", dir, err);
            }
        }
        // Explicitly ignoring an error here seems wrong, but for whatever
        // reason, this appears to fail non-deterministically in Travis because
        // the "file already exists." Does that mean `NEXT_ID` isn't really
        // atomic? Really? Very strange and baffling. ---AG
        if let Err(err) = fs::create_dir_all(&dir) {
            panic!("Could not create '{:?}': {}", dir, err);
        }
        Workdir { root: root, dir: dir, flexible: false }
    }

    pub fn flexible(mut self, yes: bool) -> Workdir {
        self.flexible = yes;
        self
    }

    pub fn create<T: Csv>(&self, name: &str, rows: T) {
        let mut wtr = match csv::Writer::from_file(&self.path(name)) {
            Ok(wtr) => wtr.flexible(self.flexible),
            Err(err) => panic!("Could not open '{:?}': {}",
                                self.path(name), err),
        };
        for row in rows.to_vecs().into_iter() {
            wtr.write(row.iter()).unwrap();
        }
        wtr.flush().unwrap();
    }

    pub fn create_indexed<T: Csv>(&self, name: &str, rows: T) {
        self.create(name, rows);

        let mut cmd = self.command("index");
        cmd.arg(name);
        self.run(&mut cmd);
    }

    pub fn read_stdout<T: Csv>(&self, cmd: &mut process::Command) -> T {
        let mut rdr = csv::Reader::from_string(self.stdout::<String>(cmd))
                                  .has_headers(false);
        Csv::from_vecs(rdr.records().collect::<Result<_, _>>().unwrap())
    }

    pub fn command(&self, sub_command: &str) -> process::Command {
        let mut cmd = process::Command::new(&self.xsv_bin());
        cmd.current_dir(&self.dir).arg(sub_command);
        cmd
    }

    pub fn output(&self, cmd: &mut process::Command) -> process::Output {
        debug!("[{}]: {:?}", self.dir.display(), cmd);
        let o = cmd.output().unwrap();
        if !o.status.success() {
            panic!("\n\n===== {:?} =====\n\
                    command failed but expected success!\
                    \n\ncwd: {}\
                    \n\nstatus: {}\
                    \n\nstdout: {}\n\nstderr: {}\
                    \n\n=====\n",
                   cmd, self.dir.display(), o.status,
                   String::from_utf8_lossy(&o.stdout),
                   String::from_utf8_lossy(&o.stderr))
        }
        o
    }

    pub fn run(&self, cmd: &mut process::Command) {
        self.output(cmd);
    }

    pub fn stdout<T: FromStr>(&self, cmd: &mut process::Command) -> T {
        let o = self.output(cmd);
        let stdout = String::from_utf8_lossy(&o.stdout);
        stdout.trim().parse().ok().expect(
            &format!("Could not convert from string: '{}'", stdout))
    }

    pub fn assert_err(&self, cmd: &mut process::Command) {
        let o = cmd.output().unwrap();
        if o.status.success() {
            panic!("\n\n===== {:?} =====\n\
                    command succeeded but expected failure!\
                    \n\ncwd: {}\
                    \n\nstatus: {}\
                    \n\nstdout: {}\n\nstderr: {}\
                    \n\n=====\n",
                   cmd, self.dir.display(), o.status,
                   String::from_utf8_lossy(&o.stdout),
                   String::from_utf8_lossy(&o.stderr));
        }
    }

    pub fn from_str<T: FromStr>(&self, name: &Path) -> T {
        let mut o = String::new();
        fs::File::open(name).unwrap().read_to_string(&mut o).unwrap();
        o.parse().ok().expect("fromstr")
    }

    pub fn path(&self, name: &str) -> PathBuf {
        self.dir.join(name)
    }

    pub fn xsv_bin(&self) -> PathBuf {
        self.root.join("xsv")
    }
}

impl fmt::Debug for Workdir {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "path={}", self.dir.display())
    }
}
<commit_msg>yet another try<commit_after>use std::env;
use std::fmt;
use std::fs::{self, PathExt};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process;
use std::str::FromStr;
use std::sync::atomic;

use csv;

use Csv;

static XSV_INTEGRATION_TEST_DIR: &'static str = "xit";

static NEXT_ID: atomic::AtomicUsize = atomic::ATOMIC_USIZE_INIT;

pub struct Workdir {
    root: PathBuf,
    dir: PathBuf,
    flexible: bool,
}

impl Workdir {
    pub fn new(name: &str) -> Workdir {
        let id = NEXT_ID.fetch_add(1, atomic::Ordering::SeqCst);
        let root = env::current_exe().unwrap()
                       .parent()
                       .expect("executable's directory")
                       .to_path_buf();
        let dir = root.join(XSV_INTEGRATION_TEST_DIR)
                      .join(name)
                      .join(&format!("test-{}", id));

        // I don't get why this is necessary, but Travis seems to need it?
        // let md = fs::metadata(&dir);
        // if fs::metadata(&dir).map(|md| md.is_dir()).unwrap_or(false) {
            // if let Err(err) = fs::remove_dir_all(&dir) {
                // panic!("Could not remove directory '{:?}': {}", dir, err);
            // }
        // }
        // if fs::metadata(&dir).map(|md| md.is_file()).unwrap_or(false) {
            // if let Err(err) = fs::remove_file(&dir) {
                // panic!("Could not remove file '{:?}': {}", dir, err);
            // }
        // }
        if let Err(err) = fs::create_dir_all(&dir) {
            panic!("Could not create '{:?}': {}", dir, err);
        }
        Workdir { root: root, dir: dir, flexible: false }
    }

    pub fn flexible(mut self, yes: bool) -> Workdir {
        self.flexible = yes;
        self
    }

    pub fn create<T: Csv>(&self, name: &str, rows: T) {
        let mut wtr = match csv::Writer::from_file(&self.path(name)) {
            Ok(wtr) => wtr.flexible(self.flexible),
            Err(err) => panic!("Could not open '{:?}': {}",
                                self.path(name), err),
        };
        for row in rows.to_vecs().into_iter() {
            wtr.write(row.iter()).unwrap();
        }
        wtr.flush().unwrap();
    }

    pub fn create_indexed<T: Csv>(&self, name: &str, rows: T) {
        self.create(name, rows);

        let mut cmd = self.command("index");
        cmd.arg(name);
        self.run(&mut cmd);
    }

    pub fn read_stdout<T: Csv>(&self, cmd: &mut process::Command) -> T {
        let mut rdr = csv::Reader::from_string(self.stdout::<String>(cmd))
                                  .has_headers(false);
        Csv::from_vecs(rdr.records().collect::<Result<_, _>>().unwrap())
    }

    pub fn command(&self, sub_command: &str) -> process::Command {
        let mut cmd = process::Command::new(&self.xsv_bin());
        cmd.current_dir(&self.dir).arg(sub_command);
        cmd
    }

    pub fn output(&self, cmd: &mut process::Command) -> process::Output {
        debug!("[{}]: {:?}", self.dir.display(), cmd);
        let o = cmd.output().unwrap();
        if !o.status.success() {
            panic!("\n\n===== {:?} =====\n\
                    command failed but expected success!\
                    \n\ncwd: {}\
                    \n\nstatus: {}\
                    \n\nstdout: {}\n\nstderr: {}\
                    \n\n=====\n",
                   cmd, self.dir.display(), o.status,
                   String::from_utf8_lossy(&o.stdout),
                   String::from_utf8_lossy(&o.stderr))
        }
        o
    }

    pub fn run(&self, cmd: &mut process::Command) {
        self.output(cmd);
    }

    pub fn stdout<T: FromStr>(&self, cmd: &mut process::Command) -> T {
        let o = self.output(cmd);
        let stdout = String::from_utf8_lossy(&o.stdout);
        stdout.trim().parse().ok().expect(
            &format!("Could not convert from string: '{}'", stdout))
    }

    pub fn assert_err(&self, cmd: &mut process::Command) {
        let o = cmd.output().unwrap();
        if o.status.success() {
            panic!("\n\n===== {:?} =====\n\
                    command succeeded but expected failure!\
                    \n\ncwd: {}\
                    \n\nstatus: {}\
                    \n\nstdout: {}\n\nstderr: {}\
                    \n\n=====\n",
                   cmd, self.dir.display(), o.status,
                   String::from_utf8_lossy(&o.stdout),
                   String::from_utf8_lossy(&o.stderr));
        }
    }

    pub fn from_str<T: FromStr>(&self, name: &Path) -> T {
        let mut o = String::new();
        fs::File::open(name).unwrap().read_to_string(&mut o).unwrap();
        o.parse().ok().expect("fromstr")
    }

    pub fn path(&self, name: &str) -> PathBuf {
        self.dir.join(name)
    }

    pub fn xsv_bin(&self) -> PathBuf {
        self.root.join("xsv")
    }
}

impl fmt::Debug for Workdir {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "path={}", self.dir.display())
    }
}
<|endoftext|>


<commit_before>use std::collections::HashSet;
use std::sync::Arc;
use super::structs::AttestationRecord;
use super::AttesterMap;
use super::attestation_parent_hashes::{
    attestation_parent_hashes,
    ParentHashesError,
};
use super::db::{
    ClientDB,
    DBError
};
use super::db::stores::{
    BlockStore,
    ValidatorStore,
};
use super::utils::types::{
    Hash256,
};
use super::message_generation::generate_signed_message;
use super::signature_verification::{
    verify_aggregate_signature_for_indices,
    SignatureVerificationError,
};

#[derive(Debug,PartialEq)]
pub enum AttestationValidationError {
    SlotTooHigh,
    SlotTooLow,
    JustifiedSlotTooHigh,
    UnknownJustifiedBlock,
    TooManyObliqueHashes,
    BadCurrentHashes,
    BadObliqueHashes,
    BadAttesterMap,
    IntWrapping,
    PublicKeyCorrupt,
    NoPublicKeyForValidator,
    BadBitfieldLength,
    InvalidBitfield,
    InvalidBitfieldEndBits,
    NoSignatures,
    NonZeroTrailingBits,
    AggregateSignatureFail,
    DBError(String),
}

pub fn validate_attestation<T>(a: &AttestationRecord,
                               block_slot: u64,
                               cycle_length: u8,
                               known_last_justified_slot: u64,
                               known_parent_hashes: &Arc<Vec<Hash256>>,
                               block_store: &Arc<BlockStore<T>>,
                               validator_store: &Arc<ValidatorStore<T>>,
                               attester_map: &Arc<AttesterMap>)
    -> Result<Option<HashSet<usize>>, AttestationValidationError>
    where T: ClientDB + Sized
{
    /*
     * The attesation slot must not be higher than the block that contained it.
     */
    if a.slot > block_slot {
        return Err(AttestationValidationError::SlotTooHigh);
    }

    /*
     * The slot of this attestation must not be more than cycle_length + 1 distance
     * from the block that contained it.
     *
     * The below code stays overflow-safe as long as cycle length is a < 64 bit integer.
     */
    if a.slot < block_slot.saturating_sub(u64::from(cycle_length) + 1) {
        return Err(AttestationValidationError::SlotTooLow);
    }

    /*
     * The attestation must indicate that its last justified slot is the same as the last justified
     * slot known to us.
     */
    if a.justified_slot > known_last_justified_slot {
        return Err(AttestationValidationError::JustifiedSlotTooHigh);
    }

    /*
     * There is no need to include more oblique parents hashes than there are blocks
     * in a cycle.
     */
    if a.oblique_parent_hashes.len() > usize::from(cycle_length) {
        return Err(AttestationValidationError::TooManyObliqueHashes);
    }

    /*
     * Retrieve the set of attestation indices for this slot and shard id.
     *
     * This is an array mapping the order that validators will appear in the bitfield to the
     * canonincal index of a validator.
     */
    let attestation_indices = attester_map.get(&(a.slot, a.shard_id))
        .ok_or(AttestationValidationError::BadAttesterMap)?;

    /*
     * The bitfield must be no longer than the minimum required to represent each validator in the
     * attestation indicies for this slot and shard id.
     */
    if a.attester_bitfield.num_bytes() !=
        bytes_for_bits(attestation_indices.len())
    {
        return Err(AttestationValidationError::BadBitfieldLength);
    }

    /*
     * If there are excess bits in the bitfield because the number of a validators in not a
     * multiple of 8, reject this attestation record.
     *
     * Allow extra set bits would permit mutliple different byte layouts (and therefore hashes) to
     * refer to the same AttesationRecord.
     */
    let last_byte =
        a.attester_bitfield.get_byte(a.attester_bitfield.num_bytes() - 1)
        .ok_or(AttestationValidationError::InvalidBitfield)?;
    if any_of_last_n_bits_are_set(*last_byte, a.attester_bitfield.len() % 8) {
        return Err(AttestationValidationError::InvalidBitfieldEndBits)
    }

    /*
     * The specified justified block hash must be known to us
     */
    if !block_store.block_exists(&a.justified_block_hash)? {
        return Err(AttestationValidationError::UnknownJustifiedBlock)
    }

    let signed_message = {
        let parent_hashes = attestation_parent_hashes(
            cycle_length,
            block_slot,
            a.slot,
            &known_parent_hashes,
            &a.oblique_parent_hashes)?;
        generate_signed_message(
            a.slot,
            &parent_hashes,
            a.shard_id,
            &a.shard_block_hash,
            a.justified_slot)
    };

    let voted_hashmap =
        verify_aggregate_signature_for_indices(
            &signed_message,
            &a.aggregate_sig,
            &attestation_indices,
            &a.attester_bitfield,
            &validator_store)?;

    Ok(voted_hashmap)
}

fn bytes_for_bits(bits: usize) -> usize {
    (bits.saturating_sub(1) / 8) + 1
}

fn any_of_last_n_bits_are_set(byte: u8, n: usize) -> bool {
    for i in 0..n {
        let mask = 0_u8 >> 8_usize - i as usize;
        if byte & mask > 0 {
            return true
        }
    }
    false
}

impl From<ParentHashesError> for AttestationValidationError {
    fn from(e: ParentHashesError) -> Self {
        match e {
            ParentHashesError::BadCurrentHashes
                => AttestationValidationError::BadCurrentHashes,
            ParentHashesError::BadObliqueHashes
                => AttestationValidationError::BadObliqueHashes,
            ParentHashesError::SlotTooLow
                => AttestationValidationError::SlotTooLow,
            ParentHashesError::SlotTooHigh
                => AttestationValidationError::SlotTooHigh,
            ParentHashesError::IntWrapping
                => AttestationValidationError::IntWrapping
        }
    }
}

impl From<DBError> for AttestationValidationError {
    fn from(e: DBError) -> Self {
        AttestationValidationError::DBError(e.message)
    }
}

impl From<SignatureVerificationError> for AttestationValidationError {
    fn from(e: SignatureVerificationError) -> Self {
        match e {
            SignatureVerificationError::BadValidatorIndex
                => AttestationValidationError::BadAttesterMap,
            SignatureVerificationError::PublicKeyCorrupt
                => AttestationValidationError::PublicKeyCorrupt,
            SignatureVerificationError::NoPublicKeyForValidator
                => AttestationValidationError::NoPublicKeyForValidator,
            SignatureVerificationError::DBError(s)
                => AttestationValidationError::DBError(s),
        }
    }
}
<commit_msg>Fix bug in attestation val. bitfield checking<commit_after>use std::collections::HashSet;
use std::sync::Arc;
use super::structs::AttestationRecord;
use super::AttesterMap;
use super::attestation_parent_hashes::{
    attestation_parent_hashes,
    ParentHashesError,
};
use super::db::{
    ClientDB,
    DBError
};
use super::db::stores::{
    BlockStore,
    ValidatorStore,
};
use super::utils::types::{
    Hash256,
};
use super::message_generation::generate_signed_message;
use super::signature_verification::{
    verify_aggregate_signature_for_indices,
    SignatureVerificationError,
};

#[derive(Debug,PartialEq)]
pub enum AttestationValidationError {
    SlotTooHigh,
    SlotTooLow,
    JustifiedSlotTooHigh,
    UnknownJustifiedBlock,
    TooManyObliqueHashes,
    BadCurrentHashes,
    BadObliqueHashes,
    BadAttesterMap,
    IntWrapping,
    PublicKeyCorrupt,
    NoPublicKeyForValidator,
    BadBitfieldLength,
    InvalidBitfield,
    InvalidBitfieldEndBits,
    NoSignatures,
    NonZeroTrailingBits,
    AggregateSignatureFail,
    DBError(String),
}

pub fn validate_attestation<T>(a: &AttestationRecord,
                               block_slot: u64,
                               cycle_length: u8,
                               known_last_justified_slot: u64,
                               known_parent_hashes: &Arc<Vec<Hash256>>,
                               block_store: &Arc<BlockStore<T>>,
                               validator_store: &Arc<ValidatorStore<T>>,
                               attester_map: &Arc<AttesterMap>)
    -> Result<Option<HashSet<usize>>, AttestationValidationError>
    where T: ClientDB + Sized
{
    /*
     * The attesation slot must not be higher than the block that contained it.
     */
    if a.slot > block_slot {
        return Err(AttestationValidationError::SlotTooHigh);
    }

    /*
     * The slot of this attestation must not be more than cycle_length + 1 distance
     * from the block that contained it.
     *
     * The below code stays overflow-safe as long as cycle length is a < 64 bit integer.
     */
    if a.slot < block_slot.saturating_sub(u64::from(cycle_length) + 1) {
        return Err(AttestationValidationError::SlotTooLow);
    }

    /*
     * The attestation must indicate that its last justified slot is the same as the last justified
     * slot known to us.
     */
    if a.justified_slot > known_last_justified_slot {
        return Err(AttestationValidationError::JustifiedSlotTooHigh);
    }

    /*
     * There is no need to include more oblique parents hashes than there are blocks
     * in a cycle.
     */
    if a.oblique_parent_hashes.len() > usize::from(cycle_length) {
        return Err(AttestationValidationError::TooManyObliqueHashes);
    }

    /*
     * Retrieve the set of attestation indices for this slot and shard id.
     *
     * This is an array mapping the order that validators will appear in the bitfield to the
     * canonincal index of a validator.
     */
    let attestation_indices = attester_map.get(&(a.slot, a.shard_id))
        .ok_or(AttestationValidationError::BadAttesterMap)?;

    /*
     * The bitfield must be no longer than the minimum required to represent each validator in the
     * attestation indicies for this slot and shard id.
     */
    if a.attester_bitfield.num_bytes() !=
        bytes_for_bits(attestation_indices.len())
    {
        return Err(AttestationValidationError::BadBitfieldLength);
    }

    /*
     * If there are excess bits in the bitfield because the number of a validators in not a
     * multiple of 8, reject this attestation record.
     *
     * Allow extra set bits would permit mutliple different byte layouts (and therefore hashes) to
     * refer to the same AttesationRecord.
     */
    let last_byte =
        a.attester_bitfield.get_byte(a.attester_bitfield.num_bytes() - 1)
        .ok_or(AttestationValidationError::InvalidBitfield)?;
    if any_of_last_n_bits_are_set(*last_byte, a.attester_bitfield.len() % 8) {
        return Err(AttestationValidationError::InvalidBitfieldEndBits)
    }

    /*
     * The specified justified block hash must be known to us
     */
    if !block_store.block_exists(&a.justified_block_hash)? {
        return Err(AttestationValidationError::UnknownJustifiedBlock)
    }

    let signed_message = {
        let parent_hashes = attestation_parent_hashes(
            cycle_length,
            block_slot,
            a.slot,
            &known_parent_hashes,
            &a.oblique_parent_hashes)?;
        generate_signed_message(
            a.slot,
            &parent_hashes,
            a.shard_id,
            &a.shard_block_hash,
            a.justified_slot)
    };

    let voted_hashmap =
        verify_aggregate_signature_for_indices(
            &signed_message,
            &a.aggregate_sig,
            &attestation_indices,
            &a.attester_bitfield,
            &validator_store)?;

    Ok(voted_hashmap)
}

fn bytes_for_bits(bits: usize) -> usize {
    (bits.saturating_sub(1) / 8) + 1
}

fn any_of_last_n_bits_are_set(byte: u8, n: usize) -> bool {
    for i in 0..n {
        let mask = 1_u8 >> 7_usize.saturating_sub(i as usize);
        if byte & mask > 0 {
            return true
        }
    }
    false
}

impl From<ParentHashesError> for AttestationValidationError {
    fn from(e: ParentHashesError) -> Self {
        match e {
            ParentHashesError::BadCurrentHashes
                => AttestationValidationError::BadCurrentHashes,
            ParentHashesError::BadObliqueHashes
                => AttestationValidationError::BadObliqueHashes,
            ParentHashesError::SlotTooLow
                => AttestationValidationError::SlotTooLow,
            ParentHashesError::SlotTooHigh
                => AttestationValidationError::SlotTooHigh,
            ParentHashesError::IntWrapping
                => AttestationValidationError::IntWrapping
        }
    }
}

impl From<DBError> for AttestationValidationError {
    fn from(e: DBError) -> Self {
        AttestationValidationError::DBError(e.message)
    }
}

impl From<SignatureVerificationError> for AttestationValidationError {
    fn from(e: SignatureVerificationError) -> Self {
        match e {
            SignatureVerificationError::BadValidatorIndex
                => AttestationValidationError::BadAttesterMap,
            SignatureVerificationError::PublicKeyCorrupt
                => AttestationValidationError::PublicKeyCorrupt,
            SignatureVerificationError::NoPublicKeyForValidator
                => AttestationValidationError::NoPublicKeyForValidator,
            SignatureVerificationError::DBError(s)
                => AttestationValidationError::DBError(s),
        }
    }
}
<|endoftext|>


<commit_before>//! This module integrates the buddy heap allocator into the Rust runtime.
use spin::Mutex;
use core::ptr;

use ::Allocator;
use super::{BuddyHeapAllocator, FreeList};

static ALLOC: Mutex<Option<BuddyHeapAllocator<'static>>>
    = Mutex::new(None);

static mut KERNEL_FREE_LISTS: [FreeList; 17]
    // TODO: I really wish there was a less awful way to do this...
    = [ FreeList::new()
      , FreeList::new()
      , FreeList::new()
      , FreeList::new()
      , FreeList::new()
      , FreeList::new()
      , FreeList::new()
      , FreeList::new()
      , FreeList::new()
      , FreeList::new()
      , FreeList::new()
      , FreeList::new()
      , FreeList::new()
      , FreeList::new()
      , FreeList::new()
      , FreeList::new()
      , FreeList::new()
      ];

pub unsafe fn init_heap(start_addr: *mut u8, heap_size: usize ) {
    trace!("init_heap() was called.");
    *(ALLOC.lock())
        = Some(BuddyHeapAllocator::new( start_addr
                                      , &mut KERNEL_FREE_LISTS
                                      , heap_size));
}

#[no_mangle]
pub extern "C" fn __rust_allocate(size: usize, align: usize) -> *mut u8 {
    trace!("__rust_allocate() was called.");
    unsafe {
        ALLOC.lock().as_mut()
             .expect("Cannot allocate memory, no system allocator exists!")
             .allocate(size, align)
             .map(|blck| {
                 trace!("__rust_allocate: allocatedd {:?}", blck);
                 blck })
             .unwrap_or(ptr::null_mut())
    }
}

#[no_mangle]
pub extern "C" fn __rust_deallocate( ptr: *mut u8, old_size: usize
                                   , align: usize ) {
    unsafe {
        ALLOC.lock().as_mut()
             .expect("Cannot deallocate memory, no system allocator exists!")
             .deallocate(ptr, old_size, align)
    }
}

#[no_mangle]
pub extern "C" fn __rust_reallocate( ptr: *mut u8, old_size: usize
                                   , size: usize, align: usize )
                                   -> *mut u8 {
    unsafe {
        ALLOC.lock().as_mut()
             .expect("Cannot reallocate memory, no system allocator exists!")
             .reallocate(ptr, old_size, size, align)
             .unwrap_or(ptr::null_mut())
     }
}

/// This is currently unsupported, so we just silently ignore it
/// and return the old size.
#[no_mangle]
pub extern "C" fn __rust_reallocate_inplace( ptr: *mut u8
                                           , old_size: usize
                                           , _: usize, _: usize )
                                           -> usize {
    old_size
}

#[no_mangle]
pub extern "C" fn __rust_usable_size(size: usize, _: usize) -> usize {
    size
}
<commit_msg>[alloc] increase number of kernel free lists to 19<commit_after>//! This module integrates the buddy heap allocator into the Rust runtime.
use spin::Mutex;
use core::ptr;

use ::Allocator;
use super::{BuddyHeapAllocator, FreeList};

pub const NUM_FREE_LISTS: usize = 19;

static ALLOC: Mutex<Option<BuddyHeapAllocator<'static>>>
    = Mutex::new(None);

static mut KERNEL_FREE_LISTS: [FreeList; NUM_FREE_LISTS]
    // TODO: I really wish there was a less awful way to do this...
    = [ FreeList::new(),  FreeList::new(), FreeList::new()
      , FreeList::new(),  FreeList::new(), FreeList::new()
      , FreeList::new(),  FreeList::new(), FreeList::new()
      , FreeList::new(),  FreeList::new(), FreeList::new()
      , FreeList::new(),  FreeList::new(), FreeList::new()
      , FreeList::new(),  FreeList::new(), FreeList::new()
      , FreeList::new()
      , ];

pub unsafe fn init_heap(start_addr: *mut u8, heap_size: usize ) {
    trace!("init_heap() was called.");
    *(ALLOC.lock())
        = Some(BuddyHeapAllocator::new( start_addr
                                      , &mut KERNEL_FREE_LISTS
                                      , heap_size));
}

#[no_mangle]
pub extern "C" fn __rust_allocate(size: usize, align: usize) -> *mut u8 {
    trace!("__rust_allocate() was called.");
    unsafe {
        ALLOC.lock().as_mut()
             .expect("Cannot allocate memory, no system allocator exists!")
             .allocate(size, align)
             .map(|blck| {
                 trace!("__rust_allocate: allocatedd {:?}", blck);
                 blck })
             .unwrap_or(ptr::null_mut())
    }
}

#[no_mangle]
pub extern "C" fn __rust_deallocate( ptr: *mut u8, old_size: usize
                                   , align: usize ) {
    unsafe {
        ALLOC.lock().as_mut()
             .expect("Cannot deallocate memory, no system allocator exists!")
             .deallocate(ptr, old_size, align)
    }
}

#[no_mangle]
pub extern "C" fn __rust_reallocate( ptr: *mut u8, old_size: usize
                                   , size: usize, align: usize )
                                   -> *mut u8 {
    unsafe {
        ALLOC.lock().as_mut()
             .expect("Cannot reallocate memory, no system allocator exists!")
             .reallocate(ptr, old_size, size, align)
             .unwrap_or(ptr::null_mut())
     }
}

/// This is currently unsupported, so we just silently ignore it
/// and return the old size.
#[no_mangle]
pub extern "C" fn __rust_reallocate_inplace( ptr: *mut u8
                                           , old_size: usize
                                           , _: usize, _: usize )
                                           -> usize {
    old_size
}

#[no_mangle]
pub extern "C" fn __rust_usable_size(size: usize, _: usize) -> usize {
    size
}
<|endoftext|>


<commit_before>extern crate gitignore;
extern crate libc;

use std::ffi::CStr;
use std::ffi::CString;
use std::mem;
use std::path::{Path, PathBuf};
use std::string::FromUtf8Error;

#[repr(C)]
pub struct RubyArray {
      len: libc::size_t,
      data: *const libc::c_void,
}

impl RubyArray {
    fn from_vec<T>(vec: Vec<T>) -> RubyArray {
        let array = RubyArray {
            data: vec.as_ptr() as *const libc::c_void,
            len: vec.len() as libc::size_t
        };
        mem::forget(vec);
        array
    }
}

#[no_mangle]
pub extern "C" fn included_files(path_ptr: *const libc::c_char) -> RubyArray {
    let path_string = ptr_to_string(path_ptr).unwrap();
    let path = Path::new(&*path_string).to_path_buf();

    let gitignore_path = path.join(".gitignore");
    let file = gitignore::File::new(&gitignore_path).unwrap();

    let files = file.included_files().unwrap();
    println!("{:?}", files);
    RubyArray::from_vec(paths_to_ptrs(files))
}

fn ptr_to_string(ptr: *const libc::c_char) -> Result<String, FromUtf8Error> {
    let cstr = unsafe { CStr::from_ptr(ptr) };
    let mut bytes = Vec::new();
    bytes.extend(cstr.to_bytes());

    String::from_utf8(bytes)
}

fn paths_to_ptrs(paths: Vec<PathBuf>) -> Vec<*const libc::c_char> {
    paths.iter().flat_map(|p| p.to_str())
                .flat_map(|s| CString::new(s))
                .map(|cs| cs.as_ptr())
                .collect()
}
<commit_msg>Correct some memory issues in the FFI layer.<commit_after>extern crate gitignore;
extern crate libc;

use std::ffi::CStr;
use std::ffi::CString;
use std::mem;
use std::path::{Path, PathBuf};
use std::string::FromUtf8Error;

#[repr(C)]
pub struct RubyArray {
      len: libc::size_t,
      data: *const libc::c_void,
}

impl RubyArray {
    fn from_vec<T>(vec: Vec<T>) -> RubyArray {
        let array = RubyArray {
            data: vec.as_ptr() as *const libc::c_void,
            len: vec.len() as libc::size_t
        };
        mem::forget(vec);
        array
    }
}

#[no_mangle]
pub extern "C" fn included_files(path_ptr: *const libc::c_char) -> RubyArray {
    let path_string = ptr_to_string(path_ptr).unwrap();
    let path = Path::new(&*path_string).to_path_buf();

    let gitignore_path = path.join(".gitignore");
    let file = gitignore::File::new(&gitignore_path).unwrap();

    let files = file.included_files().unwrap();
    println!("{:?}", files);
    RubyArray::from_vec(paths_to_ptrs(files))
}

fn ptr_to_string(ptr: *const libc::c_char) -> Result<String, FromUtf8Error> {
    let cstr = unsafe { CStr::from_ptr(ptr) };
    let mut bytes = Vec::new();
    bytes.extend(cstr.to_bytes());

    String::from_utf8(bytes)
}

fn paths_to_ptrs(paths: Vec<PathBuf>) -> Vec<*const libc::c_char> {
    paths.iter().flat_map(|p| p.to_str())
                .flat_map(|s| CString::new(s))
                .map(|cs| {
                    let ptr = cs.as_ptr();
                    mem::forget(cs);
                    ptr
                })
                .collect()
}
<|endoftext|>


<commit_before>extern crate sputnikvm;
extern crate serde_json;

mod blockchain;

pub use self::blockchain::{JSONBlock, create_block, create_context};

use serde_json::Value;
use std::str::FromStr;
use sputnikvm::{Gas, M256, U256, Address, read_hex};
use sputnikvm::vm::errors::RequireError;
use sputnikvm::vm::{VM, SeqVM, AccountCommitment, Context, Account, HashMapStorage, Patch, VMStatus};

pub fn fire_with_block(machine: &mut SeqVM, block: &JSONBlock) {
    loop {
        match machine.fire() {
            Err(RequireError::Account(address)) => {
                let account = block.request_account(address);
                machine.commit_account(account);
            },
            Err(RequireError::AccountCode(address)) => {
                let account = block.request_account_code(address);
                machine.commit_account(account);
            },
            Err(RequireError::Blockhash(number)) => {
                // The test JSON file doesn't expose any block
                // information. So those numbers are crafted by hand.
                let hash1 = M256::from_str("0xc89efdaa54c0f20c7adf612882df0950f5a951637e0307cdcb4c672f298b8bc6").unwrap();
                let hash2 = M256::from_str("0xad7c5bef027816a800da1736444fb58a807ef4c9603b7848673f7e3a68eb14a5").unwrap();
                let hash256 = M256::from_str("0x6ca54da2c4784ea43fd88b3402de07ae4bced597cbb19f323b7595857a6720ae").unwrap();

                let hash = if number == M256::from(1u64) {
                    hash1
                } else if number == M256::from(2u64) {
                    hash2
                } else if number == M256::from(256u64) {
                    hash256
                } else {
                    panic!();
                };

                machine.commit_blockhash(number, hash);
            },
            Ok(()) => return,
        }
    }
}

pub fn apply_to_block(machine: &SeqVM, block: &mut JSONBlock) {
    for account in machine.accounts() {
        let account = (*account).clone();
        block.apply_account(account);
    }
    // for log in machine.appending_logs() {
    //     let log = (*log).clone();
    //     block.apply_log(log);
    // }
}

pub fn create_machine(v: &Value, block: &JSONBlock) -> SeqVM {
    let transaction = create_context(v);

    SeqVM::new(transaction, block.block_header(), Patch::None)
}

pub fn test_machine(v: &Value, machine: &SeqVM, block: &JSONBlock, debug: bool) -> bool {
    // let ref callcreates = v["callcreates"];

    // if callcreates.as_array().is_some() {
    //     let mut i = 0;
    //     for callcreate in callcreates.as_array().unwrap() {
    //         let data = read_hex(callcreate["data"].as_str().unwrap()).unwrap();
    //         let destination = {
    //             let destination = callcreate["destination"].as_str().unwrap();
    //             if destination == "" {
    //                 None
    //             } else {
    //                 Some(Address::from_str(destination).unwrap())
    //             }
    //         };
    //         let gas_limit = Gas::from_str(callcreate["gasLimit"].as_str().unwrap()).unwrap();
    //         let value = U256::from_str(callcreate["value"].as_str().unwrap()).unwrap();

    //         if i >= machine.transactions().len() {
    //             if debug {
    //                 print!("\n");
    //                 println!("Transaction check failed, expected more than {} items.", i);
    //             }
    //             return false;
    //         }
    //         let ref transaction = machine.transactions()[i];
    //         if destination.is_none() {
    //             match transaction {
    //                 &Transaction::ContractCreation(ref create) => {
    //                     if create.gas_limit != gas_limit || create.value != value || create.init != data {
    //                         if debug {
    //                             print!("\n");
    //                             println!("Transaction mismatch. Received gas_limit 0x{:x}, value 0x{:x}", create.gas_limit, create.value);
    //                         }
    //                         return false;
    //                     }
    //                 }
    //                 &Transaction::MessageCall(ref call) => {
    //                     if debug {
    //                         print!("\n");
    //                         println!("Transaction mismatch. Received: {:?}", call);
    //                     }
    //                 }
    //             }
    //         } else {
    //             match transaction {
    //                 &Transaction::MessageCall(ref call) => {
    //                     if call.gas_limit != gas_limit || call.value != value || call.data != data || call.to != destination.unwrap() {
    //                         if debug {
    //                             print!("\n");
    //                             println!("Transaction mismatch. Received gas_limit 0x{:x}, value: 0x{:x}", call.gas_limit, call.value);
    //                         }
    //                         return false;
    //                     }
    //                 }
    //                 &Transaction::ContractCreation(ref create) => {
    //                     if debug {
    //                         print!("\n");
    //                         println!("Transaction mismatch. Received: {:?}", create);
    //                     }
    //                 }
    //             }
    //         }

    //         i = i + 1;
    //     }
    // }

    let out = v["out"].as_str();
    let gas = v["gas"].as_str();

    if out.is_some() {
        let out = read_hex(out.unwrap()).unwrap();
        let out_ref: &[u8] = out.as_ref();
        if machine.out() != out_ref {
            if debug {
                print!("\n");
                println!("Return value check failed. {:?} != {:?}", machine.out(), out_ref);
            }

            return false;
        }
    }

    if gas.is_some() {
        let gas = Gas::from_str(gas.unwrap()).unwrap();
        if machine.available_gas() != gas {
            if debug {
                print!("\n");
                println!("Gas check failed, VM returned: 0x{:x}, expected: 0x{:x}",
                         machine.available_gas(), gas);
            }

            return false;
        }
    }

    let ref post_addresses = v["post"];

    for (address, data) in post_addresses.as_object().unwrap() {
        let address = Address::from_str(address.as_str()).unwrap();
        let balance = U256::from_str(data["balance"].as_str().unwrap()).unwrap();
        let code = read_hex(data["code"].as_str().unwrap()).unwrap();
        let code_ref: &[u8] = code.as_ref();

        if code_ref != block.account_code(address) {
            if debug {
                print!("\n");
                println!("Account code check failed for address 0x{:x}.", address);
            }

            return false;
        }
        if balance != block.balance(address) {
            if debug {
                print!("\n");
                println!("Balance check failed for address 0x{:x}.", address);
            }

            return false;
        }

        let storage = data["storage"].as_object().unwrap();
        for (index, value) in storage {
            let index = M256::from_str(index.as_str()).unwrap();
            let value = M256::from_str(value.as_str().unwrap()).unwrap();
            if value != block.account_storage(address, index) {
                if debug {
                    print!("\n");
                    println!("Storage check failed for address 0x{:x} in storage index 0x{:x}",
                             address, index);
                    println!("Expected: 0x{:x}", value);
                    println!("Actual:   0x{:x}", block.account_storage(address, index));
                }
                return false;
            }
        }
    }

    let ref expect = v["expect"];

    if expect.as_object().is_some() {
        for (address, data) in expect.as_object().unwrap() {
            let address = Address::from_str(address.as_str()).unwrap();

            let storage = data["storage"].as_object().unwrap();
            for (index, value) in storage {
                let index = M256::from_str(index.as_str()).unwrap();
                let value = M256::from_str(value.as_str().unwrap()).unwrap();
                if value != block.account_storage(address, index) {
                    if debug {
                        print!("\n");
                        println!("Storage check (expect) failed for address 0x{:x} in storage index 0x{:x}",
                                 address, index);
                        println!("Expected: 0x{:x}", value);
                        println!("Actual:   0x{:x}", block.account_storage(address, index));
                    }
                    return false;
                }
            }
        }
    }

    // let ref logs = v["logs"].as_array();

    // if logs.is_some() {
    //     let logs = logs.unwrap();

    //     for log in logs {
    //         let log = log.as_object().unwrap();

    //         let address = Address::from_str(log["address"].as_str().unwrap()).unwrap();
    //         let data = read_hex(log["data"].as_str().unwrap()).unwrap();
    //         let mut topics: Vec<M256> = Vec::new();

    //         for topic in log["topics"].as_array().unwrap() {
    //             topics.push(M256::from_str(topic.as_str().unwrap()).unwrap());
    //         }

    //         if !block.find_log(address, data.as_slice(), topics.as_slice()) {
    //             if debug {
    //                 print!("\n");
    //                 println!("Log match failed.");
    //             }
    //             return false;
    //         }
    //     }
    // }

    return true;
}

fn is_ok(status: VMStatus) -> bool {
    match status {
        ExitedOk => true,
        _ => false,
    }
}

pub fn test_transaction(name: &str, v: &Value, debug: bool) -> bool {
    let mut block = create_block(v);
    let mut machine = create_machine(v, &block);
    fire_with_block(&mut machine, &block);
    apply_to_block(&machine, &mut block);

    let out = v["out"].as_str();

    if out.is_some() {
        if is_ok(machine.status()) {
            if test_machine(v, &machine, &block, debug) {
                return true;
            } else {
                return false;
            }
        } else {
            return false;
        }
    } else {
        if !is_ok(machine.status()) {
            return true;
        } else {
            return false;
        }
    }
}
<commit_msg>Problem: status is not clear Solution: print out in jsontests<commit_after>extern crate sputnikvm;
extern crate serde_json;

mod blockchain;

pub use self::blockchain::{JSONBlock, create_block, create_context};

use serde_json::Value;
use std::str::FromStr;
use sputnikvm::{Gas, M256, U256, Address, read_hex};
use sputnikvm::vm::errors::RequireError;
use sputnikvm::vm::{VM, SeqVM, AccountCommitment, Context, Account, HashMapStorage, Patch, VMStatus};

pub fn fire_with_block(machine: &mut SeqVM, block: &JSONBlock) {
    loop {
        match machine.fire() {
            Err(RequireError::Account(address)) => {
                let account = block.request_account(address);
                machine.commit_account(account);
            },
            Err(RequireError::AccountCode(address)) => {
                let account = block.request_account_code(address);
                machine.commit_account(account);
            },
            Err(RequireError::Blockhash(number)) => {
                // The test JSON file doesn't expose any block
                // information. So those numbers are crafted by hand.
                let hash1 = M256::from_str("0xc89efdaa54c0f20c7adf612882df0950f5a951637e0307cdcb4c672f298b8bc6").unwrap();
                let hash2 = M256::from_str("0xad7c5bef027816a800da1736444fb58a807ef4c9603b7848673f7e3a68eb14a5").unwrap();
                let hash256 = M256::from_str("0x6ca54da2c4784ea43fd88b3402de07ae4bced597cbb19f323b7595857a6720ae").unwrap();

                let hash = if number == M256::from(1u64) {
                    hash1
                } else if number == M256::from(2u64) {
                    hash2
                } else if number == M256::from(256u64) {
                    hash256
                } else {
                    panic!();
                };

                machine.commit_blockhash(number, hash);
            },
            Ok(()) => return,
        }
    }
}

pub fn apply_to_block(machine: &SeqVM, block: &mut JSONBlock) {
    for account in machine.accounts() {
        let account = (*account).clone();
        block.apply_account(account);
    }
    // for log in machine.appending_logs() {
    //     let log = (*log).clone();
    //     block.apply_log(log);
    // }
}

pub fn create_machine(v: &Value, block: &JSONBlock) -> SeqVM {
    let transaction = create_context(v);

    SeqVM::new(transaction, block.block_header(), Patch::None)
}

pub fn test_machine(v: &Value, machine: &SeqVM, block: &JSONBlock, debug: bool) -> bool {
    // let ref callcreates = v["callcreates"];

    // if callcreates.as_array().is_some() {
    //     let mut i = 0;
    //     for callcreate in callcreates.as_array().unwrap() {
    //         let data = read_hex(callcreate["data"].as_str().unwrap()).unwrap();
    //         let destination = {
    //             let destination = callcreate["destination"].as_str().unwrap();
    //             if destination == "" {
    //                 None
    //             } else {
    //                 Some(Address::from_str(destination).unwrap())
    //             }
    //         };
    //         let gas_limit = Gas::from_str(callcreate["gasLimit"].as_str().unwrap()).unwrap();
    //         let value = U256::from_str(callcreate["value"].as_str().unwrap()).unwrap();

    //         if i >= machine.transactions().len() {
    //             if debug {
    //                 print!("\n");
    //                 println!("Transaction check failed, expected more than {} items.", i);
    //             }
    //             return false;
    //         }
    //         let ref transaction = machine.transactions()[i];
    //         if destination.is_none() {
    //             match transaction {
    //                 &Transaction::ContractCreation(ref create) => {
    //                     if create.gas_limit != gas_limit || create.value != value || create.init != data {
    //                         if debug {
    //                             print!("\n");
    //                             println!("Transaction mismatch. Received gas_limit 0x{:x}, value 0x{:x}", create.gas_limit, create.value);
    //                         }
    //                         return false;
    //                     }
    //                 }
    //                 &Transaction::MessageCall(ref call) => {
    //                     if debug {
    //                         print!("\n");
    //                         println!("Transaction mismatch. Received: {:?}", call);
    //                     }
    //                 }
    //             }
    //         } else {
    //             match transaction {
    //                 &Transaction::MessageCall(ref call) => {
    //                     if call.gas_limit != gas_limit || call.value != value || call.data != data || call.to != destination.unwrap() {
    //                         if debug {
    //                             print!("\n");
    //                             println!("Transaction mismatch. Received gas_limit 0x{:x}, value: 0x{:x}", call.gas_limit, call.value);
    //                         }
    //                         return false;
    //                     }
    //                 }
    //                 &Transaction::ContractCreation(ref create) => {
    //                     if debug {
    //                         print!("\n");
    //                         println!("Transaction mismatch. Received: {:?}", create);
    //                     }
    //                 }
    //             }
    //         }

    //         i = i + 1;
    //     }
    // }

    let out = v["out"].as_str();
    let gas = v["gas"].as_str();

    if out.is_some() {
        let out = read_hex(out.unwrap()).unwrap();
        let out_ref: &[u8] = out.as_ref();
        if machine.out() != out_ref {
            if debug {
                print!("\n");
                println!("Return value check failed. {:?} != {:?}", machine.out(), out_ref);
            }

            return false;
        }
    }

    if gas.is_some() {
        let gas = Gas::from_str(gas.unwrap()).unwrap();
        if machine.available_gas() != gas {
            if debug {
                print!("\n");
                println!("Gas check failed, VM returned: 0x{:x}, expected: 0x{:x}",
                         machine.available_gas(), gas);
            }

            return false;
        }
    }

    let ref post_addresses = v["post"];

    for (address, data) in post_addresses.as_object().unwrap() {
        let address = Address::from_str(address.as_str()).unwrap();
        let balance = U256::from_str(data["balance"].as_str().unwrap()).unwrap();
        let code = read_hex(data["code"].as_str().unwrap()).unwrap();
        let code_ref: &[u8] = code.as_ref();

        if code_ref != block.account_code(address) {
            if debug {
                print!("\n");
                println!("Account code check failed for address 0x{:x}.", address);
            }

            return false;
        }
        if balance != block.balance(address) {
            if debug {
                print!("\n");
                println!("Balance check failed for address 0x{:x}.", address);
            }

            return false;
        }

        let storage = data["storage"].as_object().unwrap();
        for (index, value) in storage {
            let index = M256::from_str(index.as_str()).unwrap();
            let value = M256::from_str(value.as_str().unwrap()).unwrap();
            if value != block.account_storage(address, index) {
                if debug {
                    print!("\n");
                    println!("Storage check failed for address 0x{:x} in storage index 0x{:x}",
                             address, index);
                    println!("Expected: 0x{:x}", value);
                    println!("Actual:   0x{:x}", block.account_storage(address, index));
                }
                return false;
            }
        }
    }

    let ref expect = v["expect"];

    if expect.as_object().is_some() {
        for (address, data) in expect.as_object().unwrap() {
            let address = Address::from_str(address.as_str()).unwrap();

            let storage = data["storage"].as_object().unwrap();
            for (index, value) in storage {
                let index = M256::from_str(index.as_str()).unwrap();
                let value = M256::from_str(value.as_str().unwrap()).unwrap();
                if value != block.account_storage(address, index) {
                    if debug {
                        print!("\n");
                        println!("Storage check (expect) failed for address 0x{:x} in storage index 0x{:x}",
                                 address, index);
                        println!("Expected: 0x{:x}", value);
                        println!("Actual:   0x{:x}", block.account_storage(address, index));
                    }
                    return false;
                }
            }
        }
    }

    // let ref logs = v["logs"].as_array();

    // if logs.is_some() {
    //     let logs = logs.unwrap();

    //     for log in logs {
    //         let log = log.as_object().unwrap();

    //         let address = Address::from_str(log["address"].as_str().unwrap()).unwrap();
    //         let data = read_hex(log["data"].as_str().unwrap()).unwrap();
    //         let mut topics: Vec<M256> = Vec::new();

    //         for topic in log["topics"].as_array().unwrap() {
    //             topics.push(M256::from_str(topic.as_str().unwrap()).unwrap());
    //         }

    //         if !block.find_log(address, data.as_slice(), topics.as_slice()) {
    //             if debug {
    //                 print!("\n");
    //                 println!("Log match failed.");
    //             }
    //             return false;
    //         }
    //     }
    // }

    return true;
}

fn is_ok(status: VMStatus) -> bool {
    match status {
        ExitedOk => true,
        _ => false,
    }
}

pub fn test_transaction(name: &str, v: &Value, debug: bool) -> bool {
    let mut block = create_block(v);
    let mut machine = create_machine(v, &block);
    fire_with_block(&mut machine, &block);
    apply_to_block(&machine, &mut block);

    println!("status: {:?}", machine.status());
    let out = v["out"].as_str();

    if out.is_some() {
        if is_ok(machine.status()) {
            if test_machine(v, &machine, &block, debug) {
                return true;
            } else {
                return false;
            }
        } else {
            return false;
        }
    } else {
        if !is_ok(machine.status()) {
            return true;
        } else {
            return false;
        }
    }
}
<|endoftext|>


<commit_before>use desync::scheduler::*;

use super::timeout::*;

use std::thread;
use std::time::*;
use std::sync::*;
use std::sync::mpsc::*;

use futures::prelude::*;
use futures::task;
use futures::task::{ArcWake, Poll};
use futures::channel::oneshot;

#[test]
fn schedule_future() {
    timeout(|| {
        use futures::executor;

        let queue       = queue();
        let future      = future_desync(&queue, move || async {
            thread::sleep(Duration::from_millis(100));
            42
        });

        executor::block_on(async {
            assert!(future.await.unwrap() == 42);
        });
    }, 500);
}

#[test]
fn schedule_future_with_no_scheduler_threads() {
    timeout(|| {
        use futures::executor;

        let scheduler   = Scheduler::new();

        // Even with 0 threads, futures should still run (by draining on the current thread as for sync actions)
        scheduler.set_max_threads(0);
        scheduler.despawn_threads_if_overloaded();

        let queue       = queue();
        let future      = scheduler.future_desync(&queue, move || async {
            thread::sleep(Duration::from_millis(100));
            42
        });

        executor::block_on(async {
            assert!(future.await.unwrap() == 42);
        });
    }, 500);
}

#[test]
fn wake_future_with_no_scheduler_threads() {
    timeout(|| {
        use futures::executor;

        let (tx, rx)    = oneshot::channel::<i32>();
        let scheduler   = Scheduler::new();

        // Even with 0 threads, futures should still run (by draining on the current thread as for sync actions)
        scheduler.set_max_threads(0);
        scheduler.despawn_threads_if_overloaded();

        // Schedule a future that will block until we send a value
        let queue       = queue();
        let future      = scheduler.future_desync(&queue, move || async {
            rx.await.expect("Receive result")
        });

        // Schedule a thread that will send a value
        thread::spawn(move || {
            // Wait for a bit before sending the result so the future should block
            thread::sleep(Duration::from_millis(100));

            tx.send(42).expect("Send")
        });

        executor::block_on(async {
            assert!(future.await.expect("result") == 42);
        });
    }, 500);
}

#[test]
fn wait_for_future() {
    timeout(|| {
        use futures::executor;

        // We use a oneshot as our future, and a mpsc channel to track progress
        let (tx, rx)                = channel();
        let (future_tx, future_rx)  = oneshot::channel();

        let scheduler   = scheduler();
        let queue       = queue();

        // Start by sending '1' from an async
        let tx2 = tx.clone();
        desync(&queue, move || { tx2.send(1).unwrap(); });

        // Then send the value sent via our oneshot using a future
        let tx2 = tx.clone();
        let future = scheduler.after(&queue, future_rx,
            move |val| val.map(move |val| { tx2.send(val).unwrap(); 4 }));

        // Then send '3'
        let tx2 = tx.clone();
        desync(&queue, move || { tx2.send(3).unwrap(); });

        // '1' should be available, but we should otherwise be blocked on the future
        assert!(rx.recv().unwrap() == 1);
        assert!(rx.recv_timeout(Duration::from_millis(100)).is_err());

        // Send '2' to the future
        future_tx.send(2).unwrap();

        executor::block_on(async {
            // Future should resolve to 4
            assert!(future.await.unwrap() == Ok(4));

            // Should receive the '2' from the future, then 3
            assert!(rx.recv_timeout(Duration::from_millis(100)).unwrap() == 2);
            assert!(rx.recv().unwrap() == 3);
        });
    }, 500);
}

#[test]
fn future_waits_for_us() {
    timeout(|| {
        use futures::executor;

        // We use a oneshot as our future, and a mpsc channel to track progress
        let (tx, rx)                = channel();
        let (future_tx, future_rx)  = oneshot::channel();

        let scheduler   = scheduler();
        let queue       = queue();

        // Start by sending '1' from an async
        let tx2 = tx.clone();
        desync(&queue, move || { thread::sleep(Duration::from_millis(100)); tx2.send(1).unwrap(); });

        // Then send the value sent via our oneshot using a future
        let tx2 = tx.clone();
        let future = scheduler.after(&queue, future_rx,
            move |val| val.map(move |val| { tx2.send(val).unwrap(); 4 }));

        // Then send '3'
        let tx2 = tx.clone();
        desync(&queue, move || { tx2.send(3).unwrap(); });

        // Send '2' to the future
        future_tx.send(2).unwrap();

        executor::block_on(async {
            // Future should resolve to 4
            assert!(future.await.unwrap() == Ok(4));

            // '1' should be available first
            assert!(rx.recv().unwrap() == 1);

            // Should receive the '2' from the future, then 3
            assert!(rx.recv_timeout(Duration::from_millis(100)).unwrap() == 2);
            assert!(rx.recv().unwrap() == 3);
        });
    }, 500);
}

///
/// Used for polling futures to see if they've notified us yet
///
struct TestWaker {
    pub awake: Mutex<bool>
}

impl ArcWake for TestWaker {
    fn wake_by_ref(arc_self: &Arc<Self>) {
        (*arc_self.awake.lock().unwrap()) = true;
    }
}

#[test]
fn poll_two_futures_on_one_queue() {
    // 0 threads so we force the future to act in 'drain' mode
    let scheduler   = Scheduler::new();

    // Even with 0 threads, futures should still run (by draining on the current thread as for sync actions)
    scheduler.set_max_threads(0);
    scheduler.despawn_threads_if_overloaded();

    // If a single queue has a future on
    let queue           = queue();
    let (done1, recv1)  = oneshot::channel::<()>();
    let (done2, recv2)  = oneshot::channel::<()>();

    let wake1           = Arc::new(TestWaker { awake: Mutex::new(false) });
    let wake2           = Arc::new(TestWaker { awake: Mutex::new(false) });

    // Wait for done1 then done2 to signal
    let mut future_1    = scheduler.future_desync(&queue, move || {
        async move { recv1.await.ok(); }
    });

    let mut future_2    = scheduler.future_desync(&queue, move || {
        async move { recv2.await.ok(); }
    });

    // Poll future 1 then future 2 (as there are no threads to run, we'll use the 'drain on current thread' style, which will return pending as recv is pending)
    let waker_ref           = task::waker_ref(&wake1);
    let mut ctxt            = task::Context::from_waker(&waker_ref);

    assert!(future_1.poll_unpin(&mut ctxt) == Poll::Pending);

    // Only future_1 should be 'pollable' at this point (ie, is in the WaitForPoll state from the previous call)
    let waker_ref           = task::waker_ref(&wake2);
    let mut ctxt            = task::Context::from_waker(&waker_ref);

    assert!(future_2.poll_unpin(&mut ctxt) == Poll::Pending);

    // Finish both futures
    done1.send(()).unwrap();

    let waker_ref           = task::waker_ref(&wake2);
    let mut ctxt            = task::Context::from_waker(&waker_ref);

    assert!(future_2.poll_unpin(&mut ctxt) == Poll::Pending);

    done2.send(()).unwrap();

    // future_1 should be signalled for polling, future_2 should not
    assert!((*wake2.awake.lock().unwrap()) == false);
    assert!((*wake1.awake.lock().unwrap()) == true);

    // Retrieve the result for future_1
    let waker_ref           = task::waker_ref(&wake2);
    let mut ctxt            = task::Context::from_waker(&waker_ref);

    assert!(future_1.poll_unpin(&mut ctxt) == Poll::Ready(Ok(())));

    // Both future 1 and future 2 should have signalled now

    // TODO: this is a possible bug with 0 threads - the thread won't reschedule after future_1 completes, so wake2 will not yet be set
    //          (This is quite a complicated problem: if the drain continued processing jobs until it became pending instead of scheduling
    //          in the background, this would work but the return from poll could be delayed indefinitely)
    // assert!((*wake2.awake.lock().unwrap()) == true);

    // Give the scheduler a chance to run the other future (it will be queued in the background, so this is required for the notification to occur)
    scheduler.set_max_threads(1);
    for _ in 0..100 {
        if *wake2.awake.lock().unwrap() { break; }
        thread::sleep(Duration::from_millis(1));
    }
    assert!((*wake2.awake.lock().unwrap()) == true);

    let waker_ref           = task::waker_ref(&wake2);
    let mut ctxt            = task::Context::from_waker(&waker_ref);

    // Should be able to retrieve the result of future_2
    assert!(future_2.poll_unpin(&mut ctxt) == Poll::Ready(Ok(())));

    // TODO: not actually sure if this is bad behaviour or not but if future_2 is polled first, future_1 won't be available until future_2
    //      completes. This is another 0 thread only issue as future_1 will be able to send its notification when the thread pool is available.
}
<commit_msg>Test the 'desync from desync' case<commit_after>use desync::scheduler::*;

use super::timeout::*;

use std::thread;
use std::time::*;
use std::sync::*;
use std::sync::mpsc::*;

use futures::prelude::*;
use futures::task;
use futures::task::{ArcWake, Poll};
use futures::channel::oneshot;

#[test]
fn schedule_future() {
    timeout(|| {
        use futures::executor;

        let queue       = queue();
        let future      = future_desync(&queue, move || async {
            thread::sleep(Duration::from_millis(100));
            42
        });

        executor::block_on(async {
            assert!(future.await.unwrap() == 42);
        });
    }, 500);
}

#[test]
fn schedule_future_with_no_scheduler_threads() {
    timeout(|| {
        use futures::executor;

        let scheduler   = Scheduler::new();

        // Even with 0 threads, futures should still run (by draining on the current thread as for sync actions)
        scheduler.set_max_threads(0);
        scheduler.despawn_threads_if_overloaded();

        let queue       = queue();
        let future      = scheduler.future_desync(&queue, move || async {
            thread::sleep(Duration::from_millis(100));
            42
        });

        executor::block_on(async {
            assert!(future.await.unwrap() == 42);
        });
    }, 500);
}

#[test]
fn wake_future_with_no_scheduler_threads() {
    timeout(|| {
        use futures::executor;

        let (tx, rx)    = oneshot::channel::<i32>();
        let scheduler   = Scheduler::new();

        // Even with 0 threads, futures should still run (by draining on the current thread as for sync actions)
        scheduler.set_max_threads(0);
        scheduler.despawn_threads_if_overloaded();

        // Schedule a future that will block until we send a value
        let queue       = queue();
        let future      = scheduler.future_desync(&queue, move || async {
            rx.await.expect("Receive result")
        });

        // Schedule a thread that will send a value
        thread::spawn(move || {
            // Wait for a bit before sending the result so the future should block
            thread::sleep(Duration::from_millis(100));

            tx.send(42).expect("Send")
        });

        executor::block_on(async {
            assert!(future.await.expect("result") == 42);
        });
    }, 500);
}

#[test]
fn wait_for_future() {
    timeout(|| {
        use futures::executor;

        // We use a oneshot as our future, and a mpsc channel to track progress
        let (tx, rx)                = channel();
        let (future_tx, future_rx)  = oneshot::channel();

        let scheduler   = scheduler();
        let queue       = queue();

        // Start by sending '1' from an async
        let tx2 = tx.clone();
        desync(&queue, move || { tx2.send(1).unwrap(); });

        // Then send the value sent via our oneshot using a future
        let tx2 = tx.clone();
        let future = scheduler.after(&queue, future_rx,
            move |val| val.map(move |val| { tx2.send(val).unwrap(); 4 }));

        // Then send '3'
        let tx2 = tx.clone();
        desync(&queue, move || { tx2.send(3).unwrap(); });

        // '1' should be available, but we should otherwise be blocked on the future
        assert!(rx.recv().unwrap() == 1);
        assert!(rx.recv_timeout(Duration::from_millis(100)).is_err());

        // Send '2' to the future
        future_tx.send(2).unwrap();

        executor::block_on(async {
            // Future should resolve to 4
            assert!(future.await.unwrap() == Ok(4));

            // Should receive the '2' from the future, then 3
            assert!(rx.recv_timeout(Duration::from_millis(100)).unwrap() == 2);
            assert!(rx.recv().unwrap() == 3);
        });
    }, 500);
}

#[test]
fn future_waits_for_us() {
    timeout(|| {
        use futures::executor;

        // We use a oneshot as our future, and a mpsc channel to track progress
        let (tx, rx)                = channel();
        let (future_tx, future_rx)  = oneshot::channel();

        let scheduler   = scheduler();
        let queue       = queue();

        // Start by sending '1' from an async
        let tx2 = tx.clone();
        desync(&queue, move || { thread::sleep(Duration::from_millis(100)); tx2.send(1).unwrap(); });

        // Then send the value sent via our oneshot using a future
        let tx2 = tx.clone();
        let future = scheduler.after(&queue, future_rx,
            move |val| val.map(move |val| { tx2.send(val).unwrap(); 4 }));

        // Then send '3'
        let tx2 = tx.clone();
        desync(&queue, move || { tx2.send(3).unwrap(); });

        // Send '2' to the future
        future_tx.send(2).unwrap();

        executor::block_on(async {
            // Future should resolve to 4
            assert!(future.await.unwrap() == Ok(4));

            // '1' should be available first
            assert!(rx.recv().unwrap() == 1);

            // Should receive the '2' from the future, then 3
            assert!(rx.recv_timeout(Duration::from_millis(100)).unwrap() == 2);
            assert!(rx.recv().unwrap() == 3);
        });
    }, 500);
}

///
/// Used for polling futures to see if they've notified us yet
///
struct TestWaker {
    pub awake: Mutex<bool>
}

impl ArcWake for TestWaker {
    fn wake_by_ref(arc_self: &Arc<Self>) {
        (*arc_self.awake.lock().unwrap()) = true;
    }
}

#[test]
fn poll_two_futures_on_one_queue() {
    // 0 threads so we force the future to act in 'drain' mode
    let scheduler   = Scheduler::new();

    // Even with 0 threads, futures should still run (by draining on the current thread as for sync actions)
    scheduler.set_max_threads(0);
    scheduler.despawn_threads_if_overloaded();

    // If a single queue has a future on
    let queue           = queue();
    let (done1, recv1)  = oneshot::channel::<()>();
    let (done2, recv2)  = oneshot::channel::<()>();

    let wake1           = Arc::new(TestWaker { awake: Mutex::new(false) });
    let wake2           = Arc::new(TestWaker { awake: Mutex::new(false) });

    // Wait for done1 then done2 to signal
    let mut future_1    = scheduler.future_desync(&queue, move || {
        async move { recv1.await.ok(); }
    });

    let mut future_2    = scheduler.future_desync(&queue, move || {
        async move { recv2.await.ok(); }
    });

    // Poll future 1 then future 2 (as there are no threads to run, we'll use the 'drain on current thread' style, which will return pending as recv is pending)
    let waker_ref           = task::waker_ref(&wake1);
    let mut ctxt            = task::Context::from_waker(&waker_ref);

    assert!(future_1.poll_unpin(&mut ctxt) == Poll::Pending);

    // Only future_1 should be 'pollable' at this point (ie, is in the WaitForPoll state from the previous call)
    let waker_ref           = task::waker_ref(&wake2);
    let mut ctxt            = task::Context::from_waker(&waker_ref);

    assert!(future_2.poll_unpin(&mut ctxt) == Poll::Pending);

    // Finish both futures
    done1.send(()).unwrap();

    let waker_ref           = task::waker_ref(&wake2);
    let mut ctxt            = task::Context::from_waker(&waker_ref);

    assert!(future_2.poll_unpin(&mut ctxt) == Poll::Pending);

    done2.send(()).unwrap();

    // future_1 should be signalled for polling, future_2 should not
    assert!((*wake2.awake.lock().unwrap()) == false);
    assert!((*wake1.awake.lock().unwrap()) == true);

    // Retrieve the result for future_1
    let waker_ref           = task::waker_ref(&wake2);
    let mut ctxt            = task::Context::from_waker(&waker_ref);

    assert!(future_1.poll_unpin(&mut ctxt) == Poll::Ready(Ok(())));

    // Both future 1 and future 2 should have signalled now

    // TODO: this is a possible bug with 0 threads - the thread won't reschedule after future_1 completes, so wake2 will not yet be set
    //          (This is quite a complicated problem: if the drain continued processing jobs until it became pending instead of scheduling
    //          in the background, this would work but the return from poll could be delayed indefinitely)
    // assert!((*wake2.awake.lock().unwrap()) == true);

    // Give the scheduler a chance to run the other future (it will be queued in the background, so this is required for the notification to occur)
    scheduler.set_max_threads(1);
    for _ in 0..100 {
        if *wake2.awake.lock().unwrap() { break; }
        thread::sleep(Duration::from_millis(1));
    }
    assert!((*wake2.awake.lock().unwrap()) == true);

    let waker_ref           = task::waker_ref(&wake2);
    let mut ctxt            = task::Context::from_waker(&waker_ref);

    // Should be able to retrieve the result of future_2
    assert!(future_2.poll_unpin(&mut ctxt) == Poll::Ready(Ok(())));

    // TODO: not actually sure if this is bad behaviour or not but if future_2 is polled first, future_1 won't be available until future_2
    //      completes. This is another 0 thread only issue as future_1 will be able to send its notification when the thread pool is available.
}

#[test]
fn wait_for_desync_future_from_desync_future() {
    use futures::executor;

    timeout(|| {
        // This reproduces a deadlock due to a race condition, so we usually need several iterations through the test before the issue will occur
        for _i in 0..1000 {
            // We'll schedule a sync future on queue1, and wait for it from a desync future on queue2
            let queue1      = queue();
            let queue2      = queue();

            // Oneshot channel to wake the sync queue
            let (done1, recv1)  = oneshot::channel::<()>();

            let nested_future   = future_desync(&queue1, move || { async move { recv1.await.ok(); } });
            let desync_future   = future_desync(&queue2, move || { async move { nested_future.await.ok(); } });

            // Signal
            done1.send(()).unwrap();

            // Wait for the desync future in an executor
            executor::block_on(async move {
                desync_future.await.ok();
            });

            // Run sync on both queues
            sync(&queue1, move || { });
            sync(&queue2, move || { });
        }
    }, 5000);
}
<|endoftext|>


<commit_before>// Copyright 2014 Johannes Köster.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

/// A classical, flexible, q-gram index implementation.
///
/// # Example
///
/// ```
/// use bio::data_structures::qgram_index;
/// use bio::alphabets;
///
/// let text = b"ACGGCTGAGATGAT";
/// let alphabet = alphabets::dna::alphabet();
/// let q = 3;
/// let qgram_index = qgram_index::QGramIndex::new(q, text, &alphabet);
///
/// let pattern = b"GCTG";
/// let matches = qgram_index.matches(pattern, 1);
/// assert_eq!(matches, [
///     qgram_index::Match {
///         pattern: qgram_index::Interval { start: 0, stop: 4 },
///         text: qgram_index::Interval { start: 3, stop: 7 },
///         count: 2
///     }
/// ]);
/// ```

use std::collections;
use std;
use std::cmp;

use alphabets::{Alphabet, RankTransform};
use utils;

/// A classical, flexible, q-gram index implementation.
pub struct QGramIndex {
    q: u32,
    address: Vec<usize>,
    pos: Vec<usize>,
    ranks: RankTransform,
}

impl QGramIndex {
    /// Create a new q-gram index.
    /// The q has to be smaller than b / log2(|A|) with |A| being the alphabet size and b the number
    /// bits with the `usize` data type.
    pub fn new(q: u32, text: &[u8], alphabet: &Alphabet) -> Self {
        QGramIndex::with_max_count(q, text, alphabet, std::usize::MAX)
    }

    /// Create a new q-gram index, only considering q-grams that occur at most `max_count` times.
    /// The q has to be smaller than b / log2(|A|) with |A| being the alphabet size and b the number
    /// bits with the `usize` data type.
    pub fn with_max_count(q: u32, text: &[u8], alphabet: &Alphabet, max_count: usize) -> Self {
        let ranks = RankTransform::new(alphabet);

        let qgram_count = alphabet.len().pow(q as u32);
        let mut address = vec![0; qgram_count + 1];
        let mut pos = vec![0; text.len()];

        for qgram in ranks.qgrams(q, text) {
            address[qgram] += 1;
        }

        for g in 1..address.len() {
            if address[g] > max_count {
                // mask qgram
                address[g] = 0;
            }
        }

        utils::prescan(&mut address, 0, |a, b| a + b);

        {
            let mut offset = vec![0; qgram_count];
            for (i, qgram) in ranks.qgrams(q, text).enumerate() {
                let a = address[qgram as usize];
                if address[qgram as usize + 1] - a != 0 {
                    // if not masked, insert positions
                    pos[a + offset[qgram as usize]] = i;
                    offset[qgram as usize] += 1;
                }
            }
        }

        QGramIndex { q: q, address: address, pos: pos, ranks: ranks }
    }

    pub fn q(&self) -> u32 {
        self.q
    }

    /// Return text positions with matching q-gram. Complexity O(1).
    pub fn qgram_matches(&self, qgram: usize) -> &[usize] {
        &self.pos[self.address[qgram]..self.address[qgram + 1]]
    }

    /// Return matches of the given pattern.
    /// Complexity O(m + k) for pattern of length m and k being the number of matching q-grams.
    pub fn matches(&self, pattern: &[u8], min_count: usize) -> Vec<Match> {
        let q = self.q as usize;
        let mut diagonals = collections::HashMap::new();
        for (i, qgram) in self.ranks.qgrams(self.q, pattern).enumerate() {
            for &p in self.qgram_matches(qgram) {
                let diagonal = p - i;
                if !diagonals.contains_key(&diagonal) {
                    diagonals.insert(diagonal, Match {
                        pattern: Interval { start: i, stop: i + q },
                        text: Interval { start: p, stop: p + q },
                        count: 1,
                    });
                }
                else {
                    let m = diagonals.get_mut(&diagonal).unwrap();
                    m.pattern.stop = i + q;
                    m.text.stop = p + q;
                    m.count += 1;
                }
            }
        }
        diagonals.into_iter().filter_map(
            |(_, m)| if m.count >= min_count { Some(m) } else { None }
        ).collect()
    }

    /// Return exact matches (substrings) of the given pattern.
    /// Complexity O(m + k) for pattern of length m and k being the number of matching q-grams.
    pub fn exact_matches(&self, pattern: &[u8]) -> Vec<ExactMatch> {
        let q = self.q as usize;
        let mut diagonals = collections::HashMap::new();
        let mut matches = Vec::new();

        for (i, qgram) in self.ranks.qgrams(self.q, pattern).enumerate() {
            for &p in self.qgram_matches(qgram) {
                let diagonal = p - i;
                if !diagonals.contains_key(&diagonal) {
                    diagonals.insert(diagonal, ExactMatch {
                        pattern: Interval { start: i, stop: i + q },
                        text: Interval { start: p, stop: p + q },
                    });
                }
                else {
                    let m = diagonals.get_mut(&diagonal).unwrap();
                    if m.pattern.stop - q + 1 != i {
                        // discontinue match
                        matches.push(*m);
                        // start new match
                        m.pattern.start = i;
                        m.pattern.stop = i + q;
                        m.text.start = p;
                        m.text.stop = p + q;
                    }
                    else {
                        m.pattern.stop = i + q;
                        m.text.stop = p + q;
                    }
                }
            }
        }
        for (_, m) in diagonals.into_iter() {
            matches.push(m);
        }

        matches
    }
}

/// An interval, consisting of start and stop position (the latter exclusive).
#[derive(PartialEq, Eq, Debug, Copy, Clone)]
pub struct Interval {
    pub start: usize,
    pub stop: usize
}

impl Interval {
    pub fn get<'a>(&self, text: &'a [u8]) -> &'a [u8] {
        &text[self.start..self.stop]
    }
}

/// A match between the pattern and the text.
#[derive(PartialEq, Eq, Debug, Copy, Clone)]
pub struct Match {
    pub pattern: Interval,
    pub text: Interval,
    pub count: usize,
}

impl cmp::Ord for Match {
    fn cmp(&self, other: &Self) -> cmp::Ordering {
        self.count.cmp(&other.count)
    }
}

impl cmp::PartialOrd for Match {
    fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
        Some(self.cmp(other))
    }
}

/// An exact match between the pattern and the text.
#[derive(PartialEq, Debug, Copy, Clone)]
pub struct ExactMatch {
    pub pattern: Interval,
    pub text: Interval,
}

#[cfg(test)]
mod tests {
    use super::*;
    use alphabets;

    fn setup() -> (&'static [u8], alphabets::Alphabet) {
        let text = b"ACGGCTGAGATGAT";
        let alphabet = alphabets::dna::alphabet();

        (text, alphabet)
    }

    #[test]
    fn test_qgram_matches() {
        let (text, alphabet) = setup();
        let q = 3;
        let qgram_index = QGramIndex::new(q, text, &alphabet);

        let ranks = alphabets::RankTransform::new(&alphabet);

        let qgram = ranks.qgrams(q, b"TGA").next().unwrap();

        let matches = qgram_index.qgram_matches(qgram);
        assert_eq!(matches, [5, 10]);
    }

    #[test]
    fn test_matches() {
        let (text, alphabet) = setup();
        let q = 3;
        let qgram_index = QGramIndex::new(q, text, &alphabet);

        let pattern = b"GCTG";
        let matches = qgram_index.matches(pattern, 1);
        assert_eq!(matches, [
            Match {
                pattern: Interval { start: 0, stop: 4 },
                text: Interval { start: 3, stop: 7 },
                count: 2
            }
        ]);
    }

    #[test]
    fn test_exact_matches() {
        let (text, alphabet) = setup();
        let q = 3;
        let qgram_index = QGramIndex::new(q, text, &alphabet);

        let pattern = b"GCTGA";
        let exact_matches = qgram_index.exact_matches(pattern);
        assert!(exact_matches.len() == 2);
        for m in exact_matches {
            assert_eq!(m.pattern.get(pattern), m.text.get(text));
        }
    }
}
<commit_msg>Minor.<commit_after>// Copyright 2014 Johannes Köster.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! A classical, flexible, q-gram index implementation.
//!
//! # Example
//!
//! ```
//! use bio::data_structures::qgram_index;
//! use bio::alphabets;
//!
//! let text = b"ACGGCTGAGATGAT";
//! let alphabet = alphabets::dna::alphabet();
//! let q = 3;
//! let qgram_index = qgram_index::QGramIndex::new(q, text, &alphabet);
//!
//! let pattern = b"GCTG";
//! let matches = qgram_index.matches(pattern, 1);
//! assert_eq!(matches, [
//!     qgram_index::Match {
//!         pattern: qgram_index::Interval { start: 0, stop: 4 },
//!         text: qgram_index::Interval { start: 3, stop: 7 },
//!         count: 2
//!     }
//! ]);
//! ```

use std::collections;
use std;
use std::cmp;

use alphabets::{Alphabet, RankTransform};
use utils;

/// A classical, flexible, q-gram index implementation.
pub struct QGramIndex {
    q: u32,
    address: Vec<usize>,
    pos: Vec<usize>,
    ranks: RankTransform,
}

impl QGramIndex {
    /// Create a new q-gram index.
    /// The q has to be smaller than b / log2(|A|) with |A| being the alphabet size and b the number
    /// bits with the `usize` data type.
    pub fn new(q: u32, text: &[u8], alphabet: &Alphabet) -> Self {
        QGramIndex::with_max_count(q, text, alphabet, std::usize::MAX)
    }

    /// Create a new q-gram index, only considering q-grams that occur at most `max_count` times.
    /// The q has to be smaller than b / log2(|A|) with |A| being the alphabet size and b the number
    /// bits with the `usize` data type.
    pub fn with_max_count(q: u32, text: &[u8], alphabet: &Alphabet, max_count: usize) -> Self {
        let ranks = RankTransform::new(alphabet);

        let qgram_count = alphabet.len().pow(q as u32);
        let mut address = vec![0; qgram_count + 1];
        let mut pos = vec![0; text.len()];

        for qgram in ranks.qgrams(q, text) {
            address[qgram] += 1;
        }

        for g in 1..address.len() {
            if address[g] > max_count {
                // mask qgram
                address[g] = 0;
            }
        }

        utils::prescan(&mut address, 0, |a, b| a + b);

        {
            let mut offset = vec![0; qgram_count];
            for (i, qgram) in ranks.qgrams(q, text).enumerate() {
                let a = address[qgram as usize];
                if address[qgram as usize + 1] - a != 0 {
                    // if not masked, insert positions
                    pos[a + offset[qgram as usize]] = i;
                    offset[qgram as usize] += 1;
                }
            }
        }

        QGramIndex { q: q, address: address, pos: pos, ranks: ranks }
    }

    pub fn q(&self) -> u32 {
        self.q
    }

    /// Return text positions with matching q-gram. Complexity O(1).
    pub fn qgram_matches(&self, qgram: usize) -> &[usize] {
        &self.pos[self.address[qgram]..self.address[qgram + 1]]
    }

    /// Return matches of the given pattern.
    /// Complexity O(m + k) for pattern of length m and k being the number of matching q-grams.
    pub fn matches(&self, pattern: &[u8], min_count: usize) -> Vec<Match> {
        let q = self.q as usize;
        let mut diagonals = collections::HashMap::new();
        for (i, qgram) in self.ranks.qgrams(self.q, pattern).enumerate() {
            for &p in self.qgram_matches(qgram) {
                let diagonal = p - i;
                if !diagonals.contains_key(&diagonal) {
                    diagonals.insert(diagonal, Match {
                        pattern: Interval { start: i, stop: i + q },
                        text: Interval { start: p, stop: p + q },
                        count: 1,
                    });
                }
                else {
                    let m = diagonals.get_mut(&diagonal).unwrap();
                    m.pattern.stop = i + q;
                    m.text.stop = p + q;
                    m.count += 1;
                }
            }
        }
        diagonals.into_iter().filter_map(
            |(_, m)| if m.count >= min_count { Some(m) } else { None }
        ).collect()
    }

    /// Return exact matches (substrings) of the given pattern.
    /// Complexity O(m + k) for pattern of length m and k being the number of matching q-grams.
    pub fn exact_matches(&self, pattern: &[u8]) -> Vec<ExactMatch> {
        let q = self.q as usize;
        let mut diagonals = collections::HashMap::new();
        let mut matches = Vec::new();

        for (i, qgram) in self.ranks.qgrams(self.q, pattern).enumerate() {
            for &p in self.qgram_matches(qgram) {
                let diagonal = p - i;
                if !diagonals.contains_key(&diagonal) {
                    diagonals.insert(diagonal, ExactMatch {
                        pattern: Interval { start: i, stop: i + q },
                        text: Interval { start: p, stop: p + q },
                    });
                }
                else {
                    let m = diagonals.get_mut(&diagonal).unwrap();
                    if m.pattern.stop - q + 1 != i {
                        // discontinue match
                        matches.push(*m);
                        // start new match
                        m.pattern.start = i;
                        m.pattern.stop = i + q;
                        m.text.start = p;
                        m.text.stop = p + q;
                    }
                    else {
                        m.pattern.stop = i + q;
                        m.text.stop = p + q;
                    }
                }
            }
        }
        for (_, m) in diagonals.into_iter() {
            matches.push(m);
        }

        matches
    }
}

/// An interval, consisting of start and stop position (the latter exclusive).
#[derive(PartialEq, Eq, Debug, Copy, Clone)]
pub struct Interval {
    pub start: usize,
    pub stop: usize
}

impl Interval {
    pub fn get<'a>(&self, text: &'a [u8]) -> &'a [u8] {
        &text[self.start..self.stop]
    }
}

/// A match between the pattern and the text.
#[derive(PartialEq, Eq, Debug, Copy, Clone)]
pub struct Match {
    pub pattern: Interval,
    pub text: Interval,
    pub count: usize,
}

impl cmp::Ord for Match {
    fn cmp(&self, other: &Self) -> cmp::Ordering {
        self.count.cmp(&other.count)
    }
}

impl cmp::PartialOrd for Match {
    fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
        Some(self.cmp(other))
    }
}

/// An exact match between the pattern and the text.
#[derive(PartialEq, Debug, Copy, Clone)]
pub struct ExactMatch {
    pub pattern: Interval,
    pub text: Interval,
}

#[cfg(test)]
mod tests {
    use super::*;
    use alphabets;

    fn setup() -> (&'static [u8], alphabets::Alphabet) {
        let text = b"ACGGCTGAGATGAT";
        let alphabet = alphabets::dna::alphabet();

        (text, alphabet)
    }

    #[test]
    fn test_qgram_matches() {
        let (text, alphabet) = setup();
        let q = 3;
        let qgram_index = QGramIndex::new(q, text, &alphabet);

        let ranks = alphabets::RankTransform::new(&alphabet);

        let qgram = ranks.qgrams(q, b"TGA").next().unwrap();

        let matches = qgram_index.qgram_matches(qgram);
        assert_eq!(matches, [5, 10]);
    }

    #[test]
    fn test_matches() {
        let (text, alphabet) = setup();
        let q = 3;
        let qgram_index = QGramIndex::new(q, text, &alphabet);

        let pattern = b"GCTG";
        let matches = qgram_index.matches(pattern, 1);
        assert_eq!(matches, [
            Match {
                pattern: Interval { start: 0, stop: 4 },
                text: Interval { start: 3, stop: 7 },
                count: 2
            }
        ]);
    }

    #[test]
    fn test_exact_matches() {
        let (text, alphabet) = setup();
        let q = 3;
        let qgram_index = QGramIndex::new(q, text, &alphabet);

        let pattern = b"GCTGA";
        let exact_matches = qgram_index.exact_matches(pattern);
        assert!(exact_matches.len() == 2);
        for m in exact_matches {
            assert_eq!(m.pattern.get(pattern), m.text.get(text));
        }
    }
}
<|endoftext|>


<commit_before>use std::{any::Any, collections::HashMap};

use dces::prelude::Component;

/// The struct `Services` represents a global service registry. It is used to register and call
/// global services like settings service oder a database service.
#[derive(Default)]
pub struct Services {
    services: HashMap<String, Box<dyn Any>>,
}

impl Services {
    /// Creates a service registry with an empty services map.
    pub fn new() -> Self {
        Services::default()
    }

    /// Register a new services with the given key.
    pub fn register<C: Component>(&mut self, key: impl Into<String>, service: C) {
        self.services.insert(key.into(), Box::new(service));
    }

    /// Gets a service.
    ///
    /// # Panics
    ///
    /// Panics if the there is no service for the given key or the given service type is wrong.
    pub fn get<C: Component>(&self, key: impl Into<String>) -> &C {
        self.services
            .get(&key.into())
            .unwrap()
            .downcast_ref()
            .unwrap()
    }

    /// Gets a service.
    ///
    /// # Panics
    ///
    /// Panics if the there is no service for the given key or the given service type is wrong.
    // pub fn try_get<C: Component>(&self, key: impl Into<String>) -> Option<&C> {
    //     self.services
    //         .get(&key.into())
    //         .ok_or_else(|| None)
    //         .map(|c| c.downcast_ref())
    // }

    /// Gets a mutable reference of the requested service.
    ///
    /// # Panics
    ///
    /// Panics if the there is no service for the given key or the given service type is wrong.
    pub fn get_mut<C: Component>(&mut self, key: impl Into<String>) -> &C {
        self.services
            .get_mut(&key.into())
            .unwrap()
            .downcast_ref()
            .unwrap()
    }
}
<commit_msg>Rename Services to Registry. Add error handling.<commit_after><|endoftext|>