<commit_before>// Copyright 2013-2014 The rust-url developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! [*Unicode IDNA Compatibility Processing*
//! (Unicode Technical Standard #46)](http://www.unicode.org/reports/tr46/)

use self::Mapping::*;
use punycode;
use std::cmp::Ordering::{Equal, Greater, Less};
use unicode_bidi::{bidi_class, BidiClass};
use unicode_normalization::char::is_combining_mark;
use unicode_normalization::UnicodeNormalization;

include!("uts46_mapping_table.rs");

const PUNYCODE_PREFIX: &'static str = "xn--";

#[derive(Debug)]
struct StringTableSlice {
    // Store these as separate fields so the structure will have an
    // alignment of 1 and thus pack better into the Mapping enum, below.
    byte_start_lo: u8,
    byte_start_hi: u8,
    byte_len: u8,
}

fn decode_slice(slice: &StringTableSlice) -> &'static str {
    let lo = slice.byte_start_lo as usize;
    let hi = slice.byte_start_hi as usize;
    let start = (hi << 8) | lo;
    let len = slice.byte_len as usize;
    &STRING_TABLE[start..(start + len)]
}

#[repr(u8)]
#[derive(Debug)]
enum Mapping {
    Valid,
    Ignored,
    Mapped(StringTableSlice),
    Deviation(StringTableSlice),
    Disallowed,
    DisallowedStd3Valid,
    DisallowedStd3Mapped(StringTableSlice),
}

struct Range {
    from: char,
    to: char,
}

fn find_char(codepoint: char) -> &'static Mapping {
    let r = TABLE.binary_search_by(|ref range| {
        if codepoint > range.to {
            Less
        } else if codepoint < range.from {
            Greater
        } else {
            Equal
        }
    });
    r.ok()
        .map(|i| {
            const SINGLE_MARKER: u16 = 1 << 15;

            let x = INDEX_TABLE[i];
            let single = (x & SINGLE_MARKER) != 0;
            let offset = !SINGLE_MARKER & x;

            if single {
                &MAPPING_TABLE[offset as usize]
            } else {
                &MAPPING_TABLE[(offset + (codepoint as u16 - TABLE[i].from as u16)) as usize]
            }
        })
        .unwrap()
}

fn map_char(codepoint: char, config: Config, output: &mut String, errors: &mut Vec<Error>) {
    match *find_char(codepoint) {
        Mapping::Valid => output.push(codepoint),
        Mapping::Ignored => {}
        Mapping::Mapped(ref slice) => output.push_str(decode_slice(slice)),
        Mapping::Deviation(ref slice) => {
            if config.transitional_processing {
                output.push_str(decode_slice(slice))
            } else {
                output.push(codepoint)
            }
        }
        Mapping::Disallowed => {
            errors.push(Error::DissallowedCharacter);
            output.push(codepoint);
        }
        Mapping::DisallowedStd3Valid => {
            if config.use_std3_ascii_rules {
                errors.push(Error::DissallowedByStd3AsciiRules);
            }
            output.push(codepoint)
        }
        Mapping::DisallowedStd3Mapped(ref slice) => {
            if config.use_std3_ascii_rules {
                errors.push(Error::DissallowedMappedInStd3);
            }
            output.push_str(decode_slice(slice))
        }
    }
}

// http://tools.ietf.org/html/rfc5893#section-2
fn passes_bidi(label: &str, is_bidi_domain: bool) -> bool {
    // Rule 0: Bidi Rules apply to Bidi Domain Names: a name with at least one RTL label.  A label
    // is RTL if it contains at least one character of bidi class R, AL or AN.
    if !is_bidi_domain {
        return true;
    }

    let mut chars = label.chars();
    let first_char_class = match chars.next() {
        Some(c) => bidi_class(c),
        None => return true, // empty string
    };

    match first_char_class {
        // LTR label
        BidiClass::L => {
            // Rule 5
            loop {
                match chars.next() {
                    Some(c) => {
                        if !matches!(
                            bidi_class(c),
                            BidiClass::L
                                | BidiClass::EN
                                | BidiClass::ES
                                | BidiClass::CS
                                | BidiClass::ET
                                | BidiClass::ON
                                | BidiClass::BN
                                | BidiClass::NSM
                        ) {
                            return false;
                        }
                    }
                    None => {
                        break;
                    }
                }
            }

            // Rule 6
            // must end in L or EN followed by 0 or more NSM
            let mut rev_chars = label.chars().rev();
            let mut last_non_nsm = rev_chars.next();
            loop {
                match last_non_nsm {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last_non_nsm = rev_chars.next();
                        continue;
                    }
                    _ => {
                        break;
                    }
                }
            }
            match last_non_nsm {
                Some(c) if bidi_class(c) == BidiClass::L || bidi_class(c) == BidiClass::EN => {}
                Some(_) => {
                    return false;
                }
                _ => {}
            }
        }

        // RTL label
        BidiClass::R | BidiClass::AL => {
            let mut found_en = false;
            let mut found_an = false;

            // Rule 2
            loop {
                match chars.next() {
                    Some(c) => {
                        let char_class = bidi_class(c);

                        if char_class == BidiClass::EN {
                            found_en = true;
                        }
                        if char_class == BidiClass::AN {
                            found_an = true;
                        }

                        if !matches!(
                            char_class,
                            BidiClass::R
                                | BidiClass::AL
                                | BidiClass::AN
                                | BidiClass::EN
                                | BidiClass::ES
                                | BidiClass::CS
                                | BidiClass::ET
                                | BidiClass::ON
                                | BidiClass::BN
                                | BidiClass::NSM
                        ) {
                            return false;
                        }
                    }
                    None => {
                        break;
                    }
                }
            }
            // Rule 3
            let mut rev_chars = label.chars().rev();
            let mut last = rev_chars.next();
            loop {
                // must end in L or EN followed by 0 or more NSM
                match last {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last = rev_chars.next();
                        continue;
                    }
                    _ => {
                        break;
                    }
                }
            }
            match last {
                Some(c)
                    if matches!(
                        bidi_class(c),
                        BidiClass::R | BidiClass::AL | BidiClass::EN | BidiClass::AN
                    ) => {}
                _ => {
                    return false;
                }
            }

            // Rule 4
            if found_an && found_en {
                return false;
            }
        }

        // Rule 1: Should start with L or R/AL
        _ => {
            return false;
        }
    }

    return true;
}

/// http://www.unicode.org/reports/tr46/#Validity_Criteria
fn validate_full(label: &str, is_bidi_domain: bool, config: Config, errors: &mut Vec<Error>) {
    // V1: Must be in NFC form.
    if label.nfc().ne(label.chars()) {
        errors.push(Error::ValidityCriteria);
    } else {
        validate(label, is_bidi_domain, config, errors);
    }
}

fn validate(label: &str, is_bidi_domain: bool, config: Config, errors: &mut Vec<Error>) {
    let first_char = label.chars().next();
    if first_char == None {
        // Empty string, pass
    }
    // V2: No U+002D HYPHEN-MINUS in both third and fourth positions.
    //
    // NOTE: Spec says that the label must not contain a HYPHEN-MINUS character in both the
    // third and fourth positions. But nobody follows this criteria. See the spec issue below:
    // https://github.com/whatwg/url/issues/53

    // V3: neither begin nor end with a U+002D HYPHEN-MINUS
    else if config.check_hyphens && (label.starts_with("-") || label.ends_with("-")) {
        errors.push(Error::ValidityCriteria);
    }
    // V4: not contain a U+002E FULL STOP
    //
    // Here, label can't contain '.' since the input is from .split('.')

    // V5: not begin with a GC=Mark
    else if is_combining_mark(first_char.unwrap()) {
        errors.push(Error::ValidityCriteria);
    }
    // V6: Check against Mapping Table
    else if label.chars().any(|c| match *find_char(c) {
        Mapping::Valid => false,
        Mapping::Deviation(_) => config.transitional_processing,
        Mapping::DisallowedStd3Valid => config.use_std3_ascii_rules,
        _ => true,
    }) {
        errors.push(Error::ValidityCriteria);
    }
    // V7: ContextJ rules
    //
    // TODO: Implement rules and add *CheckJoiners* flag.

    // V8: Bidi rules
    //
    // TODO: Add *CheckBidi* flag
    else if !passes_bidi(label, is_bidi_domain) {
        errors.push(Error::ValidityCriteria);
    }
}

/// http://www.unicode.org/reports/tr46/#Processing
fn processing(domain: &str, config: Config, errors: &mut Vec<Error>) -> String {
    let mut mapped = String::with_capacity(domain.len());
    for c in domain.chars() {
        map_char(c, config, &mut mapped, errors)
    }
    let mut normalized = String::with_capacity(mapped.len());
    normalized.extend(mapped.nfc());

    // Find out if it's a Bidi Domain Name
    //
    // First, check for literal bidi chars
    let mut is_bidi_domain = domain
        .chars()
        .any(|c| matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN));
    if !is_bidi_domain {
        // Then check for punycode-encoded bidi chars
        for label in normalized.split('.') {
            if label.starts_with(PUNYCODE_PREFIX) {
                match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                    Some(decoded_label) => {
                        if decoded_label.chars().any(|c| {
                            matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN)
                        }) {
                            is_bidi_domain = true;
                        }
                    }
                    None => {
                        is_bidi_domain = true;
                    }
                }
            }
        }
    }

    let mut validated = String::new();
    let mut first = true;
    for label in normalized.split('.') {
        if !first {
            validated.push('.');
        }
        first = false;
        if label.starts_with(PUNYCODE_PREFIX) {
            match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                Some(decoded_label) => {
                    let config = config.transitional_processing(false);
                    validate_full(&decoded_label, is_bidi_domain, config, errors);
                    validated.push_str(&decoded_label)
                }
                None => errors.push(Error::PunycodeError),
            }
        } else {
            // `normalized` is already `NFC` so we can skip that check
            validate(label, is_bidi_domain, config, errors);
            validated.push_str(label)
        }
    }
    validated
}

#[derive(Clone, Copy)]
pub struct Config {
    use_std3_ascii_rules: bool,
    transitional_processing: bool,
    verify_dns_length: bool,
    check_hyphens: bool,
}

/// The defaults are that of https://url.spec.whatwg.org/#idna
impl Default for Config {
    fn default() -> Self {
        Config {
            use_std3_ascii_rules: false,
            transitional_processing: false,
            check_hyphens: false,
            // check_bidi: true,
            // check_joiners: true,

            // Only use for to_ascii, not to_unicode
            verify_dns_length: false,
        }
    }
}

impl Config {
    #[inline]
    pub fn use_std3_ascii_rules(mut self, value: bool) -> Self {
        self.use_std3_ascii_rules = value;
        self
    }

    #[inline]
    pub fn transitional_processing(mut self, value: bool) -> Self {
        self.transitional_processing = value;
        self
    }

    #[inline]
    pub fn verify_dns_length(mut self, value: bool) -> Self {
        self.verify_dns_length = value;
        self
    }

    #[inline]
    pub fn check_hyphens(mut self, value: bool) -> Self {
        self.check_hyphens = value;
        self
    }

    /// http://www.unicode.org/reports/tr46/#ToASCII
    pub fn to_ascii(self, domain: &str) -> Result<String, Errors> {
        let mut errors = Vec::new();
        let mut result = String::new();
        let mut first = true;
        for label in processing(domain, self, &mut errors).split('.') {
            if !first {
                result.push('.');
            }
            first = false;
            if label.is_ascii() {
                result.push_str(label);
            } else {
                match punycode::encode_str(label) {
                    Some(x) => {
                        result.push_str(PUNYCODE_PREFIX);
                        result.push_str(&x);
                    }
                    None => errors.push(Error::PunycodeError),
                }
            }
        }

        if self.verify_dns_length {
            let domain = if result.ends_with(".") {
                &result[..result.len() - 1]
            } else {
                &*result
            };
            if domain.len() < 1 || domain.split('.').any(|label| label.len() < 1) {
                errors.push(Error::TooShortForDns)
            }
            if domain.len() > 253 || domain.split('.').any(|label| label.len() > 63) {
                errors.push(Error::TooLongForDns)
            }
        }
        if errors.is_empty() {
            Ok(result)
        } else {
            Err(Errors(errors))
        }
    }

    /// http://www.unicode.org/reports/tr46/#ToUnicode
    pub fn to_unicode(self, domain: &str) -> (String, Result<(), Errors>) {
        let mut errors = Vec::new();
        let domain = processing(domain, self, &mut errors);
        let errors = if errors.is_empty() {
            Ok(())
        } else {
            Err(Errors(errors))
        };
        (domain, errors)
    }
}

#[derive(PartialEq, Eq, Clone, Copy, Debug)]
enum Error {
    PunycodeError,
    ValidityCriteria,
    DissallowedByStd3AsciiRules,
    DissallowedMappedInStd3,
    DissallowedCharacter,
    TooLongForDns,
    TooShortForDns,
}

/// Errors recorded during UTS #46 processing.
///
/// This is opaque for now, only indicating the presence of at least one error.
/// More details may be exposed in the future.
#[derive(Debug)]
pub struct Errors(Vec<Error>);
<commit_msg>Add support for '?' operator (#620)<commit_after>// Copyright 2013-2014 The rust-url developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! [*Unicode IDNA Compatibility Processing*
//! (Unicode Technical Standard #46)](http://www.unicode.org/reports/tr46/)

use self::Mapping::*;
use punycode;
use std::cmp::Ordering::{Equal, Greater, Less};
use std::{error::Error as StdError, fmt};
use unicode_bidi::{bidi_class, BidiClass};
use unicode_normalization::char::is_combining_mark;
use unicode_normalization::UnicodeNormalization;

include!("uts46_mapping_table.rs");

const PUNYCODE_PREFIX: &'static str = "xn--";

#[derive(Debug)]
struct StringTableSlice {
    // Store these as separate fields so the structure will have an
    // alignment of 1 and thus pack better into the Mapping enum, below.
    byte_start_lo: u8,
    byte_start_hi: u8,
    byte_len: u8,
}

fn decode_slice(slice: &StringTableSlice) -> &'static str {
    let lo = slice.byte_start_lo as usize;
    let hi = slice.byte_start_hi as usize;
    let start = (hi << 8) | lo;
    let len = slice.byte_len as usize;
    &STRING_TABLE[start..(start + len)]
}

#[repr(u8)]
#[derive(Debug)]
enum Mapping {
    Valid,
    Ignored,
    Mapped(StringTableSlice),
    Deviation(StringTableSlice),
    Disallowed,
    DisallowedStd3Valid,
    DisallowedStd3Mapped(StringTableSlice),
}

struct Range {
    from: char,
    to: char,
}

fn find_char(codepoint: char) -> &'static Mapping {
    let r = TABLE.binary_search_by(|ref range| {
        if codepoint > range.to {
            Less
        } else if codepoint < range.from {
            Greater
        } else {
            Equal
        }
    });
    r.ok()
        .map(|i| {
            const SINGLE_MARKER: u16 = 1 << 15;

            let x = INDEX_TABLE[i];
            let single = (x & SINGLE_MARKER) != 0;
            let offset = !SINGLE_MARKER & x;

            if single {
                &MAPPING_TABLE[offset as usize]
            } else {
                &MAPPING_TABLE[(offset + (codepoint as u16 - TABLE[i].from as u16)) as usize]
            }
        })
        .unwrap()
}

fn map_char(codepoint: char, config: Config, output: &mut String, errors: &mut Vec<Error>) {
    match *find_char(codepoint) {
        Mapping::Valid => output.push(codepoint),
        Mapping::Ignored => {}
        Mapping::Mapped(ref slice) => output.push_str(decode_slice(slice)),
        Mapping::Deviation(ref slice) => {
            if config.transitional_processing {
                output.push_str(decode_slice(slice))
            } else {
                output.push(codepoint)
            }
        }
        Mapping::Disallowed => {
            errors.push(Error::DissallowedCharacter);
            output.push(codepoint);
        }
        Mapping::DisallowedStd3Valid => {
            if config.use_std3_ascii_rules {
                errors.push(Error::DissallowedByStd3AsciiRules);
            }
            output.push(codepoint)
        }
        Mapping::DisallowedStd3Mapped(ref slice) => {
            if config.use_std3_ascii_rules {
                errors.push(Error::DissallowedMappedInStd3);
            }
            output.push_str(decode_slice(slice))
        }
    }
}

// http://tools.ietf.org/html/rfc5893#section-2
fn passes_bidi(label: &str, is_bidi_domain: bool) -> bool {
    // Rule 0: Bidi Rules apply to Bidi Domain Names: a name with at least one RTL label.  A label
    // is RTL if it contains at least one character of bidi class R, AL or AN.
    if !is_bidi_domain {
        return true;
    }

    let mut chars = label.chars();
    let first_char_class = match chars.next() {
        Some(c) => bidi_class(c),
        None => return true, // empty string
    };

    match first_char_class {
        // LTR label
        BidiClass::L => {
            // Rule 5
            loop {
                match chars.next() {
                    Some(c) => {
                        if !matches!(
                            bidi_class(c),
                            BidiClass::L
                                | BidiClass::EN
                                | BidiClass::ES
                                | BidiClass::CS
                                | BidiClass::ET
                                | BidiClass::ON
                                | BidiClass::BN
                                | BidiClass::NSM
                        ) {
                            return false;
                        }
                    }
                    None => {
                        break;
                    }
                }
            }

            // Rule 6
            // must end in L or EN followed by 0 or more NSM
            let mut rev_chars = label.chars().rev();
            let mut last_non_nsm = rev_chars.next();
            loop {
                match last_non_nsm {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last_non_nsm = rev_chars.next();
                        continue;
                    }
                    _ => {
                        break;
                    }
                }
            }
            match last_non_nsm {
                Some(c) if bidi_class(c) == BidiClass::L || bidi_class(c) == BidiClass::EN => {}
                Some(_) => {
                    return false;
                }
                _ => {}
            }
        }

        // RTL label
        BidiClass::R | BidiClass::AL => {
            let mut found_en = false;
            let mut found_an = false;

            // Rule 2
            loop {
                match chars.next() {
                    Some(c) => {
                        let char_class = bidi_class(c);

                        if char_class == BidiClass::EN {
                            found_en = true;
                        }
                        if char_class == BidiClass::AN {
                            found_an = true;
                        }

                        if !matches!(
                            char_class,
                            BidiClass::R
                                | BidiClass::AL
                                | BidiClass::AN
                                | BidiClass::EN
                                | BidiClass::ES
                                | BidiClass::CS
                                | BidiClass::ET
                                | BidiClass::ON
                                | BidiClass::BN
                                | BidiClass::NSM
                        ) {
                            return false;
                        }
                    }
                    None => {
                        break;
                    }
                }
            }
            // Rule 3
            let mut rev_chars = label.chars().rev();
            let mut last = rev_chars.next();
            loop {
                // must end in L or EN followed by 0 or more NSM
                match last {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last = rev_chars.next();
                        continue;
                    }
                    _ => {
                        break;
                    }
                }
            }
            match last {
                Some(c)
                    if matches!(
                        bidi_class(c),
                        BidiClass::R | BidiClass::AL | BidiClass::EN | BidiClass::AN
                    ) => {}
                _ => {
                    return false;
                }
            }

            // Rule 4
            if found_an && found_en {
                return false;
            }
        }

        // Rule 1: Should start with L or R/AL
        _ => {
            return false;
        }
    }

    return true;
}

/// http://www.unicode.org/reports/tr46/#Validity_Criteria
fn validate_full(label: &str, is_bidi_domain: bool, config: Config, errors: &mut Vec<Error>) {
    // V1: Must be in NFC form.
    if label.nfc().ne(label.chars()) {
        errors.push(Error::ValidityCriteria);
    } else {
        validate(label, is_bidi_domain, config, errors);
    }
}

fn validate(label: &str, is_bidi_domain: bool, config: Config, errors: &mut Vec<Error>) {
    let first_char = label.chars().next();
    if first_char == None {
        // Empty string, pass
    }
    // V2: No U+002D HYPHEN-MINUS in both third and fourth positions.
    //
    // NOTE: Spec says that the label must not contain a HYPHEN-MINUS character in both the
    // third and fourth positions. But nobody follows this criteria. See the spec issue below:
    // https://github.com/whatwg/url/issues/53

    // V3: neither begin nor end with a U+002D HYPHEN-MINUS
    else if config.check_hyphens && (label.starts_with("-") || label.ends_with("-")) {
        errors.push(Error::ValidityCriteria);
    }
    // V4: not contain a U+002E FULL STOP
    //
    // Here, label can't contain '.' since the input is from .split('.')

    // V5: not begin with a GC=Mark
    else if is_combining_mark(first_char.unwrap()) {
        errors.push(Error::ValidityCriteria);
    }
    // V6: Check against Mapping Table
    else if label.chars().any(|c| match *find_char(c) {
        Mapping::Valid => false,
        Mapping::Deviation(_) => config.transitional_processing,
        Mapping::DisallowedStd3Valid => config.use_std3_ascii_rules,
        _ => true,
    }) {
        errors.push(Error::ValidityCriteria);
    }
    // V7: ContextJ rules
    //
    // TODO: Implement rules and add *CheckJoiners* flag.

    // V8: Bidi rules
    //
    // TODO: Add *CheckBidi* flag
    else if !passes_bidi(label, is_bidi_domain) {
        errors.push(Error::ValidityCriteria);
    }
}

/// http://www.unicode.org/reports/tr46/#Processing
fn processing(domain: &str, config: Config, errors: &mut Vec<Error>) -> String {
    let mut mapped = String::with_capacity(domain.len());
    for c in domain.chars() {
        map_char(c, config, &mut mapped, errors)
    }
    let mut normalized = String::with_capacity(mapped.len());
    normalized.extend(mapped.nfc());

    // Find out if it's a Bidi Domain Name
    //
    // First, check for literal bidi chars
    let mut is_bidi_domain = domain
        .chars()
        .any(|c| matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN));
    if !is_bidi_domain {
        // Then check for punycode-encoded bidi chars
        for label in normalized.split('.') {
            if label.starts_with(PUNYCODE_PREFIX) {
                match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                    Some(decoded_label) => {
                        if decoded_label.chars().any(|c| {
                            matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN)
                        }) {
                            is_bidi_domain = true;
                        }
                    }
                    None => {
                        is_bidi_domain = true;
                    }
                }
            }
        }
    }

    let mut validated = String::new();
    let mut first = true;
    for label in normalized.split('.') {
        if !first {
            validated.push('.');
        }
        first = false;
        if label.starts_with(PUNYCODE_PREFIX) {
            match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                Some(decoded_label) => {
                    let config = config.transitional_processing(false);
                    validate_full(&decoded_label, is_bidi_domain, config, errors);
                    validated.push_str(&decoded_label)
                }
                None => errors.push(Error::PunycodeError),
            }
        } else {
            // `normalized` is already `NFC` so we can skip that check
            validate(label, is_bidi_domain, config, errors);
            validated.push_str(label)
        }
    }
    validated
}

#[derive(Clone, Copy)]
pub struct Config {
    use_std3_ascii_rules: bool,
    transitional_processing: bool,
    verify_dns_length: bool,
    check_hyphens: bool,
}

/// The defaults are that of https://url.spec.whatwg.org/#idna
impl Default for Config {
    fn default() -> Self {
        Config {
            use_std3_ascii_rules: false,
            transitional_processing: false,
            check_hyphens: false,
            // check_bidi: true,
            // check_joiners: true,

            // Only use for to_ascii, not to_unicode
            verify_dns_length: false,
        }
    }
}

impl Config {
    #[inline]
    pub fn use_std3_ascii_rules(mut self, value: bool) -> Self {
        self.use_std3_ascii_rules = value;
        self
    }

    #[inline]
    pub fn transitional_processing(mut self, value: bool) -> Self {
        self.transitional_processing = value;
        self
    }

    #[inline]
    pub fn verify_dns_length(mut self, value: bool) -> Self {
        self.verify_dns_length = value;
        self
    }

    #[inline]
    pub fn check_hyphens(mut self, value: bool) -> Self {
        self.check_hyphens = value;
        self
    }

    /// http://www.unicode.org/reports/tr46/#ToASCII
    pub fn to_ascii(self, domain: &str) -> Result<String, Errors> {
        let mut errors = Vec::new();
        let mut result = String::new();
        let mut first = true;
        for label in processing(domain, self, &mut errors).split('.') {
            if !first {
                result.push('.');
            }
            first = false;
            if label.is_ascii() {
                result.push_str(label);
            } else {
                match punycode::encode_str(label) {
                    Some(x) => {
                        result.push_str(PUNYCODE_PREFIX);
                        result.push_str(&x);
                    }
                    None => errors.push(Error::PunycodeError),
                }
            }
        }

        if self.verify_dns_length {
            let domain = if result.ends_with(".") {
                &result[..result.len() - 1]
            } else {
                &*result
            };
            if domain.len() < 1 || domain.split('.').any(|label| label.len() < 1) {
                errors.push(Error::TooShortForDns)
            }
            if domain.len() > 253 || domain.split('.').any(|label| label.len() > 63) {
                errors.push(Error::TooLongForDns)
            }
        }
        if errors.is_empty() {
            Ok(result)
        } else {
            Err(Errors(errors))
        }
    }

    /// http://www.unicode.org/reports/tr46/#ToUnicode
    pub fn to_unicode(self, domain: &str) -> (String, Result<(), Errors>) {
        let mut errors = Vec::new();
        let domain = processing(domain, self, &mut errors);
        let errors = if errors.is_empty() {
            Ok(())
        } else {
            Err(Errors(errors))
        };
        (domain, errors)
    }
}

#[derive(PartialEq, Eq, Clone, Copy, Debug)]
enum Error {
    PunycodeError,
    ValidityCriteria,
    DissallowedByStd3AsciiRules,
    DissallowedMappedInStd3,
    DissallowedCharacter,
    TooLongForDns,
    TooShortForDns,
}

/// Errors recorded during UTS #46 processing.
///
/// This is opaque for now, only indicating the presence of at least one error.
/// More details may be exposed in the future.
#[derive(Debug)]
pub struct Errors(Vec<Error>);

impl StdError for Errors {}

impl fmt::Display for Errors {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:?}", self)
    }
}
<|endoftext|>


<commit_before>// Copyright 2013 The Servo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#[link(name = "core_foundation", vers = "0.1")];

extern mod std;

pub mod array;
pub mod base;
pub mod boolean;
pub mod data;
pub mod dictionary;
pub mod number;
pub mod set;
pub mod string;
pub mod url;

#[cfg(test)]
pub mod test {
    #[test]
    fn test_stuff() {
        use boolean::CFBoolean;
        use number::number;
        use dictionary::CFDictionary;
        use string::CFString;

        /*let n = CFNumber::new_number(42 as i32);
        io::println(format!("%d", (&n).retain_count() as int));
        (&n).show();*/

        let bar = CFString::from_static_string("Bar");
        let baz = CFString::from_static_string("Baz");
        let boo = CFString::from_static_string("Boo");
        let foo = CFString::from_static_string("Foo");
        let tru = CFBoolean::true_value();
        let n42 = number(42);

        let d = CFDictionary::from_CFType_pairs([
            (bar.as_CFType(), boo.as_CFType()),
            (baz.as_CFType(), tru.as_CFType()),
            (foo.as_CFType(), n42.as_CFType()),
        ]);
        d.contents.show();
    }
}
<commit_msg>More fixes for tests.<commit_after>// Copyright 2013 The Servo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#[link(name = "core_foundation", vers = "0.1")];

extern mod std;

pub mod array;
pub mod base;
pub mod boolean;
pub mod data;
pub mod dictionary;
pub mod number;
pub mod set;
pub mod string;
pub mod url;

#[cfg(test)]
pub mod test {
    #[test]
    fn test_stuff() {
        use base::TCFType;
        use boolean::CFBoolean;
        use number::number;
        use dictionary::CFDictionary;
        use string::CFString;

        /*let n = CFNumber::new_number(42 as i32);
        io::println(format!("%d", (&n).retain_count() as int));
        (&n).show();*/

        let bar = CFString::from_static_string("Bar");
        let baz = CFString::from_static_string("Baz");
        let boo = CFString::from_static_string("Boo");
        let foo = CFString::from_static_string("Foo");
        let tru = CFBoolean::true_value();
        let n42 = number(42);

        let _d = CFDictionary::from_CFType_pairs([
            (bar.as_CFType(), boo.as_CFType()),
            (baz.as_CFType(), tru.as_CFType()),
            (foo.as_CFType(), n42.as_CFType()),
        ]);
    }
}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    account_address::AccountAddress,
    transaction::Version,
    validator_set::ValidatorSet,
    validator_verifier::{ValidatorVerifier, VerifyError},
};
use canonical_serialization::{CanonicalSerialize, CanonicalSerializer, SimpleSerializer};
use crypto::{
    hash::{CryptoHash, CryptoHasher, LedgerInfoHasher},
    HashValue, *,
};
use failure::prelude::*;
#[cfg(any(test, feature = "testing"))]
use proptest_derive::Arbitrary;
use proto_conv::{FromProto, IntoProto};
use serde::{Deserialize, Serialize};
use std::{
    collections::HashMap,
    convert::TryFrom,
    fmt::{Display, Formatter},
};

/// This structure serves a dual purpose.
///
/// First, if this structure is signed by 2f+1 validators it signifies the state of the ledger at
/// version `version` -- it contains the transaction accumulator at that version which commits to
/// all historical transactions. This structure may be expanded to include other information that
/// is derived from that accumulator (e.g. the current time according to the time contract) to
/// reduce the number of proofs a client must get.
///
/// Second, the structure contains a `consensus_data_hash` value. This is the hash of an internal
/// data structure that represents a block that is voted on in HotStuff. If 2f+1 signatures are
/// gathered on the same ledger info that represents a Quorum Certificate (QC) on the consensus
/// data.
///
/// Combining these two concepts, when a validator votes on a block, B it votes for a
/// LedgerInfo with the `version` being the latest version that will be committed if B gets 2f+1
/// votes. It sets `consensus_data_hash` to represent B so that if those 2f+1 votes are gathered a
/// QC is formed on B.
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[cfg_attr(any(test, feature = "testing"), derive(Arbitrary))]
pub struct LedgerInfo {
    /// The version of the latest transaction in the ledger.
    version: Version,

    /// The root hash of transaction accumulator that includes the latest transaction.
    transaction_accumulator_hash: HashValue,

    /// Hash of consensus specific data that is opaque to all parts of the system other than
    /// consensus.
    consensus_data_hash: HashValue,

    /// Block id of the last committed block corresponding to this LedgerInfo
    /// as reported by consensus.
    consensus_block_id: HashValue,

    /// Epoch number corresponds to the set of validators that are active for this ledger info.
    epoch_num: u64,

    /// Timestamp that represents the microseconds since the epoch (unix time) that is
    /// generated by the proposer of the block.  This is strictly increasing with every block.
    /// If a client reads a timestamp > the one they specified for transaction expiration time,
    /// they can be certain that their transaction will never be included in a block in the future
    /// (assuming that their transaction has not yet been included)
    timestamp_usecs: u64,

    /// An optional field keeping the set of new validators to start the next epoch.
    /// The very last ledger info of an epoch contains the validator set for the next one,
    /// other ledger info instances are None.
    next_validator_set: Option<ValidatorSet>,
}

impl Display for LedgerInfo {
    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
        write!(
            f,
            "LedgerInfo: [committed_block_id: {}, version: {}, epoch_num: {}, timestamp (us): {}, next_validator_set: {}]",
            self.consensus_block_id(),
            self.version(),
            self.epoch_num(),
            self.timestamp_usecs(),
            self.next_validator_set.as_ref().map_or("None".to_string(), |validator_set| format!("{}", validator_set)),
        )
    }
}

impl LedgerInfo {
    /// Constructs a `LedgerInfo` object at a specific version using a given
    /// transaction accumulator root and consensus data hash.
    pub fn new(
        version: Version,
        transaction_accumulator_hash: HashValue,
        consensus_data_hash: HashValue,
        consensus_block_id: HashValue,
        epoch_num: u64,
        timestamp_usecs: u64,
        next_validator_set: Option<ValidatorSet>,
    ) -> Self {
        LedgerInfo {
            version,
            transaction_accumulator_hash,
            consensus_data_hash,
            consensus_block_id,
            epoch_num,
            timestamp_usecs,
            next_validator_set,
        }
    }

    /// Returns the version of this `LedgerInfo`.
    pub fn version(&self) -> Version {
        self.version
    }

    /// Returns the transaction accumulator root of this `LedgerInfo`.
    pub fn transaction_accumulator_hash(&self) -> HashValue {
        self.transaction_accumulator_hash
    }

    /// Returns hash of consensus data in this `LedgerInfo`.
    pub fn consensus_data_hash(&self) -> HashValue {
        self.consensus_data_hash
    }

    pub fn consensus_block_id(&self) -> HashValue {
        self.consensus_block_id
    }

    pub fn set_consensus_data_hash(&mut self, consensus_data_hash: HashValue) {
        self.consensus_data_hash = consensus_data_hash;
    }

    pub fn epoch_num(&self) -> u64 {
        self.epoch_num
    }

    pub fn timestamp_usecs(&self) -> u64 {
        self.timestamp_usecs
    }

    /// A ledger info is nominal if it's not certifying any real version.
    pub fn is_zero(&self) -> bool {
        self.version == 0
    }

    pub fn next_validator_set(&self) -> Option<&ValidatorSet> {
        self.next_validator_set.as_ref()
    }
}

impl IntoProto for LedgerInfo {
    type ProtoType = crate::proto::ledger_info::LedgerInfo;

    fn into_proto(self) -> Self::ProtoType {
        let mut proto = Self::ProtoType::new();
        proto.set_version(self.version);
        proto.set_transaction_accumulator_hash(self.transaction_accumulator_hash.into_proto());
        proto.set_consensus_data_hash(self.consensus_data_hash.into_proto());
        proto.set_consensus_block_id(self.consensus_block_id.into_proto());
        proto.set_epoch_num(self.epoch_num);
        proto.set_timestamp_usecs(self.timestamp_usecs);
        if let Some(next_validator_set) = self.next_validator_set {
            proto.set_next_validator_set(next_validator_set.into_proto());
        }
        proto
    }
}

impl FromProto for LedgerInfo {
    type ProtoType = crate::proto::ledger_info::LedgerInfo;

    fn from_proto(proto: Self::ProtoType) -> Result<Self> {
        let version = proto.get_version();
        let transaction_accumulator_hash =
            HashValue::from_slice(proto.get_transaction_accumulator_hash())?;
        let consensus_data_hash = HashValue::from_slice(proto.get_consensus_data_hash())?;
        let consensus_block_id = HashValue::from_slice(proto.get_consensus_block_id())?;
        let epoch_num = proto.get_epoch_num();
        let timestamp_usecs = proto.get_timestamp_usecs();

        let next_validator_set =
            if let Some(validator_set_proto) = proto.next_validator_set.into_option() {
                Some(ValidatorSet::from_proto(validator_set_proto)?)
            } else {
                None
            };
        Ok(LedgerInfo::new(
            version,
            transaction_accumulator_hash,
            consensus_data_hash,
            consensus_block_id,
            epoch_num,
            timestamp_usecs,
            next_validator_set,
        ))
    }
}

impl TryFrom<crate::proto::types::LedgerInfo> for LedgerInfo {
    type Error = Error;

    fn try_from(proto: crate::proto::types::LedgerInfo) -> Result<Self> {
        let version = proto.version;
        let transaction_accumulator_hash =
            HashValue::from_slice(&proto.transaction_accumulator_hash)?;
        let consensus_data_hash = HashValue::from_slice(&proto.consensus_data_hash)?;
        let consensus_block_id = HashValue::from_slice(&proto.consensus_block_id)?;
        let epoch_num = proto.epoch_num;
        let timestamp_usecs = proto.timestamp_usecs;

        let next_validator_set = if let Some(validator_set_proto) = proto.next_validator_set {
            Some(ValidatorSet::try_from(validator_set_proto)?)
        } else {
            None
        };
        Ok(LedgerInfo::new(
            version,
            transaction_accumulator_hash,
            consensus_data_hash,
            consensus_block_id,
            epoch_num,
            timestamp_usecs,
            next_validator_set,
        ))
    }
}

impl From<LedgerInfo> for crate::proto::types::LedgerInfo {
    fn from(ledger_info: LedgerInfo) -> Self {
        Self {
            version: ledger_info.version,
            transaction_accumulator_hash: ledger_info.transaction_accumulator_hash.to_vec(),
            consensus_data_hash: ledger_info.consensus_data_hash.to_vec(),
            consensus_block_id: ledger_info.consensus_block_id.to_vec(),
            epoch_num: ledger_info.epoch_num,
            timestamp_usecs: ledger_info.timestamp_usecs,
            next_validator_set: ledger_info.next_validator_set.map(Into::into),
        }
    }
}

impl CanonicalSerialize for LedgerInfo {
    fn serialize(&self, serializer: &mut impl CanonicalSerializer) -> Result<()> {
        serializer
            .encode_u64(self.version)?
            .encode_bytes(self.transaction_accumulator_hash.as_ref())?
            .encode_bytes(self.consensus_data_hash.as_ref())?
            .encode_bytes(self.consensus_block_id.as_ref())?
            .encode_u64(self.epoch_num)?
            .encode_u64(self.timestamp_usecs)?
            .encode_optional(&self.next_validator_set)?;
        Ok(())
    }
}

impl CryptoHash for LedgerInfo {
    type Hasher = LedgerInfoHasher;

    fn hash(&self) -> HashValue {
        let mut state = Self::Hasher::default();
        state.write(
            &SimpleSerializer::<Vec<u8>>::serialize(self).expect("Serialization should work."),
        );
        state.finish()
    }
}

/// The validator node returns this structure which includes signatures
/// from validators that confirm the state.  The client needs to only pass back
/// the LedgerInfo element since the validator node doesn't need to know the signatures
/// again when the client performs a query, those are only there for the client
/// to be able to verify the state
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
pub struct LedgerInfoWithSignatures<Sig> {
    ledger_info: LedgerInfo,
    /// The validator is identified by its account address: in order to verify a signature
    /// one needs to retrieve the public key of the validator for the given epoch.
    signatures: HashMap<AccountAddress, Sig>,
}

impl<Sig> Display for LedgerInfoWithSignatures<Sig> {
    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
        write!(f, "{}", self.ledger_info)
    }
}

impl<Sig: Signature> LedgerInfoWithSignatures<Sig> {
    pub fn new(ledger_info: LedgerInfo, signatures: HashMap<AccountAddress, Sig>) -> Self {
        LedgerInfoWithSignatures {
            ledger_info,
            signatures,
        }
    }

    pub fn ledger_info(&self) -> &LedgerInfo {
        &self.ledger_info
    }

    pub fn add_signature(&mut self, validator: AccountAddress, signature: Sig) {
        self.signatures.entry(validator).or_insert(signature);
    }

    pub fn remove_signature(&mut self, validator: AccountAddress) {
        self.signatures.remove(&validator);
    }

    pub fn signatures(&self) -> &HashMap<AccountAddress, Sig> {
        &self.signatures
    }

    pub fn verify(
        &self,
        validator: &ValidatorVerifier<Sig::VerifyingKeyMaterial>,
    ) -> ::std::result::Result<(), VerifyError> {
        if self.ledger_info.is_zero() {
            // We're not trying to verify nominal ledger info that does not carry any information.
            return Ok(());
        }
        let ledger_hash = self.ledger_info().hash();
        validator.batch_verify_aggregated_signature(ledger_hash, self.signatures())
    }
}

impl<Sig: Signature> FromProto for LedgerInfoWithSignatures<Sig> {
    type ProtoType = crate::proto::ledger_info::LedgerInfoWithSignatures;

    fn from_proto(mut proto: Self::ProtoType) -> Result<Self> {
        let ledger_info = LedgerInfo::from_proto(proto.take_ledger_info())?;

        let signatures_proto = proto.take_signatures();
        let num_signatures = signatures_proto.len();
        let signatures = signatures_proto
            .into_iter()
            .map(|proto| {
                let validator_id = AccountAddress::from_proto(proto.get_validator_id().to_vec())?;
                let signature_bytes: &[u8] = proto.get_signature();
                let signature = Sig::try_from(signature_bytes)?;
                Ok((validator_id, signature))
            })
            .collect::<Result<HashMap<_, _>>>()?;
        ensure!(
            signatures.len() == num_signatures,
            "Signatures should be from different validators."
        );

        Ok(LedgerInfoWithSignatures {
            ledger_info,
            signatures,
        })
    }
}

impl<Sig: Signature> IntoProto for LedgerInfoWithSignatures<Sig> {
    type ProtoType = crate::proto::ledger_info::LedgerInfoWithSignatures;

    fn into_proto(self) -> Self::ProtoType {
        let mut proto = Self::ProtoType::new();
        proto.set_ledger_info(self.ledger_info.into_proto());
        self.signatures
            .into_iter()
            .for_each(|(validator_id, signature)| {
                let mut validator_signature = crate::proto::ledger_info::ValidatorSignature::new();
                validator_signature.set_validator_id(validator_id.into_proto());
                validator_signature.set_signature(signature.to_bytes().to_vec());
                proto.mut_signatures().push(validator_signature)
            });
        proto
    }
}
<commit_msg>[types] impl conversions from/to prost proto and LedgerInfoWithSignatures<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    account_address::AccountAddress,
    transaction::Version,
    validator_set::ValidatorSet,
    validator_verifier::{ValidatorVerifier, VerifyError},
};
use canonical_serialization::{CanonicalSerialize, CanonicalSerializer, SimpleSerializer};
use crypto::{
    hash::{CryptoHash, CryptoHasher, LedgerInfoHasher},
    HashValue, *,
};
use failure::prelude::*;
#[cfg(any(test, feature = "testing"))]
use proptest_derive::Arbitrary;
use proto_conv::{FromProto, IntoProto};
use serde::{Deserialize, Serialize};
use std::{
    collections::HashMap,
    convert::{TryFrom, TryInto},
    fmt::{Display, Formatter},
};

/// This structure serves a dual purpose.
///
/// First, if this structure is signed by 2f+1 validators it signifies the state of the ledger at
/// version `version` -- it contains the transaction accumulator at that version which commits to
/// all historical transactions. This structure may be expanded to include other information that
/// is derived from that accumulator (e.g. the current time according to the time contract) to
/// reduce the number of proofs a client must get.
///
/// Second, the structure contains a `consensus_data_hash` value. This is the hash of an internal
/// data structure that represents a block that is voted on in HotStuff. If 2f+1 signatures are
/// gathered on the same ledger info that represents a Quorum Certificate (QC) on the consensus
/// data.
///
/// Combining these two concepts, when a validator votes on a block, B it votes for a
/// LedgerInfo with the `version` being the latest version that will be committed if B gets 2f+1
/// votes. It sets `consensus_data_hash` to represent B so that if those 2f+1 votes are gathered a
/// QC is formed on B.
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[cfg_attr(any(test, feature = "testing"), derive(Arbitrary))]
pub struct LedgerInfo {
    /// The version of the latest transaction in the ledger.
    version: Version,

    /// The root hash of transaction accumulator that includes the latest transaction.
    transaction_accumulator_hash: HashValue,

    /// Hash of consensus specific data that is opaque to all parts of the system other than
    /// consensus.
    consensus_data_hash: HashValue,

    /// Block id of the last committed block corresponding to this LedgerInfo
    /// as reported by consensus.
    consensus_block_id: HashValue,

    /// Epoch number corresponds to the set of validators that are active for this ledger info.
    epoch_num: u64,

    /// Timestamp that represents the microseconds since the epoch (unix time) that is
    /// generated by the proposer of the block.  This is strictly increasing with every block.
    /// If a client reads a timestamp > the one they specified for transaction expiration time,
    /// they can be certain that their transaction will never be included in a block in the future
    /// (assuming that their transaction has not yet been included)
    timestamp_usecs: u64,

    /// An optional field keeping the set of new validators to start the next epoch.
    /// The very last ledger info of an epoch contains the validator set for the next one,
    /// other ledger info instances are None.
    next_validator_set: Option<ValidatorSet>,
}

impl Display for LedgerInfo {
    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
        write!(
            f,
            "LedgerInfo: [committed_block_id: {}, version: {}, epoch_num: {}, timestamp (us): {}, next_validator_set: {}]",
            self.consensus_block_id(),
            self.version(),
            self.epoch_num(),
            self.timestamp_usecs(),
            self.next_validator_set.as_ref().map_or("None".to_string(), |validator_set| format!("{}", validator_set)),
        )
    }
}

impl LedgerInfo {
    /// Constructs a `LedgerInfo` object at a specific version using a given
    /// transaction accumulator root and consensus data hash.
    pub fn new(
        version: Version,
        transaction_accumulator_hash: HashValue,
        consensus_data_hash: HashValue,
        consensus_block_id: HashValue,
        epoch_num: u64,
        timestamp_usecs: u64,
        next_validator_set: Option<ValidatorSet>,
    ) -> Self {
        LedgerInfo {
            version,
            transaction_accumulator_hash,
            consensus_data_hash,
            consensus_block_id,
            epoch_num,
            timestamp_usecs,
            next_validator_set,
        }
    }

    /// Returns the version of this `LedgerInfo`.
    pub fn version(&self) -> Version {
        self.version
    }

    /// Returns the transaction accumulator root of this `LedgerInfo`.
    pub fn transaction_accumulator_hash(&self) -> HashValue {
        self.transaction_accumulator_hash
    }

    /// Returns hash of consensus data in this `LedgerInfo`.
    pub fn consensus_data_hash(&self) -> HashValue {
        self.consensus_data_hash
    }

    pub fn consensus_block_id(&self) -> HashValue {
        self.consensus_block_id
    }

    pub fn set_consensus_data_hash(&mut self, consensus_data_hash: HashValue) {
        self.consensus_data_hash = consensus_data_hash;
    }

    pub fn epoch_num(&self) -> u64 {
        self.epoch_num
    }

    pub fn timestamp_usecs(&self) -> u64 {
        self.timestamp_usecs
    }

    /// A ledger info is nominal if it's not certifying any real version.
    pub fn is_zero(&self) -> bool {
        self.version == 0
    }

    pub fn next_validator_set(&self) -> Option<&ValidatorSet> {
        self.next_validator_set.as_ref()
    }
}

impl IntoProto for LedgerInfo {
    type ProtoType = crate::proto::ledger_info::LedgerInfo;

    fn into_proto(self) -> Self::ProtoType {
        let mut proto = Self::ProtoType::new();
        proto.set_version(self.version);
        proto.set_transaction_accumulator_hash(self.transaction_accumulator_hash.into_proto());
        proto.set_consensus_data_hash(self.consensus_data_hash.into_proto());
        proto.set_consensus_block_id(self.consensus_block_id.into_proto());
        proto.set_epoch_num(self.epoch_num);
        proto.set_timestamp_usecs(self.timestamp_usecs);
        if let Some(next_validator_set) = self.next_validator_set {
            proto.set_next_validator_set(next_validator_set.into_proto());
        }
        proto
    }
}

impl FromProto for LedgerInfo {
    type ProtoType = crate::proto::ledger_info::LedgerInfo;

    fn from_proto(proto: Self::ProtoType) -> Result<Self> {
        let version = proto.get_version();
        let transaction_accumulator_hash =
            HashValue::from_slice(proto.get_transaction_accumulator_hash())?;
        let consensus_data_hash = HashValue::from_slice(proto.get_consensus_data_hash())?;
        let consensus_block_id = HashValue::from_slice(proto.get_consensus_block_id())?;
        let epoch_num = proto.get_epoch_num();
        let timestamp_usecs = proto.get_timestamp_usecs();

        let next_validator_set =
            if let Some(validator_set_proto) = proto.next_validator_set.into_option() {
                Some(ValidatorSet::from_proto(validator_set_proto)?)
            } else {
                None
            };
        Ok(LedgerInfo::new(
            version,
            transaction_accumulator_hash,
            consensus_data_hash,
            consensus_block_id,
            epoch_num,
            timestamp_usecs,
            next_validator_set,
        ))
    }
}

impl TryFrom<crate::proto::types::LedgerInfo> for LedgerInfo {
    type Error = Error;

    fn try_from(proto: crate::proto::types::LedgerInfo) -> Result<Self> {
        let version = proto.version;
        let transaction_accumulator_hash =
            HashValue::from_slice(&proto.transaction_accumulator_hash)?;
        let consensus_data_hash = HashValue::from_slice(&proto.consensus_data_hash)?;
        let consensus_block_id = HashValue::from_slice(&proto.consensus_block_id)?;
        let epoch_num = proto.epoch_num;
        let timestamp_usecs = proto.timestamp_usecs;

        let next_validator_set = if let Some(validator_set_proto) = proto.next_validator_set {
            Some(ValidatorSet::try_from(validator_set_proto)?)
        } else {
            None
        };
        Ok(LedgerInfo::new(
            version,
            transaction_accumulator_hash,
            consensus_data_hash,
            consensus_block_id,
            epoch_num,
            timestamp_usecs,
            next_validator_set,
        ))
    }
}

impl From<LedgerInfo> for crate::proto::types::LedgerInfo {
    fn from(ledger_info: LedgerInfo) -> Self {
        Self {
            version: ledger_info.version,
            transaction_accumulator_hash: ledger_info.transaction_accumulator_hash.to_vec(),
            consensus_data_hash: ledger_info.consensus_data_hash.to_vec(),
            consensus_block_id: ledger_info.consensus_block_id.to_vec(),
            epoch_num: ledger_info.epoch_num,
            timestamp_usecs: ledger_info.timestamp_usecs,
            next_validator_set: ledger_info.next_validator_set.map(Into::into),
        }
    }
}

impl CanonicalSerialize for LedgerInfo {
    fn serialize(&self, serializer: &mut impl CanonicalSerializer) -> Result<()> {
        serializer
            .encode_u64(self.version)?
            .encode_bytes(self.transaction_accumulator_hash.as_ref())?
            .encode_bytes(self.consensus_data_hash.as_ref())?
            .encode_bytes(self.consensus_block_id.as_ref())?
            .encode_u64(self.epoch_num)?
            .encode_u64(self.timestamp_usecs)?
            .encode_optional(&self.next_validator_set)?;
        Ok(())
    }
}

impl CryptoHash for LedgerInfo {
    type Hasher = LedgerInfoHasher;

    fn hash(&self) -> HashValue {
        let mut state = Self::Hasher::default();
        state.write(
            &SimpleSerializer::<Vec<u8>>::serialize(self).expect("Serialization should work."),
        );
        state.finish()
    }
}

/// The validator node returns this structure which includes signatures
/// from validators that confirm the state.  The client needs to only pass back
/// the LedgerInfo element since the validator node doesn't need to know the signatures
/// again when the client performs a query, those are only there for the client
/// to be able to verify the state
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
pub struct LedgerInfoWithSignatures<Sig> {
    ledger_info: LedgerInfo,
    /// The validator is identified by its account address: in order to verify a signature
    /// one needs to retrieve the public key of the validator for the given epoch.
    signatures: HashMap<AccountAddress, Sig>,
}

impl<Sig> Display for LedgerInfoWithSignatures<Sig> {
    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
        write!(f, "{}", self.ledger_info)
    }
}

impl<Sig: Signature> LedgerInfoWithSignatures<Sig> {
    pub fn new(ledger_info: LedgerInfo, signatures: HashMap<AccountAddress, Sig>) -> Self {
        LedgerInfoWithSignatures {
            ledger_info,
            signatures,
        }
    }

    pub fn ledger_info(&self) -> &LedgerInfo {
        &self.ledger_info
    }

    pub fn add_signature(&mut self, validator: AccountAddress, signature: Sig) {
        self.signatures.entry(validator).or_insert(signature);
    }

    pub fn remove_signature(&mut self, validator: AccountAddress) {
        self.signatures.remove(&validator);
    }

    pub fn signatures(&self) -> &HashMap<AccountAddress, Sig> {
        &self.signatures
    }

    pub fn verify(
        &self,
        validator: &ValidatorVerifier<Sig::VerifyingKeyMaterial>,
    ) -> ::std::result::Result<(), VerifyError> {
        if self.ledger_info.is_zero() {
            // We're not trying to verify nominal ledger info that does not carry any information.
            return Ok(());
        }
        let ledger_hash = self.ledger_info().hash();
        validator.batch_verify_aggregated_signature(ledger_hash, self.signatures())
    }
}

impl<Sig: Signature> FromProto for LedgerInfoWithSignatures<Sig> {
    type ProtoType = crate::proto::ledger_info::LedgerInfoWithSignatures;

    fn from_proto(mut proto: Self::ProtoType) -> Result<Self> {
        let ledger_info = LedgerInfo::from_proto(proto.take_ledger_info())?;

        let signatures_proto = proto.take_signatures();
        let num_signatures = signatures_proto.len();
        let signatures = signatures_proto
            .into_iter()
            .map(|proto| {
                let validator_id = AccountAddress::from_proto(proto.get_validator_id().to_vec())?;
                let signature_bytes: &[u8] = proto.get_signature();
                let signature = Sig::try_from(signature_bytes)?;
                Ok((validator_id, signature))
            })
            .collect::<Result<HashMap<_, _>>>()?;
        ensure!(
            signatures.len() == num_signatures,
            "Signatures should be from different validators."
        );

        Ok(LedgerInfoWithSignatures {
            ledger_info,
            signatures,
        })
    }
}

impl<Sig: Signature> IntoProto for LedgerInfoWithSignatures<Sig> {
    type ProtoType = crate::proto::ledger_info::LedgerInfoWithSignatures;

    fn into_proto(self) -> Self::ProtoType {
        let mut proto = Self::ProtoType::new();
        proto.set_ledger_info(self.ledger_info.into_proto());
        self.signatures
            .into_iter()
            .for_each(|(validator_id, signature)| {
                let mut validator_signature = crate::proto::ledger_info::ValidatorSignature::new();
                validator_signature.set_validator_id(validator_id.into_proto());
                validator_signature.set_signature(signature.to_bytes().to_vec());
                proto.mut_signatures().push(validator_signature)
            });
        proto
    }
}

impl<Sig: Signature> TryFrom<crate::proto::types::LedgerInfoWithSignatures>
    for LedgerInfoWithSignatures<Sig>
{
    type Error = Error;

    fn try_from(proto: crate::proto::types::LedgerInfoWithSignatures) -> Result<Self> {
        let ledger_info = proto
            .ledger_info
            .ok_or_else(|| format_err!("Missing ledger_info"))?
            .try_into()?;

        let signatures_proto = proto.signatures;
        let num_signatures = signatures_proto.len();
        let signatures = signatures_proto
            .into_iter()
            .map(|proto| {
                let validator_id = AccountAddress::try_from(proto.validator_id)?;
                let signature_bytes: &[u8] = proto.signature.as_ref();
                let signature = Sig::try_from(signature_bytes)?;
                Ok((validator_id, signature))
            })
            .collect::<Result<HashMap<_, _>>>()?;
        ensure!(
            signatures.len() == num_signatures,
            "Signatures should be from different validators."
        );

        Ok(LedgerInfoWithSignatures {
            ledger_info,
            signatures,
        })
    }
}

impl<Sig: Signature> From<LedgerInfoWithSignatures<Sig>>
    for crate::proto::types::LedgerInfoWithSignatures
{
    fn from(ledger_info_with_sigs: LedgerInfoWithSignatures<Sig>) -> Self {
        let ledger_info = Some(ledger_info_with_sigs.ledger_info.into());
        let signatures = ledger_info_with_sigs
            .signatures
            .into_iter()
            .map(
                |(validator_id, signature)| crate::proto::types::ValidatorSignature {
                    validator_id: validator_id.to_vec(),
                    signature: signature.to_bytes().to_vec(),
                },
            )
            .collect();

        Self {
            signatures,
            ledger_info,
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2014 Johannes Köster, Christopher Schröder.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Fasta reading and writing.
//!
//! # Example
//!
//! ```
//! use std::io;
//! use bio::io::fasta;
//! let reader = fasta::Reader::new(io::stdin());
//! ```

use std::io;
use std::io::prelude::*;
use std::ascii::AsciiExt;
use std::collections;
use std::fs;
use std::path::Path;
use std::convert::AsRef;

use itertools::Itertools;

use csv;

pub struct Reader<R: io::Read> {
    reader: io::BufReader<R>,
    line: String
}

impl Reader<fs::File> {
    pub fn from_file<P: AsRef<Path>>(path: P) -> io::Result<Self> {
        fs::File::open(path).map(|f| Reader::new(f))
    }
}

impl<R: io::Read> Reader<R> {
    /// Create a new FastQ reader.
    pub fn new(reader: R) -> Self {
        Reader { reader: io::BufReader::new(reader), line: String::new() }
    }

    pub fn read(&mut self, record: &mut Record) -> io::Result<()> {
        record.clear();
        if self.line.is_empty() {
            try!(self.reader.read_line(&mut self.line));
            if self.line.is_empty() {
                return Ok(());
            }
        }

        if !self.line.starts_with(">") {
            return Err(io::Error::new(
                io::ErrorKind::Other,
                "Expected > at record start."
            ));
        }
        record.header.push_str(&self.line);
        loop {
            self.line.clear();
            try!(self.reader.read_line(&mut self.line));
            record.seq.push_str(&self.line.trim_right());
            if self.line.is_empty() || self.line.starts_with(">") {
                break;
            }
        }

        Ok(())
    }

    /// Return an iterator over the records of this FastQ file.
    pub fn records(self) -> Records<R> {
        Records { reader: self }
    }
}

pub struct Index {
    inner: collections::BTreeMap<Vec<u8>, IndexRecord>,
}

impl Index {
    pub fn new<R: io::Read>(fai: R) -> csv::Result<Self> {
        let mut inner = collections::BTreeMap::new();
        let mut fai_reader = csv::Reader::from_reader(fai).delimiter(b'\t').has_headers(false);
        for row in fai_reader.decode() {
            let (name, record): (String, IndexRecord) = try!(row);
            inner.insert(name.into_bytes(), record);
        }
        Ok(Index { inner: inner })
    }

    pub fn from_file<P: AsRef<Path>>(path: &P) -> csv::Result<Self> {
        match fs::File::open(path) {
            Ok(fai) => Self::new(fai),
            Err(e)  => Err(csv::Error::Io(e))
        }
    }

    pub fn with_fasta_file<P: AsRef<Path>>(fasta_path: &P) -> csv::Result<Self> {
        let mut ext = fasta_path.as_ref().extension().unwrap().to_str().unwrap().to_string();
        ext.push_str(".fai");
        let fai_path = fasta_path.as_ref().with_extension(ext);

        Self::from_file(&fai_path)
    }

    pub fn sequences(&self) -> Vec<Sequence> {
        self.inner.iter().map(|(name, record)| Sequence { name: name.clone(), len: record.len }).collect_vec()
    }
}

pub struct IndexedReader<R: io::Read + io::Seek> {
    reader: io::BufReader<R>,
    pub index: Index,
}

impl IndexedReader<fs::File> {
    pub fn from_file<P: AsRef<Path>>(path: &P) -> csv::Result<Self> {
        let index = try!(Index::with_fasta_file(path));

        match fs::File::open(path) {
            Ok(fasta) => Ok(IndexedReader::with_index(fasta, index)),
            Err(e)    => Err(csv::Error::Io(e))
        }
    }
}

impl<R: io::Read + io::Seek> IndexedReader<R> {
    pub fn new<I: io::Read>(fasta: R, fai: I) -> csv::Result<Self> {
        let index = try!(Index::new(fai));
        Ok(IndexedReader { reader: io::BufReader::new(fasta), index: index })
    }

    pub fn with_index(fasta: R, index: Index) -> Self {
        IndexedReader { reader: io::BufReader::new(fasta), index: index }
    }

    pub fn read_all(&mut self, seqname: &[u8], seq: &mut Vec<u8>) -> io::Result<()> {
        match self.index.inner.get(seqname) {
            Some(&idx) => self.read(seqname, 0, idx.len, seq),
            None      => Err(
                io::Error::new(
                    io::ErrorKind::Other,
                    "Unknown sequence name."
                )
            )
        }
    }

    pub fn read(&mut self, seqname: &[u8], start: u64, stop: u64, seq: &mut Vec<u8>) -> io::Result<()> {
        match self.index.inner.get(seqname) {
            Some(idx) => {
                seq.clear();
                // derived from
                // http://www.allenyu.info/item/24-quickly-fetch-sequence-from-samtools-faidx-indexed-fasta-sequences.html
                let line = start / idx.line_bases * idx.line_bytes;
                let line_offset = start % idx.line_bases;
                let offset = idx.offset + line + line_offset;
                let lines = stop / idx.line_bases * idx.line_bytes - line;
                let line_stop = stop % idx.line_bases - if lines == 0 { line_offset } else { 0 };

                try!(self.reader.seek(io::SeekFrom::Start(offset)));
                let mut buf = vec![0u8; idx.line_bases as usize];
                for _ in 0..lines {
                    // read full lines
                    try!(self.reader.read(&mut buf));
                    seq.push_all(&buf);
                }
                // read last line
                println!("linestop {}", line_stop);
                try!(self.reader.read(&mut buf[..line_stop as usize]));
                seq.push_all(&buf[..line_stop as usize]);
                Ok(())
            },
            None      => Err(
                io::Error::new(
                    io::ErrorKind::Other,
                    "Unknown sequence name."
                )
            )
        }
    }
}

#[derive(RustcDecodable, Debug, Copy, Clone)]
struct IndexRecord {
    len: u64,
    offset: u64,
    line_bases: u64,
    line_bytes: u64,
}

pub struct Sequence {
    pub name: Vec<u8>,
    pub len: u64,
}

/// A Fasta writer.
pub struct Writer<W: io::Write> {
    writer: io::BufWriter<W>,
}

impl Writer<fs::File> {
    pub fn from_file<P: AsRef<Path>>(path: P) -> io::Result<Self> {
        fs::File::create(path).map(|f| Writer::new(f))
    }
}

impl<W: io::Write> Writer<W> {
    /// Create a new Fasta writer.
    pub fn new(writer: W) -> Self {
        Writer { writer: io::BufWriter::new(writer) }
    }

    /// Directly write a Fasta record.
    pub fn write_record(&mut self, record: Record) -> io::Result<()> {
        self.write(record.id().unwrap_or(""), &record.desc(), record.seq())
    }

    /// Write a Fasta record with given values.
    ///
    /// # Arguments
    ///
    /// * `id` - the record id
    /// * `desc` - the optional descriptions
    /// * `seq` - the sequence
    pub fn write(&mut self, id: &str, desc: &[&str], seq: &[u8]) -> io::Result<()> {
        try!(self.writer.write(b">"));
        try!(self.writer.write(id.as_bytes()));
        if !desc.is_empty() {
            for d in desc {
                try!(self.writer.write(b" "));
                try!(self.writer.write(d.as_bytes()));
            }
        }
        try!(self.writer.write(b"\n"));
        try!(self.writer.write(seq));
        try!(self.writer.write(b"\n"));

        Ok(())
    }

    /// Flush the writer, ensuring that everything is written.
    pub fn flush(&mut self) -> io::Result<()> {
        self.writer.flush()
    }
}

pub struct Record {
    header: String,
    seq: String,
}

impl Record {
    pub fn new() -> Self {
        Record { header: String::new(), seq: String::new() }
    }

    pub fn is_empty(&self) -> bool {
        self.header.is_empty() && self.seq.is_empty()
    }

    /// Check validity of Fasta record.
    pub fn check(&self) -> Result<(), &str> {
        if self.id().is_none() {
            return Err("Expecting id for FastQ record.");
        }
        if !self.seq.is_ascii() {
            return Err("Non-ascii character found in sequence.");
        }

        Ok(())
    }

    /// Return the id of the record.
    pub fn id(&self) -> Option<&str> {
        self.header[1..].split_whitespace().next()
    }

    /// Return descriptions if present.
    pub fn desc(&self) -> Vec<&str> {
        self.header[1..].split_whitespace().skip(1).collect()
    }

    /// Return the sequence of the record.
    pub fn seq(&self) -> &[u8] {
        self.seq.as_bytes()
    }

    fn clear(&mut self) {
        self.header.clear();
        self.seq.clear();
    }
}

/// An iterator over the records of a Fasta file.
pub struct Records<R: io::Read> {
    reader: Reader<R>,
}

impl<R: io::Read> Iterator for Records<R> {
    type Item = io::Result<Record>;

    fn next(&mut self) -> Option<io::Result<Record>> {
        let mut record = Record::new();
        match self.reader.read(&mut record) {
            Ok(()) if record.is_empty() => None,
            Ok(())   => Some(Ok(record)),
            Err(err) => Some(Err(err))
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io;

    const FASTA_FILE: &'static [u8] = b">id desc
ACCGTAGGCTGA
";
    const FAI_FILE: &'static [u8] = b"id\t12\t9\t60\t61
";

    #[test]
    fn test_reader() {
        let reader = Reader::new(FASTA_FILE);
        let records: Vec<io::Result<Record>> = reader.records().collect();
        assert!(records.len() == 1);
        for res in records {
            let record = res.ok().unwrap();
            assert_eq!(record.check(), Ok(()));
            assert_eq!(record.id(), Some("id"));
            assert_eq!(record.desc(), ["desc"]);
            assert_eq!(record.seq(), b"ACCGTAGGCTGA");
        }
    }

    #[test]
    fn test_indexed_reader() {
        let mut reader = IndexedReader::new(io::Cursor::new(FASTA_FILE), FAI_FILE).ok().expect("Error reading index");
        let mut seq = Vec::new();
        reader.read(b"id", 1, 5, &mut seq).ok().expect("Error reading sequence.");
        assert_eq!(seq, b"CCGT");
    }

    #[test]
    fn test_writer() {
        let mut writer = Writer::new(Vec::new());
        writer.write("id", &["desc"], b"ACCGTAGGCTGA").ok().expect("Expected successful write");
        writer.flush().ok().expect("Expected successful write");
        assert_eq!(writer.writer.get_ref(), &FASTA_FILE);
    }
}
<commit_msg>Correct order for sequences in IndexedReader.<commit_after>// Copyright 2014 Johannes Köster, Christopher Schröder.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Fasta reading and writing.
//!
//! # Example
//!
//! ```
//! use std::io;
//! use bio::io::fasta;
//! let reader = fasta::Reader::new(io::stdin());
//! ```

use std::io;
use std::io::prelude::*;
use std::ascii::AsciiExt;
use std::collections;
use std::fs;
use std::path::Path;
use std::convert::AsRef;

use csv;

pub struct Reader<R: io::Read> {
    reader: io::BufReader<R>,
    line: String
}

impl Reader<fs::File> {
    pub fn from_file<P: AsRef<Path>>(path: P) -> io::Result<Self> {
        fs::File::open(path).map(|f| Reader::new(f))
    }
}

impl<R: io::Read> Reader<R> {
    /// Create a new FastQ reader.
    pub fn new(reader: R) -> Self {
        Reader { reader: io::BufReader::new(reader), line: String::new() }
    }

    pub fn read(&mut self, record: &mut Record) -> io::Result<()> {
        record.clear();
        if self.line.is_empty() {
            try!(self.reader.read_line(&mut self.line));
            if self.line.is_empty() {
                return Ok(());
            }
        }

        if !self.line.starts_with(">") {
            return Err(io::Error::new(
                io::ErrorKind::Other,
                "Expected > at record start."
            ));
        }
        record.header.push_str(&self.line);
        loop {
            self.line.clear();
            try!(self.reader.read_line(&mut self.line));
            record.seq.push_str(&self.line.trim_right());
            if self.line.is_empty() || self.line.starts_with(">") {
                break;
            }
        }

        Ok(())
    }

    /// Return an iterator over the records of this FastQ file.
    pub fn records(self) -> Records<R> {
        Records { reader: self }
    }
}

pub struct Index {
    inner: collections::HashMap<Vec<u8>, IndexRecord>,
    seqs: Vec<Vec<u8>>,
}

impl Index {
    pub fn new<R: io::Read>(fai: R) -> csv::Result<Self> {
        let mut inner = collections::HashMap::new();
        let mut seqs = vec![];
        let mut fai_reader = csv::Reader::from_reader(fai).delimiter(b'\t').has_headers(false);
        for row in fai_reader.decode() {
            let (name, record): (String, IndexRecord) = try!(row);
            seqs.push(name.clone().into_bytes());
            inner.insert(name.into_bytes(), record);
        }
        Ok(Index { inner: inner, seqs: seqs })
    }

    pub fn from_file<P: AsRef<Path>>(path: &P) -> csv::Result<Self> {
        match fs::File::open(path) {
            Ok(fai) => Self::new(fai),
            Err(e)  => Err(csv::Error::Io(e))
        }
    }

    pub fn with_fasta_file<P: AsRef<Path>>(fasta_path: &P) -> csv::Result<Self> {
        let mut ext = fasta_path.as_ref().extension().unwrap().to_str().unwrap().to_string();
        ext.push_str(".fai");
        let fai_path = fasta_path.as_ref().with_extension(ext);

        Self::from_file(&fai_path)
    }

    pub fn sequences(&self) -> Vec<Sequence> {
        self.seqs.iter().map(|name| Sequence { name: name.clone(), len: self.inner.get(name).unwrap().len }).collect()
    }
}

pub struct IndexedReader<R: io::Read + io::Seek> {
    reader: io::BufReader<R>,
    pub index: Index,
}

impl IndexedReader<fs::File> {
    pub fn from_file<P: AsRef<Path>>(path: &P) -> csv::Result<Self> {
        let index = try!(Index::with_fasta_file(path));

        match fs::File::open(path) {
            Ok(fasta) => Ok(IndexedReader::with_index(fasta, index)),
            Err(e)    => Err(csv::Error::Io(e))
        }
    }
}

impl<R: io::Read + io::Seek> IndexedReader<R> {
    pub fn new<I: io::Read>(fasta: R, fai: I) -> csv::Result<Self> {
        let index = try!(Index::new(fai));
        Ok(IndexedReader { reader: io::BufReader::new(fasta), index: index })
    }

    pub fn with_index(fasta: R, index: Index) -> Self {
        IndexedReader { reader: io::BufReader::new(fasta), index: index }
    }

    pub fn read_all(&mut self, seqname: &[u8], seq: &mut Vec<u8>) -> io::Result<()> {
        match self.index.inner.get(seqname) {
            Some(&idx) => self.read(seqname, 0, idx.len, seq),
            None      => Err(
                io::Error::new(
                    io::ErrorKind::Other,
                    "Unknown sequence name."
                )
            )
        }
    }

    pub fn read(&mut self, seqname: &[u8], start: u64, stop: u64, seq: &mut Vec<u8>) -> io::Result<()> {
        match self.index.inner.get(seqname) {
            Some(idx) => {
                seq.clear();
                // derived from
                // http://www.allenyu.info/item/24-quickly-fetch-sequence-from-samtools-faidx-indexed-fasta-sequences.html
                let line = start / idx.line_bases * idx.line_bytes;
                let line_offset = start % idx.line_bases;
                let offset = idx.offset + line + line_offset;
                let lines = stop / idx.line_bases * idx.line_bytes - line;
                let line_stop = stop % idx.line_bases - if lines == 0 { line_offset } else { 0 };

                try!(self.reader.seek(io::SeekFrom::Start(offset)));
                let mut buf = vec![0u8; idx.line_bases as usize];
                for _ in 0..lines {
                    // read full lines
                    try!(self.reader.read(&mut buf));
                    seq.push_all(&buf);
                }
                // read last line
                println!("linestop {}", line_stop);
                try!(self.reader.read(&mut buf[..line_stop as usize]));
                seq.push_all(&buf[..line_stop as usize]);
                Ok(())
            },
            None      => Err(
                io::Error::new(
                    io::ErrorKind::Other,
                    "Unknown sequence name."
                )
            )
        }
    }
}

#[derive(RustcDecodable, Debug, Copy, Clone)]
struct IndexRecord {
    len: u64,
    offset: u64,
    line_bases: u64,
    line_bytes: u64,
}

pub struct Sequence {
    pub name: Vec<u8>,
    pub len: u64,
}

/// A Fasta writer.
pub struct Writer<W: io::Write> {
    writer: io::BufWriter<W>,
}

impl Writer<fs::File> {
    pub fn from_file<P: AsRef<Path>>(path: P) -> io::Result<Self> {
        fs::File::create(path).map(|f| Writer::new(f))
    }
}

impl<W: io::Write> Writer<W> {
    /// Create a new Fasta writer.
    pub fn new(writer: W) -> Self {
        Writer { writer: io::BufWriter::new(writer) }
    }

    /// Directly write a Fasta record.
    pub fn write_record(&mut self, record: Record) -> io::Result<()> {
        self.write(record.id().unwrap_or(""), &record.desc(), record.seq())
    }

    /// Write a Fasta record with given values.
    ///
    /// # Arguments
    ///
    /// * `id` - the record id
    /// * `desc` - the optional descriptions
    /// * `seq` - the sequence
    pub fn write(&mut self, id: &str, desc: &[&str], seq: &[u8]) -> io::Result<()> {
        try!(self.writer.write(b">"));
        try!(self.writer.write(id.as_bytes()));
        if !desc.is_empty() {
            for d in desc {
                try!(self.writer.write(b" "));
                try!(self.writer.write(d.as_bytes()));
            }
        }
        try!(self.writer.write(b"\n"));
        try!(self.writer.write(seq));
        try!(self.writer.write(b"\n"));

        Ok(())
    }

    /// Flush the writer, ensuring that everything is written.
    pub fn flush(&mut self) -> io::Result<()> {
        self.writer.flush()
    }
}

pub struct Record {
    header: String,
    seq: String,
}

impl Record {
    pub fn new() -> Self {
        Record { header: String::new(), seq: String::new() }
    }

    pub fn is_empty(&self) -> bool {
        self.header.is_empty() && self.seq.is_empty()
    }

    /// Check validity of Fasta record.
    pub fn check(&self) -> Result<(), &str> {
        if self.id().is_none() {
            return Err("Expecting id for FastQ record.");
        }
        if !self.seq.is_ascii() {
            return Err("Non-ascii character found in sequence.");
        }

        Ok(())
    }

    /// Return the id of the record.
    pub fn id(&self) -> Option<&str> {
        self.header[1..].split_whitespace().next()
    }

    /// Return descriptions if present.
    pub fn desc(&self) -> Vec<&str> {
        self.header[1..].split_whitespace().skip(1).collect()
    }

    /// Return the sequence of the record.
    pub fn seq(&self) -> &[u8] {
        self.seq.as_bytes()
    }

    fn clear(&mut self) {
        self.header.clear();
        self.seq.clear();
    }
}

/// An iterator over the records of a Fasta file.
pub struct Records<R: io::Read> {
    reader: Reader<R>,
}

impl<R: io::Read> Iterator for Records<R> {
    type Item = io::Result<Record>;

    fn next(&mut self) -> Option<io::Result<Record>> {
        let mut record = Record::new();
        match self.reader.read(&mut record) {
            Ok(()) if record.is_empty() => None,
            Ok(())   => Some(Ok(record)),
            Err(err) => Some(Err(err))
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io;

    const FASTA_FILE: &'static [u8] = b">id desc
ACCGTAGGCTGA
";
    const FAI_FILE: &'static [u8] = b"id\t12\t9\t60\t61
";

    #[test]
    fn test_reader() {
        let reader = Reader::new(FASTA_FILE);
        let records: Vec<io::Result<Record>> = reader.records().collect();
        assert!(records.len() == 1);
        for res in records {
            let record = res.ok().unwrap();
            assert_eq!(record.check(), Ok(()));
            assert_eq!(record.id(), Some("id"));
            assert_eq!(record.desc(), ["desc"]);
            assert_eq!(record.seq(), b"ACCGTAGGCTGA");
        }
    }

    #[test]
    fn test_indexed_reader() {
        let mut reader = IndexedReader::new(io::Cursor::new(FASTA_FILE), FAI_FILE).ok().expect("Error reading index");
        let mut seq = Vec::new();
        reader.read(b"id", 1, 5, &mut seq).ok().expect("Error reading sequence.");
        assert_eq!(seq, b"CCGT");
    }

    #[test]
    fn test_writer() {
        let mut writer = Writer::new(Vec::new());
        writer.write("id", &["desc"], b"ACCGTAGGCTGA").ok().expect("Expected successful write");
        writer.flush().ok().expect("Expected successful write");
        assert_eq!(writer.writer.get_ref(), &FASTA_FILE);
    }
}
<|endoftext|>


<commit_before>//! Code to trace out a single lane, collecting information into the
//! lane table as we go.

use collections::Set;
use grammar::repr::*;
use lr1::core::*;
use lr1::first::FirstSets;
use lr1::lookahead::*;
use lr1::state_graph::StateGraph;

use super::table::{ConflictIndex, LaneTable};

pub struct LaneTracer<'trace, 'grammar: 'trace, L: Lookahead + 'trace> {
    states: &'trace [State<'grammar, L>],
    first_sets: FirstSets,
    state_graph: StateGraph,
    table: LaneTable<'grammar>,
}

impl<'trace, 'grammar, L: Lookahead> LaneTracer<'trace, 'grammar, L> {
    pub fn new(grammar: &'grammar Grammar,
               states: &'trace [State<'grammar, L>],
               conflicts: usize)
               -> Self {
        LaneTracer {
            states: states,
            first_sets: FirstSets::new(grammar),
            state_graph: StateGraph::new(states),
            table: LaneTable::new(grammar, conflicts),
        }
    }

    pub fn into_table(self) -> LaneTable<'grammar> {
        self.table
    }

    pub fn start_trace(&mut self,
                       state: StateIndex,
                       conflict: ConflictIndex,
                       item: Item<'grammar, L>) {
        let mut visited_set = Set::default();

        // if the conflict item is a "shift" item, then the context
        // is always the terminal to shift (and conflicts only arise
        // around shifting terminal, so it must be a terminal)
        match item.shift_symbol() {
            Some((Symbol::Terminal(term), _)) => {
                let mut token_set = TokenSet::new();
                token_set.insert(Token::Terminal(term));
                self.table.add_lookahead(state, conflict, &token_set);
            }

            Some((Symbol::Nonterminal(_), _)) => {
                panic!("invalid conflict item `{:?}`: shifts nonterminal",
                       item);
            }

            None => {
                self.continue_trace(state, conflict, item, &mut visited_set);
            }
        }
    }

    fn continue_trace(&mut self,
                      state: StateIndex,
                      conflict: ConflictIndex,
                      item: Item<'grammar, L>,
                      visited: &mut Set<(StateIndex, Item<'grammar, L>)>) {
        if !visited.insert((state, item.clone())) {
            return;
        }

        if item.index > 0 {
            // This item was reached by shifting some symbol.  We need
            // to unshift that symbol, which means we walk backwards
            // to predecessors of `state` in the state graph.
            //
            // Example:
            //
            //     X = ...p T (*) ...s
            //
            // Here we would be "unshifting" T, which means we will
            // walk to predecessors of the current state that were
            // reached by shifting T. Those predecessors will contain
            // an item like `X = ...p (*) T ...s`, which we will then
            // process in turn.
            let shifted_symbol = item.production.symbols[item.index - 1];
            let unshifted_item = Item { index: item.index - 1, ..item };
            let predecessors = self.state_graph.predecessors(state, shifted_symbol);
            for predecessor in predecessors {
                self.table.add_successor(predecessor, state);
                self.continue_trace(predecessor, conflict, unshifted_item.clone(), visited);
            }
            return;
        }

        // Either: we are in the start state, or this item was
        // reached by an epsilon transition. We have to
        // "unepsilon", which means that we search elsewhere in
        // the state for where the epsilon transition could have
        // come from.
        //
        // Example:
        //
        //     X = (*) ...
        //
        // We will search for other items in the same state like:
        //
        //     Y = ...p (*) X ...s
        //
        // We can then insert `FIRST(...s)` as lookahead for
        // `conflict`. If `...s` may derive epsilon, though, we
        // have to recurse and search with the previous item.

        let state_items = &self.states[state.0].items.vec;
        let nonterminal = item.production.nonterminal;
        for pred_item in state_items.iter()
                                    .filter(|i| i.can_shift_nonterminal(nonterminal)) {
            let symbol_sets = pred_item.symbol_sets();
            let mut first = self.first_sets.first0(symbol_sets.suffix);
            let derives_epsilon = first.take_eof();
            self.table.add_lookahead(state, conflict, &first);
            if derives_epsilon {
                self.continue_trace(state, conflict, pred_item.clone(), visited);
            }
        }
    }
}
<commit_msg>convert the lane-table construction to always use LR0 internally<commit_after>//! Code to trace out a single lane, collecting information into the
//! lane table as we go.

use collections::Set;
use grammar::repr::*;
use lr1::core::*;
use lr1::first::FirstSets;
use lr1::lookahead::*;
use lr1::state_graph::StateGraph;

use super::table::{ConflictIndex, LaneTable};

pub struct LaneTracer<'trace, 'grammar: 'trace, L: Lookahead + 'trace> {
    states: &'trace [State<'grammar, L>],
    first_sets: FirstSets,
    state_graph: StateGraph,
    table: LaneTable<'grammar>,
}

impl<'trace, 'grammar, L: Lookahead> LaneTracer<'trace, 'grammar, L> {
    pub fn new(grammar: &'grammar Grammar,
               states: &'trace [State<'grammar, L>],
               conflicts: usize)
               -> Self {
        LaneTracer {
            states: states,
            first_sets: FirstSets::new(grammar),
            state_graph: StateGraph::new(states),
            table: LaneTable::new(grammar, conflicts),
        }
    }

    pub fn into_table(self) -> LaneTable<'grammar> {
        self.table
    }

    pub fn start_trace(&mut self,
                       state: StateIndex,
                       conflict: ConflictIndex,
                       item: LR0Item<'grammar>) {
        let mut visited_set = Set::default();

        // if the conflict item is a "shift" item, then the context
        // is always the terminal to shift (and conflicts only arise
        // around shifting terminal, so it must be a terminal)
        match item.shift_symbol() {
            Some((Symbol::Terminal(term), _)) => {
                let mut token_set = TokenSet::new();
                token_set.insert(Token::Terminal(term));
                self.table.add_lookahead(state, conflict, &token_set);
            }

            Some((Symbol::Nonterminal(_), _)) => {
                panic!("invalid conflict item `{:?}`: shifts nonterminal",
                       item);
            }

            None => {
                self.continue_trace(state, conflict, item, &mut visited_set);
            }
        }
    }

    fn continue_trace(&mut self,
                      state: StateIndex,
                      conflict: ConflictIndex,
                      item: LR0Item<'grammar>,
                      visited: &mut Set<(StateIndex, LR0Item<'grammar>)>) {
        if !visited.insert((state, item)) {
            return;
        }

        if item.index > 0 {
            // This item was reached by shifting some symbol.  We need
            // to unshift that symbol, which means we walk backwards
            // to predecessors of `state` in the state graph.
            //
            // Example:
            //
            //     X = ...p T (*) ...s
            //
            // Here we would be "unshifting" T, which means we will
            // walk to predecessors of the current state that were
            // reached by shifting T. Those predecessors will contain
            // an item like `X = ...p (*) T ...s`, which we will then
            // process in turn.
            let shifted_symbol = item.production.symbols[item.index - 1];
            let unshifted_item = Item { index: item.index - 1, ..item };
            let predecessors = self.state_graph.predecessors(state, shifted_symbol);
            for predecessor in predecessors {
                self.table.add_successor(predecessor, state);
                self.continue_trace(predecessor, conflict, unshifted_item, visited);
            }
            return;
        }

        // Either: we are in the start state, or this item was
        // reached by an epsilon transition. We have to
        // "unepsilon", which means that we search elsewhere in
        // the state for where the epsilon transition could have
        // come from.
        //
        // Example:
        //
        //     X = (*) ...
        //
        // We will search for other items in the same state like:
        //
        //     Y = ...p (*) X ...s
        //
        // We can then insert `FIRST(...s)` as lookahead for
        // `conflict`. If `...s` may derive epsilon, though, we
        // have to recurse and search with the previous item.

        let state_items = &self.states[state.0].items.vec;
        let nonterminal = item.production.nonterminal;
        for pred_item in state_items.iter()
                                    .filter(|i| i.can_shift_nonterminal(nonterminal)) {
            let symbol_sets = pred_item.symbol_sets();
            let mut first = self.first_sets.first0(symbol_sets.suffix);
            let derives_epsilon = first.take_eof();
            self.table.add_lookahead(state, conflict, &first);
            if derives_epsilon {
                self.continue_trace(state, conflict, pred_item.to_lr0(), visited);
            }
        }
    }
}
<|endoftext|>