<commit_before>// Copyright 2013-2014 The rust-url developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! [*Unicode IDNA Compatibility Processing*
//! (Unicode Technical Standard #46)](http://www.unicode.org/reports/tr46/)

use self::Mapping::*;
use punycode;
#[allow(unused_imports, deprecated)]
use std::ascii::AsciiExt;
use std::cmp::Ordering::{Equal, Less, Greater};
use unicode_bidi::{BidiClass, bidi_class};
use unicode_normalization::UnicodeNormalization;
use unicode_normalization::char::is_combining_mark;

include!("uts46_mapping_table.rs");

pub static PUNYCODE_PREFIX: &'static str = "xn--";

#[derive(Debug)]
struct StringTableSlice {
    // Store these as separate fields so the structure will have an
    // alignment of 1 and thus pack better into the Mapping enum, below.
    byte_start_lo: u8,
    byte_start_hi: u8,
    byte_len: u8,
}

fn decode_slice(slice: &StringTableSlice) -> &'static str {
    let lo = slice.byte_start_lo as usize;
    let hi = slice.byte_start_hi as usize;
    let start = (hi << 8) | lo;
    let len = slice.byte_len as usize;
    &STRING_TABLE[start..(start + len)]
}

#[repr(u8)]
#[derive(Debug)]
enum Mapping {
    Valid,
    Ignored,
    Mapped(StringTableSlice),
    Deviation(StringTableSlice),
    Disallowed,
    DisallowedStd3Valid,
    DisallowedStd3Mapped(StringTableSlice),
}

struct Range {
    from: char,
    to: char,
}

fn find_char(codepoint: char) -> &'static Mapping {
    let r = TABLE.binary_search_by(|ref range| {
        if codepoint > range.to {
            Less
        } else if codepoint < range.from {
            Greater
        } else {
            Equal
        }
    });
    r.ok().map(|i| {
        const SINGLE_MARKER: u16 = 1 << 15;

        let x = INDEX_TABLE[i];
        let single = (x & SINGLE_MARKER) != 0;
        let offset = !SINGLE_MARKER & x;

        if single {
            &MAPPING_TABLE[offset as usize]
        } else {
            &MAPPING_TABLE[(offset + (codepoint as u16 - TABLE[i].from as u16)) as usize]
        }
    }).unwrap()
}

fn map_char(codepoint: char, flags: Flags, output: &mut String, errors: &mut Vec<Error>) {
    match *find_char(codepoint) {
        Mapping::Valid => output.push(codepoint),
        Mapping::Ignored => {},
        Mapping::Mapped(ref slice) => output.push_str(decode_slice(slice)),
        Mapping::Deviation(ref slice) => {
            if flags.transitional_processing {
                output.push_str(decode_slice(slice))
            } else {
                output.push(codepoint)
            }
        }
        Mapping::Disallowed => {
            errors.push(Error::DissallowedCharacter);
            output.push(codepoint);
        }
        Mapping::DisallowedStd3Valid => {
            if flags.use_std3_ascii_rules {
                errors.push(Error::DissallowedByStd3AsciiRules);
            }
            output.push(codepoint)
        }
        Mapping::DisallowedStd3Mapped(ref slice) => {
            if flags.use_std3_ascii_rules {
                errors.push(Error::DissallowedMappedInStd3);
            }
            output.push_str(decode_slice(slice))
        }
    }
}

// http://tools.ietf.org/html/rfc5893#section-2
fn passes_bidi(label: &str, is_bidi_domain: bool) -> bool {
    // Rule 0: Bidi Rules apply to Bidi Domain Names: a name with at least one RTL label.  A label
    // is RTL if it contains at least one character of bidi class R, AL or AN.
    if !is_bidi_domain {
        return true;
    }

    let mut chars = label.chars();
    let first_char_class = match chars.next() {
        Some(c) => bidi_class(c),
        None => return true, // empty string
    };

    match first_char_class {
        // LTR label
        BidiClass::L => {
            // Rule 5
            loop {
                match chars.next() {
                    Some(c) => {
                        if !matches!(bidi_class(c),
                                     BidiClass::L | BidiClass::EN |
                                     BidiClass::ES | BidiClass::CS |
                                     BidiClass::ET | BidiClass::ON |
                                     BidiClass::BN | BidiClass::NSM
                                    ) {
                            return false;
                        }
                    },
                    None => { break; },
                }
            }

            // Rule 6
            // must end in L or EN followed by 0 or more NSM
            let mut rev_chars = label.chars().rev();
            let mut last_non_nsm = rev_chars.next();
            loop {
                match last_non_nsm {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last_non_nsm = rev_chars.next();
                        continue;
                    }
                    _ => { break; },
                }
            }
            match last_non_nsm {
                Some(c) if bidi_class(c) == BidiClass::L
                    || bidi_class(c) == BidiClass::EN => {},
                Some(_) => { return false; },
                _ => {}
            }

        }

        // RTL label
        BidiClass::R | BidiClass::AL => {
            let mut found_en = false;
            let mut found_an = false;

            // Rule 2
            loop {
                match chars.next() {
                    Some(c) => {
                        let char_class = bidi_class(c);

                        if char_class == BidiClass::EN {
                            found_en = true;
                        }
                        if char_class == BidiClass::AN {
                            found_an = true;
                        }

                        if !matches!(char_class, BidiClass::R | BidiClass::AL |
                                     BidiClass::AN | BidiClass::EN |
                                     BidiClass::ES | BidiClass::CS |
                                     BidiClass::ET | BidiClass::ON |
                                     BidiClass::BN | BidiClass::NSM) {
                            return false;
                        }
                    },
                    None => { break; },
                }
            }
            // Rule 3
            let mut rev_chars = label.chars().rev();
            let mut last = rev_chars.next();
            loop { // must end in L or EN followed by 0 or more NSM
                match last {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last = rev_chars.next();
                        continue;
                    }
                    _ => { break; },
                }
            }
            match last {
                Some(c) if matches!(bidi_class(c), BidiClass::R | BidiClass::AL |
                                    BidiClass::EN | BidiClass::AN) => {},
                _ => { return false; }
            }

            // Rule 4
            if found_an && found_en {
                return false;
            }
        }

        // Rule 1: Should start with L or R/AL
        _ => {
            return false;
        }
    }

    return true;
}

/// http://www.unicode.org/reports/tr46/#Validity_Criteria
fn validate_full(label: &str, is_bidi_domain: bool, config: Config, errors: &mut Vec<Error>) {
    // V1: Must be in NFC form.
    if label.nfc().ne(label.chars()) {
        errors.push(Error::ValidityCriteria);
    } else {
        validate(label, is_bidi_domain, config, errors);
    }
}

fn validate(label: &str, is_bidi_domain: bool, config: Config, errors: &mut Vec<Error>) {
    let first_char = label.chars().next();
    if first_char == None {
        // Empty string, pass
    }

    // V2: No U+002D HYPHEN-MINUS in both third and fourth positions.
    //
    // NOTE: Spec says that the label must not contain a HYPHEN-MINUS character in both the
    // third and fourth positions. But nobody follows this criteria. See the spec issue below:
    // https://github.com/whatwg/url/issues/53
    //
    // TODO: Add *CheckHyphens* flag.

    // V3: neither begin nor end with a U+002D HYPHEN-MINUS
    else if label.starts_with("-") || label.ends_with("-") {
        errors.push(Error::ValidityCriteria);
    }

    // V4: not contain a U+002E FULL STOP
    //
    // Here, label can't contain '.' since the input is from .split('.')

    // V5: not begin with a GC=Mark
    else if is_combining_mark(first_char.unwrap()) {
        errors.push(Error::ValidityCriteria);
    }

    // V6: Check against Mapping Table
    else if label.chars().any(|c| match *find_char(c) {
        Mapping::Valid => false,
        Mapping::Deviation(_) => config.flags.transitional_processing,
        Mapping::DisallowedStd3Valid => config.flags.use_std3_ascii_rules,
        _ => true,
    }) {
        errors.push(Error::ValidityCriteria);
    }

    // V7: ContextJ rules
    //
    // TODO: Implement rules and add *CheckJoiners* flag.

    // V8: Bidi rules
    //
    // TODO: Add *CheckBidi* flag
    else if !passes_bidi(label, is_bidi_domain)
    {
        errors.push(Error::ValidityCriteria);
    }
}

/// http://www.unicode.org/reports/tr46/#Processing
fn processing(domain: &str, config: Config, errors: &mut Vec<Error>) -> String {
    let mut mapped = String::with_capacity(domain.len());
    for c in domain.chars() {
        map_char(c, config.flags, &mut mapped, errors)
    }
    let mut normalized = String::with_capacity(mapped.len());
    normalized.extend(mapped.nfc());

    // Find out if it's a Bidi Domain Name
    //
    // First, check for literal bidi chars
    let mut is_bidi_domain = domain.chars().any(|c|
        matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN)
    );
    if !is_bidi_domain {
        // Then check for punycode-encoded bidi chars
        for label in normalized.split('.') {
            if label.starts_with(PUNYCODE_PREFIX) {
                match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                    Some(decoded_label) => {
                        if decoded_label.chars().any(|c|
                            matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN)
                        ) {
                            is_bidi_domain = true;
                        }
                    }
                    None => {
                        is_bidi_domain = true;
                    }
                }
            }
        }
    }

    let mut validated = String::new();
    let mut first = true;
    for label in normalized.split('.') {
        if !first {
            validated.push('.');
        }
        first = false;
        if label.starts_with(PUNYCODE_PREFIX) {
            match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                Some(decoded_label) => {
                    let config = config.transitional_processing(false);
                    validate_full(&decoded_label, is_bidi_domain, config, errors);
                    validated.push_str(&decoded_label)
                }
                None => errors.push(Error::PunycodeError)
            }
        } else {
            // `normalized` is already `NFC` so we can skip that check
            validate(label, is_bidi_domain, config, errors);
            validated.push_str(label)
        }
    }
    validated
}

#[derive(Clone, Copy)]
pub struct Config {
    flags: Flags,
}

impl From<Flags> for Config {
    #[inline]
    fn from(flags: Flags) -> Self {
        Self { flags }
    }
}

impl Config {
    #[inline]
    pub fn use_std3_ascii_rules(mut self, value: bool) -> Self {
        self.flags.use_std3_ascii_rules = value;
        self
    }

    #[inline]
    pub fn transitional_processing(mut self, value: bool) -> Self {
        self.flags.transitional_processing = value;
        self
    }

    #[inline]
    pub fn verify_dns_length(mut self, value: bool) -> Self {
        self.flags.verify_dns_length = value;
        self
    }

    /// http://www.unicode.org/reports/tr46/#ToASCII
    pub fn to_ascii(self, domain: &str) -> Result<String, Errors> {
        let mut errors = Vec::new();
        let mut result = String::new();
        let mut first = true;
        for label in processing(domain, self, &mut errors).split('.') {
            if !first {
                result.push('.');
            }
            first = false;
            if label.is_ascii() {
                result.push_str(label);
            } else {
                match punycode::encode_str(label) {
                    Some(x) => {
                        result.push_str(PUNYCODE_PREFIX);
                        result.push_str(&x);
                    },
                    None => errors.push(Error::PunycodeError)
                }
            }
        }

        if self.flags.verify_dns_length {
            let domain = if result.ends_with(".") { &result[..result.len()-1] } else { &*result };
            if domain.len() < 1 || domain.split('.').any(|label| label.len() < 1) {
                errors.push(Error::TooShortForDns)
            }
            if domain.len() > 253 || domain.split('.').any(|label| label.len() > 63) {
                errors.push(Error::TooLongForDns)
            }
        }
        if errors.is_empty() {
            Ok(result)
        } else {
            Err(Errors(errors))
        }
    }

    /// http://www.unicode.org/reports/tr46/#ToUnicode
    pub fn to_unicode(self, domain: &str) -> (String, Result<(), Errors>) {
        let mut errors = Vec::new();
        let domain = processing(domain, self, &mut errors);
        let errors = if errors.is_empty() {
            Ok(())
        } else {
            Err(Errors(errors))
        };
        (domain, errors)
    }

}

#[derive(Copy, Clone)]
pub struct Flags {
   pub use_std3_ascii_rules: bool,
   pub transitional_processing: bool,
   pub verify_dns_length: bool,
}

#[derive(PartialEq, Eq, Clone, Copy, Debug)]
enum Error {
    PunycodeError,
    ValidityCriteria,
    DissallowedByStd3AsciiRules,
    DissallowedMappedInStd3,
    DissallowedCharacter,
    TooLongForDns,
    TooShortForDns,
}

/// Errors recorded during UTS #46 processing.
///
/// This is opaque for now, only indicating the presence of at least one error.
/// More details may be exposed in the future.
#[derive(Debug)]
pub struct Errors(Vec<Error>);

/// http://www.unicode.org/reports/tr46/#ToASCII
pub fn to_ascii(domain: &str, flags: Flags) -> Result<String, Errors> {
    Config::from(flags).to_ascii(domain)
}

/// http://www.unicode.org/reports/tr46/#ToUnicode
///
/// Only `use_std3_ascii_rules` is used in `flags`.
pub fn to_unicode(domain: &str, mut flags: Flags) -> (String, Result<(), Errors>) {
    flags.transitional_processing = false;
    Config::from(flags).to_unicode(domain)
}
<commit_msg>Introduce Config::check_hyphens in uts46<commit_after>// Copyright 2013-2014 The rust-url developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! [*Unicode IDNA Compatibility Processing*
//! (Unicode Technical Standard #46)](http://www.unicode.org/reports/tr46/)

use self::Mapping::*;
use punycode;
#[allow(unused_imports, deprecated)]
use std::ascii::AsciiExt;
use std::cmp::Ordering::{Equal, Less, Greater};
use unicode_bidi::{BidiClass, bidi_class};
use unicode_normalization::UnicodeNormalization;
use unicode_normalization::char::is_combining_mark;

include!("uts46_mapping_table.rs");

pub static PUNYCODE_PREFIX: &'static str = "xn--";

#[derive(Debug)]
struct StringTableSlice {
    // Store these as separate fields so the structure will have an
    // alignment of 1 and thus pack better into the Mapping enum, below.
    byte_start_lo: u8,
    byte_start_hi: u8,
    byte_len: u8,
}

fn decode_slice(slice: &StringTableSlice) -> &'static str {
    let lo = slice.byte_start_lo as usize;
    let hi = slice.byte_start_hi as usize;
    let start = (hi << 8) | lo;
    let len = slice.byte_len as usize;
    &STRING_TABLE[start..(start + len)]
}

#[repr(u8)]
#[derive(Debug)]
enum Mapping {
    Valid,
    Ignored,
    Mapped(StringTableSlice),
    Deviation(StringTableSlice),
    Disallowed,
    DisallowedStd3Valid,
    DisallowedStd3Mapped(StringTableSlice),
}

struct Range {
    from: char,
    to: char,
}

fn find_char(codepoint: char) -> &'static Mapping {
    let r = TABLE.binary_search_by(|ref range| {
        if codepoint > range.to {
            Less
        } else if codepoint < range.from {
            Greater
        } else {
            Equal
        }
    });
    r.ok().map(|i| {
        const SINGLE_MARKER: u16 = 1 << 15;

        let x = INDEX_TABLE[i];
        let single = (x & SINGLE_MARKER) != 0;
        let offset = !SINGLE_MARKER & x;

        if single {
            &MAPPING_TABLE[offset as usize]
        } else {
            &MAPPING_TABLE[(offset + (codepoint as u16 - TABLE[i].from as u16)) as usize]
        }
    }).unwrap()
}

fn map_char(codepoint: char, flags: Flags, output: &mut String, errors: &mut Vec<Error>) {
    match *find_char(codepoint) {
        Mapping::Valid => output.push(codepoint),
        Mapping::Ignored => {},
        Mapping::Mapped(ref slice) => output.push_str(decode_slice(slice)),
        Mapping::Deviation(ref slice) => {
            if flags.transitional_processing {
                output.push_str(decode_slice(slice))
            } else {
                output.push(codepoint)
            }
        }
        Mapping::Disallowed => {
            errors.push(Error::DissallowedCharacter);
            output.push(codepoint);
        }
        Mapping::DisallowedStd3Valid => {
            if flags.use_std3_ascii_rules {
                errors.push(Error::DissallowedByStd3AsciiRules);
            }
            output.push(codepoint)
        }
        Mapping::DisallowedStd3Mapped(ref slice) => {
            if flags.use_std3_ascii_rules {
                errors.push(Error::DissallowedMappedInStd3);
            }
            output.push_str(decode_slice(slice))
        }
    }
}

// http://tools.ietf.org/html/rfc5893#section-2
fn passes_bidi(label: &str, is_bidi_domain: bool) -> bool {
    // Rule 0: Bidi Rules apply to Bidi Domain Names: a name with at least one RTL label.  A label
    // is RTL if it contains at least one character of bidi class R, AL or AN.
    if !is_bidi_domain {
        return true;
    }

    let mut chars = label.chars();
    let first_char_class = match chars.next() {
        Some(c) => bidi_class(c),
        None => return true, // empty string
    };

    match first_char_class {
        // LTR label
        BidiClass::L => {
            // Rule 5
            loop {
                match chars.next() {
                    Some(c) => {
                        if !matches!(bidi_class(c),
                                     BidiClass::L | BidiClass::EN |
                                     BidiClass::ES | BidiClass::CS |
                                     BidiClass::ET | BidiClass::ON |
                                     BidiClass::BN | BidiClass::NSM
                                    ) {
                            return false;
                        }
                    },
                    None => { break; },
                }
            }

            // Rule 6
            // must end in L or EN followed by 0 or more NSM
            let mut rev_chars = label.chars().rev();
            let mut last_non_nsm = rev_chars.next();
            loop {
                match last_non_nsm {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last_non_nsm = rev_chars.next();
                        continue;
                    }
                    _ => { break; },
                }
            }
            match last_non_nsm {
                Some(c) if bidi_class(c) == BidiClass::L
                    || bidi_class(c) == BidiClass::EN => {},
                Some(_) => { return false; },
                _ => {}
            }

        }

        // RTL label
        BidiClass::R | BidiClass::AL => {
            let mut found_en = false;
            let mut found_an = false;

            // Rule 2
            loop {
                match chars.next() {
                    Some(c) => {
                        let char_class = bidi_class(c);

                        if char_class == BidiClass::EN {
                            found_en = true;
                        }
                        if char_class == BidiClass::AN {
                            found_an = true;
                        }

                        if !matches!(char_class, BidiClass::R | BidiClass::AL |
                                     BidiClass::AN | BidiClass::EN |
                                     BidiClass::ES | BidiClass::CS |
                                     BidiClass::ET | BidiClass::ON |
                                     BidiClass::BN | BidiClass::NSM) {
                            return false;
                        }
                    },
                    None => { break; },
                }
            }
            // Rule 3
            let mut rev_chars = label.chars().rev();
            let mut last = rev_chars.next();
            loop { // must end in L or EN followed by 0 or more NSM
                match last {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last = rev_chars.next();
                        continue;
                    }
                    _ => { break; },
                }
            }
            match last {
                Some(c) if matches!(bidi_class(c), BidiClass::R | BidiClass::AL |
                                    BidiClass::EN | BidiClass::AN) => {},
                _ => { return false; }
            }

            // Rule 4
            if found_an && found_en {
                return false;
            }
        }

        // Rule 1: Should start with L or R/AL
        _ => {
            return false;
        }
    }

    return true;
}

/// http://www.unicode.org/reports/tr46/#Validity_Criteria
fn validate_full(label: &str, is_bidi_domain: bool, config: Config, errors: &mut Vec<Error>) {
    // V1: Must be in NFC form.
    if label.nfc().ne(label.chars()) {
        errors.push(Error::ValidityCriteria);
    } else {
        validate(label, is_bidi_domain, config, errors);
    }
}

fn validate(label: &str, is_bidi_domain: bool, config: Config, errors: &mut Vec<Error>) {
    let first_char = label.chars().next();
    if first_char == None {
        // Empty string, pass
    }

    // V2: No U+002D HYPHEN-MINUS in both third and fourth positions.
    //
    // NOTE: Spec says that the label must not contain a HYPHEN-MINUS character in both the
    // third and fourth positions. But nobody follows this criteria. See the spec issue below:
    // https://github.com/whatwg/url/issues/53

    // V3: neither begin nor end with a U+002D HYPHEN-MINUS
    else if config.check_hyphens && (label.starts_with("-") || label.ends_with("-")) {
        errors.push(Error::ValidityCriteria);
    }

    // V4: not contain a U+002E FULL STOP
    //
    // Here, label can't contain '.' since the input is from .split('.')

    // V5: not begin with a GC=Mark
    else if is_combining_mark(first_char.unwrap()) {
        errors.push(Error::ValidityCriteria);
    }

    // V6: Check against Mapping Table
    else if label.chars().any(|c| match *find_char(c) {
        Mapping::Valid => false,
        Mapping::Deviation(_) => config.flags.transitional_processing,
        Mapping::DisallowedStd3Valid => config.flags.use_std3_ascii_rules,
        _ => true,
    }) {
        errors.push(Error::ValidityCriteria);
    }

    // V7: ContextJ rules
    //
    // TODO: Implement rules and add *CheckJoiners* flag.

    // V8: Bidi rules
    //
    // TODO: Add *CheckBidi* flag
    else if !passes_bidi(label, is_bidi_domain)
    {
        errors.push(Error::ValidityCriteria);
    }
}

/// http://www.unicode.org/reports/tr46/#Processing
fn processing(domain: &str, config: Config, errors: &mut Vec<Error>) -> String {
    let mut mapped = String::with_capacity(domain.len());
    for c in domain.chars() {
        map_char(c, config.flags, &mut mapped, errors)
    }
    let mut normalized = String::with_capacity(mapped.len());
    normalized.extend(mapped.nfc());

    // Find out if it's a Bidi Domain Name
    //
    // First, check for literal bidi chars
    let mut is_bidi_domain = domain.chars().any(|c|
        matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN)
    );
    if !is_bidi_domain {
        // Then check for punycode-encoded bidi chars
        for label in normalized.split('.') {
            if label.starts_with(PUNYCODE_PREFIX) {
                match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                    Some(decoded_label) => {
                        if decoded_label.chars().any(|c|
                            matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN)
                        ) {
                            is_bidi_domain = true;
                        }
                    }
                    None => {
                        is_bidi_domain = true;
                    }
                }
            }
        }
    }

    let mut validated = String::new();
    let mut first = true;
    for label in normalized.split('.') {
        if !first {
            validated.push('.');
        }
        first = false;
        if label.starts_with(PUNYCODE_PREFIX) {
            match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                Some(decoded_label) => {
                    let config = config.transitional_processing(false);
                    validate_full(&decoded_label, is_bidi_domain, config, errors);
                    validated.push_str(&decoded_label)
                }
                None => errors.push(Error::PunycodeError)
            }
        } else {
            // `normalized` is already `NFC` so we can skip that check
            validate(label, is_bidi_domain, config, errors);
            validated.push_str(label)
        }
    }
    validated
}

#[derive(Clone, Copy)]
pub struct Config {
    flags: Flags,
    check_hyphens: bool,
}

impl From<Flags> for Config {
    #[inline]
    fn from(flags: Flags) -> Self {
        Self { flags, check_hyphens: true }
    }
}

impl Config {
    #[inline]
    pub fn use_std3_ascii_rules(mut self, value: bool) -> Self {
        self.flags.use_std3_ascii_rules = value;
        self
    }

    #[inline]
    pub fn transitional_processing(mut self, value: bool) -> Self {
        self.flags.transitional_processing = value;
        self
    }

    #[inline]
    pub fn verify_dns_length(mut self, value: bool) -> Self {
        self.flags.verify_dns_length = value;
        self
    }

    #[inline]
    pub fn check_hyphens(mut self, value: bool) -> Self {
        self.check_hyphens = value;
        self
    }

    /// http://www.unicode.org/reports/tr46/#ToASCII
    pub fn to_ascii(self, domain: &str) -> Result<String, Errors> {
        let mut errors = Vec::new();
        let mut result = String::new();
        let mut first = true;
        for label in processing(domain, self, &mut errors).split('.') {
            if !first {
                result.push('.');
            }
            first = false;
            if label.is_ascii() {
                result.push_str(label);
            } else {
                match punycode::encode_str(label) {
                    Some(x) => {
                        result.push_str(PUNYCODE_PREFIX);
                        result.push_str(&x);
                    },
                    None => errors.push(Error::PunycodeError)
                }
            }
        }

        if self.flags.verify_dns_length {
            let domain = if result.ends_with(".") { &result[..result.len()-1] } else { &*result };
            if domain.len() < 1 || domain.split('.').any(|label| label.len() < 1) {
                errors.push(Error::TooShortForDns)
            }
            if domain.len() > 253 || domain.split('.').any(|label| label.len() > 63) {
                errors.push(Error::TooLongForDns)
            }
        }
        if errors.is_empty() {
            Ok(result)
        } else {
            Err(Errors(errors))
        }
    }

    /// http://www.unicode.org/reports/tr46/#ToUnicode
    pub fn to_unicode(self, domain: &str) -> (String, Result<(), Errors>) {
        let mut errors = Vec::new();
        let domain = processing(domain, self, &mut errors);
        let errors = if errors.is_empty() {
            Ok(())
        } else {
            Err(Errors(errors))
        };
        (domain, errors)
    }

}

#[derive(Copy, Clone)]
pub struct Flags {
   pub use_std3_ascii_rules: bool,
   pub transitional_processing: bool,
   pub verify_dns_length: bool,
}

#[derive(PartialEq, Eq, Clone, Copy, Debug)]
enum Error {
    PunycodeError,
    ValidityCriteria,
    DissallowedByStd3AsciiRules,
    DissallowedMappedInStd3,
    DissallowedCharacter,
    TooLongForDns,
    TooShortForDns,
}

/// Errors recorded during UTS #46 processing.
///
/// This is opaque for now, only indicating the presence of at least one error.
/// More details may be exposed in the future.
#[derive(Debug)]
pub struct Errors(Vec<Error>);

/// http://www.unicode.org/reports/tr46/#ToASCII
pub fn to_ascii(domain: &str, flags: Flags) -> Result<String, Errors> {
    Config::from(flags).to_ascii(domain)
}

/// http://www.unicode.org/reports/tr46/#ToUnicode
///
/// Only `use_std3_ascii_rules` is used in `flags`.
pub fn to_unicode(domain: &str, mut flags: Flags) -> (String, Result<(), Errors>) {
    flags.transitional_processing = false;
    Config::from(flags).to_unicode(domain)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Gfx-rs Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Various helper macros.

mod pso;
mod structure;

#[macro_export]
macro_rules! gfx_format {
    ($name:ident : $surface:ident = $container:ident<$channel:ident>) => {
        impl $crate::format::Formatted for $name {
            type Surface = $crate::format::$surface;
            type Channel = $crate::format::$channel;
            type View = $crate::format::$container<
                <$crate::format::$channel as $crate::format::ChannelTyped>::ShaderType
                >;
        }
    }
}

/// Defines vertex, constant and pipeline formats in one block
#[macro_export]
macro_rules! gfx_defines {
    ($(#[$attr:meta])* vertex $name:ident {
            $( $field:ident : $ty:ty = $e:expr, )+
    }) => {
        gfx_vertex_struct_meta!($(#[$attr])* vertex_struct_meta $name {$($field:$ty = $e,)+});
    };

    ($(#[$attr:meta])* constant $name:ident {
            $( $field:ident : $ty:ty = $e:expr, )+
    }) => {
        gfx_constant_struct_meta!($(#[$attr])* constant_struct_meta $name {$($field:$ty = $e,)+});
    };

    (pipeline $name:ident {
            $( $field:ident : $ty:ty = $e:expr, )+
    }) => {
        gfx_pipeline!($name {$($field:$ty = $e,)+});
    };

    // The recursive case for vertex structs
    ($(#[$attr:meta])* vertex $name:ident {
            $( $field:ident : $ty:ty = $e:expr, )+
    } $($tail:tt)+) => {
        gfx_defines! {
            $(#[$attr])*
            vertex $name { $($field : $ty = $e,)+ }
        }
        gfx_defines!($($tail)+);
    };

    // The recursive case for constant structs
    ($(#[$attr:meta])* constant $name:ident {
            $( $field:ident : $ty:ty = $e:expr, )+
    } $($tail:tt)+) => {
        gfx_defines! {
            $(#[$attr])*
            constant $name { $($field : $ty = $e,)+ }
        }
        gfx_defines!($($tail)+);
    };

    // The recursive case for the other keywords
    ($keyword:ident $name:ident {
            $( $field:ident : $ty:ty = $e:expr, )+
    } $($tail:tt)+) => {
        gfx_defines! {
            $keyword $name { $($field : $ty = $e,)+ }
        }
        gfx_defines!($($tail)+);
    };

    ($keyword:ident $name:ident {
            $( $field:ident : $ty:ty = $e:expr ),*
    }) => {
        gfx_defines! {
            $keyword $name {
                $($field : $ty = $e ,)*
            }
        }
    };
}
<commit_msg>Remove logic for fields without trailing commas.<commit_after>// Copyright 2015 The Gfx-rs Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Various helper macros.

mod pso;
mod structure;

#[macro_export]
macro_rules! gfx_format {
    ($name:ident : $surface:ident = $container:ident<$channel:ident>) => {
        impl $crate::format::Formatted for $name {
            type Surface = $crate::format::$surface;
            type Channel = $crate::format::$channel;
            type View = $crate::format::$container<
                <$crate::format::$channel as $crate::format::ChannelTyped>::ShaderType
                >;
        }
    }
}

/// Defines vertex, constant and pipeline formats in one block
#[macro_export]
macro_rules! gfx_defines {
    ($(#[$attr:meta])* vertex $name:ident {
            $( $field:ident : $ty:ty = $e:expr, )+
    }) => {
        gfx_vertex_struct_meta!($(#[$attr])* vertex_struct_meta $name {$($field:$ty = $e,)+});
    };

    ($(#[$attr:meta])* constant $name:ident {
            $( $field:ident : $ty:ty = $e:expr, )+
    }) => {
        gfx_constant_struct_meta!($(#[$attr])* constant_struct_meta $name {$($field:$ty = $e,)+});
    };

    (pipeline $name:ident {
            $( $field:ident : $ty:ty = $e:expr, )+
    }) => {
        gfx_pipeline!($name {$($field:$ty = $e,)+});
    };

    // The recursive case for vertex structs
    ($(#[$attr:meta])* vertex $name:ident {
            $( $field:ident : $ty:ty = $e:expr, )+
    } $($tail:tt)+) => {
        gfx_defines! {
            $(#[$attr])*
            vertex $name { $($field : $ty = $e,)+ }
        }
        gfx_defines!($($tail)+);
    };

    // The recursive case for constant structs
    ($(#[$attr:meta])* constant $name:ident {
            $( $field:ident : $ty:ty = $e:expr, )+
    } $($tail:tt)+) => {
        gfx_defines! {
            $(#[$attr])*
            constant $name { $($field : $ty = $e,)+ }
        }
        gfx_defines!($($tail)+);
    };

    // The recursive case for the other keywords
    ($keyword:ident $name:ident {
            $( $field:ident : $ty:ty = $e:expr, )+
    } $($tail:tt)+) => {
        gfx_defines! {
            $keyword $name { $($field : $ty = $e,)+ }
        }
        gfx_defines!($($tail)+);
    };
}
<|endoftext|>


<commit_before>fn main() {
    // immutable variable
    let a_variable = 1;

    // mutable variable
    let mut mutable_variable = 2;

    // Error!
    //a_variable += 1;

    // Ok
    mutable_variable += 1;
}
<commit_msg>show compiler error in variables chapter<commit_after>fn main() {
    // immutable variable
    let a_variable = 1;

    // mutable variable
    let mut mutable_variable = 2;

    // Error!
    a_variable += 1;

    // Ok
    mutable_variable += 1;
}
<|endoftext|>


<commit_before>extern crate libc;

use std::mem::transmute;
use std::mem::uninitialized;
use std::slice;

pub struct Writer {
    enc: [u8; 100]
}

extern {
    fn od_ec_enc_init(enc: *mut libc::c_void, size: u32);
    fn od_ec_enc_done(enc: *mut libc::c_void, nbytes: *mut u32) -> *const u8;
    fn od_ec_encode_cdf_q15(enc: *mut libc::c_void, s: libc::c_int, cdf: *const u16,
                            nsyms: libc::c_int);
    fn od_ec_encode_bool_q15(enc: *mut libc::c_void, val: libc::c_int, f: libc::c_uint);
}

impl Writer {
    pub fn new() -> Writer {
        unsafe {
            let enc = uninitialized();
            od_ec_enc_init(transmute(&enc), 1024);
            Writer { enc: enc }
        }
    }
    pub fn done(&mut self) -> &[u8] {
        let mut nbytes: u32 = 0;
        unsafe {
            let b = od_ec_enc_done(transmute(&self.enc), &mut nbytes);
            slice::from_raw_parts(b, nbytes as usize)
        }
    }
    pub fn cdf(&mut self, s: u32, cdf: &[u16], nsyms: u32) {
        unsafe {
            od_ec_encode_cdf_q15(transmute(&self.enc), s as libc::c_int, cdf.as_ptr(), nsyms as libc::c_int);
        }
    }
    pub fn bool(&mut self, val: bool, f: u16) {
        unsafe {
            od_ec_encode_bool_q15(transmute(&self.enc), val as libc::c_int, f as libc::c_uint);
        }
    }
    fn update_cdf(cdf: &mut [u16], val: u32, nsymbs: usize) {
        let rate = 4 + if cdf[nsymbs] > 31 { 1 } else { 0 } + (31 ^ (nsymbs as u32).leading_zeros());
        let rate2 = 5;
        let mut tmp: i32;
        let diff: i32;
        let tmp0 = 1 << rate2;
        tmp = 32768 - tmp0;
        diff = ((32768 - ((nsymbs as i32) << rate2)) >> rate) << rate;
        for i in 0..(nsymbs - 1) {
            if i as u32 == val {
                tmp -= diff;
            }
            cdf[i as usize] = ((cdf[i as usize] as i32) + ((tmp - (cdf[i as usize] as i32)) >> rate)) as u16;
            tmp -= tmp0;
        }
        if cdf[nsymbs] < 32 {
            cdf[nsymbs] += 1;
        }
    }
    pub fn symbol(&mut self, s: u32, cdf: &mut [u16], nsymbs: usize) {
        self.cdf(s, cdf, nsymbs as u32);
        Writer::update_cdf(cdf, s, nsymbs);
    }
}
<commit_msg>Simplify a little update_cdf<commit_after>extern crate libc;

use std::mem::transmute;
use std::mem::uninitialized;
use std::slice;

pub struct Writer {
    enc: [u8; 100]
}

extern {
    fn od_ec_enc_init(enc: *mut libc::c_void, size: u32);
    fn od_ec_enc_done(enc: *mut libc::c_void, nbytes: *mut u32) -> *const u8;
    fn od_ec_encode_cdf_q15(enc: *mut libc::c_void, s: libc::c_int, cdf: *const u16,
                            nsyms: libc::c_int);
    fn od_ec_encode_bool_q15(enc: *mut libc::c_void, val: libc::c_int, f: libc::c_uint);
}

impl Writer {
    pub fn new() -> Writer {
        unsafe {
            let enc = uninitialized();
            od_ec_enc_init(transmute(&enc), 1024);
            Writer { enc: enc }
        }
    }
    pub fn done(&mut self) -> &[u8] {
        let mut nbytes: u32 = 0;
        unsafe {
            let b = od_ec_enc_done(transmute(&self.enc), &mut nbytes);
            slice::from_raw_parts(b, nbytes as usize)
        }
    }
    pub fn cdf(&mut self, s: u32, cdf: &[u16], nsyms: u32) {
        unsafe {
            od_ec_encode_cdf_q15(transmute(&self.enc), s as libc::c_int, cdf.as_ptr(), nsyms as libc::c_int);
        }
    }
    pub fn bool(&mut self, val: bool, f: u16) {
        unsafe {
            od_ec_encode_bool_q15(transmute(&self.enc), val as libc::c_int, f as libc::c_uint);
        }
    }
    fn update_cdf(cdf: &mut [u16], val: u32, nsymbs: usize) {
        let rate = 4 + if cdf[nsymbs] > 31 { 1 } else { 0 } + (31 ^ (nsymbs as u32).leading_zeros());
        let rate2 = 5;
        let mut tmp: i32;
        let diff: i32;
        let tmp0 = 1 << rate2;
        tmp = 32768 - tmp0;
        diff = ((32768 - ((nsymbs as i32) << rate2)) >> rate) << rate;
        for i in 0..(nsymbs - 1) {
            if i as u32 == val {
                tmp -= diff;
            }
            let prev_cdf = cdf[i] as i32;
            cdf[i] = (prev_cdf + ((tmp - prev_cdf) >> rate)) as u16;
            tmp -= tmp0;
        }
        if cdf[nsymbs] < 32 {
            cdf[nsymbs] += 1;
        }
    }
    pub fn symbol(&mut self, s: u32, cdf: &mut [u16], nsymbs: usize) {
        self.cdf(s, cdf, nsymbs as u32);
        Writer::update_cdf(cdf, s, nsymbs);
    }
}
<|endoftext|>


<commit_before>//! Compiler plugin for Rust-PHF
//!
//! See the documentation for the `phf` crate for more details.
#[crate_id="github.com/sfackler/rust-phf/phf_mac"];
#[crate_type="lib"];
#[doc(html_root_url="http://www.rust-ci.org/sfackler/rust-phf/doc")];
#[feature(managed_boxes, macro_registrar)];

extern mod extra;
extern mod syntax;
extern mod phf;

use extra::time;
use std::hashmap::HashMap;
use std::rand;
use std::os;
use std::vec;
use syntax::ast;
use syntax::ast::{Name, TokenTree, LitStr, MutImmutable, Expr, ExprVec, ExprLit};
use syntax::codemap::Span;
use syntax::ext::base::{SyntaxExtension,
                        ExtCtxt,
                        MacResult,
                        MRExpr,
                        NormalTT,
                        SyntaxExpanderTT,
                        SyntaxExpanderTTExpanderWithoutContext};
use syntax::parse;
use syntax::parse::token;
use syntax::parse::token::{COMMA, EOF, FAT_ARROW};

static DEFAULT_LAMBDA: uint = 5;

#[macro_registrar]
#[doc(hidden)]
pub fn macro_registrar(register: |Name, SyntaxExtension|) {
    register(token::intern("phf_map"),
             NormalTT(~SyntaxExpanderTT {
                expander: SyntaxExpanderTTExpanderWithoutContext(expand_mphf_map),
                span: None
             },
             None));
}

struct Entry {
    key_str: @str,
    key: @Expr,
    value: @Expr
}

fn expand_mphf_map(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> MacResult {
    let mut parser = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(),
                                                tts.to_owned());
    let mut entries = ~[];

    let mut bad = false;
    while parser.token != EOF {
        let key = parser.parse_expr();

        let key_str = match key.node {
            ExprLit(lit) => {
                match lit.node {
                    LitStr(s, _) => s,
                    _ => {
                        cx.span_err(key.span, "expected string literal");
                        bad = true;
                        @""
                    }
                }
            }
            _ => {
                cx.span_err(key.span, "expected string literal");
                bad = true;
                @""
            }
        };

        if !parser.eat(&FAT_ARROW) {
            cx.span_err(parser.span, "expected `=>`");
            return MacResult::dummy_expr();
        }

        let value = parser.parse_expr();

        entries.push(Entry {
            key_str: key_str,
            key: key,
            value: value
        });

        if !parser.eat(&COMMA) && parser.token != EOF {
            cx.span_err(parser.span, "expected `,`");
            return MacResult::dummy_expr();
        }
    }

    if entries.len() > phf::MAX_SIZE {
        cx.span_err(parser.span,
                    format!("maps with more than {} items are not supported",
                            phf::MAX_SIZE));
        return MacResult::dummy_expr();
    }

    if bad {
        return MacResult::dummy_expr();
    }

    entries.sort_by(|a, b| a.key_str.cmp(&b.key_str));
    if check_for_duplicates(cx, sp, entries) {
        return MacResult::dummy_expr();
    }

    let start = time::precise_time_s();
    let state;
    loop {
        match generate_hash(entries) {
            Some(s) => {
                state = s;
                break;
            }
            None => {}
        }
    }
    let time = time::precise_time_s() - start;
    if os::getenv("PHF_STATS").is_some() {
        cx.parse_sess().span_diagnostic.span_note(sp, format!("PHF generation took {} seconds", time));
    }

    let len = entries.len();
    let k1 = state.k1;
    let k2 = state.k2;
    let disps = state.disps.iter().map(|&(d1, d2)| {
            quote_expr!(&*cx, ($d1, $d2))
        }).collect();
    let disps = @Expr {
        id: ast::DUMMY_NODE_ID,
        node: ExprVec(disps, MutImmutable),
        span: sp,
    };
    let entries = state.map.iter().map(|&idx| {
            match idx {
                Some(idx) => {
                    let Entry { key, value, .. } = entries[idx];
                    quote_expr!(&*cx, Some(($key, $value)))
                }
                None => quote_expr!(&*cx, None),
            }
        }).collect();
    let entries = @Expr {
        id: ast::DUMMY_NODE_ID,
        node: ExprVec(entries, MutImmutable),
        span: sp,
    };

    MRExpr(quote_expr!(cx, PhfMap {
        len: $len,
        k1: $k1,
        k2: $k2,
        disps: &'static $disps,
        entries: &'static $entries,
    }))
}

fn check_for_duplicates(cx: &mut ExtCtxt, sp: Span, entries: &[Entry]) -> bool {
    let mut dups = false;
    let mut in_dup = false;
    for window in entries.windows(2) {
        let ref a = window[0];
        let ref b = window[1];
        if a.key_str == b.key_str {
            dups = true;
            if !in_dup {
                cx.span_err(sp, format!("duplicate key \"{}\"", a.key_str));
                cx.span_note(a.key.span, "one occurrence here");
                in_dup = true;
            }
            cx.span_note(b.key.span, "one occurrence here");
        } else {
            in_dup = false;
        }
    }

    dups
}

struct HashState {
    k1: u64,
    k2: u64,
    disps: ~[(uint, uint)],
    map: ~[Option<uint>],
}

fn generate_hash(entries: &[Entry]) -> Option<HashState> {
    struct Bucket {
        idx: uint,
        keys: ~[uint],
    }

    struct Hashes {
        g: uint,
        f1: uint,
        f2: uint,
    }

    let k1 = rand::random();
    let k2 = rand::random();

    let hashes = entries.iter().map(|entry| {
            let (g, f1, f2) = phf::hash(entry.key_str, k1, k2);
            Hashes {
                g: g,
                f1: f1,
                f2: f2
            }
        }).to_owned_vec();

    let buckets_len = (entries.len() + DEFAULT_LAMBDA - 1) / DEFAULT_LAMBDA;
    let mut buckets = vec::from_fn(buckets_len,
                                   |i| Bucket { idx: i, keys: ~[] });

    for (i, hash) in hashes.iter().enumerate() {
        buckets[hash.g % buckets_len].keys.push(i);
    }

    // Sort descending
    buckets.sort_by(|a, b| b.keys.len().cmp(&a.keys.len()));

    let table_len = entries.len();
    let mut map = vec::from_elem(table_len, None);
    let mut disps = vec::from_elem(buckets_len, None);
    let mut try_map = HashMap::new();
    'buckets: for bucket in buckets.iter() {
        for d1 in range(0, table_len) {
            'disps: for d2 in range(0, table_len) {
                try_map.clear();
                for &key in bucket.keys.iter() {
                    let idx = phf::displace(hashes[key].f1, hashes[key].f2,
                                            d1, d2) % table_len;
                    if try_map.find(&idx).is_some() || map[idx].is_some() {
                        continue 'disps;
                    }
                    try_map.insert(idx, key);
                }

                // We've picked a good set of disps
                disps[bucket.idx] = Some((d1, d2));
                for (&idx, &key) in try_map.iter() {
                    map[idx] = Some(key);
                }
                continue 'buckets;
            }
        }

        // Unable to find displacements for a bucket
        return None;
    }

    let disps = disps.move_iter().map(|i| i.expect("should have a bucket")).collect();

    Some(HashState {
        k1: k1,
        k2: k2,
        disps: disps,
        map: map,
    })
}
<commit_msg>Update for API changes<commit_after>//! Compiler plugin for Rust-PHF
//!
//! See the documentation for the `phf` crate for more details.
#[crate_id="github.com/sfackler/rust-phf/phf_mac"];
#[crate_type="lib"];
#[doc(html_root_url="http://www.rust-ci.org/sfackler/rust-phf/doc")];
#[feature(managed_boxes, macro_registrar)];

extern mod extra;
extern mod syntax;
extern mod phf;

use extra::time;
use std::hashmap::HashMap;
use std::rand;
use std::os;
use std::vec;
use syntax::ast;
use syntax::ast::{Name, TokenTree, LitStr, MutImmutable, Expr, ExprVec, ExprLit};
use syntax::codemap::Span;
use syntax::ext::base::{SyntaxExtension,
                        ExtCtxt,
                        MacResult,
                        MRExpr,
                        NormalTT,
                        BasicMacroExpander};
use syntax::parse;
use syntax::parse::token;
use syntax::parse::token::{COMMA, EOF, FAT_ARROW};

static DEFAULT_LAMBDA: uint = 5;

#[macro_registrar]
#[doc(hidden)]
pub fn macro_registrar(register: |Name, SyntaxExtension|) {
    register(token::intern("phf_map"),
             NormalTT(~BasicMacroExpander {
                expander: expand_mphf_map,
                span: None
             },
             None));
}

struct Entry {
    key_str: @str,
    key: @Expr,
    value: @Expr
}

fn expand_mphf_map(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> MacResult {
    let mut parser = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(),
                                                tts.to_owned());
    let mut entries = ~[];

    let mut bad = false;
    while parser.token != EOF {
        let key = parser.parse_expr();

        let key_str = match key.node {
            ExprLit(lit) => {
                match lit.node {
                    LitStr(s, _) => s,
                    _ => {
                        cx.span_err(key.span, "expected string literal");
                        bad = true;
                        @""
                    }
                }
            }
            _ => {
                cx.span_err(key.span, "expected string literal");
                bad = true;
                @""
            }
        };

        if !parser.eat(&FAT_ARROW) {
            cx.span_err(parser.span, "expected `=>`");
            return MacResult::dummy_expr();
        }

        let value = parser.parse_expr();

        entries.push(Entry {
            key_str: key_str,
            key: key,
            value: value
        });

        if !parser.eat(&COMMA) && parser.token != EOF {
            cx.span_err(parser.span, "expected `,`");
            return MacResult::dummy_expr();
        }
    }

    if entries.len() > phf::MAX_SIZE {
        cx.span_err(parser.span,
                    format!("maps with more than {} items are not supported",
                            phf::MAX_SIZE));
        return MacResult::dummy_expr();
    }

    if bad {
        return MacResult::dummy_expr();
    }

    entries.sort_by(|a, b| a.key_str.cmp(&b.key_str));
    if check_for_duplicates(cx, sp, entries) {
        return MacResult::dummy_expr();
    }

    let start = time::precise_time_s();
    let state;
    loop {
        match generate_hash(entries) {
            Some(s) => {
                state = s;
                break;
            }
            None => {}
        }
    }
    let time = time::precise_time_s() - start;
    if os::getenv("PHF_STATS").is_some() {
        cx.parse_sess().span_diagnostic.span_note(sp, format!("PHF generation took {} seconds", time));
    }

    let len = entries.len();
    let k1 = state.k1;
    let k2 = state.k2;
    let disps = state.disps.iter().map(|&(d1, d2)| {
            quote_expr!(&*cx, ($d1, $d2))
        }).collect();
    let disps = @Expr {
        id: ast::DUMMY_NODE_ID,
        node: ExprVec(disps, MutImmutable),
        span: sp,
    };
    let entries = state.map.iter().map(|&idx| {
            match idx {
                Some(idx) => {
                    let Entry { key, value, .. } = entries[idx];
                    quote_expr!(&*cx, Some(($key, $value)))
                }
                None => quote_expr!(&*cx, None),
            }
        }).collect();
    let entries = @Expr {
        id: ast::DUMMY_NODE_ID,
        node: ExprVec(entries, MutImmutable),
        span: sp,
    };

    MRExpr(quote_expr!(cx, PhfMap {
        len: $len,
        k1: $k1,
        k2: $k2,
        disps: &'static $disps,
        entries: &'static $entries,
    }))
}

fn check_for_duplicates(cx: &mut ExtCtxt, sp: Span, entries: &[Entry]) -> bool {
    let mut dups = false;
    let mut in_dup = false;
    for window in entries.windows(2) {
        let ref a = window[0];
        let ref b = window[1];
        if a.key_str == b.key_str {
            dups = true;
            if !in_dup {
                cx.span_err(sp, format!("duplicate key \"{}\"", a.key_str));
                cx.span_note(a.key.span, "one occurrence here");
                in_dup = true;
            }
            cx.span_note(b.key.span, "one occurrence here");
        } else {
            in_dup = false;
        }
    }

    dups
}

struct HashState {
    k1: u64,
    k2: u64,
    disps: ~[(uint, uint)],
    map: ~[Option<uint>],
}

fn generate_hash(entries: &[Entry]) -> Option<HashState> {
    struct Bucket {
        idx: uint,
        keys: ~[uint],
    }

    struct Hashes {
        g: uint,
        f1: uint,
        f2: uint,
    }

    let k1 = rand::random();
    let k2 = rand::random();

    let hashes = entries.iter().map(|entry| {
            let (g, f1, f2) = phf::hash(entry.key_str, k1, k2);
            Hashes {
                g: g,
                f1: f1,
                f2: f2
            }
        }).to_owned_vec();

    let buckets_len = (entries.len() + DEFAULT_LAMBDA - 1) / DEFAULT_LAMBDA;
    let mut buckets = vec::from_fn(buckets_len,
                                   |i| Bucket { idx: i, keys: ~[] });

    for (i, hash) in hashes.iter().enumerate() {
        buckets[hash.g % buckets_len].keys.push(i);
    }

    // Sort descending
    buckets.sort_by(|a, b| b.keys.len().cmp(&a.keys.len()));

    let table_len = entries.len();
    let mut map = vec::from_elem(table_len, None);
    let mut disps = vec::from_elem(buckets_len, None);
    let mut try_map = HashMap::new();
    'buckets: for bucket in buckets.iter() {
        for d1 in range(0, table_len) {
            'disps: for d2 in range(0, table_len) {
                try_map.clear();
                for &key in bucket.keys.iter() {
                    let idx = phf::displace(hashes[key].f1, hashes[key].f2,
                                            d1, d2) % table_len;
                    if try_map.find(&idx).is_some() || map[idx].is_some() {
                        continue 'disps;
                    }
                    try_map.insert(idx, key);
                }

                // We've picked a good set of disps
                disps[bucket.idx] = Some((d1, d2));
                for (&idx, &key) in try_map.iter() {
                    map[idx] = Some(key);
                }
                continue 'buckets;
            }
        }

        // Unable to find displacements for a bucket
        return None;
    }

    let disps = disps.move_iter().map(|i| i.expect("should have a bucket")).collect();

    Some(HashState {
        k1: k1,
        k2: k2,
        disps: disps,
        map: map,
    })
}
<|endoftext|>


<commit_before>use std;
use std::sync::Arc;

use tokio_service::{self, Service as TokioService};
use jsonrpc::futures::{future, Future, Stream, Sink};
use jsonrpc::futures::sync::oneshot;
use jsonrpc::{Metadata, MetaIoHandler, Middleware, NoopMiddleware};
use jsonrpc::futures::BoxFuture;
use server_utils::tokio_core::io::Io;
use server_utils::tokio_core::reactor::Remote;
use server_utils::reactor;

use meta::{MetaExtractor, NoopExtractor, RequestContext};

/// IPC server session
pub struct Service<M: Metadata = (), S: Middleware<M> = NoopMiddleware> {
	handler: Arc<MetaIoHandler<M, S>>,
	meta: M,
}

impl<M: Metadata, S: Middleware<M>> Service<M, S> {
	/// Create new IPC server session with given handler and metadata.
	pub fn new(handler: Arc<MetaIoHandler<M, S>>, meta: M) -> Self {
		Service { handler: handler, meta: meta }
	}
}

impl<M: Metadata, S: Middleware<M>> tokio_service::Service for Service<M, S> {
	type Request = String;
	type Response = Option<String>;

	type Error = ();

	type Future = BoxFuture<Self::Response, Self::Error>;

	fn call(&self, req: Self::Request) -> Self::Future {
		trace!(target: "ipc", "Received request: {}", req);
		self.handler.handle_request(&req, self.meta.clone())
	}
}

/// IPC server builder
pub struct ServerBuilder<M: Metadata = (), S: Middleware<M> = NoopMiddleware> {
	handler: Arc<MetaIoHandler<M, S>>,
	meta_extractor: Arc<MetaExtractor<M>>,
	remote: reactor::UninitializedRemote,
}

impl<M: Metadata, S: Middleware<M>> ServerBuilder<M, S> {
	///
	pub fn new<T>(io_handler: T) -> ServerBuilder<M, S> where
		T: Into<MetaIoHandler<M, S>>,
	{
		ServerBuilder {
			handler: Arc::new(io_handler.into()),
			meta_extractor: Arc::new(NoopExtractor),
			remote: reactor::UninitializedRemote::Unspawned,
		}
	}

	/// Sets shared different event loop remote.
	pub fn event_loop_remote(mut self, remote: Remote) -> Self {
		self.remote = reactor::UninitializedRemote::Shared(remote);
		self
	}

	/// Sets session metadata extractor.
	pub fn session_metadata_extractor<X>(mut self, meta_extractor: X) -> Self where
		X: MetaExtractor<M>,
	{
		self.meta_extractor = Arc::new(meta_extractor);
		self
	}

	/// Run server (in a separate thread)
	pub fn start(self, path: &str) -> std::io::Result<Server> {
		let remote = self.remote.initialize()?;
		let rpc_handler = self.handler.clone();
		let endpoint_addr = path.to_owned();
		let meta_extractor = self.meta_extractor.clone();
		let (stop_signal, stop_receiver) = oneshot::channel();
		let (start_signal, start_receiver) = oneshot::channel();

		remote.remote().spawn(move |handle| {
			use parity_tokio_ipc::Endpoint;
			use stream_codec::StreamCodec;

			let listener = match Endpoint::new(endpoint_addr, handle) {
				Ok(l) => l,
				Err(e) => {
					start_signal.complete(Err(e));
					return future::ok(()).boxed();
				}
			};

			start_signal.complete(Ok(()));
			let remote = handle.remote().clone();
			let connections = listener.incoming();

			let server = connections.for_each(move |(io_stream, remote_id)| {
				trace!("Accepted incoming IPC connection");

				let meta = meta_extractor.extract(&RequestContext { endpoint_addr: &remote_id });
				let service = Service::new(rpc_handler.clone(), meta);
				let (writer, reader) = io_stream.framed(StreamCodec).split();
				let responses = reader.and_then(
					move |req| service.call(req).then(|response| match response {
						Err(e) => {
							warn!(target: "ipc", "Error while processing request: {:?}", e);
							future::ok(None)
						},
						Ok(None) => {
							future::ok(None)
						},
						Ok(Some(response_data)) => {
							trace!(target: "ipc", "Sent response: {}", &response_data);
							future::ok(Some(response_data))
						}
					})
				)
				.filter_map(|x| x);

				let writer = writer.send_all(responses).then(move |_| {
					trace!(target: "ipc", "Peer: service finished");
					Ok(())
				});

				remote.spawn(|_| writer);

				Ok(())
			});

			let stop = stop_receiver.map_err(|_| std::io::ErrorKind::Interrupted.into());
			server.select(stop).map(|_| ()).map_err(|_| ()).boxed()
		});

		match start_receiver.wait().expect("Message should always be sent") {
			Ok(()) => Ok(Server { path: path.to_owned(), remote: Some(remote), stop: Some(stop_signal) }),
			Err(e) => Err(e)
		}
	}
}

/// IPC Server handle
pub struct Server {
	path: String,
	remote: Option<reactor::Remote>,
	stop: Option<oneshot::Sender<()>>,
}

impl Server {
	/// Closes the server (waits for finish)
	pub fn close(mut self) {
		self.stop.take().unwrap().complete(());
		self.remote.take().unwrap().close();
		self.clear_file();
	}

	/// Wait for the server to finish
	pub fn wait(mut self) {
		self.remote.take().unwrap().wait();
	}

	/// Remove socket file
	fn clear_file(&self) {
		::std::fs::remove_file(&self.path).unwrap_or_else(|_| {}); // ignore error, file could have been gone somewhere
	}
}

impl Drop for Server {
	fn drop(&mut self) {
		self.stop.take().map(|stop| stop.complete(()));
		self.remote.take().map(|remote| remote.close());
		self.clear_file();
	}
}

#[cfg(test)]
#[cfg(not(windows))]
mod tests {
	extern crate tokio_uds;

	use std::thread;
	use super::{ServerBuilder, Server};
	use jsonrpc::{MetaIoHandler, Value};
	use jsonrpc::futures::{Future, future};
	use self::tokio_uds::UnixStream;
	use server_utils::tokio_core::reactor::Core;
	use server_utils::tokio_core::io;

	fn server_builder() -> ServerBuilder {
		let mut io = MetaIoHandler::<()>::default();
		io.add_method("say_hello", |_params| {
			Ok(Value::String("hello".to_string()))
		});
		ServerBuilder::new(io)
	}

	fn run(path: &str) -> Server {
		let builder = server_builder();
		let server = builder.start(path).expect("Server must run with no issues");
		thread::sleep(::std::time::Duration::from_millis(50));
		server
	}

	fn dummy_request(path: &str, data: &[u8]) -> Vec<u8> {
		let mut core = Core::new().expect("Tokio Core should be created with no errors");
		let mut buffer = vec![0u8; 1024];

		let stream = UnixStream::connect(path, &core.handle()).expect("Should have been connected to the server");
		let reqrep = io::write_all(stream, data)
			.and_then(|(stream, _)| {
				io::read(stream, &mut buffer)
			})
			.and_then(|(_, read_buf, len)| {
				future::ok(read_buf[0..len].to_vec())
			});
		let result = core.run(reqrep).expect("Core should run with no errors");

		result
	}

	fn dummy_request_str(path: &str, data: &[u8]) -> String {
		String::from_utf8(dummy_request(path, data)).expect("String should be utf-8")
	}

	#[test]
	fn start() {
		::logger::init_log();

		let mut io = MetaIoHandler::<()>::default();
		io.add_method("say_hello", |_params| {
			Ok(Value::String("hello".to_string()))
		});
		let server = ServerBuilder::new(io);

		let _server = server.start("/tmp/test-ipc-20000")
			.expect("Server must run with no issues");
	}

	#[test]
	fn connect() {
		::logger::init_log();
		let path = "/tmp/test-ipc-30000";
		let _server = run(path);

		let core = Core::new().expect("Tokio Core should be created with no errors");
		UnixStream::connect(path, &core.handle()).expect("Socket should connect");
	}

	#[test]
	fn request() {
		::logger::init_log();
		let path = "/tmp/test-ipc-40000";
		let _server = run(path);

		let result = dummy_request_str(
			path,
			b"{\"jsonrpc\": \"2.0\", \"method\": \"say_hello\", \"params\": [42, 23], \"id\": 1}\n",
			);

		assert_eq!(
			result,
			"{\"jsonrpc\":\"2.0\",\"result\":\"hello\",\"id\":1}\n",
			"Response does not exactly much the expected response",
			);
	}

	#[test]
	fn close() {
		::logger::init_log();
		let path = "/tmp/test-ipc-50000";
		let server = run(path);
		server.close();

		assert!(::std::fs::metadata(path).is_err(), "There should be no socket file left");
		let core = Core::new().expect("Tokio Core should be created with no errors");
		assert!(UnixStream::connect(path, &core.handle()).is_err(), "Connection to the closed socket should fail");
	}
}
<commit_msg>Try to delete socket file on unix with warning (#89)<commit_after>use std;
use std::sync::Arc;

use tokio_service::{self, Service as TokioService};
use jsonrpc::futures::{future, Future, Stream, Sink};
use jsonrpc::futures::sync::oneshot;
use jsonrpc::{Metadata, MetaIoHandler, Middleware, NoopMiddleware};
use jsonrpc::futures::BoxFuture;
use server_utils::tokio_core::io::Io;
use server_utils::tokio_core::reactor::Remote;
use server_utils::reactor;

use meta::{MetaExtractor, NoopExtractor, RequestContext};

/// IPC server session
pub struct Service<M: Metadata = (), S: Middleware<M> = NoopMiddleware> {
	handler: Arc<MetaIoHandler<M, S>>,
	meta: M,
}

impl<M: Metadata, S: Middleware<M>> Service<M, S> {
	/// Create new IPC server session with given handler and metadata.
	pub fn new(handler: Arc<MetaIoHandler<M, S>>, meta: M) -> Self {
		Service { handler: handler, meta: meta }
	}
}

impl<M: Metadata, S: Middleware<M>> tokio_service::Service for Service<M, S> {
	type Request = String;
	type Response = Option<String>;

	type Error = ();

	type Future = BoxFuture<Self::Response, Self::Error>;

	fn call(&self, req: Self::Request) -> Self::Future {
		trace!(target: "ipc", "Received request: {}", req);
		self.handler.handle_request(&req, self.meta.clone())
	}
}

/// IPC server builder
pub struct ServerBuilder<M: Metadata = (), S: Middleware<M> = NoopMiddleware> {
	handler: Arc<MetaIoHandler<M, S>>,
	meta_extractor: Arc<MetaExtractor<M>>,
	remote: reactor::UninitializedRemote,
}

impl<M: Metadata, S: Middleware<M>> ServerBuilder<M, S> {
	///
	pub fn new<T>(io_handler: T) -> ServerBuilder<M, S> where
		T: Into<MetaIoHandler<M, S>>,
	{
		ServerBuilder {
			handler: Arc::new(io_handler.into()),
			meta_extractor: Arc::new(NoopExtractor),
			remote: reactor::UninitializedRemote::Unspawned,
		}
	}

	/// Sets shared different event loop remote.
	pub fn event_loop_remote(mut self, remote: Remote) -> Self {
		self.remote = reactor::UninitializedRemote::Shared(remote);
		self
	}

	/// Sets session metadata extractor.
	pub fn session_metadata_extractor<X>(mut self, meta_extractor: X) -> Self where
		X: MetaExtractor<M>,
	{
		self.meta_extractor = Arc::new(meta_extractor);
		self
	}

	/// Run server (in a separate thread)
	pub fn start(self, path: &str) -> std::io::Result<Server> {
		let remote = self.remote.initialize()?;
		let rpc_handler = self.handler.clone();
		let endpoint_addr = path.to_owned();
		let meta_extractor = self.meta_extractor.clone();
		let (stop_signal, stop_receiver) = oneshot::channel();
		let (start_signal, start_receiver) = oneshot::channel();

		remote.remote().spawn(move |handle| {
			use parity_tokio_ipc::Endpoint;
			use stream_codec::StreamCodec;

			if cfg!(unix) {
				// warn about existing file and remove it
				if ::std::fs::remove_file(&endpoint_addr).is_ok() {
					warn!("Removed existing file '{}'.", &endpoint_addr);
				}
			}

			let listener = match Endpoint::new(endpoint_addr, handle) {
				Ok(l) => l,
				Err(e) => {
					start_signal.complete(Err(e));
					return future::ok(()).boxed();
				}
			};

			start_signal.complete(Ok(()));
			let remote = handle.remote().clone();
			let connections = listener.incoming();

			let server = connections.for_each(move |(io_stream, remote_id)| {
				trace!("Accepted incoming IPC connection");

				let meta = meta_extractor.extract(&RequestContext { endpoint_addr: &remote_id });
				let service = Service::new(rpc_handler.clone(), meta);
				let (writer, reader) = io_stream.framed(StreamCodec).split();
				let responses = reader.and_then(
					move |req| service.call(req).then(|response| match response {
						Err(e) => {
							warn!(target: "ipc", "Error while processing request: {:?}", e);
							future::ok(None)
						},
						Ok(None) => {
							future::ok(None)
						},
						Ok(Some(response_data)) => {
							trace!(target: "ipc", "Sent response: {}", &response_data);
							future::ok(Some(response_data))
						}
					})
				)
				.filter_map(|x| x);

				let writer = writer.send_all(responses).then(move |_| {
					trace!(target: "ipc", "Peer: service finished");
					Ok(())
				});

				remote.spawn(|_| writer);

				Ok(())
			});

			let stop = stop_receiver.map_err(|_| std::io::ErrorKind::Interrupted.into());
			server.select(stop).map(|_| ()).map_err(|_| ()).boxed()
		});

		match start_receiver.wait().expect("Message should always be sent") {
			Ok(()) => Ok(Server { path: path.to_owned(), remote: Some(remote), stop: Some(stop_signal) }),
			Err(e) => Err(e)
		}
	}
}

/// IPC Server handle
pub struct Server {
	path: String,
	remote: Option<reactor::Remote>,
	stop: Option<oneshot::Sender<()>>,
}

impl Server {
	/// Closes the server (waits for finish)
	pub fn close(mut self) {
		self.stop.take().unwrap().complete(());
		self.remote.take().unwrap().close();
		self.clear_file();
	}

	/// Wait for the server to finish
	pub fn wait(mut self) {
		self.remote.take().unwrap().wait();
	}

	/// Remove socket file
	fn clear_file(&self) {
		::std::fs::remove_file(&self.path).unwrap_or_else(|_| {}); // ignore error, file could have been gone somewhere
	}
}

impl Drop for Server {
	fn drop(&mut self) {
		self.stop.take().map(|stop| stop.complete(()));
		self.remote.take().map(|remote| remote.close());
		self.clear_file();
	}
}

#[cfg(test)]
#[cfg(not(windows))]
mod tests {
	extern crate tokio_uds;

	use std::thread;
	use super::{ServerBuilder, Server};
	use jsonrpc::{MetaIoHandler, Value};
	use jsonrpc::futures::{Future, future};
	use self::tokio_uds::UnixStream;
	use server_utils::tokio_core::reactor::Core;
	use server_utils::tokio_core::io;

	fn server_builder() -> ServerBuilder {
		let mut io = MetaIoHandler::<()>::default();
		io.add_method("say_hello", |_params| {
			Ok(Value::String("hello".to_string()))
		});
		ServerBuilder::new(io)
	}

	fn run(path: &str) -> Server {
		let builder = server_builder();
		let server = builder.start(path).expect("Server must run with no issues");
		thread::sleep(::std::time::Duration::from_millis(50));
		server
	}

	fn dummy_request(path: &str, data: &[u8]) -> Vec<u8> {
		let mut core = Core::new().expect("Tokio Core should be created with no errors");
		let mut buffer = vec![0u8; 1024];

		let stream = UnixStream::connect(path, &core.handle()).expect("Should have been connected to the server");
		let reqrep = io::write_all(stream, data)
			.and_then(|(stream, _)| {
				io::read(stream, &mut buffer)
			})
			.and_then(|(_, read_buf, len)| {
				future::ok(read_buf[0..len].to_vec())
			});
		let result = core.run(reqrep).expect("Core should run with no errors");

		result
	}

	fn dummy_request_str(path: &str, data: &[u8]) -> String {
		String::from_utf8(dummy_request(path, data)).expect("String should be utf-8")
	}

	#[test]
	fn start() {
		::logger::init_log();

		let mut io = MetaIoHandler::<()>::default();
		io.add_method("say_hello", |_params| {
			Ok(Value::String("hello".to_string()))
		});
		let server = ServerBuilder::new(io);

		let _server = server.start("/tmp/test-ipc-20000")
			.expect("Server must run with no issues");
	}

	#[test]
	fn connect() {
		::logger::init_log();
		let path = "/tmp/test-ipc-30000";
		let _server = run(path);

		let core = Core::new().expect("Tokio Core should be created with no errors");
		UnixStream::connect(path, &core.handle()).expect("Socket should connect");
	}

	#[test]
	fn request() {
		::logger::init_log();
		let path = "/tmp/test-ipc-40000";
		let _server = run(path);

		let result = dummy_request_str(
			path,
			b"{\"jsonrpc\": \"2.0\", \"method\": \"say_hello\", \"params\": [42, 23], \"id\": 1}\n",
			);

		assert_eq!(
			result,
			"{\"jsonrpc\":\"2.0\",\"result\":\"hello\",\"id\":1}\n",
			"Response does not exactly much the expected response",
			);
	}

	#[test]
	fn close() {
		::logger::init_log();
		let path = "/tmp/test-ipc-50000";
		let server = run(path);
		server.close();

		assert!(::std::fs::metadata(path).is_err(), "There should be no socket file left");
		let core = Core::new().expect("Tokio Core should be created with no errors");
		assert!(UnixStream::connect(path, &core.handle()).is_err(), "Connection to the closed socket should fail");
	}
}
<|endoftext|>


<commit_before>use error::Error;
use summary::Summarizer;

pub struct LinearRegression {
    intercept: f64,
    r: f64,
    slope: f64,
    standard_error: f64,
}

impl LinearRegression {
    pub fn new(data: &[(f64, f64)]) -> Result<Self, Error> {
        if data.is_empty() {
            return Err(Error::EmptySample);
        }

        LinearRegression::simple_lr(data)
    }

    pub fn intercept(&self) -> f64 {
        self.intercept
    }

    pub fn r(&self) -> f64 {
        self.r
    }

    pub fn slope(&self) -> f64 {
        self.slope
    }

    pub fn standard_error(&self) -> f64 {
        self.standard_error
    }

    fn simple_lr(data: &[(f64, f64)]) -> Result<Self, Error> {
        let n = data.len() as f64;

        let (x, y): (Vec<_,>, Vec<_>) = data.iter().cloned().unzip();

        let summ_x = Summarizer::new(&x)?;
        let summ_y = Summarizer::new(&y)?;

        let mean_x = summ_x.mean();
        let mean_y = summ_y.mean();

        let std_x = summ_x.standard_deviation();
        let std_y = summ_y.standard_deviation();

        let r_num: f64 = (0..x.len())
            .map(|i| (x[i] - mean_x) * (y[i] - mean_y))
            .sum();
        let r_den = (n - 1.0) * std_x * std_y;
        let r = r_num / r_den;

        let slope = r * (std_y / std_x);
        let intercept = mean_y - slope * mean_x;

        let df = n - 2.0;
        let standard_error = (slope / df.sqrt()) * (1.0 / r.powi(2) - 1.0).sqrt();

        Ok(LinearRegression {
            intercept,
            r,
            slope,
            standard_error,
        })
    }
}
<commit_msg>Document `lr` module<commit_after>use error::Error;
use summary::Summarizer;

/// The results of a simple linear regression with one predictor variable and
/// one response variable.
pub struct LinearRegression {
    intercept: f64,
    r: f64,
    slope: f64,
    standard_error: f64,
}

impl LinearRegression {
    /// Fit the sample data to a linear model `Y = αX + β`, where `X` is the
    /// predictor variable and `Y` is the response variable.
    ///
    /// The sample data points are pairs of the form `(x, y)`, where each `x` is
    /// interpreted as an observed value of the predictor variable and `y` is a
    /// value of the response variable.
    pub fn new(data: &[(f64, f64)]) -> Result<Self, Error> {
        if data.is_empty() {
            return Err(Error::EmptySample);
        }

        LinearRegression::simple_lr(data)
    }

    /// Intercept `β` of the fitted linear model `Y = αX + β`.
    pub fn intercept(&self) -> f64 {
        self.intercept
    }

    /// Pearson's correlation coefficient.
    pub fn r(&self) -> f64 {
        self.r
    }

    /// Slope coefficient `α` of the fitted linear model `Y = αX + β`.
    pub fn slope(&self) -> f64 {
        self.slope
    }

    /// Standard error of the estimate.
    pub fn standard_error(&self) -> f64 {
        self.standard_error
    }

    fn simple_lr(data: &[(f64, f64)]) -> Result<Self, Error> {
        let n = data.len() as f64;

        let (x, y): (Vec<_,>, Vec<_>) = data.iter().cloned().unzip();

        let summ_x = Summarizer::new(&x)?;
        let summ_y = Summarizer::new(&y)?;

        let mean_x = summ_x.mean();
        let mean_y = summ_y.mean();

        let std_x = summ_x.standard_deviation();
        let std_y = summ_y.standard_deviation();

        let r_num: f64 = (0..x.len())
            .map(|i| (x[i] - mean_x) * (y[i] - mean_y))
            .sum();
        let r_den = (n - 1.0) * std_x * std_y;
        let r = r_num / r_den;

        let slope = r * (std_y / std_x);
        let intercept = mean_y - slope * mean_x;

        let df = n - 2.0;
        let standard_error = (slope / df.sqrt()) * (1.0 / r.powi(2) - 1.0).sqrt();

        Ok(LinearRegression {
            intercept,
            r,
            slope,
            standard_error,
        })
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2017-2018, The rav1e contributors. All rights reserved
//
// This source code is subject to the terms of the BSD 2 Clause License and
// the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
// was not distributed with this source code in the LICENSE file, you can
// obtain it at www.aomedia.org/license/software. If the Alliance for Open
// Media Patent License 1.0 was not distributed with this source code in the
// PATENTS file, you can obtain it at www.aomedia.org/license/patent.

use context::BlockOffset;
use context::BLOCK_TO_PLANE_SHIFT;
use context::MI_SIZE;
use partition::*;
use plane::*;
use FrameInvariants;
use FrameState;

#[inline(always)]
pub fn get_sad(
  plane_org: &PlaneSlice, plane_ref: &PlaneSlice, blk_h: usize,
  blk_w: usize
) -> u32 {
  let mut sum = 0 as u32;

  let org_iter = plane_org.iter_width(blk_w);
  let ref_iter = plane_ref.iter_width(blk_w);

  for (slice_org, slice_ref) in org_iter.take(blk_h).zip(ref_iter) {
      sum += slice_org
        .iter()
        .zip(slice_ref)
        .map(|(&a, &b)| (a as i32 - b as i32).abs() as u32)
        .sum::<u32>();
  }

  sum
}
pub fn motion_estimation(
  fi: &FrameInvariants, fs: &FrameState, bsize: BlockSize,
  bo: &BlockOffset, ref_frame: usize, pmv: &MotionVector
) -> MotionVector {
  match fi.rec_buffer.frames[fi.ref_frames[ref_frame - LAST_FRAME]] {
    Some(ref rec) => {
      let po = PlaneOffset {
        x: (bo.x as isize) << BLOCK_TO_PLANE_SHIFT,
        y: (bo.y as isize) << BLOCK_TO_PLANE_SHIFT
      };
      let range = 32 as isize;
      let blk_w = bsize.width();
      let blk_h = bsize.height();
      let border_w = 128 + blk_w as isize * 8;
      let border_h = 128 + blk_h as isize * 8;
      let mvx_min = -(bo.x as isize) * (8 * MI_SIZE) as isize - border_w;
      let mvx_max = (fi.w_in_b - bo.x - blk_w / MI_SIZE) as isize * (8 * MI_SIZE) as isize + border_w;
      let mvy_min = -(bo.y as isize) * (8 * MI_SIZE) as isize - border_h;
      let mvy_max = (fi.h_in_b - bo.y - blk_h / MI_SIZE) as isize * (8 * MI_SIZE) as isize + border_h;
      let x_lo = po.x + ((-range + (pmv.col / 8) as isize).max(mvx_min / 8));
      let x_hi = po.x + ((range + (pmv.col / 8) as isize).min(mvx_max / 8));
      let y_lo = po.y + ((-range + (pmv.row / 8) as isize).max(mvy_min / 8));
      let y_hi = po.y + ((range + (pmv.row / 8) as isize).min(mvy_max / 8));

      let mut lowest_sad = 128 * 128 * 4096 as u32;
      let mut best_mv = MotionVector { row: 0, col: 0 };

      for y in (y_lo..y_hi).step_by(2) {
        for x in (x_lo..x_hi).step_by(2) {
          let plane_org = fs.input.planes[0].slice(&po);
          let plane_ref = rec.frame.planes[0].slice(&PlaneOffset { x, y });

          let sad = get_sad(&plane_org, &plane_ref, blk_h, blk_w);

          if sad < lowest_sad {
            lowest_sad = sad;
            best_mv = MotionVector {
              row: 8 * (y as i16 - po.y as i16),
              col: 8 * (x as i16 - po.x as i16)
            }
          }
        }
      }

      let mode = PredictionMode::NEWMV;
      let mut tmp_plane = Plane::new(blk_w, blk_h, 0, 0, 0, 0);

      let mut steps = vec![8, 4, 2];
      if fi.allow_high_precision_mv {
        steps.push(1);
      }

      for step in steps {
        let center_mv_h = best_mv;
        for i in 0..3 {
          for j in 0..3 {
            // Skip the center point that was already tested
            if i == 1 && j == 1 {
              continue;
            }

            let cand_mv = MotionVector {
              row: center_mv_h.row + step * (i as i16 - 1),
              col: center_mv_h.col + step * (j as i16 - 1)
            };

            if (cand_mv.col as isize) < mvx_min || (cand_mv.col as isize) > mvx_max {
              continue;
            }
            if (cand_mv.row as isize) < mvy_min || (cand_mv.row as isize) > mvy_max {
              continue;
            }

            {
              let tmp_slice =
                &mut tmp_plane.mut_slice(&PlaneOffset { x: 0, y: 0 });

              mode.predict_inter(
                fi, 0, &po, tmp_slice, blk_w, blk_h, ref_frame, &cand_mv, 8,
              );
            }

            let plane_org = fs.input.planes[0].slice(&po);
            let plane_ref = tmp_plane.slice(&PlaneOffset { x: 0, y: 0 });

            let sad = get_sad(&plane_org, &plane_ref, blk_h, blk_w);

            if sad < lowest_sad {
              lowest_sad = sad;
              best_mv = cand_mv;
            }
          }
        }
      }

      best_mv
    }

    None => MotionVector { row: 0, col: 0 }
  }
}

#[cfg(test)]
pub mod test {
  use super::*;

  // Generate plane data for get_sad_same()
  fn setup_sad() -> (Plane, Plane) {
    let mut input_plane = Plane::new(640, 480, 0, 0, 128 + 8, 128 + 8);
    let mut rec_plane = input_plane.clone();

    for (i, row) in input_plane.data.chunks_mut(input_plane.cfg.stride).enumerate() {
      for (j, mut pixel) in row.into_iter().enumerate() {
        *pixel = ((j + i) % 256) as u16;
      }
    }

    for (i, row) in rec_plane.data.chunks_mut(rec_plane.cfg.stride).enumerate() {
      for (j, mut pixel) in row.into_iter().enumerate() {
        *pixel = ((j as i32 - i as i32) % 256) as u16;
      }
    }

    (input_plane, rec_plane)
  }

  // Regression and validation test for SAD computation
  #[test]
  fn get_sad_same() {
    use partition::BlockSize;
    use partition::BlockSize::*;

    let blocks: Vec<(BlockSize, u32)> = vec![
      (BLOCK_4X4, 393592),
      (BLOCK_4X8, 395176),
      (BLOCK_8X4, 1440456),
      (BLOCK_8X8, 1835664),
      (BLOCK_8X16, 1842256),
      (BLOCK_16X8, 6022352),
      (BLOCK_16X16, 7864736),
      (BLOCK_16X32, 7893152),
      (BLOCK_32X16, 24605344),
      (BLOCK_32X32, 32499008),
      (BLOCK_32X64, 32629056),
      (BLOCK_64X32, 99412288),
      (BLOCK_64X64, 132043392),
      (BLOCK_64X128, 132621248),
      (BLOCK_128X64, 399430272),
      (BLOCK_128X128, 532067552),
      (BLOCK_4X16, 398344),
      (BLOCK_16X4, 3533800),
      (BLOCK_8X32, 1855440),
      (BLOCK_32X8, 14392656),
      (BLOCK_16X64, 7949984),
      (BLOCK_64X16, 58061984),
    ];

    let (input_plane, rec_plane) = setup_sad();

    for block in blocks {
        let bsw = block.0.width();
        let bsh = block.0.height();
        let po = PlaneOffset { x: 40, y: 40 };

        let mut input_slice = input_plane.slice(&po);
        let mut rec_slice = rec_plane.slice(&po);

        assert_eq!(block.1, get_sad(&mut input_slice, &mut rec_slice, bsw, bsh));
    }
  }
}
<commit_msg>Use modulo instead of remainder in SAD test (#642)<commit_after>// Copyright (c) 2017-2018, The rav1e contributors. All rights reserved
//
// This source code is subject to the terms of the BSD 2 Clause License and
// the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
// was not distributed with this source code in the LICENSE file, you can
// obtain it at www.aomedia.org/license/software. If the Alliance for Open
// Media Patent License 1.0 was not distributed with this source code in the
// PATENTS file, you can obtain it at www.aomedia.org/license/patent.

use context::BlockOffset;
use context::BLOCK_TO_PLANE_SHIFT;
use context::MI_SIZE;
use partition::*;
use plane::*;
use FrameInvariants;
use FrameState;

#[inline(always)]
pub fn get_sad(
  plane_org: &PlaneSlice, plane_ref: &PlaneSlice, blk_h: usize,
  blk_w: usize
) -> u32 {
  let mut sum = 0 as u32;

  let org_iter = plane_org.iter_width(blk_w);
  let ref_iter = plane_ref.iter_width(blk_w);

  for (slice_org, slice_ref) in org_iter.take(blk_h).zip(ref_iter) {
      sum += slice_org
        .iter()
        .zip(slice_ref)
        .map(|(&a, &b)| (a as i32 - b as i32).abs() as u32)
        .sum::<u32>();
  }

  sum
}
pub fn motion_estimation(
  fi: &FrameInvariants, fs: &FrameState, bsize: BlockSize,
  bo: &BlockOffset, ref_frame: usize, pmv: &MotionVector
) -> MotionVector {
  match fi.rec_buffer.frames[fi.ref_frames[ref_frame - LAST_FRAME]] {
    Some(ref rec) => {
      let po = PlaneOffset {
        x: (bo.x as isize) << BLOCK_TO_PLANE_SHIFT,
        y: (bo.y as isize) << BLOCK_TO_PLANE_SHIFT
      };
      let range = 32 as isize;
      let blk_w = bsize.width();
      let blk_h = bsize.height();
      let border_w = 128 + blk_w as isize * 8;
      let border_h = 128 + blk_h as isize * 8;
      let mvx_min = -(bo.x as isize) * (8 * MI_SIZE) as isize - border_w;
      let mvx_max = (fi.w_in_b - bo.x - blk_w / MI_SIZE) as isize * (8 * MI_SIZE) as isize + border_w;
      let mvy_min = -(bo.y as isize) * (8 * MI_SIZE) as isize - border_h;
      let mvy_max = (fi.h_in_b - bo.y - blk_h / MI_SIZE) as isize * (8 * MI_SIZE) as isize + border_h;
      let x_lo = po.x + ((-range + (pmv.col / 8) as isize).max(mvx_min / 8));
      let x_hi = po.x + ((range + (pmv.col / 8) as isize).min(mvx_max / 8));
      let y_lo = po.y + ((-range + (pmv.row / 8) as isize).max(mvy_min / 8));
      let y_hi = po.y + ((range + (pmv.row / 8) as isize).min(mvy_max / 8));

      let mut lowest_sad = 128 * 128 * 4096 as u32;
      let mut best_mv = MotionVector { row: 0, col: 0 };

      for y in (y_lo..y_hi).step_by(2) {
        for x in (x_lo..x_hi).step_by(2) {
          let plane_org = fs.input.planes[0].slice(&po);
          let plane_ref = rec.frame.planes[0].slice(&PlaneOffset { x, y });

          let sad = get_sad(&plane_org, &plane_ref, blk_h, blk_w);

          if sad < lowest_sad {
            lowest_sad = sad;
            best_mv = MotionVector {
              row: 8 * (y as i16 - po.y as i16),
              col: 8 * (x as i16 - po.x as i16)
            }
          }
        }
      }

      let mode = PredictionMode::NEWMV;
      let mut tmp_plane = Plane::new(blk_w, blk_h, 0, 0, 0, 0);

      let mut steps = vec![8, 4, 2];
      if fi.allow_high_precision_mv {
        steps.push(1);
      }

      for step in steps {
        let center_mv_h = best_mv;
        for i in 0..3 {
          for j in 0..3 {
            // Skip the center point that was already tested
            if i == 1 && j == 1 {
              continue;
            }

            let cand_mv = MotionVector {
              row: center_mv_h.row + step * (i as i16 - 1),
              col: center_mv_h.col + step * (j as i16 - 1)
            };

            if (cand_mv.col as isize) < mvx_min || (cand_mv.col as isize) > mvx_max {
              continue;
            }
            if (cand_mv.row as isize) < mvy_min || (cand_mv.row as isize) > mvy_max {
              continue;
            }

            {
              let tmp_slice =
                &mut tmp_plane.mut_slice(&PlaneOffset { x: 0, y: 0 });

              mode.predict_inter(
                fi, 0, &po, tmp_slice, blk_w, blk_h, ref_frame, &cand_mv, 8,
              );
            }

            let plane_org = fs.input.planes[0].slice(&po);
            let plane_ref = tmp_plane.slice(&PlaneOffset { x: 0, y: 0 });

            let sad = get_sad(&plane_org, &plane_ref, blk_h, blk_w);

            if sad < lowest_sad {
              lowest_sad = sad;
              best_mv = cand_mv;
            }
          }
        }
      }

      best_mv
    }

    None => MotionVector { row: 0, col: 0 }
  }
}

#[cfg(test)]
pub mod test {
  use super::*;

  // Generate plane data for get_sad_same()
  fn setup_sad() -> (Plane, Plane) {
    let mut input_plane = Plane::new(640, 480, 0, 0, 128 + 8, 128 + 8);
    let mut rec_plane = input_plane.clone();

    for (i, row) in input_plane.data.chunks_mut(input_plane.cfg.stride).enumerate() {
      for (j, mut pixel) in row.into_iter().enumerate() {
        let val = ((j + i) as i32 & 255i32) as u16;
        assert!(val >= u8::min_value().into() &&
            val <= u8::max_value().into());
        *pixel = val;
      }
    }

    for (i, row) in rec_plane.data.chunks_mut(rec_plane.cfg.stride).enumerate() {
      for (j, mut pixel) in row.into_iter().enumerate() {
        let val = (j as i32 - i as i32 & 255i32) as u16;
        assert!(val >= u8::min_value().into() &&
            val <= u8::max_value().into());
        *pixel = val;
      }
    }

    (input_plane, rec_plane)
  }

  // Regression and validation test for SAD computation
  #[test]
  fn get_sad_same() {
    use partition::BlockSize;
    use partition::BlockSize::*;

    let blocks: Vec<(BlockSize, u32)> = vec![
      (BLOCK_4X4, 1912),
      (BLOCK_4X8, 3496),
      (BLOCK_8X4, 4296),
      (BLOCK_8X8, 7824),
      (BLOCK_8X16, 14416),
      (BLOCK_16X8, 16592),
      (BLOCK_16X16, 31136),
      (BLOCK_16X32, 59552),
      (BLOCK_32X16, 60064),
      (BLOCK_32X32, 120128),
      (BLOCK_32X64, 250176),
      (BLOCK_64X32, 186688),
      (BLOCK_64X64, 438912),
      (BLOCK_64X128, 1016768),
      (BLOCK_128X64, 654272),
      (BLOCK_128X128, 1689792),
      (BLOCK_4X16, 6664),
      (BLOCK_16X4, 8680),
      (BLOCK_8X32, 27600),
      (BLOCK_32X8, 31056),
      (BLOCK_16X64, 116384),
      (BLOCK_64X16, 93344),
    ];

    let (input_plane, rec_plane) = setup_sad();

    for block in blocks {
        let bsw = block.0.width();
        let bsh = block.0.height();
        let po = PlaneOffset { x: 40, y: 40 };

        let mut input_slice = input_plane.slice(&po);
        let mut rec_slice = rec_plane.slice(&po);

        assert_eq!(block.1, get_sad(&mut input_slice, &mut rec_slice, bsw, bsh));
    }
  }
}
<|endoftext|>