<commit_before>/// Literal kind.
///
/// E.g. `"foo"`, `42`, `12.34` or `bool`
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum Lit {
    /// A string literal (`"foo"`)
    Str(String, StrStyle),
    /// A byte string (`b"foo"`)
    ByteStr(Vec<u8>, StrStyle),
    /// A byte char (`b'f'`)
    Byte(u8),
    /// A character literal (`'a'`)
    Char(char),
    /// An integer literal (`1`)
    Int(u64, IntTy),
    /// A float literal (`1f64` or `1E10f64` or `1.0E10`)
    Float(String, FloatTy),
    /// A boolean literal
    Bool(bool),
}

#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum StrStyle {
    /// A regular string, like `"foo"`
    Cooked,
    /// A raw string, like `r##"foo"##`
    ///
    /// The uint is the number of `#` symbols used
    Raw(usize),
}

impl From<String> for Lit {
    fn from(input: String) -> Lit {
        Lit::Str(input, StrStyle::Cooked)
    }
}

impl<'a> From<&'a str> for Lit {
    fn from(input: &str) -> Lit {
        Lit::Str(input.into(), StrStyle::Cooked)
    }
}

impl From<Vec<u8>> for Lit {
    fn from(input: Vec<u8>) -> Lit {
        Lit::ByteStr(input, StrStyle::Cooked)
    }
}

impl<'a> From<&'a [u8]> for Lit {
    fn from(input: &[u8]) -> Lit {
        Lit::ByteStr(input.into(), StrStyle::Cooked)
    }
}

impl From<char> for Lit {
    fn from(input: char) -> Lit {
        Lit::Char(input)
    }
}

impl From<bool> for Lit {
    fn from(input: bool) -> Lit {
        Lit::Bool(input)
    }
}

#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum IntTy {
    Isize,
    I8,
    I16,
    I32,
    I64,
    Usize,
    U8,
    U16,
    U32,
    U64,
    Unsuffixed,
}

#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum FloatTy {
    F32,
    F64,
    Unsuffixed,
}

macro_rules! impl_from_for_lit {
    (Int, [$($rust_type:ty => $syn_type:expr),+]) => {
        $(
            impl From<$rust_type> for Lit {
                fn from(input: $rust_type) -> Lit {
                    Lit::Int(input as u64, $syn_type)
                }
            }
        )+
    };
    (Float, [$($rust_type:ty => $syn_type:expr),+]) => {
        $(
            impl From<$rust_type> for Lit {
                fn from(input: $rust_type) -> Lit {
                    Lit::Float(format!("{}", input), $syn_type)
                }
            }
        )+
    };
}

impl_from_for_lit! {Int, [
    isize => IntTy::Isize,
    i8 => IntTy::I8,
    i16 => IntTy::I16,
    i32 => IntTy::I32,
    i64 => IntTy::I64,
    usize => IntTy::Usize,
    u8 => IntTy::U8,
    u16 => IntTy::U16,
    u32 => IntTy::U32,
    u64 => IntTy::U64
]}

impl_from_for_lit! {Float, [
    f32 => FloatTy::F32,
    f64 => FloatTy::F64
]}

#[cfg(feature = "parsing")]
pub mod parsing {
    use super::*;
    use escape::{cooked_char, cooked_string, raw_string};
    use space::whitespace;
    use nom::IResult;
    use unicode_xid::UnicodeXID;

    named!(pub lit -> Lit, alt!(
        string
        |
        byte_string
        |
        byte
        |
        character
        |
        float // must be before int
        |
        int => { |(value, ty)| Lit::Int(value, ty) }
        |
        boolean
    ));

    named!(string -> Lit, alt!(
        quoted_string => { |s| Lit::Str(s, StrStyle::Cooked) }
        |
        preceded!(
            punct!("r"),
            raw_string
        ) => { |(s, n)| Lit::Str(s, StrStyle::Raw(n)) }
    ));

    named!(pub quoted_string -> String, delimited!(
        punct!("\""),
        cooked_string,
        tag!("\"")
    ));

    named!(byte_string -> Lit, alt!(
        delimited!(
            punct!("b\""),
            cooked_string,
            tag!("\"")
        ) => { |s: String| Lit::ByteStr(s.into_bytes(), StrStyle::Cooked) }
        |
        preceded!(
            punct!("br"),
            raw_string
        ) => { |(s, n): (String, _)| Lit::ByteStr(s.into_bytes(), StrStyle::Raw(n)) }
    ));

    named!(byte -> Lit, do_parse!(
        punct!("b") >>
        tag!("'") >>
        ch: cooked_char >>
        tag!("'") >>
        (Lit::Byte(ch as u8))
    ));

    named!(character -> Lit, do_parse!(
        punct!("'") >>
        ch: cooked_char >>
        tag!("'") >>
        (Lit::Char(ch))
    ));

    named!(float -> Lit, do_parse!(
        option!(whitespace) >>
        value: float_string >>
        suffix: alt!(
            tag!("f32") => { |_| FloatTy::F32 }
            |
            tag!("f64") => { |_| FloatTy::F64 }
            |
            epsilon!() => { |_| FloatTy::Unsuffixed }
        ) >>
        (Lit::Float(value, suffix))
    ));

    named!(pub int -> (u64, IntTy), tuple!(
        preceded!(
            option!(whitespace),
            digits
        ),
        alt!(
            tag!("isize") => { |_| IntTy::Isize }
            |
            tag!("i8") => { |_| IntTy::I8 }
            |
            tag!("i16") => { |_| IntTy::I16 }
            |
            tag!("i32") => { |_| IntTy::I32 }
            |
            tag!("i64") => { |_| IntTy::I64 }
            |
            tag!("usize") => { |_| IntTy::Usize }
            |
            tag!("u8") => { |_| IntTy::U8 }
            |
            tag!("u16") => { |_| IntTy::U16 }
            |
            tag!("u32") => { |_| IntTy::U32 }
            |
            tag!("u64") => { |_| IntTy::U64 }
            |
            epsilon!() => { |_| IntTy::Unsuffixed }
        )
    ));

    named!(boolean -> Lit, alt!(
        keyword!("true") => { |_| Lit::Bool(true) }
        |
        keyword!("false") => { |_| Lit::Bool(false) }
    ));

    fn float_string(input: &str) -> IResult<&str, String> {
        let mut chars = input.chars().peekable();
        match chars.next() {
            Some(ch) if ch >= '0' && ch <= '9' => {}
            _ => return IResult::Error,
        }

        let mut len = 1;
        let mut has_dot = false;
        let mut has_exp = false;
        while let Some(&ch) = chars.peek() {
            match ch {
                '0'...'9' | '_' => {
                    chars.next();
                    len += 1;
                }
                '.' => {
                    if has_dot {
                        break;
                    }
                    chars.next();
                    if chars.peek()
                        .map(|&ch| ch == '.' || UnicodeXID::is_xid_start(ch))
                        .unwrap_or(false) {
                        return IResult::Error;
                    }
                    len += 1;
                    has_dot = true;
                }
                'e' | 'E' => {
                    chars.next();
                    len += 1;
                    has_exp = true;
                    break;
                }
                _ => break,
            }
        }

        let rest = &input[len..];
        if !(has_dot || has_exp || rest.starts_with("f32") || rest.starts_with("f64")) {
            return IResult::Error;
        }

        if has_exp {
            let mut has_exp_value = false;
            while let Some(&ch) = chars.peek() {
                match ch {
                    '+' | '-' => {
                        if has_exp_value {
                            break;
                        }
                        chars.next();
                        len += 1;
                    }
                    '0'...'9' => {
                        chars.next();
                        len += 1;
                        has_exp_value = true;
                    }
                    '_' => {
                        chars.next();
                        len += 1;
                    }
                    _ => break,
                }
            }
            if !has_exp_value {
                return IResult::Error;
            }
        }

        IResult::Done(&input[len..], input[..len].into())
    }

    pub fn digits(input: &str) -> IResult<&str, u64> {
        let mut value = 0u64;
        let mut len = 0;
        let mut bytes = input.bytes().peekable();
        while let Some(&b) = bytes.peek() {
            match b {
                b'0'...b'9' => {
                    value = match value.checked_mul(10) {
                        Some(value) => value,
                        None => return IResult::Error,
                    };
                    value = match value.checked_add((b - b'0') as u64) {
                        Some(value) => value,
                        None => return IResult::Error,
                    };
                    bytes.next();
                    len += 1;
                }
                _ => break,
            }
        }
        if len > 0 {
            IResult::Done(&input[len..], value)
        } else {
            IResult::Error
        }
    }
}

#[cfg(feature = "printing")]
mod printing {
    use super::*;
    use quote::{Tokens, ToTokens};
    use std::{ascii, iter};
    use std::fmt::{self, Display};
    use std::str;

    impl ToTokens for Lit {
        fn to_tokens(&self, tokens: &mut Tokens) {
            match *self {
                Lit::Str(ref s, StrStyle::Cooked) => s.to_tokens(tokens),
                Lit::Str(ref s, StrStyle::Raw(n)) => {
                    tokens.append(&format!("r{delim}\"{string}\"{delim}",
                        delim = iter::repeat("#").take(n).collect::<String>(),
                        string = s));
                }
                Lit::ByteStr(ref v, StrStyle::Cooked) => {
                    let mut escaped = "b\"".to_string();
                    for &ch in v.iter() {
                        escaped.extend(ascii::escape_default(ch).map(|c| c as char));
                    }
                    escaped.push('"');
                    tokens.append(&escaped);
                }
                Lit::ByteStr(ref vec, StrStyle::Raw(n)) => {
                    tokens.append(&format!("br{delim}\"{string}\"{delim}",
                        delim = iter::repeat("#").take(n).collect::<String>(),
                        string = str::from_utf8(vec).unwrap()));
                }
                Lit::Byte(b) => tokens.append(&format!("b{:?}", b as char)),
                Lit::Char(ch) => ch.to_tokens(tokens),
                Lit::Int(value, ty) => tokens.append(&format!("{}{}", value, ty)),
                Lit::Float(ref value, ty) => tokens.append(&format!("{}{}", value, ty)),
                Lit::Bool(true) => tokens.append("true"),
                Lit::Bool(false) => tokens.append("false"),
            }
        }
    }

    impl Display for IntTy {
        fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> {
            match *self {
                IntTy::Isize => formatter.write_str("isize"),
                IntTy::I8 => formatter.write_str("i8"),
                IntTy::I16 => formatter.write_str("i16"),
                IntTy::I32 => formatter.write_str("i32"),
                IntTy::I64 => formatter.write_str("i64"),
                IntTy::Usize => formatter.write_str("usize"),
                IntTy::U8 => formatter.write_str("u8"),
                IntTy::U16 => formatter.write_str("u16"),
                IntTy::U32 => formatter.write_str("u32"),
                IntTy::U64 => formatter.write_str("u64"),
                IntTy::Unsuffixed => Ok(()),
            }
        }
    }

    impl Display for FloatTy {
        fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> {
            match *self {
                FloatTy::F32 => formatter.write_str("f32"),
                FloatTy::F64 => formatter.write_str("f64"),
                FloatTy::Unsuffixed => Ok(()),
            }
        }
    }
}
<commit_msg>Integers with underscores<commit_after>/// Literal kind.
///
/// E.g. `"foo"`, `42`, `12.34` or `bool`
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum Lit {
    /// A string literal (`"foo"`)
    Str(String, StrStyle),
    /// A byte string (`b"foo"`)
    ByteStr(Vec<u8>, StrStyle),
    /// A byte char (`b'f'`)
    Byte(u8),
    /// A character literal (`'a'`)
    Char(char),
    /// An integer literal (`1`)
    Int(u64, IntTy),
    /// A float literal (`1f64` or `1E10f64` or `1.0E10`)
    Float(String, FloatTy),
    /// A boolean literal
    Bool(bool),
}

#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum StrStyle {
    /// A regular string, like `"foo"`
    Cooked,
    /// A raw string, like `r##"foo"##`
    ///
    /// The uint is the number of `#` symbols used
    Raw(usize),
}

impl From<String> for Lit {
    fn from(input: String) -> Lit {
        Lit::Str(input, StrStyle::Cooked)
    }
}

impl<'a> From<&'a str> for Lit {
    fn from(input: &str) -> Lit {
        Lit::Str(input.into(), StrStyle::Cooked)
    }
}

impl From<Vec<u8>> for Lit {
    fn from(input: Vec<u8>) -> Lit {
        Lit::ByteStr(input, StrStyle::Cooked)
    }
}

impl<'a> From<&'a [u8]> for Lit {
    fn from(input: &[u8]) -> Lit {
        Lit::ByteStr(input.into(), StrStyle::Cooked)
    }
}

impl From<char> for Lit {
    fn from(input: char) -> Lit {
        Lit::Char(input)
    }
}

impl From<bool> for Lit {
    fn from(input: bool) -> Lit {
        Lit::Bool(input)
    }
}

#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum IntTy {
    Isize,
    I8,
    I16,
    I32,
    I64,
    Usize,
    U8,
    U16,
    U32,
    U64,
    Unsuffixed,
}

#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum FloatTy {
    F32,
    F64,
    Unsuffixed,
}

macro_rules! impl_from_for_lit {
    (Int, [$($rust_type:ty => $syn_type:expr),+]) => {
        $(
            impl From<$rust_type> for Lit {
                fn from(input: $rust_type) -> Lit {
                    Lit::Int(input as u64, $syn_type)
                }
            }
        )+
    };
    (Float, [$($rust_type:ty => $syn_type:expr),+]) => {
        $(
            impl From<$rust_type> for Lit {
                fn from(input: $rust_type) -> Lit {
                    Lit::Float(format!("{}", input), $syn_type)
                }
            }
        )+
    };
}

impl_from_for_lit! {Int, [
    isize => IntTy::Isize,
    i8 => IntTy::I8,
    i16 => IntTy::I16,
    i32 => IntTy::I32,
    i64 => IntTy::I64,
    usize => IntTy::Usize,
    u8 => IntTy::U8,
    u16 => IntTy::U16,
    u32 => IntTy::U32,
    u64 => IntTy::U64
]}

impl_from_for_lit! {Float, [
    f32 => FloatTy::F32,
    f64 => FloatTy::F64
]}

#[cfg(feature = "parsing")]
pub mod parsing {
    use super::*;
    use escape::{cooked_char, cooked_string, raw_string};
    use space::whitespace;
    use nom::IResult;
    use unicode_xid::UnicodeXID;

    named!(pub lit -> Lit, alt!(
        string
        |
        byte_string
        |
        byte
        |
        character
        |
        float // must be before int
        |
        int => { |(value, ty)| Lit::Int(value, ty) }
        |
        boolean
    ));

    named!(string -> Lit, alt!(
        quoted_string => { |s| Lit::Str(s, StrStyle::Cooked) }
        |
        preceded!(
            punct!("r"),
            raw_string
        ) => { |(s, n)| Lit::Str(s, StrStyle::Raw(n)) }
    ));

    named!(pub quoted_string -> String, delimited!(
        punct!("\""),
        cooked_string,
        tag!("\"")
    ));

    named!(byte_string -> Lit, alt!(
        delimited!(
            punct!("b\""),
            cooked_string,
            tag!("\"")
        ) => { |s: String| Lit::ByteStr(s.into_bytes(), StrStyle::Cooked) }
        |
        preceded!(
            punct!("br"),
            raw_string
        ) => { |(s, n): (String, _)| Lit::ByteStr(s.into_bytes(), StrStyle::Raw(n)) }
    ));

    named!(byte -> Lit, do_parse!(
        punct!("b") >>
        tag!("'") >>
        ch: cooked_char >>
        tag!("'") >>
        (Lit::Byte(ch as u8))
    ));

    named!(character -> Lit, do_parse!(
        punct!("'") >>
        ch: cooked_char >>
        tag!("'") >>
        (Lit::Char(ch))
    ));

    named!(float -> Lit, do_parse!(
        option!(whitespace) >>
        value: float_string >>
        suffix: alt!(
            tag!("f32") => { |_| FloatTy::F32 }
            |
            tag!("f64") => { |_| FloatTy::F64 }
            |
            epsilon!() => { |_| FloatTy::Unsuffixed }
        ) >>
        (Lit::Float(value, suffix))
    ));

    named!(pub int -> (u64, IntTy), tuple!(
        preceded!(
            option!(whitespace),
            digits
        ),
        alt!(
            tag!("isize") => { |_| IntTy::Isize }
            |
            tag!("i8") => { |_| IntTy::I8 }
            |
            tag!("i16") => { |_| IntTy::I16 }
            |
            tag!("i32") => { |_| IntTy::I32 }
            |
            tag!("i64") => { |_| IntTy::I64 }
            |
            tag!("usize") => { |_| IntTy::Usize }
            |
            tag!("u8") => { |_| IntTy::U8 }
            |
            tag!("u16") => { |_| IntTy::U16 }
            |
            tag!("u32") => { |_| IntTy::U32 }
            |
            tag!("u64") => { |_| IntTy::U64 }
            |
            epsilon!() => { |_| IntTy::Unsuffixed }
        )
    ));

    named!(boolean -> Lit, alt!(
        keyword!("true") => { |_| Lit::Bool(true) }
        |
        keyword!("false") => { |_| Lit::Bool(false) }
    ));

    fn float_string(input: &str) -> IResult<&str, String> {
        let mut chars = input.chars().peekable();
        match chars.next() {
            Some(ch) if ch >= '0' && ch <= '9' => {}
            _ => return IResult::Error,
        }

        let mut len = 1;
        let mut has_dot = false;
        let mut has_exp = false;
        while let Some(&ch) = chars.peek() {
            match ch {
                '0'...'9' | '_' => {
                    chars.next();
                    len += 1;
                }
                '.' => {
                    if has_dot {
                        break;
                    }
                    chars.next();
                    if chars.peek()
                        .map(|&ch| ch == '.' || UnicodeXID::is_xid_start(ch))
                        .unwrap_or(false) {
                        return IResult::Error;
                    }
                    len += 1;
                    has_dot = true;
                }
                'e' | 'E' => {
                    chars.next();
                    len += 1;
                    has_exp = true;
                    break;
                }
                _ => break,
            }
        }

        let rest = &input[len..];
        if !(has_dot || has_exp || rest.starts_with("f32") || rest.starts_with("f64")) {
            return IResult::Error;
        }

        if has_exp {
            let mut has_exp_value = false;
            while let Some(&ch) = chars.peek() {
                match ch {
                    '+' | '-' => {
                        if has_exp_value {
                            break;
                        }
                        chars.next();
                        len += 1;
                    }
                    '0'...'9' => {
                        chars.next();
                        len += 1;
                        has_exp_value = true;
                    }
                    '_' => {
                        chars.next();
                        len += 1;
                    }
                    _ => break,
                }
            }
            if !has_exp_value {
                return IResult::Error;
            }
        }

        IResult::Done(&input[len..], input[..len].into())
    }

    pub fn digits(input: &str) -> IResult<&str, u64> {
        let mut value = 0u64;
        let mut len = 0;
        let mut empty = true;
        for b in input.bytes() {
            match b {
                b'0'...b'9' => {
                    value = match value.checked_mul(10) {
                        Some(value) => value,
                        None => return IResult::Error,
                    };
                    value = match value.checked_add((b - b'0') as u64) {
                        Some(value) => value,
                        None => return IResult::Error,
                    };
                    len += 1;
                    empty = false;
                }
                b'_' => {
                    if empty {
                        return IResult::Error;
                    }
                    len += 1;
                }
                _ => break,
            }
        }
        if empty {
            IResult::Error
        } else {
            IResult::Done(&input[len..], value)
        }
    }
}

#[cfg(feature = "printing")]
mod printing {
    use super::*;
    use quote::{Tokens, ToTokens};
    use std::{ascii, iter};
    use std::fmt::{self, Display};
    use std::str;

    impl ToTokens for Lit {
        fn to_tokens(&self, tokens: &mut Tokens) {
            match *self {
                Lit::Str(ref s, StrStyle::Cooked) => s.to_tokens(tokens),
                Lit::Str(ref s, StrStyle::Raw(n)) => {
                    tokens.append(&format!("r{delim}\"{string}\"{delim}",
                        delim = iter::repeat("#").take(n).collect::<String>(),
                        string = s));
                }
                Lit::ByteStr(ref v, StrStyle::Cooked) => {
                    let mut escaped = "b\"".to_string();
                    for &ch in v.iter() {
                        escaped.extend(ascii::escape_default(ch).map(|c| c as char));
                    }
                    escaped.push('"');
                    tokens.append(&escaped);
                }
                Lit::ByteStr(ref vec, StrStyle::Raw(n)) => {
                    tokens.append(&format!("br{delim}\"{string}\"{delim}",
                        delim = iter::repeat("#").take(n).collect::<String>(),
                        string = str::from_utf8(vec).unwrap()));
                }
                Lit::Byte(b) => tokens.append(&format!("b{:?}", b as char)),
                Lit::Char(ch) => ch.to_tokens(tokens),
                Lit::Int(value, ty) => tokens.append(&format!("{}{}", value, ty)),
                Lit::Float(ref value, ty) => tokens.append(&format!("{}{}", value, ty)),
                Lit::Bool(true) => tokens.append("true"),
                Lit::Bool(false) => tokens.append("false"),
            }
        }
    }

    impl Display for IntTy {
        fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> {
            match *self {
                IntTy::Isize => formatter.write_str("isize"),
                IntTy::I8 => formatter.write_str("i8"),
                IntTy::I16 => formatter.write_str("i16"),
                IntTy::I32 => formatter.write_str("i32"),
                IntTy::I64 => formatter.write_str("i64"),
                IntTy::Usize => formatter.write_str("usize"),
                IntTy::U8 => formatter.write_str("u8"),
                IntTy::U16 => formatter.write_str("u16"),
                IntTy::U32 => formatter.write_str("u32"),
                IntTy::U64 => formatter.write_str("u64"),
                IntTy::Unsuffixed => Ok(()),
            }
        }
    }

    impl Display for FloatTy {
        fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> {
            match *self {
                FloatTy::F32 => formatter.write_str("f32"),
                FloatTy::F64 => formatter.write_str("f64"),
                FloatTy::Unsuffixed => Ok(()),
            }
        }
    }
}
<|endoftext|>


<commit_before>use std::collections::BTreeSet;
use std::cmp;
use rand::Rng;
use std::ops::Range;
use std::iter::Rev;
use super::{Learning, Genetic};
use std::marker::PhantomData;

/*
Defines an opcode for the Mep. Every opcode contains an instruction and two parameter indices. These specify which
previous opcodes produced the result required as inputs to this opcode. These parameters can also come from the inputs
to the program, which sequentially preceed the internal instructions.
*/
struct Opcode<Ins> {
    instruction: Ins,
    first: usize,
    second: usize,
}

impl<Ins> Clone for Opcode<Ins> where Ins: Clone {
    fn clone(&self) -> Self {
        Opcode{
            instruction: self.instruction.clone(),
            first: self.first,
            second: self.second,
        }
    }
}

/*
A multi-expression program represented using a series of operations that can reuse results of previous operations.
*/
pub struct Mep<Ins, R, Param, F1, F2>
    where R: Rng, F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param
{
    program: Vec<Opcode<Ins>>,
    unit_mutate_size: usize,
    crossover_points: usize,
    inputs: usize,
    mutator: F1,
    processor: F2,
    phantom: (PhantomData<R>, PhantomData<Param>),
}

struct ResultIterator<'a, Ins: 'a, R: 'a, Param: 'a, F1: 'a, F2: 'a>
    where F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param, R: Rng, Param: Clone
{
    mep: &'a Mep<Ins, R, Param, F1, F2>,
    buff: Vec<Option<Param>>,
    solve_iter: Rev<Range<usize>>,
    inputs: &'a [Param]
}

impl<Ins, R, Param, F1, F2> Clone for Mep<Ins, R, Param, F1, F2>
    where F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param, R: Rng, Ins: Clone
{
    fn clone(&self) -> Self {
        Mep{
            program: self.program.clone(),
            unit_mutate_size: self.unit_mutate_size,
            crossover_points: self.crossover_points,
            inputs: self.inputs,
            mutator: self.mutator.clone(),
            processor: self.processor.clone(),
            phantom: (PhantomData, PhantomData),
        }
    }
}

impl<Ins, R, Param, F1, F2> Mep<Ins, R, Param, F1, F2>
    where F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param, R: Rng
{
    /*
    Generates a new Mep with a particular size and takes a closure to generate random instructions.
    Takes an RNG as well to generate random internal data for each instruction. This Rng is different from the Mep's
    Rng, thus it is parameterized separately here.
    */
    pub fn new<I, Rg>(inputs: usize, unit_mutate_size: usize, crossover_points: usize, rng: &mut Rg, instruction_iter: I,
        mutator: F1, processor: F2) -> Self where I: Iterator<Item=Ins>, Rg: Rng {
        Mep{
            program: instruction_iter.enumerate()
                .map(|(index, ins)| Opcode{
                        instruction: ins,
                        first: rng.gen_range(0, index + inputs),
                        second: rng.gen_range(0, index + inputs)
                    }
                ).collect(),
            unit_mutate_size: unit_mutate_size,
            crossover_points: crossover_points,
            inputs: inputs,
            mutator: mutator,
            processor: processor,
            phantom: (PhantomData, PhantomData),
        }
    }
}

impl<Ins, R, Param, F1, F2> Learning<R, Param, Param> for Mep<Ins, R, Param, F1, F2>
    where F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param, R: Rng, Param: Clone
{
    fn compute<'a>(&'a self, inputs: &'a [Param], outputs: usize) -> Box<Iterator<Item=Param> + 'a> {
        //Ensure we have enough opcodes to produce the desired amount of outputs, otherwise the programmer has failed
        assert!(outputs <= self.program.len());
        Box::new(ResultIterator{
            mep: self,
            buff: vec![None; self.program.len()],
            solve_iter: (self.program.len() + self.inputs - outputs..self.program.len() + self.inputs).rev(),
            inputs: inputs,
        })
    }

    fn train(&self, _: &[Param], _: &[Param], _: &mut R) {
        //TODO: Implement fully
    }
}

impl<Ins, R, Param, F1, F2> Genetic<R, Param, Param> for Mep<Ins, R, Param, F1, F2>
    where F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param, R: Rng, Ins: Clone, Param: Clone
{
    fn mate(parents: (&Self, &Self), rng: &mut R) -> Self {
        //Each Mep must have the same amount of inputs
        //TODO: Once Rust implements generic values, this can be made explicit and is not needed
        assert!(parents.0.inputs == parents.1.inputs);
        //Get the smallest of the two lengths
        let total_instructions = cmp::min(parents.0.program.len(), parents.1.program.len());
        Mep{program:
            //Generate a randomly sized sequence between 1 and half of the total possible crossover points
            (0..rng.gen_range(1, total_instructions / 2))
            //Map these to random crossover points
            .map(|_| rng.gen_range(0, total_instructions))
            //Add total_instructions at the end so we can generate a range with it
            .chain(Some(total_instructions))
            //Sort them by value into BTree, which removes duplicates and orders them
            .fold(BTreeSet::new(), |mut set, i| {set.insert(i); set})
            //Iterate over the sorted values
            .iter()
            //Turn every copy of two, prepending a 0, into a range
            .scan(0, |prev, x| {let out = Some(*prev..*x); *prev = *x; out})
            //Enumerate by index to differentiate odd and even values
            .enumerate()
            //Map even pairs to ranges in parent 0 and odd ones to ranges in parent 1 and expand the ranges
            .flat_map(|(index, range)| {
                {if index % 2 == 0 {parents.0} else {parents.1}}.program[range].iter().cloned()
            })
            //Collect all the instruction ranges from each parent
            .collect(),

            unit_mutate_size: if parents.0.unit_mutate_size < parents.1.unit_mutate_size {
                rng.gen_range(parents.0.unit_mutate_size, parents.1.unit_mutate_size + 1)
            } else {
                rng.gen_range(parents.1.unit_mutate_size, parents.0.unit_mutate_size + 1)
            },

            crossover_points: if parents.0.crossover_points < parents.1.crossover_points {
                rng.gen_range(parents.0.crossover_points, parents.1.crossover_points + 1)
            } else {
                rng.gen_range(parents.1.crossover_points, parents.0.crossover_points + 1)
            },

            inputs: parents.0.inputs,
            mutator: {if rng.gen_range(0, 2) == 0 {parents.0} else {parents.1}}.mutator.clone(),
            processor: {if rng.gen_range(0, 2) == 0 {parents.0} else {parents.1}}.processor.clone(),
            phantom: (PhantomData, PhantomData),
        }
    }

    /*
    The Mep mutate function operates using the unit_mutate_size. This variable specifies the amount of instructions for
    which a single mutation is expect to occour every time mutate is called. This variable can be mutated inside of
    mutate, in which case it may never go below 1, but may tend towards infinity in increments of 1. This variable is
    implemented as a u64 to permit it to expand unbounded to mutation levels that are so low that mutations virtually
    never happen. Allowing this to mutate allows species to find the equilibrium between genomic adaptability and
    stability. If a species develops information gathering, then it can adapt intellegently, making it possibly more
    beneficial to operate at lower mutation rates. Setting the default mutation rate for species properly, or allowing
    it to adapt as the simulation continues, permits species to survive more frequently that are randomly generated.

    Likewise, the functions for random instruction mutation can be adapted as the simulation continues
    to optimize the generation of more desireable random mutations. For instance, instructions that
    occur more frequently should be generated randomly more frequently.
    */
    fn mutate(&mut self, rng: &mut R) {
        //Mutate unit_mutate_size
        if rng.gen_range(0, self.unit_mutate_size) == 0 {
            //Make it possibly go up or down by 1
            match rng.gen_range(0, 2) {
                0 => self.unit_mutate_size += 1,
                1 => if self.unit_mutate_size > 1 {self.unit_mutate_size -= 1},
                _ => unreachable!(),
            }
        }
        //Mutate crossover_points
        if rng.gen_range(0, self.unit_mutate_size) == 0 {
            //Make it possibly go up or down by 1
            match rng.gen_range(0, 2) {
                0 => self.crossover_points += 1,
                1 => if self.crossover_points > 1 {self.crossover_points -= 1},
                _ => unreachable!(),
            }
        }

        //Mutate the instructions using the mutator
        loop {
            //Choose a random location in the instructions and then add a random value up to the unit_mutate_size
            let choice = rng.gen_range(0, self.program.len()) + rng.gen_range(0, self.unit_mutate_size);
            //Whenever we choose a location outside the vector reject the choice and end mutation
            if choice >= self.program.len() {
                break;
            }
            let op = &mut self.program[choice];
            //Randomly mutate only one of the things contained here
            match rng.gen_range(0, 3) {
                0 => (self.mutator)(&mut op.instruction, rng),
                1 => op.first = rng.gen_range(0, choice + self.inputs),
                2 => op.second = rng.gen_range(0, choice + self.inputs),
                _ => unreachable!(),
            }
        }
    }
}

impl<'a, Ins, R, Param, F1, F2> ResultIterator<'a, Ins, R, Param, F1, F2>
    where F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param, R: Rng, Param: Clone
{
    fn op_solved(&mut self, i: usize) -> Param {
        //If this is an input, it is already solved, so return the result immediately
        if i < self.mep.inputs {
            return self.inputs[i].clone();
        }
        //Check if this has been evaluated or not
        let possible = self.buff[i - self.mep.inputs].clone();
        match possible {
            //If it has, return the value
            Some(x) => x,
            //If it hasnt been solved
            None => {
                //Get a reference to the opcode
                let op = &self.mep.program[i];
                //Compute the result of the operation, ensuring the inputs are solved beforehand
                let result = (self.mep.processor)(&op.instruction, self.op_solved(op.first), self.op_solved(op.second));
                //Properly store the Some result to the buffer
                self.buff[i - self.mep.inputs] = Some(result.clone());
                //Return the result
                result
            }
        }
    }
}

impl<'a, Ins, R, Param, F1, F2> Iterator for ResultIterator<'a, Ins, R, Param, F1, F2>
    where F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param, R: Rng, Param: Clone
{
    type Item = Param;
    fn next(&mut self) -> Option<Param> {
        match self.solve_iter.next() {
            None => None,
            Some(i) => Some(self.op_solved(i)),
        }
    }
}

#[cfg(test)]
mod tests {
    use rand::{Isaac64Rng, SeedableRng, Rng};
    use super::*;
    use super::super::{Genetic, Learning};

    #[test]
    fn mep_new() {
        let a: Mep<u32> = Mep::new(3, 3, 0..8);

        assert_eq!(a.instructions, (0..8).collect::<Vec<_>>());
    }

    #[test]
    fn mep_crossover() {
        let mut rng = Isaac64Rng::from_seed(&[1, 2, 3, 4]);
        let (a, b) = {
            let mut clos = || Mep::new(3, 3, rng.gen_iter::<u32>().map(|x| x % 10).take(10));
            (clos(), clos())
        };
        let old_rngs: Vec<_> = rng.clone().gen_iter::<u32>().take(5).collect();
        let mut c = Mep::mate((&a, &b), &mut rng);
        //Ensure that rng was borrowed mutably
        assert!(rng.clone().gen_iter::<u32>().take(5).collect::<Vec<_>>() != old_rngs);

        c.mutate(&mut rng, |ins: &mut u32| *ins = 2);
        c.call(|_, (_, _)| {});

        assert_eq!(c.instructions, [0, 7, 5, 4, 2, 8, 5, 6, 0, 2]);
    }
}
<commit_msg>fixed phantom data warning and tests<commit_after>use std::collections::BTreeSet;
use std::cmp;
use rand::Rng;
use std::ops::Range;
use std::iter::Rev;
use super::{Learning, Genetic};
use std::marker::PhantomData;

/*
Defines an opcode for the Mep. Every opcode contains an instruction and two parameter indices. These specify which
previous opcodes produced the result required as inputs to this opcode. These parameters can also come from the inputs
to the program, which sequentially preceed the internal instructions.
*/
struct Opcode<Ins> {
    instruction: Ins,
    first: usize,
    second: usize,
}

impl<Ins> Clone for Opcode<Ins> where Ins: Clone {
    fn clone(&self) -> Self {
        Opcode{
            instruction: self.instruction.clone(),
            first: self.first,
            second: self.second,
        }
    }
}

/*
A multi-expression program represented using a series of operations that can reuse results of previous operations.
*/
pub struct Mep<Ins, R, Param, F1, F2>
    where R: Rng, F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param
{
    program: Vec<Opcode<Ins>>,
    unit_mutate_size: usize,
    crossover_points: usize,
    inputs: usize,
    mutator: F1,
    processor: F2,
    _phantom: (PhantomData<R>, PhantomData<Param>),
}

struct ResultIterator<'a, Ins: 'a, R: 'a, Param: 'a, F1: 'a, F2: 'a>
    where F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param, R: Rng, Param: Clone
{
    mep: &'a Mep<Ins, R, Param, F1, F2>,
    buff: Vec<Option<Param>>,
    solve_iter: Rev<Range<usize>>,
    inputs: &'a [Param]
}

impl<Ins, R, Param, F1, F2> Clone for Mep<Ins, R, Param, F1, F2>
    where F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param, R: Rng, Ins: Clone
{
    fn clone(&self) -> Self {
        Mep{
            program: self.program.clone(),
            unit_mutate_size: self.unit_mutate_size,
            crossover_points: self.crossover_points,
            inputs: self.inputs,
            mutator: self.mutator.clone(),
            processor: self.processor.clone(),
            phantom: (PhantomData, PhantomData),
        }
    }
}

impl<Ins, R, Param, F1, F2> Mep<Ins, R, Param, F1, F2>
    where F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param, R: Rng
{
    /*
    Generates a new Mep with a particular size and takes a closure to generate random instructions.
    Takes an RNG as well to generate random internal data for each instruction. This Rng is different from the Mep's
    Rng, thus it is parameterized separately here.
    */
    pub fn new<I, Rg>(inputs: usize, unit_mutate_size: usize, crossover_points: usize, rng: &mut Rg, instruction_iter: I,
        mutator: F1, processor: F2) -> Self where I: Iterator<Item=Ins>, Rg: Rng {
        Mep{
            program: instruction_iter.enumerate()
                .map(|(index, ins)| Opcode{
                        instruction: ins,
                        first: rng.gen_range(0, index + inputs),
                        second: rng.gen_range(0, index + inputs)
                    }
                ).collect(),
            unit_mutate_size: unit_mutate_size,
            crossover_points: crossover_points,
            inputs: inputs,
            mutator: mutator,
            processor: processor,
            phantom: (PhantomData, PhantomData),
        }
    }
}

impl<Ins, R, Param, F1, F2> Learning<R, Param, Param> for Mep<Ins, R, Param, F1, F2>
    where F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param, R: Rng, Param: Clone
{
    fn compute<'a>(&'a self, inputs: &'a [Param], outputs: usize) -> Box<Iterator<Item=Param> + 'a> {
        //Ensure we have enough opcodes to produce the desired amount of outputs, otherwise the programmer has failed
        assert!(outputs <= self.program.len());
        Box::new(ResultIterator{
            mep: self,
            buff: vec![None; self.program.len()],
            solve_iter: (self.program.len() + self.inputs - outputs..self.program.len() + self.inputs).rev(),
            inputs: inputs,
        })
    }

    fn train(&self, _: &[Param], _: &[Param], _: &mut R) {
        //TODO: Implement fully
    }
}

impl<Ins, R, Param, F1, F2> Genetic<R, Param, Param> for Mep<Ins, R, Param, F1, F2>
    where F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param, R: Rng, Ins: Clone, Param: Clone
{
    fn mate(parents: (&Self, &Self), rng: &mut R) -> Self {
        //Each Mep must have the same amount of inputs
        //TODO: Once Rust implements generic values, this can be made explicit and is not needed
        assert!(parents.0.inputs == parents.1.inputs);
        //Get the smallest of the two lengths
        let total_instructions = cmp::min(parents.0.program.len(), parents.1.program.len());
        Mep{program:
            //Generate a randomly sized sequence between 1 and half of the total possible crossover points
            (0..rng.gen_range(1, total_instructions / 2))
            //Map these to random crossover points
            .map(|_| rng.gen_range(0, total_instructions))
            //Add total_instructions at the end so we can generate a range with it
            .chain(Some(total_instructions))
            //Sort them by value into BTree, which removes duplicates and orders them
            .fold(BTreeSet::new(), |mut set, i| {set.insert(i); set})
            //Iterate over the sorted values
            .iter()
            //Turn every copy of two, prepending a 0, into a range
            .scan(0, |prev, x| {let out = Some(*prev..*x); *prev = *x; out})
            //Enumerate by index to differentiate odd and even values
            .enumerate()
            //Map even pairs to ranges in parent 0 and odd ones to ranges in parent 1 and expand the ranges
            .flat_map(|(index, range)| {
                {if index % 2 == 0 {parents.0} else {parents.1}}.program[range].iter().cloned()
            })
            //Collect all the instruction ranges from each parent
            .collect(),

            unit_mutate_size: if parents.0.unit_mutate_size < parents.1.unit_mutate_size {
                rng.gen_range(parents.0.unit_mutate_size, parents.1.unit_mutate_size + 1)
            } else {
                rng.gen_range(parents.1.unit_mutate_size, parents.0.unit_mutate_size + 1)
            },

            crossover_points: if parents.0.crossover_points < parents.1.crossover_points {
                rng.gen_range(parents.0.crossover_points, parents.1.crossover_points + 1)
            } else {
                rng.gen_range(parents.1.crossover_points, parents.0.crossover_points + 1)
            },

            inputs: parents.0.inputs,
            mutator: {if rng.gen_range(0, 2) == 0 {parents.0} else {parents.1}}.mutator.clone(),
            processor: {if rng.gen_range(0, 2) == 0 {parents.0} else {parents.1}}.processor.clone(),
            phantom: (PhantomData, PhantomData),
        }
    }

    /*
    The Mep mutate function operates using the unit_mutate_size. This variable specifies the amount of instructions for
    which a single mutation is expect to occour every time mutate is called. This variable can be mutated inside of
    mutate, in which case it may never go below 1, but may tend towards infinity in increments of 1. This variable is
    implemented as a u64 to permit it to expand unbounded to mutation levels that are so low that mutations virtually
    never happen. Allowing this to mutate allows species to find the equilibrium between genomic adaptability and
    stability. If a species develops information gathering, then it can adapt intellegently, making it possibly more
    beneficial to operate at lower mutation rates. Setting the default mutation rate for species properly, or allowing
    it to adapt as the simulation continues, permits species to survive more frequently that are randomly generated.

    Likewise, the functions for random instruction mutation can be adapted as the simulation continues
    to optimize the generation of more desireable random mutations. For instance, instructions that
    occur more frequently should be generated randomly more frequently.
    */
    fn mutate(&mut self, rng: &mut R) {
        //Mutate unit_mutate_size
        if rng.gen_range(0, self.unit_mutate_size) == 0 {
            //Make it possibly go up or down by 1
            match rng.gen_range(0, 2) {
                0 => self.unit_mutate_size += 1,
                1 => if self.unit_mutate_size > 1 {self.unit_mutate_size -= 1},
                _ => unreachable!(),
            }
        }
        //Mutate crossover_points
        if rng.gen_range(0, self.unit_mutate_size) == 0 {
            //Make it possibly go up or down by 1
            match rng.gen_range(0, 2) {
                0 => self.crossover_points += 1,
                1 => if self.crossover_points > 1 {self.crossover_points -= 1},
                _ => unreachable!(),
            }
        }

        //Mutate the instructions using the mutator
        loop {
            //Choose a random location in the instructions and then add a random value up to the unit_mutate_size
            let choice = rng.gen_range(0, self.program.len()) + rng.gen_range(0, self.unit_mutate_size);
            //Whenever we choose a location outside the vector reject the choice and end mutation
            if choice >= self.program.len() {
                break;
            }
            let op = &mut self.program[choice];
            //Randomly mutate only one of the things contained here
            match rng.gen_range(0, 3) {
                0 => (self.mutator)(&mut op.instruction, rng),
                1 => op.first = rng.gen_range(0, choice + self.inputs),
                2 => op.second = rng.gen_range(0, choice + self.inputs),
                _ => unreachable!(),
            }
        }
    }
}

impl<'a, Ins, R, Param, F1, F2> ResultIterator<'a, Ins, R, Param, F1, F2>
    where F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param, R: Rng, Param: Clone
{
    fn op_solved(&mut self, i: usize) -> Param {
        //If this is an input, it is already solved, so return the result immediately
        if i < self.mep.inputs {
            return self.inputs[i].clone();
        }
        //Check if this has been evaluated or not
        let possible = self.buff[i - self.mep.inputs].clone();
        match possible {
            //If it has, return the value
            Some(x) => x,
            //If it hasnt been solved
            None => {
                //Get a reference to the opcode
                let op = &self.mep.program[i];
                //Compute the result of the operation, ensuring the inputs are solved beforehand
                let result = (self.mep.processor)(&op.instruction, self.op_solved(op.first), self.op_solved(op.second));
                //Properly store the Some result to the buffer
                self.buff[i - self.mep.inputs] = Some(result.clone());
                //Return the result
                result
            }
        }
    }
}

impl<'a, Ins, R, Param, F1, F2> Iterator for ResultIterator<'a, Ins, R, Param, F1, F2>
    where F1: Clone + Fn(&mut Ins, &mut R), F2: Clone + Fn(&Ins, Param, Param) -> Param, R: Rng, Param: Clone
{
    type Item = Param;
    fn next(&mut self) -> Option<Param> {
        match self.solve_iter.next() {
            None => None,
            Some(i) => Some(self.op_solved(i)),
        }
    }
}

#[cfg(test)]
mod tests {
    use rand::{Isaac64Rng, SeedableRng, Rng};
    use super::*;
    use super::super::{Genetic, Learning};

    #[test]
    fn mep_new() {
        let a: Mep<u32> = Mep::new(3, 3, 0..8);

        assert_eq!(a.instructions, (0..8).collect::<Vec<_>>());
    }

    #[test]
    fn mep_crossover() {
        let mut rng = Isaac64Rng::from_seed(&[1, 2, 3, 4]);
        let (a, b) = {
            let mut clos = || Mep::new(3, 3, rng.gen_iter::<u32>().map(|x| x % 10).take(10));
            (clos(), clos())
        };
        let old_rngs: Vec<_> = rng.clone().gen_iter::<u32>().take(5).collect();
        let mut c = Mep::mate((&a, &b), &mut rng);
        //Ensure that rng was borrowed mutably
        assert!(rng.clone().gen_iter::<u32>().take(5).collect::<Vec<_>>() != old_rngs);

        c.mutate(&mut rng, |ins: &mut u32| *ins = 2);
        c.call(|_, (_, _)| {});

        assert_eq!(c.instructions, [0, 7, 5, 4, 2, 8, 5, 6, 0, 2]);
    }
}
<|endoftext|>


<commit_before>// Copyright (C) 2019, Cloudflare, Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#[macro_use]
extern crate log;

use std::net::ToSocketAddrs;

use ring::rand::*;

const MAX_DATAGRAM_SIZE: usize = 1350;

const USAGE: &str = "Usage:
  http3-client [options] URL
  http3-client -h | --help

Options:
  --max-data BYTES         Connection-wide flow control limit [default: 10000000].
  --max-stream-data BYTES  Per-stream flow control limit [default: 1000000].
  --wire-version VERSION   The version number to send to the server [default: babababa].
  --no-verify              Don't verify server's certificate.
  --no-grease              Don't send GREASE.
  -h --help                Show this screen.
";

fn main() {
    let mut buf = [0; 65535];
    let mut out = [0; MAX_DATAGRAM_SIZE];

    env_logger::builder()
        .default_format_timestamp_nanos(true)
        .init();

    let args = docopt::Docopt::new(USAGE)
        .and_then(|dopt| dopt.parse())
        .unwrap_or_else(|e| e.exit());

    let max_data = args.get_str("--max-data");
    let max_data = u64::from_str_radix(max_data, 10).unwrap();

    let max_stream_data = args.get_str("--max-stream-data");
    let max_stream_data = u64::from_str_radix(max_stream_data, 10).unwrap();

    let version = args.get_str("--wire-version");
    let version = u32::from_str_radix(version, 16).unwrap();

    let url = url::Url::parse(args.get_str("URL")).unwrap();

    // Setup the event loop.
    let poll = mio::Poll::new().unwrap();
    let mut events = mio::Events::with_capacity(1024);

    // Resolve server address.
    let peer_addr = url.to_socket_addrs().unwrap().next().unwrap();
    info!("connecting to {:}", peer_addr);

    // Bind to INADDR_ANY or IN6ADDR_ANY depending on the IP family of the
    // server address. This is needed on macOS and BSD variants that don't
    // support binding to IN6ADDR_ANY for both v4 and v6.
    let bind_addr = match peer_addr {
        std::net::SocketAddr::V4(_) => "0.0.0.0:0",
        std::net::SocketAddr::V6(_) => "[::]:0",
    };

    // Create the UDP socket backing the QUIC connection, and register it with
    // the event loop.
    let socket = std::net::UdpSocket::bind(bind_addr).unwrap();
    socket.connect(peer_addr).unwrap();

    let socket = mio::net::UdpSocket::from_socket(socket).unwrap();
    poll.register(
        &socket,
        mio::Token(0),
        mio::Ready::readable(),
        mio::PollOpt::edge(),
    )
    .unwrap();

    // Create the configuration for the QUIC connection.
    let mut config = quiche::Config::new(version).unwrap();

    config.verify_peer(true);

    config
        .set_application_protos(quiche::h3::APPLICATION_PROTOCOL)
        .unwrap();

    config.set_idle_timeout(5000);
    config.set_max_packet_size(MAX_DATAGRAM_SIZE as u64);
    config.set_initial_max_data(max_data);
    config.set_initial_max_stream_data_bidi_local(max_stream_data);
    config.set_initial_max_stream_data_bidi_remote(max_stream_data);
    config.set_initial_max_stream_data_uni(max_stream_data);
    config.set_initial_max_streams_bidi(100);
    config.set_initial_max_streams_uni(100);
    config.set_disable_migration(true);

    let mut http3_conn = None;

    if args.get_bool("--no-verify") {
        config.verify_peer(false);
    }

    if args.get_bool("--no-grease") {
        config.grease(false);
    }

    if std::env::var_os("SSLKEYLOGFILE").is_some() {
        config.log_keys();
    }

    // Generate a random source connection ID for the connection.
    let mut scid = [0; quiche::MAX_CONN_ID_LEN];
    SystemRandom::new().fill(&mut scid[..]).unwrap();

    // Create a QUIC connection and initiate handshake.
    let mut conn = quiche::connect(url.domain(), &scid, &mut config).unwrap();

    let write = match conn.send(&mut out) {
        Ok(v) => v,

        Err(e) => panic!("initial send failed: {:?}", e),
    };

    while let Err(e) = socket.send(&out[..write]) {
        if e.kind() == std::io::ErrorKind::WouldBlock {
            debug!("send() would block");
            continue;
        }

        panic!("send() failed: {:?}", e);
    }

    debug!("written {}", write);

    let req_start = std::time::Instant::now();

    loop {
        poll.poll(&mut events, conn.timeout()).unwrap();

        // Read incoming UDP packets from the socket and feed them to quiche,
        // until there are no more packets to read.
        'read: loop {
            // If the event loop reported no events, it means that the timeout
            // has expired, so handle it without attempting to read packets. We
            // will then proceed with the send loop.
            if events.is_empty() {
                debug!("timed out");

                conn.on_timeout();

                break 'read;
            }

            let len = match socket.recv(&mut buf) {
                Ok(v) => v,

                Err(e) => {
                    // There are no more UDP packets to read, so end the read
                    // loop.
                    if e.kind() == std::io::ErrorKind::WouldBlock {
                        debug!("recv() would block");
                        break 'read;
                    }

                    panic!("recv() failed: {:?}", e);
                },
            };

            debug!("got {} bytes", len);

            // Process potentially coalesced packets.
            let read = match conn.recv(&mut buf[..len]) {
                Ok(v) => v,

                Err(quiche::Error::Done) => {
                    debug!("done reading");
                    break;
                },

                Err(e) => {
                    error!("recv failed: {:?}", e);
                    break 'read;
                },
            };

            debug!("processed {} bytes", read);
        }

        if conn.is_closed() {
            info!("connection closed, {:?}", conn.stats());
            break;
        }

        // Create a new HTTP/3 connection and end an HTTP request as soon as
        // the QUIC connection is established.
        if conn.is_established() && http3_conn.is_none() {
            let h3_config = quiche::h3::Config::new(0, 1024, 0, 0).unwrap();

            let mut h3_conn =
                quiche::h3::Connection::with_transport(&mut conn, &h3_config)
                    .unwrap();

            let mut path = String::from(url.path());

            if let Some(query) = url.query() {
                path.push('?');
                path.push_str(query);
            }

            let req = vec![
                quiche::h3::Header::new(":method", "GET"),
                quiche::h3::Header::new(":scheme", url.scheme()),
                quiche::h3::Header::new(":authority", url.host_str().unwrap()),
                quiche::h3::Header::new(":path", &path),
                quiche::h3::Header::new("user-agent", "quiche"),
            ];

            info!("sending HTTP request {:?}", req);

            if let Err(e) = h3_conn.send_request(&mut conn, &req, true) {
                error!("failed to send request {:?}", e);
                break;
            }

            http3_conn = Some(h3_conn);
        }

        if let Some(http3_conn) = &mut http3_conn {
            // Process HTTP/3 events.
            loop {
                match http3_conn.poll(&mut conn) {
                    Ok((stream_id, quiche::h3::Event::Headers(headers))) => {
                        info!(
                            "got response headers {:?} on stream id {}",
                            headers, stream_id
                        );
                    },

                    Ok((stream_id, quiche::h3::Event::Data)) => {
                        if let Ok(read) =
                            http3_conn.recv_body(&mut conn, stream_id, &mut buf)
                        {
                            debug!(
                                "got {} bytes of response data on stream {}",
                                read, stream_id
                            );

                            print!("{}", unsafe {
                                std::str::from_utf8_unchecked(&buf[..read])
                            });
                        }
                    },

                    Ok((_stream_id, quiche::h3::Event::Finished)) => {
                        info!(
                            "response received in {:?}, closing...",
                            req_start.elapsed()
                        );

                        match conn.close(true, 0x00, b"kthxbye") {
                            // Already closed.
                            Ok(_) | Err(quiche::Error::Done) => (),

                            Err(e) => panic!("error closing conn: {:?}", e),
                        }

                        break;
                    },

                    Err(quiche::h3::Error::Done) => {
                        break;
                    },

                    Err(e) => {
                        error!("HTTP/3 processing failed: {:?}", e);

                        break;
                    },
                }
            }
        }

        // Generate outgoing QUIC packets and send them on the UDP socket, until
        // quiche reports that there are no more packets to be sent.
        loop {
            let write = match conn.send(&mut out) {
                Ok(v) => v,

                Err(quiche::Error::Done) => {
                    debug!("done writing");
                    break;
                },

                Err(e) => {
                    error!("send failed: {:?}", e);
                    conn.close(false, 0x1, b"fail").ok();
                    break;
                },
            };

            if let Err(e) = socket.send(&out[..write]) {
                if e.kind() == std::io::ErrorKind::WouldBlock {
                    debug!("send() would block");
                    break;
                }

                panic!("send() failed: {:?}", e);
            }

            debug!("written {}", write);
        }

        if conn.is_closed() {
            info!("connection closed, {:?}", conn.stats());
            break;
        }
    }
}
<commit_msg>http3-client: support setting HTTP methods other than GET<commit_after>// Copyright (C) 2019, Cloudflare, Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#[macro_use]
extern crate log;

use std::net::ToSocketAddrs;

use ring::rand::*;

const MAX_DATAGRAM_SIZE: usize = 1350;

const USAGE: &str = "Usage:
  http3-client [options] URL
  http3-client -h | --help

Options:
  --method METHOD          Use the given HTTP request method [default: GET].
  --max-data BYTES         Connection-wide flow control limit [default: 10000000].
  --max-stream-data BYTES  Per-stream flow control limit [default: 1000000].
  --wire-version VERSION   The version number to send to the server [default: babababa].
  --no-verify              Don't verify server's certificate.
  --no-grease              Don't send GREASE.
  -h --help                Show this screen.
";

fn main() {
    let mut buf = [0; 65535];
    let mut out = [0; MAX_DATAGRAM_SIZE];

    env_logger::builder()
        .default_format_timestamp_nanos(true)
        .init();

    let args = docopt::Docopt::new(USAGE)
        .and_then(|dopt| dopt.parse())
        .unwrap_or_else(|e| e.exit());

    let max_data = args.get_str("--max-data");
    let max_data = u64::from_str_radix(max_data, 10).unwrap();

    let max_stream_data = args.get_str("--max-stream-data");
    let max_stream_data = u64::from_str_radix(max_stream_data, 10).unwrap();

    let version = args.get_str("--wire-version");
    let version = u32::from_str_radix(version, 16).unwrap();

    let url = url::Url::parse(args.get_str("URL")).unwrap();

    // Setup the event loop.
    let poll = mio::Poll::new().unwrap();
    let mut events = mio::Events::with_capacity(1024);

    // Resolve server address.
    let peer_addr = url.to_socket_addrs().unwrap().next().unwrap();
    info!("connecting to {:}", peer_addr);

    // Bind to INADDR_ANY or IN6ADDR_ANY depending on the IP family of the
    // server address. This is needed on macOS and BSD variants that don't
    // support binding to IN6ADDR_ANY for both v4 and v6.
    let bind_addr = match peer_addr {
        std::net::SocketAddr::V4(_) => "0.0.0.0:0",
        std::net::SocketAddr::V6(_) => "[::]:0",
    };

    // Create the UDP socket backing the QUIC connection, and register it with
    // the event loop.
    let socket = std::net::UdpSocket::bind(bind_addr).unwrap();
    socket.connect(peer_addr).unwrap();

    let socket = mio::net::UdpSocket::from_socket(socket).unwrap();
    poll.register(
        &socket,
        mio::Token(0),
        mio::Ready::readable(),
        mio::PollOpt::edge(),
    )
    .unwrap();

    // Create the configuration for the QUIC connection.
    let mut config = quiche::Config::new(version).unwrap();

    config.verify_peer(true);

    config
        .set_application_protos(quiche::h3::APPLICATION_PROTOCOL)
        .unwrap();

    config.set_idle_timeout(5000);
    config.set_max_packet_size(MAX_DATAGRAM_SIZE as u64);
    config.set_initial_max_data(max_data);
    config.set_initial_max_stream_data_bidi_local(max_stream_data);
    config.set_initial_max_stream_data_bidi_remote(max_stream_data);
    config.set_initial_max_stream_data_uni(max_stream_data);
    config.set_initial_max_streams_bidi(100);
    config.set_initial_max_streams_uni(100);
    config.set_disable_migration(true);

    let mut http3_conn = None;

    if args.get_bool("--no-verify") {
        config.verify_peer(false);
    }

    if args.get_bool("--no-grease") {
        config.grease(false);
    }

    if std::env::var_os("SSLKEYLOGFILE").is_some() {
        config.log_keys();
    }

    // Generate a random source connection ID for the connection.
    let mut scid = [0; quiche::MAX_CONN_ID_LEN];
    SystemRandom::new().fill(&mut scid[..]).unwrap();

    // Create a QUIC connection and initiate handshake.
    let mut conn = quiche::connect(url.domain(), &scid, &mut config).unwrap();

    let write = match conn.send(&mut out) {
        Ok(v) => v,

        Err(e) => panic!("initial send failed: {:?}", e),
    };

    while let Err(e) = socket.send(&out[..write]) {
        if e.kind() == std::io::ErrorKind::WouldBlock {
            debug!("send() would block");
            continue;
        }

        panic!("send() failed: {:?}", e);
    }

    debug!("written {}", write);

    let req_start = std::time::Instant::now();

    loop {
        poll.poll(&mut events, conn.timeout()).unwrap();

        // Read incoming UDP packets from the socket and feed them to quiche,
        // until there are no more packets to read.
        'read: loop {
            // If the event loop reported no events, it means that the timeout
            // has expired, so handle it without attempting to read packets. We
            // will then proceed with the send loop.
            if events.is_empty() {
                debug!("timed out");

                conn.on_timeout();

                break 'read;
            }

            let len = match socket.recv(&mut buf) {
                Ok(v) => v,

                Err(e) => {
                    // There are no more UDP packets to read, so end the read
                    // loop.
                    if e.kind() == std::io::ErrorKind::WouldBlock {
                        debug!("recv() would block");
                        break 'read;
                    }

                    panic!("recv() failed: {:?}", e);
                },
            };

            debug!("got {} bytes", len);

            // Process potentially coalesced packets.
            let read = match conn.recv(&mut buf[..len]) {
                Ok(v) => v,

                Err(quiche::Error::Done) => {
                    debug!("done reading");
                    break;
                },

                Err(e) => {
                    error!("recv failed: {:?}", e);
                    break 'read;
                },
            };

            debug!("processed {} bytes", read);
        }

        if conn.is_closed() {
            info!("connection closed, {:?}", conn.stats());
            break;
        }

        // Create a new HTTP/3 connection and end an HTTP request as soon as
        // the QUIC connection is established.
        if conn.is_established() && http3_conn.is_none() {
            let h3_config = quiche::h3::Config::new(0, 1024, 0, 0).unwrap();

            let mut h3_conn =
                quiche::h3::Connection::with_transport(&mut conn, &h3_config)
                    .unwrap();

            let mut path = String::from(url.path());

            if let Some(query) = url.query() {
                path.push('?');
                path.push_str(query);
            }

            let req = vec![
                quiche::h3::Header::new(":method", args.get_str("--method")),
                quiche::h3::Header::new(":scheme", url.scheme()),
                quiche::h3::Header::new(":authority", url.host_str().unwrap()),
                quiche::h3::Header::new(":path", &path),
                quiche::h3::Header::new("user-agent", "quiche"),
            ];

            info!("sending HTTP request {:?}", req);

            if let Err(e) = h3_conn.send_request(&mut conn, &req, true) {
                error!("failed to send request {:?}", e);
                break;
            }

            http3_conn = Some(h3_conn);
        }

        if let Some(http3_conn) = &mut http3_conn {
            // Process HTTP/3 events.
            loop {
                match http3_conn.poll(&mut conn) {
                    Ok((stream_id, quiche::h3::Event::Headers(headers))) => {
                        info!(
                            "got response headers {:?} on stream id {}",
                            headers, stream_id
                        );
                    },

                    Ok((stream_id, quiche::h3::Event::Data)) => {
                        if let Ok(read) =
                            http3_conn.recv_body(&mut conn, stream_id, &mut buf)
                        {
                            debug!(
                                "got {} bytes of response data on stream {}",
                                read, stream_id
                            );

                            print!("{}", unsafe {
                                std::str::from_utf8_unchecked(&buf[..read])
                            });
                        }
                    },

                    Ok((_stream_id, quiche::h3::Event::Finished)) => {
                        info!(
                            "response received in {:?}, closing...",
                            req_start.elapsed()
                        );

                        match conn.close(true, 0x00, b"kthxbye") {
                            // Already closed.
                            Ok(_) | Err(quiche::Error::Done) => (),

                            Err(e) => panic!("error closing conn: {:?}", e),
                        }

                        break;
                    },

                    Err(quiche::h3::Error::Done) => {
                        break;
                    },

                    Err(e) => {
                        error!("HTTP/3 processing failed: {:?}", e);

                        break;
                    },
                }
            }
        }

        // Generate outgoing QUIC packets and send them on the UDP socket, until
        // quiche reports that there are no more packets to be sent.
        loop {
            let write = match conn.send(&mut out) {
                Ok(v) => v,

                Err(quiche::Error::Done) => {
                    debug!("done writing");
                    break;
                },

                Err(e) => {
                    error!("send failed: {:?}", e);
                    conn.close(false, 0x1, b"fail").ok();
                    break;
                },
            };

            if let Err(e) = socket.send(&out[..write]) {
                if e.kind() == std::io::ErrorKind::WouldBlock {
                    debug!("send() would block");
                    break;
                }

                panic!("send() failed: {:?}", e);
            }

            debug!("written {}", write);
        }

        if conn.is_closed() {
            info!("connection closed, {:?}", conn.stats());
            break;
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

//! Networking primitives not covered by the standard library.

#[cfg(target_os = "linux")]
pub mod linux;

#[cfg(target_os = "macos")]
pub mod macos;

#[cfg(target_os = "windows")]
pub mod windows;

// TODO(panhania@): Add support for Windows.

/// A MAC address.
///
/// MAC addresses are defined as 48-bit numbers in a IEEE 802 standard [1].
///
/// [1]: https://standards.ieee.org/wp-content/uploads/import/documents/tutorials/macgrp.pdf
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct MacAddr {
    /// Octets forming up the 48-bit MAC address number.
    octets: [u8; 6],
}

impl MacAddr {

    /// Return the six 8-bit integers that make up this address.
    pub fn octets(&self) -> [u8; 6] {
        self.octets
    }
}

impl From<[u8; 6]> for MacAddr {

    fn from(octets: [u8; 6]) -> MacAddr {
        MacAddr { octets }
    }
}

/// A network interface information.
pub struct Interface {
    /// A name of this interface as reported by the system.
    name: std::ffi::OsString,
    /// IP addresses associated with this interface.
    ip_addrs: Vec<std::net::IpAddr>,
    /// The MAC address associated with this interface.
    mac_addr: Option<MacAddr>,
}

impl Interface {

    /// Returns the name of this interface as reported by the system.
    pub fn name(&self) -> &std::ffi::OsStr {
        self.name.as_os_str()
    }

    /// Returns the IP addresses associated with this interface.
    pub fn ip_addrs(&self) -> &[std::net::IpAddr] {
        self.ip_addrs.as_slice()
    }

    /// Returns the MAC address associated with this interface (if any).
    pub fn mac_addr(&self) -> Option<&MacAddr> {
        self.mac_addr.as_ref()
    }
}

/// Collects information about available network interfaces.
///
/// The information collected by this mathod can be more or less complete,
/// depending on the operating system.
///
/// # Errors
///
/// This function will fail if there was some kind of issue (e.g. insufficient
/// permissions to make certain system calls) during information collection.
///
/// # Examples
///
/// ```
/// let ifaces = rrg::net::interfaces().unwrap();
/// for iface in ifaces {
///     let name = iface.name().to_string_lossy();
///     println!("{} ({} IP addresses)", name, iface.ip_addrs().len());
/// }
/// ```
pub fn interfaces() -> std::io::Result<impl Iterator<Item = Interface>> {
    #[cfg(target_os = "linux")]
    use self::linux::interfaces;

    #[cfg(target_os = "macos")]
    use self::macos::interfaces;

    #[cfg(target_os = "windows")]
    use self::windows::interfaces;

    interfaces()
}
<commit_msg>Derive basic traits for the net interface type.<commit_after>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

//! Networking primitives not covered by the standard library.

#[cfg(target_os = "linux")]
pub mod linux;

#[cfg(target_os = "macos")]
pub mod macos;

#[cfg(target_os = "windows")]
pub mod windows;

// TODO(panhania@): Add support for Windows.

/// A MAC address.
///
/// MAC addresses are defined as 48-bit numbers in a IEEE 802 standard [1].
///
/// [1]: https://standards.ieee.org/wp-content/uploads/import/documents/tutorials/macgrp.pdf
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct MacAddr {
    /// Octets forming up the 48-bit MAC address number.
    octets: [u8; 6],
}

impl MacAddr {

    /// Return the six 8-bit integers that make up this address.
    pub fn octets(&self) -> [u8; 6] {
        self.octets
    }
}

impl From<[u8; 6]> for MacAddr {

    fn from(octets: [u8; 6]) -> MacAddr {
        MacAddr { octets }
    }
}

/// A network interface information.
#[derive(Clone, Debug)]
pub struct Interface {
    /// A name of this interface as reported by the system.
    name: std::ffi::OsString,
    /// IP addresses associated with this interface.
    ip_addrs: Vec<std::net::IpAddr>,
    /// The MAC address associated with this interface.
    mac_addr: Option<MacAddr>,
}

impl Interface {

    /// Returns the name of this interface as reported by the system.
    pub fn name(&self) -> &std::ffi::OsStr {
        self.name.as_os_str()
    }

    /// Returns the IP addresses associated with this interface.
    pub fn ip_addrs(&self) -> &[std::net::IpAddr] {
        self.ip_addrs.as_slice()
    }

    /// Returns the MAC address associated with this interface (if any).
    pub fn mac_addr(&self) -> Option<&MacAddr> {
        self.mac_addr.as_ref()
    }
}

/// Collects information about available network interfaces.
///
/// The information collected by this mathod can be more or less complete,
/// depending on the operating system.
///
/// # Errors
///
/// This function will fail if there was some kind of issue (e.g. insufficient
/// permissions to make certain system calls) during information collection.
///
/// # Examples
///
/// ```
/// let ifaces = rrg::net::interfaces().unwrap();
/// for iface in ifaces {
///     let name = iface.name().to_string_lossy();
///     println!("{} ({} IP addresses)", name, iface.ip_addrs().len());
/// }
/// ```
pub fn interfaces() -> std::io::Result<impl Iterator<Item = Interface>> {
    #[cfg(target_os = "linux")]
    use self::linux::interfaces;

    #[cfg(target_os = "macos")]
    use self::macos::interfaces;

    #[cfg(target_os = "windows")]
    use self::windows::interfaces;

    interfaces()
}
<|endoftext|>