<commit_before>extern crate proc_macro;
extern crate proc_macro2;
#[macro_use]
extern crate syn;
#[macro_use]
extern crate quote;

use std::collections::HashMap;

use proc_macro::TokenStream;
use proc_macro2::Ident;
use syn::{ItemFn, Stmt, UseTree, ItemUse, Item, Block, Expr,
          parse::{self, Parse, ParseStream},
          punctuated::Punctuated,
          fold::Fold};

/// Parses a list of variable names separated by commas
///
///     a, b, c
///
/// This is how the compiler passes in arguments to our attribute -- it is
/// everything inside the delimiters after the attribute name.
///
///     #[wlroots_dehandle(a, b, c)]
///
struct Args {
    vars: HashMap<Ident, bool>
}

impl Parse for Args {
    fn parse(input: ParseStream) -> parse::Result<Self> {
        let vars = Punctuated::<Ident, Token![,]>::parse_terminated(input)?;
        Ok(Args {
            vars: vars.into_iter().map(|k| (k, false)).collect(),
        })
    }
}

impl Fold for Args {
    fn fold_block(&mut self, block: Block) -> Block {
        build_block(block.stmts.iter(), self)
    }
}

impl Args {
    fn is_handle(&mut self, name: Ident) -> bool {
        if let Some(seen) = self.vars.get_mut(&name) {
            *seen = true;
            true
        } else {
            false
        }
    }
}

/// Attribute to automatically call the `run` method on handles with the
/// remaining block of code.
///
/// The name of the variable you want to use as the upgraded handle should be
/// provided as an argument to the attribute. It does not need to be the same
/// as the handle variable.
///
/// The syntax in the code should be `use $handle as $upgraded_handle`.
/// E.g the variable in the code that stores the handle should go on the
/// **left** and the variable you used in the attribute declaration should
/// go on the **right**.
///
/// # Panics
/// If the handle is invalid (e.g. default constructed, or is a dangling
/// handle) then your code will `panic!`.
///
/// If this is undesirable, please use the non-proc macro `with_handles!`.
///
/// # Example
///
/// ```
/// impl InputManagerHandler for InputManager {
///     #[wlroots_dehandle(compositor, keyboard, seat)]
///     fn keyboard_added(&mut self,
///                       compositor_handle: CompositorHandle,
///                       keyboard: KeyboardHandle)
///                       -> Option<Box<Keyboard Handler>> {
///         {
///             use compositor_handle as compositor;
///             use keyboard as keyboard;
///             let server: &mut ::Server = compositor.into();
///             server.keyboards.push(keyboard.weak_reference());
///             // Now that we have at least one keyboard, update the seat capabilities.
///             let server_seat = &server.seat.seat;
///             use server_seat as seat;
///             let mut capabilities = seat.capabilities();
///             capabilities.insert(Capability::Keyboard);
///             seat.set_capabilities(capabilities);
///             seat.set_keyboard(keyboard.input_device());
///         }
///         // Due to some weird closure inference rules, this has to be outside
///         // of the above block.
///         Some(Box::new(::Keyboard))
///     }
/// }
/// ```
#[proc_macro_attribute]
pub fn wlroots_dehandle(args: TokenStream, input: TokenStream) -> TokenStream {
    let input = parse_macro_input!(input as ItemFn);
    let mut args = parse_macro_input!(args as Args);
    if args.vars.len() == 0 {
        panic!("wlroots_dehandle requires at least one argument")
    }
    let output = args.fold_item_fn(input);
    for (arg, seen) in args.vars {
        if !seen {
            panic!(format!("Must use all declared handles, didn't use `{}`",
                           arg))
        }
    }
    TokenStream::from(quote!(#output))
}

fn build_block(mut input: std::slice::Iter<Stmt>, args: &mut Args) -> Block {
    let mut output = vec![];
    let mut inner = None;
    while let Some(stmt) = input.next().cloned() {
        use {Item::Use, UseTree::Rename};
        match stmt.clone() {
            Stmt::Item(Use(ItemUse { tree: Rename(use_stmt), ..})) => {
                if args.is_handle(use_stmt.rename.clone()) {
                    inner = Some((use_stmt.ident, use_stmt.rename));
                    break
                }
                output.push(stmt)
            },
            // Recurse into function body
            Stmt::Item(Item::Fn(mut function)) => {
                let inner_block = function.block.clone();
                *function.block = build_block(inner_block.stmts.iter(), args);
                output.push(Stmt::Item(Item::Fn(function)))
            },
            // Recurse into let call
            Stmt::Local(mut local) => {
                if let Some((_, body)) = local.init.clone() {
                    let body = build_block_expr(*body.clone(), args);
                    let body = parse_quote!(#body);
                    local.init.as_mut().unwrap().1 = body;
                    output.push(Stmt::Local(local))
                }
            },
            Stmt::Expr(expr) => {
                let body = build_block_expr(expr, args);
                output.push(parse_quote!({#body}))
            }
            Stmt::Semi(expr, _) => {
                let body = build_block_expr(expr, args);
                output.push(parse_quote!({#body;}))
            }
            _ => output.push(stmt)
        }
    }
    if let Some((handle, dehandle)) = inner {
        let inner_block = build_block(input, args);
        let handle_call = parse_quote!(
            {(#handle).run(|#dehandle|{
                #inner_block
            }).expect(concat!("Could not upgrade handle ",
                              stringify!(#handle), " to ",
                              stringify!(#dehandle)))}
        );
        output.push(handle_call);
    }
    parse_quote!({#(#output)*})
}

/// Tries to build a block from the expression.
fn build_block_expr(expr: Expr, args: &mut Args) -> Expr {
    match expr {
        Expr::Block(block) => {
            let block = build_block(block.block.stmts.iter(), args);
            parse_quote!(#block)
        }
        Expr::Let(mut let_expr) => {
            *let_expr.expr = build_block_expr(*let_expr.expr.clone(), args);
            Expr::Let(let_expr)
        },
        Expr::If(mut if_expr) => {
            let then_branch = if_expr.then_branch.clone();
            let then_branch = build_block_expr(parse_quote!(#then_branch), args);
            if_expr.then_branch = parse_quote!(#then_branch);
            if_expr.else_branch = match if_expr.else_branch.clone() {
                None => if_expr.else_branch,
                Some((token, else_branch)) => {
                    Some((token, Box::new( build_block_expr(*else_branch, args))))
                }
            };
            Expr::If(if_expr)
        },
        Expr::While(mut while_expr) => {
            let body = while_expr.body.clone();
            let body = build_block_expr(parse_quote!(#body), args);
            while_expr.body = parse_quote!(#body);
            Expr::While(while_expr)
        },
        Expr::ForLoop(mut for_expr) => {
            let body = for_expr.body.clone();
            let body = build_block_expr(parse_quote!(#body), args);
            for_expr.body = parse_quote!(#body);
            Expr::ForLoop(for_expr)
        },
        Expr::Loop(mut loop_expr) => {
            let body = loop_expr.body.clone();
            let body = build_block_expr(parse_quote!(#body), args);
            loop_expr.body = parse_quote!(#body);
            Expr::Loop(loop_expr)
        },
        Expr::Match(mut match_expr) => {
            for arm in &mut match_expr.arms {
                *arm.body = build_block_expr(*arm.body.clone(), args)
            }
            Expr::Match(match_expr)
        },
        Expr::Struct(mut struct_expr) => {
            for field in &mut struct_expr.fields {
                field.expr = build_block_expr(field.expr.clone(), args);
            }
            Expr::Struct(struct_expr)
        },
        Expr::Call(mut call_expr) => {
            for arg in &mut call_expr.args {
                *arg = build_block_expr(arg.clone(), args);
            }
            Expr::Call(call_expr)
        },
        Expr::MethodCall(mut call_expr) => {
            for arg in &mut call_expr.args {
                *arg = build_block_expr(arg.clone(), args);
            }
            Expr::MethodCall(call_expr)
        },
        Expr::Closure(mut closure_expr) => {
            *closure_expr.body = build_block_expr(*closure_expr.body.clone(),
                                                  args);
            Expr::Closure(closure_expr)
        },
        Expr::Unsafe(mut unsafe_expr) => {
            unsafe_expr.block = build_block(unsafe_expr.block.stmts.iter(),
                                            args);
            Expr::Unsafe(unsafe_expr)
        },
        Expr::Assign(mut assign_expr) => {
            *assign_expr.right = build_block_expr(*assign_expr.right.clone(),
                                                  args);
            Expr::Assign(assign_expr)
        },
        Expr::AssignOp(mut assign_expr) => {
            *assign_expr.right = build_block_expr(*assign_expr.right.clone(),
                                                  args);
            Expr::AssignOp(assign_expr)
        },
        Expr::Break(mut break_expr) => {
            match break_expr.expr {
                None => {},
                Some(ref mut expr) => {
                    **expr = build_block_expr(*expr.clone(), args);
                }
            }
            Expr::Break(break_expr)
        },
        Expr::Return(mut return_expr) => {
            match return_expr.expr {
                None => {},
                Some(ref mut expr) => {
                    **expr = build_block_expr(*expr.clone(), args);
                }
            }
            Expr::Return(return_expr)
        },
        Expr::Reference(mut reference_expr) => {
            *reference_expr.expr = build_block_expr(*reference_expr.expr.clone(),
                                                    args);
            Expr::Reference(reference_expr)
        },
        Expr::Paren(mut paren_expr) => {
            *paren_expr.expr = build_block_expr(*paren_expr.expr.clone(), args);
            Expr::Paren(paren_expr)
        },
        Expr::Unary(mut unary_expr) => {
            *unary_expr.expr = build_block_expr(*unary_expr.expr.clone(), args);
            Expr::Unary(unary_expr)
        },
        Expr::Binary(mut binary_expr) => {
            *binary_expr.left = build_block_expr(*binary_expr.left.clone(), args);
            *binary_expr.right = build_block_expr(*binary_expr.right.clone(), args);
            Expr::Binary(binary_expr)
        },
        v => {
            v
        }
    }
}
<commit_msg>Ignore code for test<commit_after>extern crate proc_macro;
extern crate proc_macro2;
#[macro_use]
extern crate syn;
#[macro_use]
extern crate quote;

use std::collections::HashMap;

use proc_macro::TokenStream;
use proc_macro2::Ident;
use syn::{ItemFn, Stmt, UseTree, ItemUse, Item, Block, Expr,
          parse::{self, Parse, ParseStream},
          punctuated::Punctuated,
          fold::Fold};

/// Parses a list of variable names separated by commas
///
/// This is how the compiler passes in arguments to our attribute -- it is
/// everything inside the delimiters after the attribute name.
///```rust,ignore
///     #[wlroots_dehandle(a, b, c)]
///```
struct Args {
    vars: HashMap<Ident, bool>
}

impl Parse for Args {
    fn parse(input: ParseStream) -> parse::Result<Self> {
        let vars = Punctuated::<Ident, Token![,]>::parse_terminated(input)?;
        Ok(Args {
            vars: vars.into_iter().map(|k| (k, false)).collect(),
        })
    }
}

impl Fold for Args {
    fn fold_block(&mut self, block: Block) -> Block {
        build_block(block.stmts.iter(), self)
    }
}

impl Args {
    fn is_handle(&mut self, name: Ident) -> bool {
        if let Some(seen) = self.vars.get_mut(&name) {
            *seen = true;
            true
        } else {
            false
        }
    }
}

/// Attribute to automatically call the `run` method on handles with the
/// remaining block of code.
///
/// The name of the variable you want to use as the upgraded handle should be
/// provided as an argument to the attribute. It does not need to be the same
/// as the handle variable.
///
/// The syntax in the code should be `use $handle as $upgraded_handle`.
/// E.g the variable in the code that stores the handle should go on the
/// **left** and the variable you used in the attribute declaration should
/// go on the **right**.
///
/// # Panics
/// If the handle is invalid (e.g. default constructed, or is a dangling
/// handle) then your code will `panic!`.
///
/// If this is undesirable, please use the non-proc macro `with_handles!`.
///
/// # Example
///
/// ```rust,ignore
/// impl InputManagerHandler for InputManager {
///     #[wlroots_dehandle(compositor, keyboard, seat)]
///     fn keyboard_added(&mut self,
///                       compositor_handle: CompositorHandle,
///                       keyboard: KeyboardHandle)
///                       -> Option<Box<Keyboard Handler>> {
///         {
///             use compositor_handle as compositor;
///             use keyboard as keyboard;
///             let server: &mut ::Server = compositor.into();
///             server.keyboards.push(keyboard.weak_reference());
///             // Now that we have at least one keyboard, update the seat capabilities.
///             let server_seat = &server.seat.seat;
///             use server_seat as seat;
///             let mut capabilities = seat.capabilities();
///             capabilities.insert(Capability::Keyboard);
///             seat.set_capabilities(capabilities);
///             seat.set_keyboard(keyboard.input_device());
///         }
///         // Due to some weird closure inference rules, this has to be outside
///         // of the above block.
///         Some(Box::new(::Keyboard))
///     }
/// }
/// ```
#[proc_macro_attribute]
pub fn wlroots_dehandle(args: TokenStream, input: TokenStream) -> TokenStream {
    let input = parse_macro_input!(input as ItemFn);
    let mut args = parse_macro_input!(args as Args);
    if args.vars.len() == 0 {
        panic!("wlroots_dehandle requires at least one argument")
    }
    let output = args.fold_item_fn(input);
    for (arg, seen) in args.vars {
        if !seen {
            panic!(format!("Must use all declared handles, didn't use `{}`",
                           arg))
        }
    }
    TokenStream::from(quote!(#output))
}

fn build_block(mut input: std::slice::Iter<Stmt>, args: &mut Args) -> Block {
    let mut output = vec![];
    let mut inner = None;
    while let Some(stmt) = input.next().cloned() {
        use {Item::Use, UseTree::Rename};
        match stmt.clone() {
            Stmt::Item(Use(ItemUse { tree: Rename(use_stmt), ..})) => {
                if args.is_handle(use_stmt.rename.clone()) {
                    inner = Some((use_stmt.ident, use_stmt.rename));
                    break
                }
                output.push(stmt)
            },
            // Recurse into function body
            Stmt::Item(Item::Fn(mut function)) => {
                let inner_block = function.block.clone();
                *function.block = build_block(inner_block.stmts.iter(), args);
                output.push(Stmt::Item(Item::Fn(function)))
            },
            // Recurse into let call
            Stmt::Local(mut local) => {
                if let Some((_, body)) = local.init.clone() {
                    let body = build_block_expr(*body.clone(), args);
                    let body = parse_quote!(#body);
                    local.init.as_mut().unwrap().1 = body;
                    output.push(Stmt::Local(local))
                }
            },
            Stmt::Expr(expr) => {
                let body = build_block_expr(expr, args);
                output.push(parse_quote!({#body}))
            }
            Stmt::Semi(expr, _) => {
                let body = build_block_expr(expr, args);
                output.push(parse_quote!({#body;}))
            }
            _ => output.push(stmt)
        }
    }
    if let Some((handle, dehandle)) = inner {
        let inner_block = build_block(input, args);
        let handle_call = parse_quote!(
            {(#handle).run(|#dehandle|{
                #inner_block
            }).expect(concat!("Could not upgrade handle ",
                              stringify!(#handle), " to ",
                              stringify!(#dehandle)))}
        );
        output.push(handle_call);
    }
    parse_quote!({#(#output)*})
}

/// Tries to build a block from the expression.
fn build_block_expr(expr: Expr, args: &mut Args) -> Expr {
    match expr {
        Expr::Block(block) => {
            let block = build_block(block.block.stmts.iter(), args);
            parse_quote!(#block)
        }
        Expr::Let(mut let_expr) => {
            *let_expr.expr = build_block_expr(*let_expr.expr.clone(), args);
            Expr::Let(let_expr)
        },
        Expr::If(mut if_expr) => {
            let then_branch = if_expr.then_branch.clone();
            let then_branch = build_block_expr(parse_quote!(#then_branch), args);
            if_expr.then_branch = parse_quote!(#then_branch);
            if_expr.else_branch = match if_expr.else_branch.clone() {
                None => if_expr.else_branch,
                Some((token, else_branch)) => {
                    Some((token, Box::new( build_block_expr(*else_branch, args))))
                }
            };
            Expr::If(if_expr)
        },
        Expr::While(mut while_expr) => {
            let body = while_expr.body.clone();
            let body = build_block_expr(parse_quote!(#body), args);
            while_expr.body = parse_quote!(#body);
            Expr::While(while_expr)
        },
        Expr::ForLoop(mut for_expr) => {
            let body = for_expr.body.clone();
            let body = build_block_expr(parse_quote!(#body), args);
            for_expr.body = parse_quote!(#body);
            Expr::ForLoop(for_expr)
        },
        Expr::Loop(mut loop_expr) => {
            let body = loop_expr.body.clone();
            let body = build_block_expr(parse_quote!(#body), args);
            loop_expr.body = parse_quote!(#body);
            Expr::Loop(loop_expr)
        },
        Expr::Match(mut match_expr) => {
            for arm in &mut match_expr.arms {
                *arm.body = build_block_expr(*arm.body.clone(), args)
            }
            Expr::Match(match_expr)
        },
        Expr::Struct(mut struct_expr) => {
            for field in &mut struct_expr.fields {
                field.expr = build_block_expr(field.expr.clone(), args);
            }
            Expr::Struct(struct_expr)
        },
        Expr::Call(mut call_expr) => {
            for arg in &mut call_expr.args {
                *arg = build_block_expr(arg.clone(), args);
            }
            Expr::Call(call_expr)
        },
        Expr::MethodCall(mut call_expr) => {
            for arg in &mut call_expr.args {
                *arg = build_block_expr(arg.clone(), args);
            }
            Expr::MethodCall(call_expr)
        },
        Expr::Closure(mut closure_expr) => {
            *closure_expr.body = build_block_expr(*closure_expr.body.clone(),
                                                  args);
            Expr::Closure(closure_expr)
        },
        Expr::Unsafe(mut unsafe_expr) => {
            unsafe_expr.block = build_block(unsafe_expr.block.stmts.iter(),
                                            args);
            Expr::Unsafe(unsafe_expr)
        },
        Expr::Assign(mut assign_expr) => {
            *assign_expr.right = build_block_expr(*assign_expr.right.clone(),
                                                  args);
            Expr::Assign(assign_expr)
        },
        Expr::AssignOp(mut assign_expr) => {
            *assign_expr.right = build_block_expr(*assign_expr.right.clone(),
                                                  args);
            Expr::AssignOp(assign_expr)
        },
        Expr::Break(mut break_expr) => {
            match break_expr.expr {
                None => {},
                Some(ref mut expr) => {
                    **expr = build_block_expr(*expr.clone(), args);
                }
            }
            Expr::Break(break_expr)
        },
        Expr::Return(mut return_expr) => {
            match return_expr.expr {
                None => {},
                Some(ref mut expr) => {
                    **expr = build_block_expr(*expr.clone(), args);
                }
            }
            Expr::Return(return_expr)
        },
        Expr::Reference(mut reference_expr) => {
            *reference_expr.expr = build_block_expr(*reference_expr.expr.clone(),
                                                    args);
            Expr::Reference(reference_expr)
        },
        Expr::Paren(mut paren_expr) => {
            *paren_expr.expr = build_block_expr(*paren_expr.expr.clone(), args);
            Expr::Paren(paren_expr)
        },
        Expr::Unary(mut unary_expr) => {
            *unary_expr.expr = build_block_expr(*unary_expr.expr.clone(), args);
            Expr::Unary(unary_expr)
        },
        Expr::Binary(mut binary_expr) => {
            *binary_expr.left = build_block_expr(*binary_expr.left.clone(), args);
            *binary_expr.right = build_block_expr(*binary_expr.right.clone(), args);
            Expr::Binary(binary_expr)
        },
        v => {
            v
        }
    }
}
<|endoftext|>


<commit_before>use std::sync::{Arc, atomic::{AtomicBool, Ordering}};
use log::{info, warn, error, debug};
use std::io::{Error, ErrorKind};
use futures::sync::oneshot;
use tokio::net::TcpStream;
use std::net::SocketAddr;
use tokio_codec::Framed;
use tokio::timer::Delay;
use tokio::prelude::*;
use rand;

use crate::fut_util::*;

use super::peer::*;
use super::rpc::*;

mod state;

pub mod event;
pub use self::event::*;

///
/// Connects to the `SocketAddr` with the specified `PeerType`.
///
pub fn connect(addr: SocketAddr, peer_type: PeerType) -> impl Future<Item = Peer, Error = Error> {
    TcpStream::connect(&addr).and_then(move |stream| {
        let frame = Framed::new(stream, codec::RpcCodec::new());
        let msg = RpcPayload {
            id: 0,
            msg: Some(RpcMsg::Handshake(peer_type))
        };

        debug!("[{}] Sending handshake: {:?}", addr, &msg);
        frame.send(msg)
    }).and_then(move |frame| {
        let (resp, frame) = frame.into_future().map_err(|(e, _)| e).wait()?;
        let resp = resp.ok_or_else(|| Error::from(ErrorKind::UnexpectedEof))?;
        debug!("[{}] Received handshake message: {:?}", addr, &resp);
        if resp.id != 0 {
            return Err(Error::new(ErrorKind::InvalidData, "expected id to be 0"))
        } else if resp.msg.is_some() {
            return Err(Error::new(ErrorKind::InvalidData, "expected msg to be empty"))
        }
        Ok(Peer::new(peer_type, addr, frame))
    })
}

///
/// Creates a persistent connection to the `SocketAddr` provided. Events are
/// emitted via the `ClientReceiver`. When sending a `ClientEvent::Disconnect`
/// event the returned `ClientReceiver` stream will be closed.
///
/// See [`ClientEvent`] for the type of events to send and receive.
///
/// [`ClientEvent`]: ./event/enum.ClientEvent.html
///
/// # Panics
///
/// When attempting to send a message that requires the connection to be open.
///
pub fn connect_loop(addr: SocketAddr, peer_type: PeerType) -> (ClientSender, ClientReceiver) {
    let (out_tx, out_rx) = channel::unbounded();
    let state = state::ConnectState::new(addr, peer_type);
    start_connect_loop(state.clone(), out_tx, 0);

    let tracker = channel::tracked::<ClientEvent, Option<oneshot::Receiver<RpcPayload>>, _>({
        let stay_connected = Arc::clone(&state.stay_connected);
        let inner = Arc::clone(&state.inner);
        move |msg| {
            match msg {
                ClientEvent::Message(payload) => {
                    let inner = inner.lock();
                    let inner = inner.as_ref().expect("must be connected to send payload");
                    let send_tracked = {
                        if let Some(msg) = &payload.msg {
                            match msg {
                                RpcMsg::Properties(var) => {
                                    var.req_ref().is_some()
                                },
                                RpcMsg::Block(var) => {
                                    var.req_ref().is_some()
                                },
                                RpcMsg::Balance(var) => {
                                    var.req_ref().is_some()
                                },
                                RpcMsg::TotalFee(var) => {
                                    var.req_ref().is_some()
                                },
                                _ => false
                            }
                        } else {
                            false
                        }
                    };
                    if send_tracked {
                        return Some(inner.sender.send(*payload))
                    } else {
                        inner.sender.send_untracked(*payload);
                    }
                },
                ClientEvent::Connect => panic!("cannot connect from event channel"),
                ClientEvent::Disconnect => {
                    stay_connected.store(false, Ordering::Release);
                    let inner = inner.lock();
                    let inner = inner.as_ref().expect("must be connected to disconnect");
                    inner.notifier.send(());
                }
            }
            None
        }
    });

    (tracker, out_rx)
}

fn start_connect_loop(state: state::ConnectState, out_tx: ChannelSender<ClientEvent>, mut tries: u8) {
    if !state.stay_connected.load(Ordering::Acquire) { return }
    let c = connect(state.addr, state.peer_type);
    tokio::spawn(c.and_then({
        let out_tx = out_tx.clone();
        let state = state.clone();
        move |peer| {
            if !state.stay_connected.load(Ordering::Acquire) { return Ok(()) }
            info!("[{}] Connected to peer", state.addr);

            let connected = Arc::new(AtomicBool::new(true));
            out_tx.send(ClientEvent::Connect);
            tries = 0;

            let rx = {
                let (tx, rx) = channel::unbounded();
                let guard = &mut *state.inner.lock();
                *guard = Some(state::InternalState {
                    sender: peer.get_sender(),
                    notifier: tx
                });
                rx.map_err(|_| {
                    Error::new(ErrorKind::Other, "rx error")
                })
            };

            tokio::spawn(ZipEither::new(peer, rx).take_while({
                let stay_connected = Arc::clone(&state.stay_connected);
                let connected = Arc::clone(&connected);
                move |_| {
                    let stay_connected = stay_connected.load(Ordering::Acquire);
                    let connected = connected.load(Ordering::Acquire);
                    Ok(stay_connected && connected)
                }
            }).for_each({
                let out_tx = out_tx.clone();
                move |(rpc, _)| {
                    if let Some(rpc) = rpc {
                        out_tx.send(ClientEvent::Message(Box::new(rpc)));
                    }
                    Ok(())
                }
            }).and_then({
                let addr = state.addr;
                move |_| {
                    warn!("[{}] Peer disconnected", addr);
                    Ok(())
                }
            }).or_else({
                let addr = state.addr;
                move |e| -> Result<_, ()> {
                    error!("[{}] Peer frame processing error: {}", addr, e);
                    Ok(())
                }
            }).and_then({
                let stay_connected = Arc::clone(&state.stay_connected);
                move |_| {
                    out_tx.send(ClientEvent::Disconnect);
                    {
                        let guard = &mut *state.inner.lock();
                        *guard = None;
                    }
                    if stay_connected.load(Ordering::Acquire) {
                        try_connect(state, out_tx, tries.saturating_add(1));
                    }
                    Ok(())
                }
            }));

            Ok(())
        }
    }).map_err(move |e| {
        if state.stay_connected.load(Ordering::Acquire) {
            error!("[{}] Failed to connect to peer: {:?}", state.addr, e);
            try_connect(state, out_tx, tries.saturating_add(1));
        }
    }));
}

fn try_connect(state: state::ConnectState, out_tx: ChannelSender<ClientEvent>, tries: u8) {
    use std::time::{Duration, Instant};

    let ms = backoff(tries);
    info!("[{}] Attempting to reconnect to peer in {} ms (tries: {})", state.addr, ms, tries);
    let d = Instant::now() + Duration::from_millis(ms);
    tokio::spawn(Delay::new(d).map(move |_| {
        start_connect_loop(state, out_tx, tries);
    }).map_err(|e| {
        error!("Connection timer error: {}", e);
    }));
}

fn backoff(tries: u8) -> u64 {
    use rand::Rng;
    let tries = f64::from(tries);
    let max = 15000f64;
    let rand = rand::thread_rng().gen_range::<f64>(0.2f64, 1f64);
    max.min((1.25f64.powf(tries) * 1000f64 * rand).floor()) as u64
}
<commit_msg>Avoid waiting on the future when connecting to a server peer<commit_after>use std::sync::{Arc, atomic::{AtomicBool, Ordering}};
use log::{info, warn, error, debug};
use std::io::{Error, ErrorKind};
use futures::sync::oneshot;
use tokio::net::TcpStream;
use std::net::SocketAddr;
use tokio_codec::Framed;
use tokio::timer::Delay;
use tokio::prelude::*;
use rand;

use crate::fut_util::*;

use super::peer::*;
use super::rpc::*;

mod state;

pub mod event;
pub use self::event::*;

///
/// Connects to the `SocketAddr` with the specified `PeerType`.
///
pub fn connect(addr: SocketAddr, peer_type: PeerType) -> impl Future<Item = Peer, Error = Error> {
    TcpStream::connect(&addr).and_then(move |stream| {
        let frame = Framed::new(stream, codec::RpcCodec::new());
        let msg = RpcPayload {
            id: 0,
            msg: Some(RpcMsg::Handshake(peer_type))
        };

        debug!("[{}] Sending handshake: {:?}", addr, &msg);
        frame.send(msg)
    }).and_then(move |frame| {
        frame.into_future().map_err(|(e, _)| e)
    }).and_then(move |(resp, frame)| {
        let resp = resp.ok_or_else(|| Error::from(ErrorKind::UnexpectedEof))?;
        debug!("[{}] Received handshake message: {:?}", addr, &resp);
        if resp.id != 0 {
            return Err(Error::new(ErrorKind::InvalidData, "expected id to be 0"))
        } else if resp.msg.is_some() {
            return Err(Error::new(ErrorKind::InvalidData, "expected msg to be empty"))
        }
        Ok(Peer::new(peer_type, addr, frame))
    })
}

///
/// Creates a persistent connection to the `SocketAddr` provided. Events are
/// emitted via the `ClientReceiver`. When sending a `ClientEvent::Disconnect`
/// event the returned `ClientReceiver` stream will be closed.
///
/// See [`ClientEvent`] for the type of events to send and receive.
///
/// [`ClientEvent`]: ./event/enum.ClientEvent.html
///
/// # Panics
///
/// When attempting to send a message that requires the connection to be open.
///
pub fn connect_loop(addr: SocketAddr, peer_type: PeerType) -> (ClientSender, ClientReceiver) {
    let (out_tx, out_rx) = channel::unbounded();
    let state = state::ConnectState::new(addr, peer_type);
    start_connect_loop(state.clone(), out_tx, 0);

    let tracker = channel::tracked::<ClientEvent, Option<oneshot::Receiver<RpcPayload>>, _>({
        let stay_connected = Arc::clone(&state.stay_connected);
        let inner = Arc::clone(&state.inner);
        move |msg| {
            match msg {
                ClientEvent::Message(payload) => {
                    let inner = inner.lock();
                    let inner = inner.as_ref().expect("must be connected to send payload");
                    let send_tracked = {
                        if let Some(msg) = &payload.msg {
                            match msg {
                                RpcMsg::Properties(var) => {
                                    var.req_ref().is_some()
                                },
                                RpcMsg::Block(var) => {
                                    var.req_ref().is_some()
                                },
                                RpcMsg::Balance(var) => {
                                    var.req_ref().is_some()
                                },
                                RpcMsg::TotalFee(var) => {
                                    var.req_ref().is_some()
                                },
                                _ => false
                            }
                        } else {
                            false
                        }
                    };
                    if send_tracked {
                        return Some(inner.sender.send(*payload))
                    } else {
                        inner.sender.send_untracked(*payload);
                    }
                },
                ClientEvent::Connect => panic!("cannot connect from event channel"),
                ClientEvent::Disconnect => {
                    stay_connected.store(false, Ordering::Release);
                    let inner = inner.lock();
                    let inner = inner.as_ref().expect("must be connected to disconnect");
                    inner.notifier.send(());
                }
            }
            None
        }
    });

    (tracker, out_rx)
}

fn start_connect_loop(state: state::ConnectState, out_tx: ChannelSender<ClientEvent>, mut tries: u8) {
    if !state.stay_connected.load(Ordering::Acquire) { return }
    let c = connect(state.addr, state.peer_type);
    tokio::spawn(c.and_then({
        let out_tx = out_tx.clone();
        let state = state.clone();
        move |peer| {
            if !state.stay_connected.load(Ordering::Acquire) { return Ok(()) }
            info!("[{}] Connected to peer", state.addr);

            let connected = Arc::new(AtomicBool::new(true));
            out_tx.send(ClientEvent::Connect);
            tries = 0;

            let rx = {
                let (tx, rx) = channel::unbounded();
                let guard = &mut *state.inner.lock();
                *guard = Some(state::InternalState {
                    sender: peer.get_sender(),
                    notifier: tx
                });
                rx.map_err(|_| {
                    Error::new(ErrorKind::Other, "rx error")
                })
            };

            tokio::spawn(ZipEither::new(peer, rx).take_while({
                let stay_connected = Arc::clone(&state.stay_connected);
                let connected = Arc::clone(&connected);
                move |_| {
                    let stay_connected = stay_connected.load(Ordering::Acquire);
                    let connected = connected.load(Ordering::Acquire);
                    Ok(stay_connected && connected)
                }
            }).for_each({
                let out_tx = out_tx.clone();
                move |(rpc, _)| {
                    if let Some(rpc) = rpc {
                        out_tx.send(ClientEvent::Message(Box::new(rpc)));
                    }
                    Ok(())
                }
            }).and_then({
                let addr = state.addr;
                move |_| {
                    warn!("[{}] Peer disconnected", addr);
                    Ok(())
                }
            }).or_else({
                let addr = state.addr;
                move |e| -> Result<_, ()> {
                    error!("[{}] Peer frame processing error: {}", addr, e);
                    Ok(())
                }
            }).and_then({
                let stay_connected = Arc::clone(&state.stay_connected);
                move |_| {
                    out_tx.send(ClientEvent::Disconnect);
                    {
                        let guard = &mut *state.inner.lock();
                        *guard = None;
                    }
                    if stay_connected.load(Ordering::Acquire) {
                        try_connect(state, out_tx, tries.saturating_add(1));
                    }
                    Ok(())
                }
            }));

            Ok(())
        }
    }).map_err(move |e| {
        if state.stay_connected.load(Ordering::Acquire) {
            error!("[{}] Failed to connect to peer: {:?}", state.addr, e);
            try_connect(state, out_tx, tries.saturating_add(1));
        }
    }));
}

fn try_connect(state: state::ConnectState, out_tx: ChannelSender<ClientEvent>, tries: u8) {
    use std::time::{Duration, Instant};

    let ms = backoff(tries);
    info!("[{}] Attempting to reconnect to peer in {} ms (tries: {})", state.addr, ms, tries);
    let d = Instant::now() + Duration::from_millis(ms);
    tokio::spawn(Delay::new(d).map(move |_| {
        start_connect_loop(state, out_tx, tries);
    }).map_err(|e| {
        error!("Connection timer error: {}", e);
    }));
}

fn backoff(tries: u8) -> u64 {
    use rand::Rng;
    let tries = f64::from(tries);
    let max = 15000f64;
    let rand = rand::thread_rng().gen_range::<f64>(0.2f64, 1f64);
    max.min((1.25f64.powf(tries) * 1000f64 * rand).floor()) as u64
}
<|endoftext|>


<commit_before>//! # OpenTelemetry Propagator interface
//!
//! Propagators API consists of two main formats:
//!
//! - `BinaryFormat` is used to serialize and deserialize a value
//! into a binary representation.
//! - `HttpTextFormat` is used to inject and extract a value as
//! text into injectors and extractors that travel in-band across process boundaries.
//!
//! Deserializing must set `is_remote` to true on the returned
//! `SpanContext`.
//!
//! ## Binary Format
//!
//! `BinaryFormat` is a formatter to serialize and deserialize a value
//! into a binary format.
//!
//! `BinaryFormat` MUST expose the APIs that serializes values into bytes,
//! and deserializes values from bytes.
//!
//! ### ToBytes
//!
//! Serializes the given value into the on-the-wire representation.
//!
//! Required arguments:
//!
//! - the value to serialize, can be `SpanContext` or `DistributedContext`.
//!
//! Returns the on-the-wire byte representation of the value.
//!
//! ### FromBytes
//!
//! Creates a value from the given on-the-wire encoded representation.
//!
//! If the value could not be parsed, the underlying implementation
//! SHOULD decide to return ether an empty value, an invalid value, or
//! a valid value.
//!
//! Required arguments:
//!
//! - on-the-wire byte representation of the value.
//!
//! Returns a value deserialized from bytes.
//!
//! ## HTTP Text Format
//!
//! `HttpTextFormat` is a formatter that injects and extracts a value
//! as text into injectors and extractors that travel in-band across process boundaries.
//!
//! Encoding is expected to conform to the HTTP Header Field semantics.
//! Values are often encoded as RPC/HTTP request headers.
//!
//! The carrier of propagated data on both the client (injector) and
//! server (extractor) side is usually a http request. Propagation is
//! usually implemented via library-specific request interceptors, where
//! the client-side injects values and the server-side extracts them.
//!
//! `HttpTextFormat` MUST expose the APIs that injects values into injectors,
//! and extracts values from extractors.
//!
//! ### Fields
//!
//! The propagation fields defined. If your injector is reused, you should
//! delete the fields here before calling `inject`.
//!
//! For example, if the injector is a single-use or immutable request object,
//! you don't need to clear fields as they couldn't have been set before.
//! If it is a mutable, retryable object, successive calls should clear
//! these fields first.
//!
//! The use cases of this are:
//!
//! - allow pre-allocation of fields, especially in systems like gRPC
//! Metadata
//! - allow a single-pass over an iterator
//!
//! Returns list of fields that will be used by this formatter.
//!
//! ### Inject
//!
//! Injects the value downstream. For example, as http headers.
//!
//! Required arguments:
//!
//! - the `SpanContext` to be injected.
//! - the injector that holds propagation fields. For example, an outgoing
//! message or http request.
//! - the `Setter` invoked for each propagation key to add or remove.
//!
//! #### Setter argument
//!
//! Setter is an argument in `Inject` that puts value into given field.
//!
//! `Setter` allows a `HttpTextFormat` to set propagated fields into a
//! injector.
//!
//! `Setter` MUST be stateless and allowed to be saved as a constant to
//! avoid runtime allocations. One of the ways to implement it is `Setter`
//! class with `Put` method as described below.
//!
//! ##### Put
//!
//! Replaces a propagated field with the given value.
//!
//! Required arguments:
//!
//! - the injector holds propagation fields. For example, an outgoing message
//! or http request.
//! - the key of the field.
//! - the value of the field.
//!
//! The implementation SHOULD preserve casing (e.g. it should not transform
//! `Content-Type` to `content-type`) if the used protocol is case insensitive,
//! otherwise it MUST preserve casing.
//!
//! ### Extract
//!
//! Extracts the value from upstream. For example, as http headers.
//!
//! If the value could not be parsed, the underlying implementation will
//! decide to return an object representing either an empty value, an invalid
//! value, or a valid value.
//!
//! Required arguments:
//!
//! - the extractor holds propagation fields. For example, an outgoing message
//! or http request.
//! - the instance of `Getter` invoked for each propagation key to get.
//!
//! Returns the non-null extracted value.
//!
//! #### Getter argument
//!
//! Getter is an argument in `Extract` that get value from given field
//!
//! `Getter` allows a `HttpTextFormat` to read propagated fields from a
//! extractor.
//!
//! `Getter` MUST be stateless and allowed to be saved as a constant to avoid
//! runtime allocations. One of the ways to implement it is `Getter` class
//! with `Get` method as described below.
//!
//! ##### Get
//!
//!  The Get function MUST return the first value of the given propagation
//! key or return `None` if the key doesn't exist.
//!
//! Required arguments:
//!
//! - the extractor of propagation fields, such as an HTTP request.
//! - the key of the field.
//!
//! The `get` function is responsible for handling case sensitivity. If
//! the getter is intended to work with an HTTP request object, the getter
//! MUST be case insensitive. To improve compatibility with other text-based
//! protocols, text format implementations MUST ensure to always use the
//! canonical casing for their attributes. NOTE: Canonical casing for HTTP
//! headers is usually title case (e.g. `Content-Type` instead of `content-type`).
//!
use crate::api;
use std::collections::HashMap;

pub mod composite_propagator;
pub mod text_propagator;

/// Injector provides an interface for adding fields from an underlying struct like `HashMap`
pub trait Injector {
    /// Add a key and value to the underlying.
    fn set(&mut self, key: &str, value: String);
}

/// Extractor provides an interface for removing fields from an underlying struct like `HashMap`
pub trait Extractor {
    /// Get a value from a key from the underlying data.
    fn get(&self, key: &str) -> Option<&str>;
}

impl<S: std::hash::BuildHasher> api::Injector for HashMap<String, String, S> {
    /// Set a key and value in the HashMap.
    fn set(&mut self, key: &str, value: String) {
        self.insert(String::from(key), value);
    }
}

impl<S: std::hash::BuildHasher> api::Extractor for HashMap<String, String, S> {
    /// Get a value for a key from the HashMap.
    fn get(&self, key: &str) -> Option<&str> {
        self.get(key).map(|v| v.as_str())
    }
}

#[cfg(feature = "http")]
impl api::Injector for http::HeaderMap {
    /// Set a key and value in the HeaderMap.  Does nothing if the key or value are not valid inputs.
    fn set(&mut self, key: &str, value: String) {
        if let Ok(name) = http::header::HeaderName::from_bytes(key.as_bytes()) {
            if let Ok(val) = http::header::HeaderValue::from_str(&value) {
                self.insert(name, val);
            }
        }
    }
}

#[cfg(feature = "http")]
impl api::Extractor for http::HeaderMap {
    /// Get a value for a key from the HeaderMap.  If the value is not valid ASCII, returns None.
    fn get(&self, key: &str) -> Option<&str> {
        match self.get(key) {
            Some(val) => match val.to_str() {
                Ok(ascii) => Some(ascii),
                Err(_) => None,
            },
            None => None,
        }
    }
}

#[cfg(feature = "tonic")]
impl api::Injector for tonic::metadata::MetadataMap {
    /// Set a key and value in the MetadataMap.  Does nothing if the key or value are not valid inputs
    fn set(&mut self, key: &str, value: String) {
        if let Ok(key) = tonic::metadata::MetadataKey::from_bytes(key.to_lowercase().as_bytes()) {
            if let Ok(val) = tonic::metadata::MetadataValue::from_str(&value) {
                self.insert(key, val);
            }
        }
    }
}

#[cfg(feature = "tonic")]
impl api::Extractor for tonic::metadata::MetadataMap {
    /// Get a value for a key from the MetadataMap.  If the value can't be converted to &str, returns None
    fn get(&self, key: &str) -> Option<&str> {
        self.get(key).and_then(|metadata| metadata.to_str().ok())
    }
}
<commit_msg>Ensure extractors are case insensitive (#193)<commit_after>//! # OpenTelemetry Propagator interface
//!
//! Propagators API consists of two main formats:
//!
//! - `BinaryFormat` is used to serialize and deserialize a value
//! into a binary representation.
//! - `HttpTextFormat` is used to inject and extract a value as
//! text into injectors and extractors that travel in-band across process boundaries.
//!
//! Deserializing must set `is_remote` to true on the returned
//! `SpanContext`.
//!
//! ## Binary Format
//!
//! `BinaryFormat` is a formatter to serialize and deserialize a value
//! into a binary format.
//!
//! `BinaryFormat` MUST expose the APIs that serializes values into bytes,
//! and deserializes values from bytes.
//!
//! ### ToBytes
//!
//! Serializes the given value into the on-the-wire representation.
//!
//! Required arguments:
//!
//! - the value to serialize, can be `SpanContext` or `DistributedContext`.
//!
//! Returns the on-the-wire byte representation of the value.
//!
//! ### FromBytes
//!
//! Creates a value from the given on-the-wire encoded representation.
//!
//! If the value could not be parsed, the underlying implementation
//! SHOULD decide to return ether an empty value, an invalid value, or
//! a valid value.
//!
//! Required arguments:
//!
//! - on-the-wire byte representation of the value.
//!
//! Returns a value deserialized from bytes.
//!
//! ## HTTP Text Format
//!
//! `HttpTextFormat` is a formatter that injects and extracts a value
//! as text into injectors and extractors that travel in-band across process boundaries.
//!
//! Encoding is expected to conform to the HTTP Header Field semantics.
//! Values are often encoded as RPC/HTTP request headers.
//!
//! The carrier of propagated data on both the client (injector) and
//! server (extractor) side is usually a http request. Propagation is
//! usually implemented via library-specific request interceptors, where
//! the client-side injects values and the server-side extracts them.
//!
//! `HttpTextFormat` MUST expose the APIs that injects values into injectors,
//! and extracts values from extractors.
//!
//! ### Fields
//!
//! The propagation fields defined. If your injector is reused, you should
//! delete the fields here before calling `inject`.
//!
//! For example, if the injector is a single-use or immutable request object,
//! you don't need to clear fields as they couldn't have been set before.
//! If it is a mutable, retryable object, successive calls should clear
//! these fields first.
//!
//! The use cases of this are:
//!
//! - allow pre-allocation of fields, especially in systems like gRPC
//! Metadata
//! - allow a single-pass over an iterator
//!
//! Returns list of fields that will be used by this formatter.
//!
//! ### Inject
//!
//! Injects the value downstream. For example, as http headers.
//!
//! Required arguments:
//!
//! - the `SpanContext` to be injected.
//! - the injector that holds propagation fields. For example, an outgoing
//! message or http request.
//! - the `Setter` invoked for each propagation key to add or remove.
//!
//! #### Setter argument
//!
//! Setter is an argument in `Inject` that puts value into given field.
//!
//! `Setter` allows a `HttpTextFormat` to set propagated fields into a
//! injector.
//!
//! `Setter` MUST be stateless and allowed to be saved as a constant to
//! avoid runtime allocations. One of the ways to implement it is `Setter`
//! class with `Put` method as described below.
//!
//! ##### Put
//!
//! Replaces a propagated field with the given value.
//!
//! Required arguments:
//!
//! - the injector holds propagation fields. For example, an outgoing message
//! or http request.
//! - the key of the field.
//! - the value of the field.
//!
//! The implementation SHOULD preserve casing (e.g. it should not transform
//! `Content-Type` to `content-type`) if the used protocol is case insensitive,
//! otherwise it MUST preserve casing.
//!
//! ### Extract
//!
//! Extracts the value from upstream. For example, as http headers.
//!
//! If the value could not be parsed, the underlying implementation will
//! decide to return an object representing either an empty value, an invalid
//! value, or a valid value.
//!
//! Required arguments:
//!
//! - the extractor holds propagation fields. For example, an outgoing message
//! or http request.
//! - the instance of `Getter` invoked for each propagation key to get.
//!
//! Returns the non-null extracted value.
//!
//! #### Getter argument
//!
//! Getter is an argument in `Extract` that get value from given field
//!
//! `Getter` allows a `HttpTextFormat` to read propagated fields from a
//! extractor.
//!
//! `Getter` MUST be stateless and allowed to be saved as a constant to avoid
//! runtime allocations. One of the ways to implement it is `Getter` class
//! with `Get` method as described below.
//!
//! ##### Get
//!
//!  The Get function MUST return the first value of the given propagation
//! key or return `None` if the key doesn't exist.
//!
//! Required arguments:
//!
//! - the extractor of propagation fields, such as an HTTP request.
//! - the key of the field.
//!
//! The `get` function is responsible for handling case sensitivity. If
//! the getter is intended to work with an HTTP request object, the getter
//! MUST be case insensitive. To improve compatibility with other text-based
//! protocols, text format implementations MUST ensure to always use the
//! canonical casing for their attributes. NOTE: Canonical casing for HTTP
//! headers is usually title case (e.g. `Content-Type` instead of `content-type`).
//!
use crate::api;
use std::collections::HashMap;

pub mod composite_propagator;
pub mod text_propagator;

/// Injector provides an interface for adding fields from an underlying struct like `HashMap`
pub trait Injector {
    /// Add a key and value to the underlying.
    fn set(&mut self, key: &str, value: String);
}

/// Extractor provides an interface for removing fields from an underlying struct like `HashMap`
pub trait Extractor {
    /// Get a value from a key from the underlying data.
    fn get(&self, key: &str) -> Option<&str>;
}

impl<S: std::hash::BuildHasher> api::Injector for HashMap<String, String, S> {
    /// Set a key and value in the HashMap.
    fn set(&mut self, key: &str, value: String) {
        self.insert(key.to_lowercase(), value);
    }
}

impl<S: std::hash::BuildHasher> api::Extractor for HashMap<String, String, S> {
    /// Get a value for a key from the HashMap.
    fn get(&self, key: &str) -> Option<&str> {
        self.get(&key.to_lowercase()).map(|v| v.as_str())
    }
}

#[cfg(feature = "http")]
impl api::Injector for http::HeaderMap {
    /// Set a key and value in the HeaderMap.  Does nothing if the key or value are not valid inputs.
    fn set(&mut self, key: &str, value: String) {
        if let Ok(name) = http::header::HeaderName::from_bytes(key.as_bytes()) {
            if let Ok(val) = http::header::HeaderValue::from_str(&value) {
                self.insert(name, val);
            }
        }
    }
}

#[cfg(feature = "http")]
impl api::Extractor for http::HeaderMap {
    /// Get a value for a key from the HeaderMap.  If the value is not valid ASCII, returns None.
    fn get(&self, key: &str) -> Option<&str> {
        self.get(key).and_then(|value| value.to_str().ok())
    }
}

#[cfg(feature = "tonic")]
impl api::Injector for tonic::metadata::MetadataMap {
    /// Set a key and value in the MetadataMap.  Does nothing if the key or value are not valid inputs
    fn set(&mut self, key: &str, value: String) {
        if let Ok(key) = tonic::metadata::MetadataKey::from_bytes(key.as_bytes()) {
            if let Ok(val) = tonic::metadata::MetadataValue::from_str(&value) {
                self.insert(key, val);
            }
        }
    }
}

#[cfg(feature = "tonic")]
impl api::Extractor for tonic::metadata::MetadataMap {
    /// Get a value for a key from the MetadataMap.  If the value can't be converted to &str, returns None
    fn get(&self, key: &str) -> Option<&str> {
        self.get(key).and_then(|metadata| metadata.to_str().ok())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::collections::HashMap;

    #[test]
    fn hash_map() {
        let mut carrier = HashMap::new();
        carrier.set("headerName", "value".to_string());

        assert_eq!(
            Extractor::get(&carrier, "HEADERNAME"),
            Some("value"),
            "case insensitive extraction"
        )
    }

    #[test]
    #[cfg(feature = "http")]
    fn http_headers() {
        let mut carrier = http::HeaderMap::new();
        carrier.set("headerName", "value".to_string());

        assert_eq!(
            Extractor::get(&carrier, "HEADERNAME"),
            Some("value"),
            "case insensitive extraction"
        )
    }

    #[test]
    #[cfg(feature = "tonic")]
    fn tonic_headers() {
        let mut carrier = tonic::metadata::MetadataMap::new();
        carrier.set("headerName", "value".to_string());

        assert_eq!(
            Extractor::get(&carrier, "HEADERNAME"),
            Some("value"),
            "case insensitive extraction"
        )
    }
}
<|endoftext|>


<commit_before>//! `mdbook`'s low level rendering interface.
//!
//! # Note
//!
//! You usually don't need to work with this module directly. If you want to
//! implement your own backend, then check out the [For Developers] section of
//! the user guide.
//!
//! The definition for [RenderContext] may be useful though.
//!
//! [For Developers]: https://rust-lang-nursery.github.io/mdBook/lib/index.html
//! [RenderContext]: struct.RenderContext.html

pub use self::html_handlebars::HtmlHandlebars;

mod html_handlebars;

use serde_json;
use shlex::Shlex;
use std::fs;
use std::io::{self, Read};
use std::path::PathBuf;
use std::process::{Command, Stdio};

use book::Book;
use config::Config;
use errors::*;

/// An arbitrary `mdbook` backend.
///
/// Although it's quite possible for you to import `mdbook` as a library and
/// provide your own renderer, there are two main renderer implementations that
/// 99% of users will ever use:
///
/// - [HtmlHandlebars] - the built-in HTML renderer
/// - [CmdRenderer] - a generic renderer which shells out to a program to do the
///   actual rendering
///
/// [HtmlHandlebars]: struct.HtmlHandlebars.html
/// [CmdRenderer]: struct.CmdRenderer.html
pub trait Renderer {
    /// The `Renderer`'s name.
    fn name(&self) -> &str;

    /// Invoke the `Renderer`, passing in all the necessary information for
    /// describing a book.
    fn render(&self, ctx: &RenderContext) -> Result<()>;
}

/// The context provided to all renderers.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct RenderContext {
    /// Which version of `mdbook` did this come from (as written in `mdbook`'s
    /// `Cargo.toml`). Useful if you know the renderer is only compatible with
    /// certain versions of `mdbook`.
    pub version: String,
    /// The book's root directory.
    pub root: PathBuf,
    /// A loaded representation of the book itself.
    pub book: Book,
    /// The loaded configuration file.
    pub config: Config,
    /// Where the renderer *must* put any build artefacts generated. To allow
    /// renderers to cache intermediate results, this directory is not
    /// guaranteed to be empty or even exist.
    pub destination: PathBuf,
    #[serde(skip)]
    __non_exhaustive: (),
}

impl RenderContext {
    /// Create a new `RenderContext`.
    pub fn new<P, Q>(root: P, book: Book, config: Config, destination: Q) -> RenderContext
    where
        P: Into<PathBuf>,
        Q: Into<PathBuf>,
    {
        RenderContext {
            book: book,
            config: config,
            version: ::MDBOOK_VERSION.to_string(),
            root: root.into(),
            destination: destination.into(),
            __non_exhaustive: (),
        }
    }

    /// Get the source directory's (absolute) path on disk.
    pub fn source_dir(&self) -> PathBuf {
        self.root.join(&self.config.book.src)
    }

    /// Load a `RenderContext` from its JSON representation.
    pub fn from_json<R: Read>(reader: R) -> Result<RenderContext> {
        serde_json::from_reader(reader).chain_err(|| "Unable to deserialize the `RenderContext`")
    }
}

/// A generic renderer which will shell out to an arbitrary executable.
///
/// # Rendering Protocol
///
/// When the renderer's `render()` method is invoked, `CmdRenderer` will spawn
/// the `cmd` as a subprocess. The `RenderContext` is passed to the subprocess
/// as a JSON string (using `serde_json`).
///
/// > **Note:** The command used doesn't necessarily need to be a single
/// > executable (i.e. `/path/to/renderer`). The `cmd` string lets you pass
/// > in command line arguments, so there's no reason why it couldn't be
/// > `python /path/to/renderer --from mdbook --to epub`.
///
/// Anything the subprocess writes to `stdin` or `stdout` will be passed through
/// to the user. While this gives the renderer maximum flexibility to output
/// whatever it wants, to avoid spamming users it is recommended to avoid
/// unnecessary output.
///
/// To help choose the appropriate output level, the `RUST_LOG` environment
/// variable will be passed through to the subprocess, if set.
///
/// If the subprocess wishes to indicate that rendering failed, it should exit
/// with a non-zero return code.
#[derive(Debug, Clone, PartialEq)]
pub struct CmdRenderer {
    name: String,
    cmd: String,
}

impl CmdRenderer {
    /// Create a new `CmdRenderer` which will invoke the provided `cmd` string.
    pub fn new(name: String, cmd: String) -> CmdRenderer {
        CmdRenderer { name, cmd }
    }

    fn compose_command(&self) -> Result<Command> {
        let mut words = Shlex::new(&self.cmd);
        let executable = match words.next() {
            Some(e) => e,
            None => bail!("Command string was empty"),
        };

        let mut cmd = Command::new(executable);

        for arg in words {
            cmd.arg(arg);
        }

        Ok(cmd)
    }
}

impl Renderer for CmdRenderer {
    fn name(&self) -> &str {
        &self.name
    }

    fn render(&self, ctx: &RenderContext) -> Result<()> {
        info!("Invoking the \"{}\" renderer", self.name);

        let _ = fs::create_dir_all(&ctx.destination);

        let mut child = match self
            .compose_command()?
            .stdin(Stdio::piped())
            .stdout(Stdio::inherit())
            .stderr(Stdio::inherit())
            .current_dir(&ctx.destination)
            .spawn()
        {
            Ok(c) => c,
            Err(ref e) if e.kind() == io::ErrorKind::NotFound => {
                warn!(
                    "The command wasn't found, is the \"{}\" backend installed?",
                    self.name
                );
                warn!("\tCommand: {}", self.cmd);
                return Ok(());
            }
            Err(e) => {
                return Err(e).chain_err(|| "Unable to start the backend")?;
            }
        };

        {
            let mut stdin = child.stdin.take().expect("Child has stdin");
            if let Err(e) = serde_json::to_writer(&mut stdin, &ctx) {
                // Looks like the backend hung up before we could finish
                // sending it the render context. Log the error and keep going
                warn!("Error writing the RenderContext to the backend, {}", e);
            }

            // explicitly close the `stdin` file handle
            drop(stdin);
        }

        let status = child
            .wait()
            .chain_err(|| "Error waiting for the backend to complete")?;

        trace!("{} exited with output: {:?}", self.cmd, status);

        if !status.success() {
            error!("Renderer exited with non-zero return code.");
            bail!("The \"{}\" renderer failed", self.name);
        } else {
            Ok(())
        }
    }
}
<commit_msg>Fix invalid url in renderer documentation<commit_after>//! `mdbook`'s low level rendering interface.
//!
//! # Note
//!
//! You usually don't need to work with this module directly. If you want to
//! implement your own backend, then check out the [For Developers] section of
//! the user guide.
//!
//! The definition for [RenderContext] may be useful though.
//!
//! [For Developers]: https://rust-lang-nursery.github.io/mdBook/for_developers/index.html
//! [RenderContext]: struct.RenderContext.html

pub use self::html_handlebars::HtmlHandlebars;

mod html_handlebars;

use serde_json;
use shlex::Shlex;
use std::fs;
use std::io::{self, Read};
use std::path::PathBuf;
use std::process::{Command, Stdio};

use book::Book;
use config::Config;
use errors::*;

/// An arbitrary `mdbook` backend.
///
/// Although it's quite possible for you to import `mdbook` as a library and
/// provide your own renderer, there are two main renderer implementations that
/// 99% of users will ever use:
///
/// - [HtmlHandlebars] - the built-in HTML renderer
/// - [CmdRenderer] - a generic renderer which shells out to a program to do the
///   actual rendering
///
/// [HtmlHandlebars]: struct.HtmlHandlebars.html
/// [CmdRenderer]: struct.CmdRenderer.html
pub trait Renderer {
    /// The `Renderer`'s name.
    fn name(&self) -> &str;

    /// Invoke the `Renderer`, passing in all the necessary information for
    /// describing a book.
    fn render(&self, ctx: &RenderContext) -> Result<()>;
}

/// The context provided to all renderers.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct RenderContext {
    /// Which version of `mdbook` did this come from (as written in `mdbook`'s
    /// `Cargo.toml`). Useful if you know the renderer is only compatible with
    /// certain versions of `mdbook`.
    pub version: String,
    /// The book's root directory.
    pub root: PathBuf,
    /// A loaded representation of the book itself.
    pub book: Book,
    /// The loaded configuration file.
    pub config: Config,
    /// Where the renderer *must* put any build artefacts generated. To allow
    /// renderers to cache intermediate results, this directory is not
    /// guaranteed to be empty or even exist.
    pub destination: PathBuf,
    #[serde(skip)]
    __non_exhaustive: (),
}

impl RenderContext {
    /// Create a new `RenderContext`.
    pub fn new<P, Q>(root: P, book: Book, config: Config, destination: Q) -> RenderContext
    where
        P: Into<PathBuf>,
        Q: Into<PathBuf>,
    {
        RenderContext {
            book: book,
            config: config,
            version: ::MDBOOK_VERSION.to_string(),
            root: root.into(),
            destination: destination.into(),
            __non_exhaustive: (),
        }
    }

    /// Get the source directory's (absolute) path on disk.
    pub fn source_dir(&self) -> PathBuf {
        self.root.join(&self.config.book.src)
    }

    /// Load a `RenderContext` from its JSON representation.
    pub fn from_json<R: Read>(reader: R) -> Result<RenderContext> {
        serde_json::from_reader(reader).chain_err(|| "Unable to deserialize the `RenderContext`")
    }
}

/// A generic renderer which will shell out to an arbitrary executable.
///
/// # Rendering Protocol
///
/// When the renderer's `render()` method is invoked, `CmdRenderer` will spawn
/// the `cmd` as a subprocess. The `RenderContext` is passed to the subprocess
/// as a JSON string (using `serde_json`).
///
/// > **Note:** The command used doesn't necessarily need to be a single
/// > executable (i.e. `/path/to/renderer`). The `cmd` string lets you pass
/// > in command line arguments, so there's no reason why it couldn't be
/// > `python /path/to/renderer --from mdbook --to epub`.
///
/// Anything the subprocess writes to `stdin` or `stdout` will be passed through
/// to the user. While this gives the renderer maximum flexibility to output
/// whatever it wants, to avoid spamming users it is recommended to avoid
/// unnecessary output.
///
/// To help choose the appropriate output level, the `RUST_LOG` environment
/// variable will be passed through to the subprocess, if set.
///
/// If the subprocess wishes to indicate that rendering failed, it should exit
/// with a non-zero return code.
#[derive(Debug, Clone, PartialEq)]
pub struct CmdRenderer {
    name: String,
    cmd: String,
}

impl CmdRenderer {
    /// Create a new `CmdRenderer` which will invoke the provided `cmd` string.
    pub fn new(name: String, cmd: String) -> CmdRenderer {
        CmdRenderer { name, cmd }
    }

    fn compose_command(&self) -> Result<Command> {
        let mut words = Shlex::new(&self.cmd);
        let executable = match words.next() {
            Some(e) => e,
            None => bail!("Command string was empty"),
        };

        let mut cmd = Command::new(executable);

        for arg in words {
            cmd.arg(arg);
        }

        Ok(cmd)
    }
}

impl Renderer for CmdRenderer {
    fn name(&self) -> &str {
        &self.name
    }

    fn render(&self, ctx: &RenderContext) -> Result<()> {
        info!("Invoking the \"{}\" renderer", self.name);

        let _ = fs::create_dir_all(&ctx.destination);

        let mut child = match self
            .compose_command()?
            .stdin(Stdio::piped())
            .stdout(Stdio::inherit())
            .stderr(Stdio::inherit())
            .current_dir(&ctx.destination)
            .spawn()
        {
            Ok(c) => c,
            Err(ref e) if e.kind() == io::ErrorKind::NotFound => {
                warn!(
                    "The command wasn't found, is the \"{}\" backend installed?",
                    self.name
                );
                warn!("\tCommand: {}", self.cmd);
                return Ok(());
            }
            Err(e) => {
                return Err(e).chain_err(|| "Unable to start the backend")?;
            }
        };

        {
            let mut stdin = child.stdin.take().expect("Child has stdin");
            if let Err(e) = serde_json::to_writer(&mut stdin, &ctx) {
                // Looks like the backend hung up before we could finish
                // sending it the render context. Log the error and keep going
                warn!("Error writing the RenderContext to the backend, {}", e);
            }

            // explicitly close the `stdin` file handle
            drop(stdin);
        }

        let status = child
            .wait()
            .chain_err(|| "Error waiting for the backend to complete")?;

        trace!("{} exited with output: {:?}", self.cmd, status);

        if !status.success() {
            error!("Renderer exited with non-zero return code.");
            bail!("The \"{}\" renderer failed", self.name);
        } else {
            Ok(())
        }
    }
}
<|endoftext|>


<commit_before>use std::marker;

trait Vector {
  fn size(&self) -> usize;
}

struct ArrayVector {
  array: [i32; 1]
}

impl Vector for ArrayVector {
 fn size(&self) -> usize {
  self.array.len()
 }
}

trait RowBlock<'b> {
  fn vector(&'b self, cid: usize) -> &'b Vector;
}

struct XRowBlock<'b> {
  vectors: Vec<Box<&'b Vector>>
}

impl<'b> RowBlock<'b> for XRowBlock<'b> {
 fn vector(&self, cid: usize) -> &'b Vector {
   *self.vectors[cid]
 }
}

struct YRowBlock<'a> {
  vectors: Vec<Box<Vector>>,
  _marker: marker::PhantomData<&'a ()>
}

impl<'a> YRowBlock<'a> {
  fn set_vector(&mut self, v: Box<Vector>) {
    self.vectors.push(v);
  }
}

impl<'a> RowBlock<'a> for YRowBlock<'a> {
 fn vector(&'a self, cid: usize) -> &'a Vector {
   &*self.vectors[cid]
 }
}

#[test]
fn test_yrowblock() {
  let mut y = YRowBlock {vectors: Vec::new(), _marker: marker::PhantomData};
  let vector = ArrayVector {array: [0]};
  y.set_vector(Box::new(vector));
  let vec: &Vector = y.vector(0);
}<commit_msg>Implemented a sample code.<commit_after>use std::marker;

trait Vector {
  fn size(&self) -> usize;

  //fn as_array<T: Sized>() -> &[T]
}

struct ArrayVector {
  array: [i32; 1]
}

impl Vector for ArrayVector {
 fn size(&self) -> usize {
  self.array.len()
 }
}

trait RowBlock<'b> {
  fn vector(&'b self, cid: usize) -> &'b Vector;
}

struct AssemblyRowBlock<'b> {
  vectors: Vec<Box<&'b Vector>>
}

impl<'b> AssemblyRowBlock<'b> {
  fn set_vector(&mut self, v: &'b Vector) {
    self.vectors.push(Box::new(v));
  }
}

impl<'b> RowBlock<'b> for AssemblyRowBlock<'b> {
 fn vector(&self, cid: usize) -> &'b Vector {
   *self.vectors[cid]
 }
}

struct YRowBlock<'a> {
  vectors: Vec<Box<Vector>>,
  _marker: marker::PhantomData<&'a ()>
}

impl<'a> YRowBlock<'a> {
  fn set_vector(&mut self, v: Box<Vector>) {
    self.vectors.push(v);
  }
}

impl<'a> RowBlock<'a> for YRowBlock<'a> {
 fn vector(&'a self, cid: usize) -> &'a Vector {
   &*self.vectors[cid]
 }
}

#[test]
fn test_yrowblock() {
  let mut y = YRowBlock {vectors: Vec::new(), _marker: marker::PhantomData};
  let vector = ArrayVector {array: [0]};
  y.set_vector(Box::new(vector));

  let mut ass = AssemblyRowBlock {vectors: Vec::new()};
  ass.set_vector(y.vector(0));
  let v: &Vector = ass.vector(0);
}<|endoftext|>


<commit_before>use buffer::{Lexeme, Position, Token};
use syntect::parsing::{ParseState, Scope, ScopeStack, SyntaxDefinition};
use buffer::token::line_iterator::LineIterator;
use std::vec::IntoIter;

pub struct TokenIterator<'a> {
    scopes: ScopeStack,
    parser: ParseState,
    line_tokens: Option<IntoIter<Token<'a>>>,
    lines: LineIterator<'a>
}

impl<'a> TokenIterator<'a> {
    pub fn new(data: &'a str, def: &SyntaxDefinition) -> TokenIterator<'a> {
        TokenIterator{
            scopes: ScopeStack::new(),
            parser: ParseState::new(def),
            line_tokens: None,
            lines: LineIterator::new(data)
        }
    }

    fn next_token(&mut self) -> Option<Token<'a>> {
        // Try to fetch a token from the current line.
        if let Some(ref mut tokens) = self.line_tokens {
            if let Some(token) = tokens.next() {
                return Some(token)
            }
        }

        // We're done with this line; on to the next.
        self.parse_next_line();

        // If this returns none, we're done.
        if let Some(ref mut tokens) = self.line_tokens {
            tokens.next()
        } else {
            None
        }
    }

    fn parse_next_line(&mut self) {
        let mut tokens = Vec::new();
        let mut offset = 0;

        if let Some((line_number, line)) = self.lines.next() {
            if line_number > 0 {
                // We've found another line, so push a newline token.
                tokens.push(Token::Newline);
            }

            for (change_offset, scope_change) in self.parser.parse_line(line) {
                // We only want to capture the deepest scope for a given token,
                // so we apply all of them and only capture once we move on to
                // another token/offset.
                if change_offset > offset {
                    tokens.push(
                        Token::Lexeme(Lexeme{
                            value: &line[offset..change_offset],
                            scope: self.scopes.as_slice().last().map(|s| s.clone()),
                            position: Position{
                                line: line_number,
                                offset: offset
                            }
                        })
                    );
                    offset = change_offset;
                }

                // Apply the scope and keep a reference to it, so
                // that we can pair it with a token later on.
                self.scopes.apply(&scope_change);

            }

            // We already have discrete variant for newlines,
            // so exclude them when considering content length.
            let line_length = line_length(line);
            if offset < line_length {
                // The rest of the line hasn't triggered a scope
                // change; categorize it with the last known scope.
                tokens.push(
                    Token::Lexeme(Lexeme{
                        value: &line[offset..line_length],
                        scope: self.scopes.as_slice().last().map(|s| s.clone()),
                        position: Position{
                            line: line_number,
                            offset: offset
                        }
                    })
                );
            }

            self.line_tokens = Some(tokens.into_iter());
        } else {
            self.line_tokens = None;
        }
    }
}

fn line_length(line: &str) -> usize {
    if line.chars().last() == Some('\n') {
        line.len() - 1
    } else {
        line.len()
    }
}

impl<'a> Iterator for TokenIterator<'a> {
    type Item = Token<'a>;

    fn next(&mut self) -> Option<Self::Item> {
        if let Some(token) = self.next_token() {
            return Some(token)
        }

        self.parse_next_line();
        self.next_token()
    }
}

#[cfg(test)]
mod tests {
    use super::TokenIterator;
    use buffer::{Lexeme, Position, Scope, Token};
    use syntect::parsing::SyntaxSet;

    #[test]
    fn token_iterator_returns_correct_tokens() {
        let syntax_set = SyntaxSet::load_defaults_newlines();
        let def = syntax_set.find_syntax_by_extension("rs");
        let iterator = TokenIterator::new("struct Buffer {\n  data: String\n}garbage\n\n", def.unwrap());
        let expected_tokens = vec![
            Token::Lexeme(Lexeme{
                value: "struct",
                scope: Some(Scope::new("storage.type.struct.rust").unwrap()),
                position: Position{ line: 0, offset: 0 }
            }),
            Token::Lexeme(Lexeme{
                value: " ",
                scope: Some(Scope::new("meta.struct.rust").unwrap()),
                position: Position{ line: 0, offset: 6 }
            }),
            Token::Lexeme(Lexeme{
                value: "Buffer",
                scope: Some(Scope::new("entity.name.struct.rust").unwrap()),
                position: Position{ line: 0, offset: 7 }
            }),
            Token::Lexeme(Lexeme{
                value: " ",
                scope: Some(Scope::new("meta.struct.rust").unwrap()),
                position: Position{ line: 0, offset: 13 }
            }),
            Token::Lexeme(Lexeme{
                value: "{",
                scope: Some(Scope::new("punctuation.definition.block.begin.rust").unwrap()),
                position: Position{ line: 0, offset: 14 }
            }),
            Token::Newline,
            Token::Lexeme(Lexeme{
                value: "  ",
                scope: Some(Scope::new("meta.block.rust").unwrap()),
                position: Position{ line: 1, offset: 0 }
            }),
            Token::Lexeme(Lexeme{
                value: "data",
                scope: Some(Scope::new("variable.other.property.rust").unwrap()),
                position: Position{ line: 1, offset: 2 }
            }),
            Token::Lexeme(Lexeme{
                value: ":",
                scope: Some(Scope::new("punctuation.separator.rust").unwrap()),
                position: Position{ line: 1, offset: 6 }
            }),
            Token::Lexeme(Lexeme{
                value: " String",
                scope: Some(Scope::new("meta.block.rust").unwrap()),
                position: Position{ line: 1, offset: 7 }
            }),
            Token::Newline,
            Token::Lexeme(Lexeme{
                value: "}",
                scope: Some(Scope::new("punctuation.definition.block.end.rust").unwrap()),
                position: Position{ line: 2, offset: 0 }
            }),
            Token::Lexeme(Lexeme{
                value: "garbage",
                scope: Some(Scope::new("source.rust").unwrap()),
                position: Position{ line: 2, offset: 1 }
            }),
            Token::Newline,
            Token::Newline
        ];
        let actual_tokens: Vec<Token> = iterator.collect();
        println!("{:?}", actual_tokens);
        for (index, token) in expected_tokens.into_iter().enumerate() {
            assert_eq!(token, actual_tokens[index]);
        }

        //assert_eq!(expected_tokens, actual_tokens);
    }

    #[test]
    fn token_iterator_handles_content_without_trailing_newline() {
        let syntax_set = SyntaxSet::load_defaults_newlines();
        let def = syntax_set.find_syntax_by_extension("rs");
        let iterator = TokenIterator::new("struct", def.unwrap());
        let expected_tokens = vec![
            Token::Lexeme(Lexeme{
                value: "struct",
                scope: Some(Scope::new("storage.type.struct.rust").unwrap()),
                position: Position{ line: 0, offset: 0 }
            })
        ];
        let actual_tokens: Vec<Token> = iterator.collect();
        for (index, token) in expected_tokens.into_iter().enumerate() {
            assert_eq!(token, actual_tokens[index]);
        }

        //assert_eq!(expected_tokens, actual_tokens);
    }
}
<commit_msg>Improve newline handling in line parser<commit_after>use buffer::{Lexeme, Position, Token};
use syntect::parsing::{ParseState, Scope, ScopeStack, SyntaxDefinition};
use buffer::token::line_iterator::LineIterator;
use std::vec::IntoIter;

pub struct TokenIterator<'a> {
    scopes: ScopeStack,
    parser: ParseState,
    line_tokens: Option<IntoIter<Token<'a>>>,
    lines: LineIterator<'a>
}

impl<'a> TokenIterator<'a> {
    pub fn new(data: &'a str, def: &SyntaxDefinition) -> TokenIterator<'a> {
        TokenIterator{
            scopes: ScopeStack::new(),
            parser: ParseState::new(def),
            line_tokens: None,
            lines: LineIterator::new(data)
        }
    }

    fn next_token(&mut self) -> Option<Token<'a>> {
        // Try to fetch a token from the current line.
        if let Some(ref mut tokens) = self.line_tokens {
            if let Some(token) = tokens.next() {
                return Some(token)
            }
        }

        // We're done with this line; on to the next.
        self.parse_next_line();

        // If this returns none, we're done.
        if let Some(ref mut tokens) = self.line_tokens {
            tokens.next()
        } else {
            None
        }
    }

    fn parse_next_line(&mut self) {
        let mut tokens = Vec::new();
        let mut offset = 0;
        let mut last_scope: Option<Scope> = None;

        if let Some((line_number, line)) = self.lines.next() {
            if line_number > 0 {
                // We've found another line, so push a newline token.
                tokens.push(Token::Newline);
            }

            for (change_offset, scope_change) in self.parser.parse_line(line) {
                // We only want to capture the deepest scope for a given token,
                // so we apply all of them and only capture once we move on to
                // another token/offset.
                if change_offset > offset {
                    tokens.push(
                        Token::Lexeme(Lexeme{
                            value: &line[offset..change_offset],
                            scope: last_scope.clone(),
                            position: Position{
                                line: line_number,
                                offset: offset
                            }
                        })
                    );
                    offset = change_offset;
                }

                // Apply the scope and keep a reference to it, so
                // that we can pair it with a token later on.
                self.scopes.apply(&scope_change);
                last_scope = self.scopes.as_slice().last().map(|s| s.clone());

            }

            // We already have discrete variant for newlines,
            // so exclude them when considering content length.
            if let Some(end_of_line) = line.len().checked_sub(1) {
                if offset < end_of_line {
                    // The rest of the line hasn't triggered a scope
                    // change; categorize it with the last known scope.
                    tokens.push(
                        Token::Lexeme(Lexeme{
                            value: &line[offset..end_of_line],
                            scope: last_scope.clone(),
                            position: Position{
                                line: line_number,
                                offset: offset
                            }
                        })
                    );
                }
            };

            self.line_tokens = Some(tokens.into_iter());
        } else {
            self.line_tokens = None;
        }
    }
}

impl<'a> Iterator for TokenIterator<'a> {
    type Item = Token<'a>;

    fn next(&mut self) -> Option<Self::Item> {
        if let Some(token) = self.next_token() {
            return Some(token)
        }

        self.parse_next_line();
        self.next_token()
    }
}

#[cfg(test)]
mod tests {
    use super::TokenIterator;
    use buffer::{Lexeme, Position, Scope, Token};
    use syntect::parsing::SyntaxSet;

    #[test]
    fn token_iterator_returns_correct_tokens() {
        let syntax_set = SyntaxSet::load_defaults_newlines();
        let def = syntax_set.find_syntax_by_extension("rs");
        let iterator = TokenIterator::new("struct Buffer {\n// comment\n  data: String\n}garbage\n\n", def.unwrap());
        let expected_tokens = vec![
            Token::Lexeme(Lexeme{
                value: "struct",
                scope: Some(Scope::new("storage.type.struct.rust").unwrap()),
                position: Position{ line: 0, offset: 0 }
            }),
            Token::Lexeme(Lexeme{
                value: " ",
                scope: Some(Scope::new("meta.struct.rust").unwrap()),
                position: Position{ line: 0, offset: 6 }
            }),
            Token::Lexeme(Lexeme{
                value: "Buffer",
                scope: Some(Scope::new("entity.name.struct.rust").unwrap()),
                position: Position{ line: 0, offset: 7 }
            }),
            Token::Lexeme(Lexeme{
                value: " ",
                scope: Some(Scope::new("meta.struct.rust").unwrap()),
                position: Position{ line: 0, offset: 13 }
            }),
            Token::Lexeme(Lexeme{
                value: "{",
                scope: Some(Scope::new("punctuation.definition.block.begin.rust").unwrap()),
                position: Position{ line: 0, offset: 14 }
            }),
            Token::Newline,
            Token::Lexeme(Lexeme{
                value: "// comment",
                scope: Some(Scope::new("comment.line.double-slash.rust").unwrap()),
                position: Position{ line: 1, offset: 0 }
            }),
            Token::Newline,
            Token::Lexeme(Lexeme{
                value: "  ",
                scope: None,
                position: Position{ line: 2, offset: 0 }
            }),
            Token::Lexeme(Lexeme{
                value: "data",
                scope: Some(Scope::new("variable.other.property.rust").unwrap()),
                position: Position{ line: 2, offset: 2 }
            }),
            Token::Lexeme(Lexeme{
                value: ":",
                scope: Some(Scope::new("punctuation.separator.rust").unwrap()),
                position: Position{ line: 2, offset: 6 }
            }),
            Token::Lexeme(Lexeme{
                value: " String",
                scope: Some(Scope::new("meta.block.rust").unwrap()),
                position: Position{ line: 2, offset: 7 }
            }),
            Token::Newline,
            Token::Lexeme(Lexeme{
                value: "}",
                scope: Some(Scope::new("punctuation.definition.block.end.rust").unwrap()),
                position: Position{ line: 3, offset: 0 }
            }),
            Token::Lexeme(Lexeme{
                value: "garbage",
                scope: Some(Scope::new("source.rust").unwrap()),
                position: Position{ line: 3, offset: 1 }
            }),
            Token::Newline,
            Token::Newline
        ];
        let actual_tokens: Vec<Token> = iterator.collect();
        println!("{:?}", actual_tokens);
        for (index, token) in expected_tokens.into_iter().enumerate() {
            assert_eq!(token, actual_tokens[index]);
        }

        //assert_eq!(expected_tokens, actual_tokens);
    }

    #[test]
    fn token_iterator_handles_content_without_trailing_newline() {
        let syntax_set = SyntaxSet::load_defaults_newlines();
        let def = syntax_set.find_syntax_by_extension("rs");
        let iterator = TokenIterator::new("struct", def.unwrap());
        let expected_tokens = vec![
            Token::Lexeme(Lexeme{
                value: "struct",
                scope: Some(Scope::new("storage.type.struct.rust").unwrap()),
                position: Position{ line: 0, offset: 0 }
            })
        ];
        let actual_tokens: Vec<Token> = iterator.collect();
        for (index, token) in expected_tokens.into_iter().enumerate() {
            assert_eq!(token, actual_tokens[index]);
        }

        //assert_eq!(expected_tokens, actual_tokens);
    }
}
<|endoftext|>


<commit_before>//! Endpoints for account registration and management.

pub mod add_3pid;
pub mod bind_3pid;
pub mod change_password;
pub mod deactivate;
pub mod delete_3pid;
pub mod get_username_availability;
pub mod register;
pub mod request_3pid_management_token_via_email;
pub mod request_3pid_management_token_via_msisdn;
pub mod request_openid_token;
pub mod request_password_change_token_via_email;
pub mod request_password_change_token_via_msisdn;
pub mod request_registration_token_via_email;
pub mod request_registration_token_via_msisdn;
pub mod unbind_3pid;

pub mod whoami;

use ruma_serde::Outgoing;
use serde::{Deserialize, Serialize};

/// Additional authentication information for requestToken endpoints.
#[derive(Clone, Debug, Outgoing, Serialize)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
pub struct IdentityServerInfo<'a> {
    /// The ID server to send the onward request to as a hostname with an
    /// appended colon and port number if the port is not the default.
    pub id_server: &'a str,

    /// Access token previously registered with identity server.
    pub id_access_token: &'a str,
}

impl<'a> IdentityServerInfo<'a> {
    /// Creates a new `IdentityServerInfo` with the given server name and access token.
    pub fn new(id_server: &'a str, id_access_token: &'a str) -> Self {
        Self { id_server, id_access_token }
    }
}

/// Possible values for deleting or unbinding 3PIDs
#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum ThirdPartyIdRemovalStatus {
    /// Either the homeserver couldn't determine the right identity server to contact, or the
    /// identity server refused the operation.
    NoSupport,

    /// Success.
    Success,
}
<commit_msg>client-api: Allow custom values for ThirdPartyIdRemovalStatus<commit_after>//! Endpoints for account registration and management.

pub mod add_3pid;
pub mod bind_3pid;
pub mod change_password;
pub mod deactivate;
pub mod delete_3pid;
pub mod get_username_availability;
pub mod register;
pub mod request_3pid_management_token_via_email;
pub mod request_3pid_management_token_via_msisdn;
pub mod request_openid_token;
pub mod request_password_change_token_via_email;
pub mod request_password_change_token_via_msisdn;
pub mod request_registration_token_via_email;
pub mod request_registration_token_via_msisdn;
pub mod unbind_3pid;

pub mod whoami;

use ruma_serde::{Outgoing, StringEnum};
use serde::Serialize;

/// Additional authentication information for requestToken endpoints.
#[derive(Clone, Debug, Outgoing, Serialize)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
pub struct IdentityServerInfo<'a> {
    /// The ID server to send the onward request to as a hostname with an
    /// appended colon and port number if the port is not the default.
    pub id_server: &'a str,

    /// Access token previously registered with identity server.
    pub id_access_token: &'a str,
}

impl<'a> IdentityServerInfo<'a> {
    /// Creates a new `IdentityServerInfo` with the given server name and access token.
    pub fn new(id_server: &'a str, id_access_token: &'a str) -> Self {
        Self { id_server, id_access_token }
    }
}

/// Possible values for deleting or unbinding 3PIDs.
#[derive(Clone, Debug, StringEnum)]
#[ruma_enum(rename_all = "kebab-case")]
pub enum ThirdPartyIdRemovalStatus {
    /// Either the homeserver couldn't determine the right identity server to contact, or the
    /// identity server refused the operation.
    NoSupport,

    /// Success.
    Success,

    #[doc(hidden)]
    _Custom(String),
}
<|endoftext|>