<commit_before>//! Advertises the capabilities of the LSP Server.
use std::env;

use lsp_types::{
    CallHierarchyServerCapability, ClientCapabilities, CodeActionKind, CodeActionOptions,
    CodeActionProviderCapability, CodeLensOptions, CompletionOptions,
    DocumentOnTypeFormattingOptions, FileOperationFilter, FileOperationPattern,
    FileOperationPatternKind, FileOperationRegistrationOptions, FoldingRangeProviderCapability,
    HoverProviderCapability, ImplementationProviderCapability, OneOf, RenameOptions, SaveOptions,
    SelectionRangeProviderCapability, SemanticTokensFullOptions, SemanticTokensLegend,
    SemanticTokensOptions, ServerCapabilities, SignatureHelpOptions, TextDocumentSyncCapability,
    TextDocumentSyncKind, TextDocumentSyncOptions, TypeDefinitionProviderCapability,
    WorkDoneProgressOptions, WorkspaceFileOperationsServerCapabilities,
    WorkspaceServerCapabilities,
};
use serde_json::json;

use crate::semantic_tokens;

pub fn server_capabilities(client_caps: &ClientCapabilities) -> ServerCapabilities {
    ServerCapabilities {
        text_document_sync: Some(TextDocumentSyncCapability::Options(TextDocumentSyncOptions {
            open_close: Some(true),
            change: Some(if env::var("RA_NO_INCREMENTAL_SYNC").is_ok() {
                TextDocumentSyncKind::Full
            } else {
                TextDocumentSyncKind::Incremental
            }),
            will_save: None,
            will_save_wait_until: None,
            save: Some(SaveOptions::default().into()),
        })),
        hover_provider: Some(HoverProviderCapability::Simple(true)),
        completion_provider: Some(CompletionOptions {
            resolve_provider: completions_resolve_provider(client_caps),
            trigger_characters: Some(vec![":".to_string(), ".".to_string(), "'".to_string()]),
            all_commit_characters: None,
            completion_item: None,
            work_done_progress_options: WorkDoneProgressOptions { work_done_progress: None },
        }),
        signature_help_provider: Some(SignatureHelpOptions {
            trigger_characters: Some(vec!["(".to_string(), ",".to_string()]),
            retrigger_characters: None,
            work_done_progress_options: WorkDoneProgressOptions { work_done_progress: None },
        }),
        declaration_provider: None,
        definition_provider: Some(OneOf::Left(true)),
        type_definition_provider: Some(TypeDefinitionProviderCapability::Simple(true)),
        implementation_provider: Some(ImplementationProviderCapability::Simple(true)),
        references_provider: Some(OneOf::Left(true)),
        document_highlight_provider: Some(OneOf::Left(true)),
        document_symbol_provider: Some(OneOf::Left(true)),
        workspace_symbol_provider: Some(OneOf::Left(true)),
        code_action_provider: Some(code_action_capabilities(client_caps)),
        code_lens_provider: Some(CodeLensOptions { resolve_provider: Some(true) }),
        document_formatting_provider: Some(OneOf::Left(true)),
        document_range_formatting_provider: Some(OneOf::Left(true)),
        document_on_type_formatting_provider: Some(DocumentOnTypeFormattingOptions {
            first_trigger_character: "=".to_string(),
            more_trigger_character: Some(vec![".".to_string(), ">".to_string(), "{".to_string()]),
        }),
        selection_range_provider: Some(SelectionRangeProviderCapability::Simple(true)),
        folding_range_provider: Some(FoldingRangeProviderCapability::Simple(true)),
        rename_provider: Some(OneOf::Right(RenameOptions {
            prepare_provider: Some(true),
            work_done_progress_options: WorkDoneProgressOptions { work_done_progress: None },
        })),
        linked_editing_range_provider: None,
        document_link_provider: None,
        color_provider: None,
        execute_command_provider: None,
        workspace: Some(WorkspaceServerCapabilities {
            workspace_folders: None,
            file_operations: Some(WorkspaceFileOperationsServerCapabilities {
                did_create: None,
                will_create: None,
                did_rename: None,
                will_rename: Some(FileOperationRegistrationOptions {
                    filters: vec![
                        FileOperationFilter {
                            scheme: Some(String::from("file")),
                            pattern: FileOperationPattern {
                                glob: String::from("**/*.rs"),
                                matches: Some(FileOperationPatternKind::File),
                                options: None,
                            },
                        },
                        FileOperationFilter {
                            scheme: Some(String::from("file")),
                            pattern: FileOperationPattern {
                                glob: String::from("**"),
                                matches: Some(FileOperationPatternKind::Folder),
                                options: None,
                            },
                        },
                    ],
                }),
                did_delete: None,
                will_delete: None,
            }),
        }),
        call_hierarchy_provider: Some(CallHierarchyServerCapability::Simple(true)),
        semantic_tokens_provider: Some(
            SemanticTokensOptions {
                legend: SemanticTokensLegend {
                    token_types: semantic_tokens::SUPPORTED_TYPES.to_vec(),
                    token_modifiers: semantic_tokens::SUPPORTED_MODIFIERS.to_vec(),
                },

                full: Some(SemanticTokensFullOptions::Delta { delta: Some(true) }),
                range: Some(true),
                work_done_progress_options: Default::default(),
            }
            .into(),
        ),
        moniker_provider: None,
        experimental: Some(json!({
            "joinLines": true,
            "ssr": true,
            "onEnter": true,
            "parentModule": true,
            "runnables": {
                "kinds": [ "cargo" ],
            },
            "workspaceSymbolScopeKindFiltering": true,
        })),
    }
}

fn completions_resolve_provider(client_caps: &ClientCapabilities) -> Option<bool> {
    if completion_item_edit_resolve(client_caps) {
        Some(true)
    } else {
        log::info!("No `additionalTextEdits` completion resolve capability was found in the client capabilities, autoimport completion is disabled");
        None
    }
}

/// Parses client capabilities and returns all completion resolve capabilities rust-analyzer supports.
pub(crate) fn completion_item_edit_resolve(caps: &ClientCapabilities) -> bool {
    (|| {
        Some(
            caps.text_document
                .as_ref()?
                .completion
                .as_ref()?
                .completion_item
                .as_ref()?
                .resolve_support
                .as_ref()?
                .properties
                .iter()
                .any(|cap_string| cap_string.as_str() == "additionalTextEdits"),
        )
    })() == Some(true)
}

fn code_action_capabilities(client_caps: &ClientCapabilities) -> CodeActionProviderCapability {
    client_caps
        .text_document
        .as_ref()
        .and_then(|it| it.code_action.as_ref())
        .and_then(|it| it.code_action_literal_support.as_ref())
        .map_or(CodeActionProviderCapability::Simple(true), |_| {
            CodeActionProviderCapability::Options(CodeActionOptions {
                // Advertise support for all built-in CodeActionKinds.
                // Ideally we would base this off of the client capabilities
                // but the client is supposed to fall back gracefully for unknown values.
                code_action_kinds: Some(vec![
                    CodeActionKind::EMPTY,
                    CodeActionKind::QUICKFIX,
                    CodeActionKind::REFACTOR,
                    CodeActionKind::REFACTOR_EXTRACT,
                    CodeActionKind::REFACTOR_INLINE,
                    CodeActionKind::REFACTOR_REWRITE,
                ]),
                resolve_provider: Some(true),
                work_done_progress_options: Default::default(),
            })
        })
}
<commit_msg>Remove undocumented `TextDocumentSyncKind::Full` support<commit_after>//! Advertises the capabilities of the LSP Server.
use lsp_types::{
    CallHierarchyServerCapability, ClientCapabilities, CodeActionKind, CodeActionOptions,
    CodeActionProviderCapability, CodeLensOptions, CompletionOptions,
    DocumentOnTypeFormattingOptions, FileOperationFilter, FileOperationPattern,
    FileOperationPatternKind, FileOperationRegistrationOptions, FoldingRangeProviderCapability,
    HoverProviderCapability, ImplementationProviderCapability, OneOf, RenameOptions, SaveOptions,
    SelectionRangeProviderCapability, SemanticTokensFullOptions, SemanticTokensLegend,
    SemanticTokensOptions, ServerCapabilities, SignatureHelpOptions, TextDocumentSyncCapability,
    TextDocumentSyncKind, TextDocumentSyncOptions, TypeDefinitionProviderCapability,
    WorkDoneProgressOptions, WorkspaceFileOperationsServerCapabilities,
    WorkspaceServerCapabilities,
};
use serde_json::json;

use crate::semantic_tokens;

pub fn server_capabilities(client_caps: &ClientCapabilities) -> ServerCapabilities {
    ServerCapabilities {
        text_document_sync: Some(TextDocumentSyncCapability::Options(TextDocumentSyncOptions {
            open_close: Some(true),
            change: Some(TextDocumentSyncKind::Incremental),
            will_save: None,
            will_save_wait_until: None,
            save: Some(SaveOptions::default().into()),
        })),
        hover_provider: Some(HoverProviderCapability::Simple(true)),
        completion_provider: Some(CompletionOptions {
            resolve_provider: completions_resolve_provider(client_caps),
            trigger_characters: Some(vec![":".to_string(), ".".to_string(), "'".to_string()]),
            all_commit_characters: None,
            completion_item: None,
            work_done_progress_options: WorkDoneProgressOptions { work_done_progress: None },
        }),
        signature_help_provider: Some(SignatureHelpOptions {
            trigger_characters: Some(vec!["(".to_string(), ",".to_string()]),
            retrigger_characters: None,
            work_done_progress_options: WorkDoneProgressOptions { work_done_progress: None },
        }),
        declaration_provider: None,
        definition_provider: Some(OneOf::Left(true)),
        type_definition_provider: Some(TypeDefinitionProviderCapability::Simple(true)),
        implementation_provider: Some(ImplementationProviderCapability::Simple(true)),
        references_provider: Some(OneOf::Left(true)),
        document_highlight_provider: Some(OneOf::Left(true)),
        document_symbol_provider: Some(OneOf::Left(true)),
        workspace_symbol_provider: Some(OneOf::Left(true)),
        code_action_provider: Some(code_action_capabilities(client_caps)),
        code_lens_provider: Some(CodeLensOptions { resolve_provider: Some(true) }),
        document_formatting_provider: Some(OneOf::Left(true)),
        document_range_formatting_provider: Some(OneOf::Left(true)),
        document_on_type_formatting_provider: Some(DocumentOnTypeFormattingOptions {
            first_trigger_character: "=".to_string(),
            more_trigger_character: Some(vec![".".to_string(), ">".to_string(), "{".to_string()]),
        }),
        selection_range_provider: Some(SelectionRangeProviderCapability::Simple(true)),
        folding_range_provider: Some(FoldingRangeProviderCapability::Simple(true)),
        rename_provider: Some(OneOf::Right(RenameOptions {
            prepare_provider: Some(true),
            work_done_progress_options: WorkDoneProgressOptions { work_done_progress: None },
        })),
        linked_editing_range_provider: None,
        document_link_provider: None,
        color_provider: None,
        execute_command_provider: None,
        workspace: Some(WorkspaceServerCapabilities {
            workspace_folders: None,
            file_operations: Some(WorkspaceFileOperationsServerCapabilities {
                did_create: None,
                will_create: None,
                did_rename: None,
                will_rename: Some(FileOperationRegistrationOptions {
                    filters: vec![
                        FileOperationFilter {
                            scheme: Some(String::from("file")),
                            pattern: FileOperationPattern {
                                glob: String::from("**/*.rs"),
                                matches: Some(FileOperationPatternKind::File),
                                options: None,
                            },
                        },
                        FileOperationFilter {
                            scheme: Some(String::from("file")),
                            pattern: FileOperationPattern {
                                glob: String::from("**"),
                                matches: Some(FileOperationPatternKind::Folder),
                                options: None,
                            },
                        },
                    ],
                }),
                did_delete: None,
                will_delete: None,
            }),
        }),
        call_hierarchy_provider: Some(CallHierarchyServerCapability::Simple(true)),
        semantic_tokens_provider: Some(
            SemanticTokensOptions {
                legend: SemanticTokensLegend {
                    token_types: semantic_tokens::SUPPORTED_TYPES.to_vec(),
                    token_modifiers: semantic_tokens::SUPPORTED_MODIFIERS.to_vec(),
                },

                full: Some(SemanticTokensFullOptions::Delta { delta: Some(true) }),
                range: Some(true),
                work_done_progress_options: Default::default(),
            }
            .into(),
        ),
        moniker_provider: None,
        experimental: Some(json!({
            "joinLines": true,
            "ssr": true,
            "onEnter": true,
            "parentModule": true,
            "runnables": {
                "kinds": [ "cargo" ],
            },
            "workspaceSymbolScopeKindFiltering": true,
        })),
    }
}

fn completions_resolve_provider(client_caps: &ClientCapabilities) -> Option<bool> {
    if completion_item_edit_resolve(client_caps) {
        Some(true)
    } else {
        log::info!("No `additionalTextEdits` completion resolve capability was found in the client capabilities, autoimport completion is disabled");
        None
    }
}

/// Parses client capabilities and returns all completion resolve capabilities rust-analyzer supports.
pub(crate) fn completion_item_edit_resolve(caps: &ClientCapabilities) -> bool {
    (|| {
        Some(
            caps.text_document
                .as_ref()?
                .completion
                .as_ref()?
                .completion_item
                .as_ref()?
                .resolve_support
                .as_ref()?
                .properties
                .iter()
                .any(|cap_string| cap_string.as_str() == "additionalTextEdits"),
        )
    })() == Some(true)
}

fn code_action_capabilities(client_caps: &ClientCapabilities) -> CodeActionProviderCapability {
    client_caps
        .text_document
        .as_ref()
        .and_then(|it| it.code_action.as_ref())
        .and_then(|it| it.code_action_literal_support.as_ref())
        .map_or(CodeActionProviderCapability::Simple(true), |_| {
            CodeActionProviderCapability::Options(CodeActionOptions {
                // Advertise support for all built-in CodeActionKinds.
                // Ideally we would base this off of the client capabilities
                // but the client is supposed to fall back gracefully for unknown values.
                code_action_kinds: Some(vec![
                    CodeActionKind::EMPTY,
                    CodeActionKind::QUICKFIX,
                    CodeActionKind::REFACTOR,
                    CodeActionKind::REFACTOR_EXTRACT,
                    CodeActionKind::REFACTOR_INLINE,
                    CodeActionKind::REFACTOR_REWRITE,
                ]),
                resolve_provider: Some(true),
                work_done_progress_options: Default::default(),
            })
        })
}
<|endoftext|>


<commit_before>#![cfg(exhaustive)]

extern crate num_cpus;
extern crate ryu;

use std::str;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::thread;

#[test]
fn test_exhaustive() {
    const BATCH_SIZE: u32 = 1_000_000;
    let counter = Arc::new(AtomicUsize::new(0));
    let finished = Arc::new(AtomicUsize::new(0));

    let mut workers = Vec::new();
    for _ in 0..num_cpus::get() {
        let counter = counter.clone();
        let finished = finished.clone();
        workers.push(thread::spawn(move || loop {
            let batch = counter.fetch_add(1, Ordering::SeqCst) as u32;
            if batch > u32::max_value() / BATCH_SIZE {
                return;
            }

            let min = batch * BATCH_SIZE;
            let max = if batch == u32::max_value() / BATCH_SIZE {
                u32::max_value()
            } else {
                min + BATCH_SIZE - 1
            };

            let mut bytes = [0u8; 24];
            let mut buffer = ryu::Buffer::new();
            for u in min..=max {
                let f = f32::from_bits(u);
                if !f.is_finite() {
                    continue;
                }
                let n = unsafe { ryu::f2s_buffered_n(f, &mut bytes[0]) };
                assert_eq!(f, str::from_utf8(&bytes[..n]).unwrap().parse().unwrap());
                assert_eq!(f, buffer.format(f).parse().unwrap());
            }

            let increment = (max - min + 1) as usize;
            let update = finished.fetch_add(increment, Ordering::SeqCst);
            println!("{}", update + increment);
        }));
    }

    for w in workers {
        w.join().unwrap();
    }
}
<commit_msg>Preserve error messages in exhaustive test<commit_after>#![cfg(exhaustive)]

extern crate num_cpus;
extern crate ryu;

use std::str;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::thread;

#[test]
fn test_exhaustive() {
    const BATCH_SIZE: u32 = 1_000_000;
    let counter = Arc::new(AtomicUsize::new(0));
    let finished = Arc::new(AtomicUsize::new(0));

    let mut workers = Vec::new();
    for _ in 0..num_cpus::get() {
        let counter = counter.clone();
        let finished = finished.clone();
        workers.push(thread::spawn(move || loop {
            let batch = counter.fetch_add(1, Ordering::SeqCst) as u32;
            if batch > u32::max_value() / BATCH_SIZE {
                return;
            }

            let min = batch * BATCH_SIZE;
            let max = if batch == u32::max_value() / BATCH_SIZE {
                u32::max_value()
            } else {
                min + BATCH_SIZE - 1
            };

            let mut bytes = [0u8; 24];
            let mut buffer = ryu::Buffer::new();
            for u in min..=max {
                let f = f32::from_bits(u);
                if !f.is_finite() {
                    continue;
                }
                let n = unsafe { ryu::f2s_buffered_n(f, &mut bytes[0]) };
                assert_eq!(Ok(Ok(f)), str::from_utf8(&bytes[..n]).map(str::parse));
                assert_eq!(Ok(f), buffer.format(f).parse());
            }

            let increment = (max - min + 1) as usize;
            let update = finished.fetch_add(increment, Ordering::SeqCst);
            println!("{}", update + increment);
        }));
    }

    for w in workers {
        w.join().unwrap();
    }
}
<|endoftext|>


<commit_before>#![allow(
    clippy::decimal_literal_representation,
    clippy::derive_partial_eq_without_eq,
    clippy::unreadable_literal,
    clippy::shadow_unrelated
)]

use indoc::indoc;
use serde::ser::SerializeMap;
use serde_derive::{Deserialize, Serialize};
use serde_yaml::{Mapping, Number, Value};
use std::collections::BTreeMap;
use std::fmt::Debug;

fn test_serde<T>(thing: &T, yaml: &str)
where
    T: serde::Serialize + serde::de::DeserializeOwned + PartialEq + Debug,
{
    let serialized = serde_yaml::to_string(&thing).unwrap();
    assert_eq!(yaml, serialized);

    let value = serde_yaml::to_value(&thing).unwrap();
    let serialized = serde_yaml::to_string(&value).unwrap();
    assert_eq!(yaml, serialized);

    let deserialized: T = serde_yaml::from_str(yaml).unwrap();
    assert_eq!(*thing, deserialized);

    let value: Value = serde_yaml::from_str(yaml).unwrap();
    let deserialized = T::deserialize(&value).unwrap();
    assert_eq!(*thing, deserialized);

    let deserialized: T = serde_yaml::from_value(value).unwrap();
    assert_eq!(*thing, deserialized);

    serde_yaml::from_str::<serde::de::IgnoredAny>(yaml).unwrap();
}

#[test]
fn test_default() {
    assert_eq!(Value::default(), Value::Null);
}

#[test]
fn test_int() {
    let thing = 256;
    let yaml = indoc! {"
        256
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_int_max_u64() {
    let thing = u64::MAX;
    let yaml = indoc! {"
        18446744073709551615
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_int_min_i64() {
    let thing = i64::MIN;
    let yaml = indoc! {"
        -9223372036854775808
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_int_max_i64() {
    let thing = i64::MAX;
    let yaml = indoc! {"
        9223372036854775807
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_i128_small() {
    let thing: i128 = -256;
    let yaml = indoc! {"
        -256
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_u128_small() {
    let thing: u128 = 256;
    let yaml = indoc! {"
        256
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_float() {
    let thing = 25.6;
    let yaml = indoc! {"
        25.6
    "};
    test_serde(&thing, yaml);

    let thing = 25.;
    let yaml = indoc! {"
        25.0
    "};
    test_serde(&thing, yaml);

    let thing = f64::INFINITY;
    let yaml = indoc! {"
        .inf
    "};
    test_serde(&thing, yaml);

    let thing = f64::NEG_INFINITY;
    let yaml = indoc! {"
        -.inf
    "};
    test_serde(&thing, yaml);

    let float: f64 = serde_yaml::from_str(indoc! {"
        .nan
    "})
    .unwrap();
    assert!(float.is_nan());
}

#[test]
fn test_float32() {
    let thing: f32 = 25.5;
    let yaml = indoc! {"
        25.5
    "};
    test_serde(&thing, yaml);

    let thing = f32::INFINITY;
    let yaml = indoc! {"
        .inf
    "};
    test_serde(&thing, yaml);

    let thing = f32::NEG_INFINITY;
    let yaml = indoc! {"
        -.inf
    "};
    test_serde(&thing, yaml);

    let single_float: f32 = serde_yaml::from_str(indoc! {"
        .nan
    "})
    .unwrap();
    assert!(single_float.is_nan());
}

#[test]
fn test_char() {
    let ch = '.';
    let yaml = indoc! {"
        '.'
    "};
    assert_eq!(yaml, serde_yaml::to_string(&ch).unwrap());

    let ch = '#';
    let yaml = indoc! {"
        '#'
    "};
    assert_eq!(yaml, serde_yaml::to_string(&ch).unwrap());

    let ch = '-';
    let yaml = indoc! {"
        '-'
    "};
    assert_eq!(yaml, serde_yaml::to_string(&ch).unwrap());
}

#[test]
fn test_vec() {
    let thing = vec![1, 2, 3];
    let yaml = indoc! {"
        - 1
        - 2
        - 3
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_map() {
    let mut thing = BTreeMap::new();
    thing.insert("x".to_owned(), 1);
    thing.insert("y".to_owned(), 2);
    let yaml = indoc! {"
        x: 1
        y: 2
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_map_key_value() {
    struct Map;

    impl serde::Serialize for Map {
        fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
        where
            S: serde::Serializer,
        {
            // Test maps which do not serialize using serialize_entry.
            let mut map = serializer.serialize_map(Some(1))?;
            map.serialize_key("k")?;
            map.serialize_value("v")?;
            map.end()
        }
    }

    let yaml = indoc! {"
        k: v
    "};
    assert_eq!(yaml, serde_yaml::to_string(&Map).unwrap());
}

#[test]
fn test_basic_struct() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Basic {
        x: isize,
        y: String,
        z: bool,
    }
    let thing = Basic {
        x: -4,
        y: "hi\tquoted".to_owned(),
        z: true,
    };
    let yaml = indoc! {r#"
        x: -4
        y: "hi\tquoted"
        z: true
    "#};
    test_serde(&thing, yaml);
}

#[test]
fn test_multiline_string() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Struct {
        trailing_newline: String,
        no_trailing_newline: String,
    }
    let thing = Struct {
        trailing_newline: "aaa\nbbb\n".to_owned(),
        no_trailing_newline: "aaa\nbbb".to_owned(),
    };
    let yaml = indoc! {r#"
        trailing_newline: |
          aaa
          bbb
        no_trailing_newline: |-
          aaa
          bbb
    "#};
    test_serde(&thing, yaml);
}

#[test]
fn test_strings_needing_quote() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Struct {
        boolean: String,
        integer: String,
        void: String,
    }
    let thing = Struct {
        boolean: "true".to_owned(),
        integer: "1".to_owned(),
        void: "null".to_owned(),
    };
    let yaml = indoc! {r#"
        boolean: 'true'
        integer: '1'
        void: 'null'
    "#};
    test_serde(&thing, yaml);
}

#[test]
fn test_nested_vec() {
    let thing = vec![vec![1, 2, 3], vec![4, 5, 6]];
    let yaml = indoc! {"
        - - 1
          - 2
          - 3
        - - 4
          - 5
          - 6
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_nested_struct() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Outer {
        inner: Inner,
    }
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Inner {
        v: u16,
    }
    let thing = Outer {
        inner: Inner { v: 512 },
    };
    let yaml = indoc! {"
        inner:
          v: 512
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_nested_enum() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    enum Outer {
        Inner(Inner),
    }
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    enum Inner {
        Unit,
    }
    let thing = Outer::Inner(Inner::Unit);
    let yaml = indoc! {"
        !Inner Unit
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_option() {
    let thing = vec![Some(1), None, Some(3)];
    let yaml = indoc! {"
        - 1
        - null
        - 3
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_unit() {
    let thing = vec![(), ()];
    let yaml = indoc! {"
        - null
        - null
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_unit_struct() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Foo;
    let thing = Foo;
    let yaml = indoc! {"
        null
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_unit_variant() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    enum Variant {
        First,
        Second,
    }
    let thing = Variant::First;
    let yaml = indoc! {"
        First
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_newtype_struct() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct OriginalType {
        v: u16,
    }
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct NewType(OriginalType);
    let thing = NewType(OriginalType { v: 1 });
    let yaml = indoc! {"
        v: 1
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_newtype_variant() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    enum Variant {
        Size(usize),
    }
    let thing = Variant::Size(127);
    let yaml = indoc! {"
        !Size 127
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_tuple_variant() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    enum Variant {
        Rgb(u8, u8, u8),
    }
    let thing = Variant::Rgb(32, 64, 96);
    let yaml = indoc! {"
        !Rgb
        - 32
        - 64
        - 96
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_struct_variant() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    enum Variant {
        Color { r: u8, g: u8, b: u8 },
    }
    let thing = Variant::Color {
        r: 32,
        g: 64,
        b: 96,
    };
    let yaml = indoc! {"
        !Color
        r: 32
        g: 64
        b: 96
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_tagged_map_value() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Bindings {
        profile: Profile,
    }
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    enum Profile {
        ClassValidator { class_name: String },
    }
    let thing = Bindings {
        profile: Profile::ClassValidator {
            class_name: "ApplicationConfig".to_owned(),
        },
    };
    let yaml = indoc! {"
        profile: !ClassValidator
          class_name: ApplicationConfig
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_value() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    pub struct GenericInstructions {
        #[serde(rename = "type")]
        pub typ: String,
        pub config: Value,
    }
    let thing = GenericInstructions {
        typ: "primary".to_string(),
        config: Value::Sequence(vec![
            Value::Null,
            Value::Bool(true),
            Value::Number(Number::from(65535)),
            Value::Number(Number::from(0.54321)),
            Value::String("s".into()),
            Value::Mapping(Mapping::new()),
        ]),
    };
    let yaml = indoc! {"
        type: primary
        config:
        - null
        - true
        - 65535
        - 0.54321
        - s
        - {}
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_mapping() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Data {
        pub substructure: Mapping,
    }

    let mut thing = Data {
        substructure: Mapping::new(),
    };
    thing.substructure.insert(
        Value::String("a".to_owned()),
        Value::String("foo".to_owned()),
    );
    thing.substructure.insert(
        Value::String("b".to_owned()),
        Value::String("bar".to_owned()),
    );

    let yaml = indoc! {"
        substructure:
          a: foo
          b: bar
    "};

    test_serde(&thing, yaml);
}
<commit_msg>Resolve needless_borrow clippy lint in test<commit_after>#![allow(
    clippy::decimal_literal_representation,
    clippy::derive_partial_eq_without_eq,
    clippy::unreadable_literal,
    clippy::shadow_unrelated
)]

use indoc::indoc;
use serde::ser::SerializeMap;
use serde_derive::{Deserialize, Serialize};
use serde_yaml::{Mapping, Number, Value};
use std::collections::BTreeMap;
use std::fmt::Debug;

fn test_serde<T>(thing: &T, yaml: &str)
where
    T: serde::Serialize + serde::de::DeserializeOwned + PartialEq + Debug,
{
    let serialized = serde_yaml::to_string(&thing).unwrap();
    assert_eq!(yaml, serialized);

    let value = serde_yaml::to_value(thing).unwrap();
    let serialized = serde_yaml::to_string(&value).unwrap();
    assert_eq!(yaml, serialized);

    let deserialized: T = serde_yaml::from_str(yaml).unwrap();
    assert_eq!(*thing, deserialized);

    let value: Value = serde_yaml::from_str(yaml).unwrap();
    let deserialized = T::deserialize(&value).unwrap();
    assert_eq!(*thing, deserialized);

    let deserialized: T = serde_yaml::from_value(value).unwrap();
    assert_eq!(*thing, deserialized);

    serde_yaml::from_str::<serde::de::IgnoredAny>(yaml).unwrap();
}

#[test]
fn test_default() {
    assert_eq!(Value::default(), Value::Null);
}

#[test]
fn test_int() {
    let thing = 256;
    let yaml = indoc! {"
        256
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_int_max_u64() {
    let thing = u64::MAX;
    let yaml = indoc! {"
        18446744073709551615
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_int_min_i64() {
    let thing = i64::MIN;
    let yaml = indoc! {"
        -9223372036854775808
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_int_max_i64() {
    let thing = i64::MAX;
    let yaml = indoc! {"
        9223372036854775807
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_i128_small() {
    let thing: i128 = -256;
    let yaml = indoc! {"
        -256
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_u128_small() {
    let thing: u128 = 256;
    let yaml = indoc! {"
        256
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_float() {
    let thing = 25.6;
    let yaml = indoc! {"
        25.6
    "};
    test_serde(&thing, yaml);

    let thing = 25.;
    let yaml = indoc! {"
        25.0
    "};
    test_serde(&thing, yaml);

    let thing = f64::INFINITY;
    let yaml = indoc! {"
        .inf
    "};
    test_serde(&thing, yaml);

    let thing = f64::NEG_INFINITY;
    let yaml = indoc! {"
        -.inf
    "};
    test_serde(&thing, yaml);

    let float: f64 = serde_yaml::from_str(indoc! {"
        .nan
    "})
    .unwrap();
    assert!(float.is_nan());
}

#[test]
fn test_float32() {
    let thing: f32 = 25.5;
    let yaml = indoc! {"
        25.5
    "};
    test_serde(&thing, yaml);

    let thing = f32::INFINITY;
    let yaml = indoc! {"
        .inf
    "};
    test_serde(&thing, yaml);

    let thing = f32::NEG_INFINITY;
    let yaml = indoc! {"
        -.inf
    "};
    test_serde(&thing, yaml);

    let single_float: f32 = serde_yaml::from_str(indoc! {"
        .nan
    "})
    .unwrap();
    assert!(single_float.is_nan());
}

#[test]
fn test_char() {
    let ch = '.';
    let yaml = indoc! {"
        '.'
    "};
    assert_eq!(yaml, serde_yaml::to_string(&ch).unwrap());

    let ch = '#';
    let yaml = indoc! {"
        '#'
    "};
    assert_eq!(yaml, serde_yaml::to_string(&ch).unwrap());

    let ch = '-';
    let yaml = indoc! {"
        '-'
    "};
    assert_eq!(yaml, serde_yaml::to_string(&ch).unwrap());
}

#[test]
fn test_vec() {
    let thing = vec![1, 2, 3];
    let yaml = indoc! {"
        - 1
        - 2
        - 3
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_map() {
    let mut thing = BTreeMap::new();
    thing.insert("x".to_owned(), 1);
    thing.insert("y".to_owned(), 2);
    let yaml = indoc! {"
        x: 1
        y: 2
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_map_key_value() {
    struct Map;

    impl serde::Serialize for Map {
        fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
        where
            S: serde::Serializer,
        {
            // Test maps which do not serialize using serialize_entry.
            let mut map = serializer.serialize_map(Some(1))?;
            map.serialize_key("k")?;
            map.serialize_value("v")?;
            map.end()
        }
    }

    let yaml = indoc! {"
        k: v
    "};
    assert_eq!(yaml, serde_yaml::to_string(&Map).unwrap());
}

#[test]
fn test_basic_struct() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Basic {
        x: isize,
        y: String,
        z: bool,
    }
    let thing = Basic {
        x: -4,
        y: "hi\tquoted".to_owned(),
        z: true,
    };
    let yaml = indoc! {r#"
        x: -4
        y: "hi\tquoted"
        z: true
    "#};
    test_serde(&thing, yaml);
}

#[test]
fn test_multiline_string() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Struct {
        trailing_newline: String,
        no_trailing_newline: String,
    }
    let thing = Struct {
        trailing_newline: "aaa\nbbb\n".to_owned(),
        no_trailing_newline: "aaa\nbbb".to_owned(),
    };
    let yaml = indoc! {r#"
        trailing_newline: |
          aaa
          bbb
        no_trailing_newline: |-
          aaa
          bbb
    "#};
    test_serde(&thing, yaml);
}

#[test]
fn test_strings_needing_quote() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Struct {
        boolean: String,
        integer: String,
        void: String,
    }
    let thing = Struct {
        boolean: "true".to_owned(),
        integer: "1".to_owned(),
        void: "null".to_owned(),
    };
    let yaml = indoc! {r#"
        boolean: 'true'
        integer: '1'
        void: 'null'
    "#};
    test_serde(&thing, yaml);
}

#[test]
fn test_nested_vec() {
    let thing = vec![vec![1, 2, 3], vec![4, 5, 6]];
    let yaml = indoc! {"
        - - 1
          - 2
          - 3
        - - 4
          - 5
          - 6
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_nested_struct() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Outer {
        inner: Inner,
    }
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Inner {
        v: u16,
    }
    let thing = Outer {
        inner: Inner { v: 512 },
    };
    let yaml = indoc! {"
        inner:
          v: 512
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_nested_enum() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    enum Outer {
        Inner(Inner),
    }
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    enum Inner {
        Unit,
    }
    let thing = Outer::Inner(Inner::Unit);
    let yaml = indoc! {"
        !Inner Unit
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_option() {
    let thing = vec![Some(1), None, Some(3)];
    let yaml = indoc! {"
        - 1
        - null
        - 3
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_unit() {
    let thing = vec![(), ()];
    let yaml = indoc! {"
        - null
        - null
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_unit_struct() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Foo;
    let thing = Foo;
    let yaml = indoc! {"
        null
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_unit_variant() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    enum Variant {
        First,
        Second,
    }
    let thing = Variant::First;
    let yaml = indoc! {"
        First
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_newtype_struct() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct OriginalType {
        v: u16,
    }
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct NewType(OriginalType);
    let thing = NewType(OriginalType { v: 1 });
    let yaml = indoc! {"
        v: 1
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_newtype_variant() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    enum Variant {
        Size(usize),
    }
    let thing = Variant::Size(127);
    let yaml = indoc! {"
        !Size 127
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_tuple_variant() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    enum Variant {
        Rgb(u8, u8, u8),
    }
    let thing = Variant::Rgb(32, 64, 96);
    let yaml = indoc! {"
        !Rgb
        - 32
        - 64
        - 96
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_struct_variant() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    enum Variant {
        Color { r: u8, g: u8, b: u8 },
    }
    let thing = Variant::Color {
        r: 32,
        g: 64,
        b: 96,
    };
    let yaml = indoc! {"
        !Color
        r: 32
        g: 64
        b: 96
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_tagged_map_value() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Bindings {
        profile: Profile,
    }
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    enum Profile {
        ClassValidator { class_name: String },
    }
    let thing = Bindings {
        profile: Profile::ClassValidator {
            class_name: "ApplicationConfig".to_owned(),
        },
    };
    let yaml = indoc! {"
        profile: !ClassValidator
          class_name: ApplicationConfig
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_value() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    pub struct GenericInstructions {
        #[serde(rename = "type")]
        pub typ: String,
        pub config: Value,
    }
    let thing = GenericInstructions {
        typ: "primary".to_string(),
        config: Value::Sequence(vec![
            Value::Null,
            Value::Bool(true),
            Value::Number(Number::from(65535)),
            Value::Number(Number::from(0.54321)),
            Value::String("s".into()),
            Value::Mapping(Mapping::new()),
        ]),
    };
    let yaml = indoc! {"
        type: primary
        config:
        - null
        - true
        - 65535
        - 0.54321
        - s
        - {}
    "};
    test_serde(&thing, yaml);
}

#[test]
fn test_mapping() {
    #[derive(Serialize, Deserialize, PartialEq, Debug)]
    struct Data {
        pub substructure: Mapping,
    }

    let mut thing = Data {
        substructure: Mapping::new(),
    };
    thing.substructure.insert(
        Value::String("a".to_owned()),
        Value::String("foo".to_owned()),
    );
    thing.substructure.insert(
        Value::String("b".to_owned()),
        Value::String("bar".to_owned()),
    );

    let yaml = indoc! {"
        substructure:
          a: foo
          b: bar
    "};

    test_serde(&thing, yaml);
}
<|endoftext|>


<commit_before>extern crate tinyrsh;

use std::collections::HashMap;
use std::net::{TcpListener, TcpStream};
use std::io::{Write};
use std::io;
use std::process;
use std::os::unix::io::{RawFd};
use tinyrsh::child::PersistentChild;
use tinyrsh::fdstore::{FdStore, OnceAccept, OnceRead};

#[derive(Debug)]
struct AuxData {
    total_cons: u64,
    rbytes: u64,
    wbytes: u64,
}

fn main() {
    println!("Hello, world!");

    let mut aux = AuxData{ total_cons: 0, rbytes:0, wbytes:0 };

    let child = PersistentChild::new("../py-ptysh/ptysh.py");

    println!("child terminated? {}", child.has_terminated());

    // TODO linebuffered? how is child output buffered at all?
    let listener = TcpListener::bind("127.0.0.1:6699").unwrap();

    let mut fdstore = FdStore::new(listener, child, aux);

   println!("select-looping");
   loop {
       //let readfds = fdstore.all_fdset.readfds_select();
       //for fd in readfds {
       let srvacceptfun = |  a: OnceAccept, clients: &HashMap<RawFd, TcpStream>, aux: &mut AuxData | -> io::Result<TcpStream> {
              println!("Accepting new connection");
              aux.total_cons += 1;
              println!("{:?}", aux);
              a.do_accept().map( | (mut stream, addr) | {
                      println!("new client: {:?}", addr);
                      assert_eq!(stream.write(b"hello\n").expect("greet client failed"), 6);
                      stream.write_all(b"Your fellow peers:").expect("client write");
                      let mut v = vec![];
                      for other in clients.keys() {
                          v.push(format!("fd {}", other));
                      }
                      stream.write_all(v.join(", ").as_bytes()).expect("client write");
                      stream.write(b"\n").expect("client write");
                      for mut client in clients.values() {
                          client.write_all(b"new client connected\n").expect("client write");
                      }
                      stream
              })
           };
       let clientfun = | client: OnceRead<TcpStream>, to: &mut process::ChildStdin, aux: &mut AuxData | -> bool {
            let mut buf = [0; 10];
            let n = client.do_read(&mut buf).expect("copy_to read");
            let written = to.write(&buf[..n]).expect("copy_to write");
            to.flush();
            assert_eq!(n, written);
            aux.rbytes += n as u64;
            n != 0
           };
       let childstdoutfun = | stdout: OnceRead<process::ChildStdout>, clients: &HashMap<RawFd, TcpStream>, aux: &mut AuxData | -> bool {
            println!("child (stdout) got active");
            let mut buf = [0; 10];
            let n = stdout.do_read(&mut buf).expect("read_child");
            let child_eof = n == 0;
            if clients.is_empty() {
                println!("Warning: child output but no remote connected");
                println!("{:?} ({} bytes) `{}'", buf, n, String::from_utf8_lossy(&buf[..n]));
            }
            for mut remote in clients.values(){
                //println!("forwarding to {}", remote.peer_addr().unwrap());
                let written = remote.write(&buf[..n]).expect("remote write");
                assert_eq!(n, written);
                aux.wbytes += written as u64;
            }
            !child_eof
           };
       let childstderrfun = | stderr: OnceRead<process::ChildStderr>, clients: &HashMap<RawFd, TcpStream>, aux: &mut AuxData | -> bool {
            println!("child (stderr) got active");
            let mut buf = [0; 10];
            let n = stderr.do_read(&mut buf).expect("read_child");
            println!("({} bytes) `{}'", n, String::from_utf8_lossy(&buf[..n]));
            n != 0
           };

       //do all the stuff
       fdstore.select(&srvacceptfun, &clientfun, &childstdoutfun, &childstderrfun);
   } //loop
}
<commit_msg>safer debugging<commit_after>extern crate tinyrsh;

use std::collections::HashMap;
use std::net::{TcpListener, TcpStream};
use std::io::{Write};
use std::io;
use std::process;
use std::os::unix::io::{RawFd};
use tinyrsh::child::PersistentChild;
use tinyrsh::fdstore::{FdStore, OnceAccept, OnceRead};

#[derive(Debug)]
struct AuxData {
    total_cons: u64,
    rbytes: u64,
    wbytes: u64,
}

//print only safe ascii characters for debugging. In particular don't print shell escapes.
fn safe_ascii_debug(v: &[u8]) -> String {
    let mut s = String::new();
    for &c in v.iter() {
        if c >= 0x20u8 && c <= 0x7Eu8 && c != 0x5Cu8 {
            s.push(c as char);
        } else {
            s.push('X')
        }
    }
    s
}

fn main() {
    println!("Hello, world!");

    let mut aux = AuxData{ total_cons: 0, rbytes:0, wbytes:0 };

    let child = PersistentChild::new("../py-ptysh/ptysh.py");

    println!("child terminated? {}", child.has_terminated());

    // TODO linebuffered? how is child output buffered at all?
    let listener = TcpListener::bind("127.0.0.1:6699").unwrap();

    let mut fdstore = FdStore::new(listener, child, aux);

   println!("select-looping");
   loop {
       //let readfds = fdstore.all_fdset.readfds_select();
       //for fd in readfds {
       let srvacceptfun = |  a: OnceAccept, clients: &HashMap<RawFd, TcpStream>, aux: &mut AuxData | -> io::Result<TcpStream> {
              println!("Accepting new connection");
              aux.total_cons += 1;
              println!("{:?}", aux);
              a.do_accept().map( | (mut stream, addr) | {
                      println!("new client: {:?}", addr);
                      assert_eq!(stream.write(b"hello\n").expect("greet client failed"), 6);
                      stream.write_all(b"Your fellow peers:").expect("client write");
                      let mut v = vec![];
                      for other in clients.keys() {
                          v.push(format!("fd {}", other));
                      }
                      stream.write_all(v.join(", ").as_bytes()).expect("client write");
                      stream.write(b"\n").expect("client write");
                      for mut client in clients.values() {
                          client.write_all(b"new client connected\n").expect("client write");
                      }
                      stream
              })
           };
       let clientfun = | client: OnceRead<TcpStream>, to: &mut process::ChildStdin, aux: &mut AuxData | -> bool {
            let mut buf = [0; 1024];
            let n = client.do_read(&mut buf).expect("copy_to read");
            let written = to.write(&buf[..n]).expect("copy_to write");
            to.flush();
            assert_eq!(n, written);
            aux.rbytes += n as u64;
            n != 0
           };
       let childstdoutfun = | stdout: OnceRead<process::ChildStdout>, clients: &HashMap<RawFd, TcpStream>, aux: &mut AuxData | -> bool {
            println!("child (stdout) got active");
            let mut buf = [0; 1024];
            let n = stdout.do_read(&mut buf).expect("read_child");
            let child_eof = n == 0;
            if clients.is_empty() {
                println!("Warning: child output but no remote connected");
                println!("({} bytes) `{}'", n, safe_ascii_debug(&buf[..n]));
            }
            for mut remote in clients.values(){
                //println!("forwarding to {}", remote.peer_addr().unwrap());
                let written = remote.write(&buf[..n]).expect("remote write");
                assert_eq!(n, written);
                aux.wbytes += written as u64;
            }
            !child_eof
           };
       let childstderrfun = | stderr: OnceRead<process::ChildStderr>, clients: &HashMap<RawFd, TcpStream>, aux: &mut AuxData | -> bool {
            println!("child (stderr) got active");
            let mut buf = [0; 1024];
            let n = stderr.do_read(&mut buf).expect("read_child");
            println!("({} bytes) `{}'", n, safe_ascii_debug(&buf[..n]));
            n != 0
           };

       //do all the stuff
       fdstore.select(&srvacceptfun, &clientfun, &childstdoutfun, &childstderrfun);
   } //loop
}
<|endoftext|>


<commit_before>extern crate tinyrsh;

use std::collections::HashMap;
use std::net::{TcpListener, TcpStream};
use std::io::{Read, Write};
use std::os::unix::io::{RawFd, AsRawFd, IntoRawFd};
use tinyrsh::select as select;
use tinyrsh::select::FdSet;
use std::process::{Command, Stdio};

fn greet_client(mut stream: &TcpStream) {
    let remote = stream.peer_addr().unwrap();
    println!("connected: {}", remote);

    match stream.write(b"hello") {
        Err(e) => println!("{}", e),
        Ok(n) => println!("wrote {} bytes", n),
    };
}

//what the develry! mut stream as parameter??
fn read_client(mut stream: &TcpStream) -> bool {
    let remote = stream.peer_addr().unwrap();

    let mut buffer = [0; 10];
    let cont = match stream.read(&mut buffer) {
        Err(e) => {
                println!("{}", e);
                false
            },
        Ok(n) => {
                println!("{}: {:?} ({} bytes)", remote, buffer, n);
                n != 0
            },
    };
    cont
}

fn debug_fdset(fdset: &FdSet) {
    let maxfds = select::FD_SETSIZE;
    for i in 0 .. maxfds {
        if fdset.contains(i) {
            println!("contains {}", i);
        }
    }
}

fn read_child<T: Read>(readt: &mut T) {
    let mut buf = [0; 10];
    let n = readt.read(&mut buf).expect("read_child");
    println!("{:?} ({} bytes) `{}'", buf, n, String::from_utf8_lossy(&buf[..n]));
}

fn copy_to<T: Read, U: Write>(from: &mut T, to: &mut U) -> bool {
    let mut buf = [0; 10];
    let n = from.read(&mut buf).expect("copy_to read");
    println!("copy_to read {:?} ({} bytes) `{}'", buf, n, String::from_utf8_lossy(&buf[..n]));
    let written = to.write(&buf[..n]).expect("copy_to write");
    to.flush();
    assert_eq!(n, written);

    n != 0
}

fn main() {
    println!("Hello, world!");

    let child = Command::new("../py-ptysh/ptysh.py")
            //.arg("-a")
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .spawn()
            .expect("failed to execute process");
    println!("spawned child with pid {}", child.id());

    let mut child_stdin  = child.stdin.unwrap();
    let mut child_stdout = child.stdout.unwrap();
    let mut child_stderr = child.stderr.unwrap();

        // test read output!!! not linebuffered??
        let mut buffer = [0; 10];
        let n = child_stdout.read(&mut buffer).expect("child stdout read");
        println!("childstdout: {:?} ({} bytes)", buffer, n);

    let listener = TcpListener::bind("127.0.0.1:6699").unwrap();

    let mut all_fdset = FdSet::new();
    let srv_fd = listener.as_raw_fd(); // not consumed

    //high fd for select() call
    let mut high_fd = 0;

    for fd in [srv_fd, child_stdout.as_raw_fd(), child_stderr.as_raw_fd()].iter() {
        println!("inserting fd {}", fd);
        all_fdset.insert(*fd);
        high_fd = std::cmp::max(high_fd, *fd);
    };

    let mut clients: HashMap<RawFd, TcpStream> = HashMap::new(); //could also use FromRawFd

    fn add_client(all_fdset: &mut FdSet, clients: &mut HashMap<RawFd, TcpStream>, stream: TcpStream) -> RawFd {
        let stream_fd: RawFd = stream.as_raw_fd(); //not consume
        println!("inserted fd {}", stream_fd);
        clients.insert(stream_fd, stream);
        all_fdset.insert(stream_fd);
        stream_fd
    };
    fn del_client(all_fdset: &mut FdSet, clients: &mut HashMap<RawFd, TcpStream>, stream_fd: RawFd) {
        assert!(clients.contains_key(&stream_fd));
        //let stream = clients.get(&stream_fd);
        println!("removing fd {}", stream_fd);
        assert!(all_fdset.contains(stream_fd));
        all_fdset.remove(stream_fd);
        assert!(clients.contains_key(&stream_fd));
        clients.remove(&stream_fd);
    };

    for stream in listener.incoming() {
        match stream {
            Ok(mut stream) => {
                greet_client(&mut stream);
                high_fd = std::cmp::max(high_fd, add_client(&mut all_fdset, &mut clients, stream));
                println!("new high fd: {}", high_fd);
            }
            Err(e) => { /* connection failed */ }
        }
        println!("selecting");
        loop {
            let mut fdset = all_fdset.clone();
            let res = select::select(high_fd + 1, Some(&mut fdset), None, None, None);
            assert!(res > 0);
            println!("selected returned {}", res);
            debug_fdset(&fdset);

            let mut accept_new = false;

            let mut handled_fds = 0;
            for i in 0 .. high_fd + 1 {
                if fdset.contains(i) {
                    if i == srv_fd {
                        println!("need to accept new connection");
                        accept_new = true;
                    } else if i == child_stdout.as_raw_fd() {
                        println!("child fd {} (stdout) got active", i);
                        read_child(&mut child_stdout);
                    } else if i == child_stderr.as_raw_fd() {
                        println!("child fd {} (stderr) got active", i);
                        read_child(&mut child_stderr);
                    } else {
                        assert!(clients.contains_key(&i));
                        let cont = {
                            //let ref stream = clients[&i];
                            let stream: &mut TcpStream= clients.get_mut(&i).unwrap();
                            //read_client(stream)
                            copy_to(stream, &mut child_stdin)
                        };
                        if !cont {
                            del_client(&mut all_fdset, &mut clients, i);
                        };
                    }
                    handled_fds += 1;
                }
            }
            assert_eq!(handled_fds, res);
            if accept_new {break}
        }
    }
}
<commit_msg>copy childsh stdout<commit_after>extern crate tinyrsh;

use std::collections::HashMap;
use std::net::{TcpListener, TcpStream};
use std::io::{Read, Write};
use std::os::unix::io::{RawFd, AsRawFd, IntoRawFd};
use tinyrsh::select as select;
use tinyrsh::select::FdSet;
use std::process::{Command, Stdio};

fn greet_client(mut stream: &TcpStream) {
    let remote = stream.peer_addr().unwrap();
    println!("connected: {}", remote);

    match stream.write(b"hello") {
        Err(e) => println!("{}", e),
        Ok(n) => println!("wrote {} bytes", n),
    };
}

//what the develry! mut stream as parameter??
fn read_client(mut stream: &TcpStream) -> bool {
    let remote = stream.peer_addr().unwrap();

    let mut buffer = [0; 10];
    let cont = match stream.read(&mut buffer) {
        Err(e) => {
                println!("{}", e);
                false
            },
        Ok(n) => {
                println!("{}: {:?} ({} bytes)", remote, buffer, n);
                n != 0
            },
    };
    cont
}

fn debug_fdset(fdset: &FdSet) {
    let maxfds = select::FD_SETSIZE;
    for i in 0 .. maxfds {
        if fdset.contains(i) {
            println!("contains {}", i);
        }
    }
}

fn read_child<T: Read>(readt: &mut T) {
    let mut buf = [0; 10];
    let n = readt.read(&mut buf).expect("read_child");
    println!("{:?} ({} bytes) `{}'", buf, n, String::from_utf8_lossy(&buf[..n]));
}

fn forward_to_remotes<T: Read, S>(child_out: &mut T, clients: &HashMap<S, TcpStream>) -> ()
    where S : std::cmp::Eq, S : std::hash::Hash
{
    let mut buf = [0; 10];
    let n = child_out.read(&mut buf).expect("read_child");
    println!("{:?} ({} bytes) `{}'", buf, n, String::from_utf8_lossy(&buf[..n]));
    assert!(n != 0); //TODO eof
    for mut remote in clients.values(){
        println!("forwarding to {}", remote.peer_addr().unwrap());
        let written = remote.write(&buf[..n]).expect("remote write");
        assert_eq!(n, written);
    }
}

fn copy_to<T: Read, U: Write>(from: &mut T, to: &mut U) -> bool {
    let mut buf = [0; 10];
    let n = from.read(&mut buf).expect("copy_to read");
    println!("copy_to read {:?} ({} bytes) `{}'", buf, n, String::from_utf8_lossy(&buf[..n]));
    let written = to.write(&buf[..n]).expect("copy_to write");
    to.flush();
    assert_eq!(n, written);

    n != 0
}

fn main() {
    println!("Hello, world!");

    let child = Command::new("../py-ptysh/ptysh.py")
            //.arg("-a")
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .spawn()
            .expect("failed to execute process");
    println!("spawned child with pid {}", child.id());

    let mut child_stdin  = child.stdin.unwrap();
    let mut child_stdout = child.stdout.unwrap();
    let mut child_stderr = child.stderr.unwrap();
    // stderr == stdout? no, but py-ptysh copies everything to std out
    assert!(child_stdout.as_raw_fd() != child_stderr.as_raw_fd());

        // test read output!!! not linebuffered??
        let mut buffer = [0; 10];
        let n = child_stdout.read(&mut buffer).expect("child stdout read");
        println!("childstdout: {:?} ({} bytes)", buffer, n);

    let listener = TcpListener::bind("127.0.0.1:6699").unwrap();

    let mut all_fdset = FdSet::new();
    let srv_fd = listener.as_raw_fd(); // not consumed

    //high fd for select() call
    let mut high_fd = 0;

    for fd in [srv_fd, child_stdout.as_raw_fd(), child_stderr.as_raw_fd()].iter() {
        println!("inserting fd {}", fd);
        all_fdset.insert(*fd);
        high_fd = std::cmp::max(high_fd, *fd);
    };

    let mut clients: HashMap<RawFd, TcpStream> = HashMap::new(); //could also use FromRawFd

    fn add_client(all_fdset: &mut FdSet, clients: &mut HashMap<RawFd, TcpStream>, stream: TcpStream) -> RawFd {
        let stream_fd: RawFd = stream.as_raw_fd(); //not consume
        println!("inserted fd {}", stream_fd);
        clients.insert(stream_fd, stream);
        all_fdset.insert(stream_fd);
        stream_fd
    };
    fn del_client(all_fdset: &mut FdSet, clients: &mut HashMap<RawFd, TcpStream>, stream_fd: RawFd) {
        assert!(clients.contains_key(&stream_fd));
        //let stream = clients.get(&stream_fd);
        println!("removing fd {}", stream_fd);
        assert!(all_fdset.contains(stream_fd));
        all_fdset.remove(stream_fd);
        assert!(clients.contains_key(&stream_fd));
        clients.remove(&stream_fd);
    };

    for stream in listener.incoming() {
        match stream {
            Ok(mut stream) => {
                greet_client(&mut stream);
                high_fd = std::cmp::max(high_fd, add_client(&mut all_fdset, &mut clients, stream));
                println!("new high fd: {}", high_fd);
            }
            Err(e) => { /* connection failed */ }
        };
        let stream = (); //TODO get rid of listener iterator
        println!("selecting");
        loop {
            let mut fdset = all_fdset.clone();
            let res = select::select(high_fd + 1, Some(&mut fdset), None, None, None);
            assert!(res > 0);
            println!("selected returned {}", res);
            debug_fdset(&fdset);

            let mut accept_new = false;

            let mut handled_fds = 0;
            for i in 0 .. high_fd + 1 {
                if fdset.contains(i) {
                    if i == srv_fd {
                        println!("need to accept new connection");
                        accept_new = true;
                    } else if i == child_stdout.as_raw_fd() {
                        println!("child fd {} (stdout) got active", i);
                        forward_to_remotes(&mut child_stdout, &clients);
                    } else if i == child_stderr.as_raw_fd() {
                        println!("child fd {} (stderr) got active", i);
                        read_child(&mut child_stderr);
                    } else {
                        assert!(clients.contains_key(&i));
                        let cont = {
                            //let ref stream = clients[&i];
                            let stream: &mut TcpStream= clients.get_mut(&i).unwrap();
                            //read_client(stream)
                            copy_to(stream, &mut child_stdin)
                        };
                        if !cont {
                            del_client(&mut all_fdset, &mut clients, i);
                        };
                    }
                    handled_fds += 1;
                }
            }
            assert_eq!(handled_fds, res);
            if accept_new {break}
        }
    }
}
<|endoftext|>


<commit_before>#![allow(dead_code)]

use std::rc::Rc;

use rational::Rational64;
use data::audiosample::{Soniton, ChannelMap};
use data::pixel::Formaton;

#[derive(Clone,Debug,PartialEq)]
pub enum CodecID {
    VP9,
    Opus
}

#[derive(Clone,Debug,PartialEq)]
pub struct VideoInfo {
    pub width: usize,
    pub height: usize,
    pub format: Option<Rc<Formaton>>,
}

#[derive(Clone,Debug,PartialEq)]
pub struct AudioInfo {
    pub samples: usize,
    pub rate: usize,
    pub map: Option<ChannelMap>,
    pub format: Option<Rc<Soniton>>,
}

#[derive(Clone,Debug,PartialEq)]
pub enum MediaKind {
    Video(VideoInfo),
    Audio(AudioInfo),
}

#[derive(Clone,Debug,PartialEq)]
pub struct CodecParams {
    pub kind: Option<MediaKind>,
    pub codec_id: Option<CodecID>,
    pub extradata: Option<Vec<u8>>,
//    pub tag: Option<u32>,
    pub bit_rate: usize,
//    pub bits_per_coded_sample: usize,
}

#[derive(Clone,Debug,PartialEq)]
pub struct Stream {
    pub id: usize,
    pub index: usize,
    pub params : CodecParams,
    pub start: Option<u64>,
    pub duration: Option<u64>,
    pub timebase : Rational64,
//  seek_index : SeekIndex
}

pub struct StreamGroup<'a> {
    pub id: usize,
    pub start: u64,
    pub end: u64,
    pub streams: &'a [Stream],
}
<commit_msg>format: Update the API<commit_after>#![allow(dead_code)]

use std::rc::Rc;

use rational::Rational64;
use data::audiosample::{Soniton, ChannelMap};
use data::pixel::Formaton;

#[derive(Clone,Debug,PartialEq)]
pub struct VideoInfo {
    pub width: usize,
    pub height: usize,
    pub format: Option<Rc<Formaton>>,
}

#[derive(Clone,Debug,PartialEq)]
pub struct AudioInfo {
    pub rate: usize,
    pub map: Option<ChannelMap>,
    pub format: Option<Rc<Soniton>>,
}

#[derive(Clone,Debug,PartialEq)]
pub enum MediaKind {
    Video(VideoInfo),
    Audio(AudioInfo),
}

#[derive(Clone,Debug,PartialEq)]
pub struct CodecParams {
    pub kind: Option<MediaKind>,
    pub codec_id: Option<String>,
    pub extradata: Option<Vec<u8>>,
//    pub tag: Option<u32>,
    pub bit_rate: usize,
//    pub bits_per_coded_sample: usize,
}

#[derive(Clone,Debug,PartialEq)]
pub struct Stream {
    pub id: usize,
    pub index: usize,
    pub params : CodecParams,
    pub start: Option<u64>,
    pub duration: Option<u64>,
    pub timebase : Rational64,
//  seek_index : SeekIndex
}

pub struct StreamGroup<'a> {
    pub id: usize,
    pub start: u64,
    pub end: u64,
    pub streams: &'a [Stream],
}
<|endoftext|>


<commit_before>use std::io::stdio::{stdout, stderr};
use std::io::fs::File;

use serialize::json::from_reader;
use J = serialize::json;
use argparse::{ArgumentParser, Store, List};

use super::super::env::Environ;

pub fn extract_json(_env: &mut Environ, args: Vec<String>)
    -> Result<int, String>
{
    let mut filename = Path::new(".");
    let mut columns = Vec::new();
    {
        let mut ap = ArgumentParser::new();
        ap.refer(&mut filename)
            .add_argument("filename", box Store::<Path>,
                "A JSON file to parse")
            .required();
        ap.refer(&mut columns)
            .add_argument("column", box List::<String>,
                "A columns to extract");
        match ap.parse(args, &mut stdout(), &mut stderr()) {
            Ok(()) => {}
            Err(0) => return Ok(0),
            Err(_) => return Ok(122),
        }
    }
    let mut file = try!(File::open(&filename)
        .map_err(|e| format!("Can't open file {}: {}",
                             filename.display(), e)));
    let json = try!(from_reader(&mut file)
        .map_err(|e| format!("Can't parse {}: {}", filename.display(), e)));
    let lst = match json {
        J::List(lst) => lst,
        _ => return Err(format!("Root entitity is not a list")),
    };
    let mut out = stdout();
    let ncols = columns.len();
    for obj in lst.iter() {
        let tree = match obj {
            &J::Object(ref tmap) => tmap,
            _ => return Err(format!("Not an object in a list")),
        };
        for (idx, col) in columns.iter().enumerate() {
            match tree.find(col) {
                Some(&J::String(ref x)) => try!(out.write_str(x.as_slice())
                    .map_err(|e| format!("Error writing to stdout: {}", e))),
                _ => {},
            };
            if idx < ncols - 1 {
                try!(out.write_char('\t')
                    .map_err(|e| format!("Error writing to stdout: {}", e)));
            }
        }
        try!(out.write_char('\n')
            .map_err(|e| format!("Error writing to stdout: {}", e)));
    }
    return Ok(0);
}
<commit_msg>Implement empty json field<commit_after>use std::io::stdio::{stdout, stderr};
use std::io::fs::File;

use serialize::json::from_reader;
use J = serialize::json;
use argparse::{ArgumentParser, Store, List};

use super::super::env::Environ;

pub fn extract_json(_env: &mut Environ, args: Vec<String>)
    -> Result<int, String>
{
    let mut filename = Path::new(".");
    let mut columns: Vec<String> = Vec::new();
    {
        let mut ap = ArgumentParser::new();
        ap.refer(&mut filename)
            .add_argument("filename", box Store::<Path>,
                "A JSON file to parse")
            .required();
        ap.refer(&mut columns)
            .add_argument("column", box List::<String>,
                "A columns to extract");
        match ap.parse(args, &mut stdout(), &mut stderr()) {
            Ok(()) => {}
            Err(0) => return Ok(0),
            Err(_) => return Ok(122),
        }
    }
    let mut file = try!(File::open(&filename)
        .map_err(|e| format!("Can't open file {}: {}",
                             filename.display(), e)));
    let json = try!(from_reader(&mut file)
        .map_err(|e| format!("Can't parse {}: {}", filename.display(), e)));
    let lst = match json {
        J::List(lst) => lst,
        _ => return Err(format!("Root entitity is not a list")),
    };
    let mut out = stdout();
    let ncols = columns.len();
    for obj in lst.iter() {
        let tree = match obj {
            &J::String(ref val) if columns.get(0).len() == 0 => {
                try!(out.write_str(val.as_slice()).and(out.write_char('\n'))
                    .map_err(|e| format!("Error writing to stdout: {}", e)));
                continue;
            }
            &J::Object(ref tmap) => tmap,
            _ => return Err(format!("Not an object in a list")),
        };
        for (idx, col) in columns.iter().enumerate() {
            match tree.find(col) {
                Some(&J::String(ref x)) => try!(out.write_str(x.as_slice())
                    .map_err(|e| format!("Error writing to stdout: {}", e))),
                _ => {},
            };
            if idx < ncols - 1 {
                try!(out.write_char('\t')
                    .map_err(|e| format!("Error writing to stdout: {}", e)));
            }
        }
        try!(out.write_char('\n')
            .map_err(|e| format!("Error writing to stdout: {}", e)));
    }
    return Ok(0);
}
<|endoftext|>


<commit_before>use crate::parser;
use crate::{Error, Record};

use std::fs;
use std::io;
use std::io::{BufRead, BufReader, BufWriter, Write};
use std::path::Path;

#[cfg(feature = "gzip")]
use libflate::gzip::{Decoder as GzipReader, Encoder as GzipWriter};

const KB: usize = 1_024;
const MB: usize = 1_048_576;

pub struct WarcReader<R> {
    reader: R,
}

pub struct WarcWriter<W> {
    writer: W,
}

impl<W: Write> WarcWriter<W> {
    pub fn new(w: W) -> Self {
        WarcWriter { writer: w }
    }

    pub fn write(&mut self, record: &Record) -> io::Result<usize> {
        let mut bytes_written = 0;

        bytes_written += self.writer.write(&[87, 65, 82, 67, 47])?;
        bytes_written += self.writer.write(record.version.as_bytes())?;
        bytes_written += self.writer.write(&[13, 10])?;

        for (token, value) in record.headers.iter() {
            bytes_written += self.writer.write(token.as_bytes())?;
            bytes_written += self.writer.write(&[58, 32])?;
            bytes_written += self.writer.write(&value)?;
            bytes_written += self.writer.write(&[13, 10])?;
        }
        bytes_written += self.writer.write(&[13, 10])?;

        bytes_written += self.writer.write(&record.body)?;
        bytes_written += self.writer.write(&[13, 10])?;
        bytes_written += self.writer.write(&[13, 10])?;

        Ok(bytes_written)
    }
}

impl<W: Write> WarcWriter<BufWriter<W>> {
    pub fn into_inner(self) -> Result<W, std::io::IntoInnerError<BufWriter<W>>> {
        self.writer.into_inner()
    }
}

impl WarcWriter<BufWriter<fs::File>> {
    pub fn from_path<P: AsRef<Path>>(path: P) -> io::Result<Self> {
        let file = fs::OpenOptions::new()
            .read(true)
            .write(true)
            .create(true)
            .open(&path)?;
        let writer = BufWriter::with_capacity(1 * MB, file);

        Ok(WarcWriter::new(writer))
    }
}

#[cfg(feature = "gzip")]
impl WarcWriter<BufWriter<GzipWriter<std::fs::File>>> {
    pub fn from_path_gzip<P: AsRef<Path>>(path: P) -> io::Result<Self> {
        let file = fs::OpenOptions::new()
            .read(true)
            .write(true)
            .create(true)
            .open(&path)?;
        let gzip_stream = GzipWriter::new(file)?;
        let writer = BufWriter::with_capacity(1 * MB, gzip_stream);

        Ok(WarcWriter::new(writer))
    }
}

impl<R: BufRead> WarcReader<R> {
    pub fn new(r: R) -> Self {
        WarcReader { reader: r }
    }
}

impl WarcReader<BufReader<fs::File>> {
    pub fn from_path<P: AsRef<Path>>(path: P) -> io::Result<Self> {
        let file = fs::OpenOptions::new()
            .read(true)
            .write(true)
            .create(true)
            .open(&path)?;
        let reader = BufReader::with_capacity(1 * MB, file);

        Ok(WarcReader::new(reader))
    }
}

#[cfg(feature = "gzip")]
impl WarcReader<BufReader<GzipReader<std::fs::File>>> {
    pub fn from_path_gzip<P: AsRef<Path>>(path: P) -> io::Result<Self> {
        let file = fs::OpenOptions::new()
            .read(true)
            .write(true)
            .create(true)
            .open(&path)?;
        let gzip_stream = GzipReader::new(file)?;
        let reader = BufReader::with_capacity(1 * MB, gzip_stream);

        Ok(WarcReader::new(reader))
    }
}

impl<R: BufRead> Iterator for WarcReader<R> {
    type Item = Result<Record, Error>;

    fn next(&mut self) -> Option<Self::Item> {
        let mut header_buffer: Vec<u8> = Vec::with_capacity(64 * KB);
        let mut found_headers = false;
        while !found_headers {
            let bytes_read = match self.reader.read_until(b'\n', &mut header_buffer) {
                Err(_) => return Some(Err(Error::ReadData)),
                Ok(len) => len,
            };

            if bytes_read == 0 {
                return None;
            }

            if bytes_read == 2 {
                let last_two_chars = header_buffer.len() - 2;
                if &header_buffer[last_two_chars..] == b"\r\n" {
                    found_headers = true;
                }
            }
        }

        let headers_parsed = match parser::headers(&header_buffer) {
            Err(_) => return Some(Err(Error::ParseHeaders)),
            Ok(parsed) => parsed.1,
        };
        let version_ref = headers_parsed.0;
        let headers_ref = headers_parsed.1;
        let expected_body_len = headers_parsed.2;

        let mut body_buffer: Vec<u8> = Vec::with_capacity(1 * MB);
        let mut found_body = expected_body_len == 0;
        let mut body_bytes_read = 0;
        let maximum_read_range = expected_body_len + 4;
        while !found_body {
            let bytes_read = match self.reader.read_until(b'\n', &mut body_buffer) {
                Err(_) => return Some(Err(Error::ReadData)),
                Ok(len) => len,
            };

            body_bytes_read += bytes_read;

            // we expect 4 characters (\r\n\r\n) after the body
            if bytes_read == 2 && body_bytes_read == maximum_read_range {
                found_body = true;
            }

            if bytes_read == 0 {
                return Some(Err(Error::UnexpectedEOB));
            }

            if body_bytes_read > maximum_read_range {
                return Some(Err(Error::ReadOverflow));
            }
        }

        let body_ref = &body_buffer[..expected_body_len];

        let record = Record {
            version: version_ref.to_owned(),
            headers: headers_ref
                .into_iter()
                .map(|(token, value)| (token.to_owned(), value.to_owned()))
                .collect(),
            body: body_ref.to_owned(),
        };

        return Some(Ok(record));
    }
}
<commit_msg>normalize header tokens (#15)<commit_after>use crate::parser;
use crate::{Error, Record};

use std::fs;
use std::io;
use std::io::{BufRead, BufReader, BufWriter, Write};
use std::path::Path;

#[cfg(feature = "gzip")]
use libflate::gzip::{Decoder as GzipReader, Encoder as GzipWriter};

const KB: usize = 1_024;
const MB: usize = 1_048_576;

pub struct WarcReader<R> {
    reader: R,
}

pub struct WarcWriter<W> {
    writer: W,
}

impl<W: Write> WarcWriter<W> {
    pub fn new(w: W) -> Self {
        WarcWriter { writer: w }
    }

    pub fn write(&mut self, record: &Record) -> io::Result<usize> {
        let mut bytes_written = 0;

        bytes_written += self.writer.write(&[87, 65, 82, 67, 47])?;
        bytes_written += self.writer.write(record.version.as_bytes())?;
        bytes_written += self.writer.write(&[13, 10])?;

        for (token, value) in record.headers.iter() {
            bytes_written += self.writer.write(token.as_bytes())?;
            bytes_written += self.writer.write(&[58, 32])?;
            bytes_written += self.writer.write(&value)?;
            bytes_written += self.writer.write(&[13, 10])?;
        }
        bytes_written += self.writer.write(&[13, 10])?;

        bytes_written += self.writer.write(&record.body)?;
        bytes_written += self.writer.write(&[13, 10])?;
        bytes_written += self.writer.write(&[13, 10])?;

        Ok(bytes_written)
    }
}

impl<W: Write> WarcWriter<BufWriter<W>> {
    pub fn into_inner(self) -> Result<W, std::io::IntoInnerError<BufWriter<W>>> {
        self.writer.into_inner()
    }
}

impl WarcWriter<BufWriter<fs::File>> {
    pub fn from_path<P: AsRef<Path>>(path: P) -> io::Result<Self> {
        let file = fs::OpenOptions::new()
            .read(true)
            .write(true)
            .create(true)
            .open(&path)?;
        let writer = BufWriter::with_capacity(1 * MB, file);

        Ok(WarcWriter::new(writer))
    }
}

#[cfg(feature = "gzip")]
impl WarcWriter<BufWriter<GzipWriter<std::fs::File>>> {
    pub fn from_path_gzip<P: AsRef<Path>>(path: P) -> io::Result<Self> {
        let file = fs::OpenOptions::new()
            .read(true)
            .write(true)
            .create(true)
            .open(&path)?;
        let gzip_stream = GzipWriter::new(file)?;
        let writer = BufWriter::with_capacity(1 * MB, gzip_stream);

        Ok(WarcWriter::new(writer))
    }
}

impl<R: BufRead> WarcReader<R> {
    pub fn new(r: R) -> Self {
        WarcReader { reader: r }
    }
}

impl WarcReader<BufReader<fs::File>> {
    pub fn from_path<P: AsRef<Path>>(path: P) -> io::Result<Self> {
        let file = fs::OpenOptions::new()
            .read(true)
            .write(true)
            .create(true)
            .open(&path)?;
        let reader = BufReader::with_capacity(1 * MB, file);

        Ok(WarcReader::new(reader))
    }
}

#[cfg(feature = "gzip")]
impl WarcReader<BufReader<GzipReader<std::fs::File>>> {
    pub fn from_path_gzip<P: AsRef<Path>>(path: P) -> io::Result<Self> {
        let file = fs::OpenOptions::new()
            .read(true)
            .write(true)
            .create(true)
            .open(&path)?;
        let gzip_stream = GzipReader::new(file)?;
        let reader = BufReader::with_capacity(1 * MB, gzip_stream);

        Ok(WarcReader::new(reader))
    }
}

impl<R: BufRead> Iterator for WarcReader<R> {
    type Item = Result<Record, Error>;

    fn next(&mut self) -> Option<Self::Item> {
        let mut header_buffer: Vec<u8> = Vec::with_capacity(64 * KB);
        let mut found_headers = false;
        while !found_headers {
            let bytes_read = match self.reader.read_until(b'\n', &mut header_buffer) {
                Err(_) => return Some(Err(Error::ReadData)),
                Ok(len) => len,
            };

            if bytes_read == 0 {
                return None;
            }

            if bytes_read == 2 {
                let last_two_chars = header_buffer.len() - 2;
                if &header_buffer[last_two_chars..] == b"\r\n" {
                    found_headers = true;
                }
            }
        }

        let headers_parsed = match parser::headers(&header_buffer) {
            Err(_) => return Some(Err(Error::ParseHeaders)),
            Ok(parsed) => parsed.1,
        };
        let version_ref = headers_parsed.0;
        let headers_ref = headers_parsed.1;
        let expected_body_len = headers_parsed.2;

        let mut body_buffer: Vec<u8> = Vec::with_capacity(1 * MB);
        let mut found_body = expected_body_len == 0;
        let mut body_bytes_read = 0;
        let maximum_read_range = expected_body_len + 4;
        while !found_body {
            let bytes_read = match self.reader.read_until(b'\n', &mut body_buffer) {
                Err(_) => return Some(Err(Error::ReadData)),
                Ok(len) => len,
            };

            body_bytes_read += bytes_read;

            // we expect 4 characters (\r\n\r\n) after the body
            if bytes_read == 2 && body_bytes_read == maximum_read_range {
                found_body = true;
            }

            if bytes_read == 0 {
                return Some(Err(Error::UnexpectedEOB));
            }

            if body_bytes_read > maximum_read_range {
                return Some(Err(Error::ReadOverflow));
            }
        }

        let body_ref = &body_buffer[..expected_body_len];

        let record = Record {
            version: version_ref.to_owned(),
            headers: headers_ref
                .into_iter()
                .map(|(token, value)| (token.to_lowercase().to_owned(), value.to_owned()))
                .collect(),
            body: body_ref.to_owned(),
        };

        return Some(Ok(record));
    }
}
<|endoftext|>


<commit_before>use hyper::header::ContentType;
use hyper::mime::Mime;
use hyper::server::{Server, Request, Response};
use hyper::status::StatusCode;
use hyper::uri::RequestUri::AbsolutePath;
use prometheus::{self, Encoder, TextEncoder};
use varz::{StartInstant, Varz};
use std::io;
use std::sync::Arc;
use std::sync::mpsc;
use std::thread::spawn;

use super::RPDNSContext;
use super::WEBSERVICE_THREADS;

pub struct WebService {
    varz: Arc<Varz>,
}

impl WebService {
    fn new(rpdns_context: &RPDNSContext) -> WebService {
        WebService { varz: rpdns_context.varz.clone() }
    }

    fn handler(&self, req: Request, mut res: Response) {
        match req.uri {
            AbsolutePath(ref path) if path == "/metrics" => path,
            _ => {
                *res.status_mut() = StatusCode::NotFound;
                return;
            }
        };
        let StartInstant(start_instant) = self.varz.start_instant;
        let uptime = start_instant.elapsed().as_secs();
        self.varz.uptime.set(uptime as f64);
        let client_queries = self.varz.client_queries_udp.get() +
                             self.varz.client_queries_tcp.get();
        self.varz.client_queries.set(client_queries);
        let metric_families = prometheus::gather();
        let mut buffer = vec![];
        let encoder = TextEncoder::new();
        encoder.encode(&metric_families, &mut buffer).unwrap();
        res.headers_mut()
            .set(ContentType(encoder.format_type().parse::<Mime>().unwrap()));
        res.send(&buffer).unwrap();
    }

    pub fn spawn(rpdns_context: &RPDNSContext,
                 service_ready_tx: mpsc::SyncSender<u8>)
                 -> io::Result<()> {
        let listen_addr = rpdns_context.config.webservice_listen_addr.to_owned();
        let web_service = WebService::new(rpdns_context);
        spawn(move || {
            let mut server = Server::http(&*listen_addr).expect("Unable to spawn the webservice");
            server.keep_alive(None);
            info!("Webservice started on {}", listen_addr);
            server.handle_threads(move |req: Request, res: Response| {
                                    web_service.handler(req, res)
                                },
                                WEBSERVICE_THREADS)
                .expect("Unable to start the webservice");
        });
        Ok(())
    }
}
<commit_msg>Make the webservice thread more consistent with other threads<commit_after>use hyper::header::ContentType;
use hyper::mime::Mime;
use hyper::server::{Server, Request, Response};
use hyper::status::StatusCode;
use hyper::uri::RequestUri::AbsolutePath;
use prometheus::{self, Encoder, TextEncoder};
use varz::{StartInstant, Varz};
use std::io;
use std::sync::Arc;
use std::sync::mpsc;
use std::thread;

use super::RPDNSContext;
use super::WEBSERVICE_THREADS;

pub struct WebService {
    varz: Arc<Varz>,
}

impl WebService {
    fn new(rpdns_context: &RPDNSContext) -> WebService {
        WebService { varz: rpdns_context.varz.clone() }
    }

    fn handler(&self, req: Request, mut res: Response) {
        match req.uri {
            AbsolutePath(ref path) if path == "/metrics" => path,
            _ => {
                *res.status_mut() = StatusCode::NotFound;
                return;
            }
        };
        let StartInstant(start_instant) = self.varz.start_instant;
        let uptime = start_instant.elapsed().as_secs();
        self.varz.uptime.set(uptime as f64);
        let client_queries = self.varz.client_queries_udp.get() +
                             self.varz.client_queries_tcp.get();
        self.varz.client_queries.set(client_queries);
        let metric_families = prometheus::gather();
        let mut buffer = vec![];
        let encoder = TextEncoder::new();
        encoder.encode(&metric_families, &mut buffer).unwrap();
        res.headers_mut()
            .set(ContentType(encoder.format_type().parse::<Mime>().unwrap()));
        res.send(&buffer).unwrap();
    }

    pub fn spawn(rpdns_context: &RPDNSContext,
                 service_ready_tx: mpsc::SyncSender<u8>)
                 -> io::Result<thread::JoinHandle<()>> {
        let listen_addr = rpdns_context.config.webservice_listen_addr.to_owned();
        let web_service = WebService::new(rpdns_context);
        let webservice_th = thread::spawn(move || {
            let mut server = Server::http(&*listen_addr).expect("Unable to spawn the webservice");
            server.keep_alive(None);
            service_ready_tx.send(2).unwrap();
            info!("Webservice started on {}", listen_addr);
            server.handle_threads(move |req: Request, res: Response| {
                                    web_service.handler(req, res)
                                },
                                WEBSERVICE_THREADS)
                .expect("Unable to start the webservice");
        });
        Ok(webservice_th)
    }
}
<|endoftext|>