<commit_before>mod sourcegen_tests;
mod sourcegen_ast;
mod ast_src;

use std::{
    fmt::Write,
    fs,
    path::{Path, PathBuf},
};

use ast::NameOwner;
use expect_test::expect_file;
use rayon::prelude::*;
use test_utils::{bench, bench_fixture, project_root, skip_slow_tests};

use crate::{ast, fuzz, tokenize, AstNode, SourceFile, SyntaxError, TextRange, TextSize, Token};

#[test]
fn lexer_tests() {
    // FIXME:
    // * Add tests for unicode escapes in byte-character and [raw]-byte-string literals
    // * Add tests for unescape errors

    dir_tests(&test_data_dir(), &["lexer/ok"], "txt", |text, path| {
        let (tokens, errors) = tokenize(text);
        assert_errors_are_absent(&errors, path);
        dump_tokens_and_errors(&tokens, &errors, text)
    });
    dir_tests(&test_data_dir(), &["lexer/err"], "txt", |text, path| {
        let (tokens, errors) = tokenize(text);
        assert_errors_are_present(&errors, path);
        dump_tokens_and_errors(&tokens, &errors, text)
    });
}

#[test]
fn parse_smoke_test() {
    let code = r##"
fn main() {
    println!("Hello, world!")
}
    "##;

    let parse = SourceFile::parse(code);
    // eprintln!("{:#?}", parse.syntax_node());
    assert!(parse.ok().is_ok());
}

#[test]
fn benchmark_parser() {
    if skip_slow_tests() {
        return;
    }
    let data = bench_fixture::glorious_old_parser();
    let tree = {
        let _b = bench("parsing");
        let p = SourceFile::parse(&data);
        assert!(p.errors.is_empty());
        assert_eq!(p.tree().syntax.text_range().len(), 352474.into());
        p.tree()
    };

    {
        let _b = bench("tree traversal");
        let fn_names =
            tree.syntax().descendants().filter_map(ast::Fn::cast).filter_map(|f| f.name()).count();
        assert_eq!(fn_names, 268);
    }
}

#[test]
fn parser_tests() {
    dir_tests(&test_data_dir(), &["parser/inline/ok", "parser/ok"], "rast", |text, path| {
        let parse = SourceFile::parse(text);
        let errors = parse.errors();
        assert_errors_are_absent(errors, path);
        parse.debug_dump()
    });
    dir_tests(&test_data_dir(), &["parser/err", "parser/inline/err"], "rast", |text, path| {
        let parse = SourceFile::parse(text);
        let errors = parse.errors();
        assert_errors_are_present(errors, path);
        parse.debug_dump()
    });
}

#[test]
fn expr_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/expr/ok"],
        &["parser/fragments/expr/err"],
        crate::ast::Expr::parse,
    );
}

#[test]
fn path_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/path/ok"],
        &["parser/fragments/path/err"],
        crate::ast::Path::parse,
    );
}

#[test]
fn pattern_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/pattern/ok"],
        &["parser/fragments/pattern/err"],
        crate::ast::Pat::parse,
    );
}

#[test]
fn item_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/item/ok"],
        &["parser/fragments/item/err"],
        crate::ast::Item::parse,
    );
}

#[test]
fn type_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/type/ok"],
        &["parser/fragments/type/err"],
        crate::ast::Type::parse,
    );
}

#[test]
fn stmt_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/stmt/ok"],
        &["parser/fragments/stmt/err"],
        crate::ast::Stmt::parse,
    );
}

#[test]
fn parser_fuzz_tests() {
    for (_, text) in collect_rust_files(&test_data_dir(), &["parser/fuzz-failures"]) {
        fuzz::check_parser(&text)
    }
}

#[test]
fn reparse_fuzz_tests() {
    for (_, text) in collect_rust_files(&test_data_dir(), &["reparse/fuzz-failures"]) {
        let check = fuzz::CheckReparse::from_data(text.as_bytes()).unwrap();
        check.run();
    }
}

/// Test that Rust-analyzer can parse and validate the rust-analyzer
#[test]
fn self_hosting_parsing() {
    let crates_dir = project_root().join("crates");

    let mut files = ::sourcegen::list_rust_files(&crates_dir);
    files.retain(|path| {
        // Get all files which are not in the crates/syntax/test_data folder
        !path.components().any(|component| component.as_os_str() == "test_data")
    });

    assert!(
        files.len() > 100,
        "self_hosting_parsing found too few files - is it running in the right directory?"
    );

    let errors = files
        .into_par_iter()
        .filter_map(|file| {
            let text = read_text(&file);
            match SourceFile::parse(&text).ok() {
                Ok(_) => None,
                Err(err) => Some((file, err)),
            }
        })
        .collect::<Vec<_>>();

    if !errors.is_empty() {
        let errors = errors
            .into_iter()
            .map(|(path, err)| format!("{}: {:?}\n", path.display(), err[0]))
            .collect::<String>();
        panic!("Parsing errors:\n{}\n", errors);
    }
}

fn test_data_dir() -> PathBuf {
    project_root().join("crates/syntax/test_data")
}

fn assert_errors_are_present(errors: &[SyntaxError], path: &Path) {
    assert!(!errors.is_empty(), "There should be errors in the file {:?}", path.display());
}
fn assert_errors_are_absent(errors: &[SyntaxError], path: &Path) {
    assert_eq!(
        errors,
        &[] as &[SyntaxError],
        "There should be no errors in the file {:?}",
        path.display(),
    );
}

fn dump_tokens_and_errors(tokens: &[Token], errors: &[SyntaxError], text: &str) -> String {
    let mut acc = String::new();
    let mut offset: TextSize = 0.into();
    for token in tokens {
        let token_len = token.len;
        let token_text = &text[TextRange::at(offset, token.len)];
        offset += token.len;
        writeln!(acc, "{:?} {:?} {:?}", token.kind, token_len, token_text).unwrap();
    }
    for err in errors {
        writeln!(acc, "> error{:?} token({:?}) msg({})", err.range(), &text[err.range()], err)
            .unwrap();
    }
    acc
}

fn fragment_parser_dir_test<T, F>(ok_paths: &[&str], err_paths: &[&str], f: F)
where
    T: crate::AstNode,
    F: Fn(&str) -> Result<T, ()>,
{
    dir_tests(&test_data_dir(), ok_paths, "rast", |text, path| {
        if let Ok(node) = f(text) {
            format!("{:#?}", crate::ast::AstNode::syntax(&node))
        } else {
            panic!("Failed to parse '{:?}'", path);
        }
    });
    dir_tests(&test_data_dir(), err_paths, "rast", |text, path| {
        if f(text).is_ok() {
            panic!("'{:?}' successfully parsed when it should have errored", path);
        } else {
            "ERROR\n".to_owned()
        }
    });
}

/// Calls callback `f` with input code and file paths for each `.rs` file in `test_data_dir`
/// subdirectories defined by `paths`.
///
/// If the content of the matching output file differs from the output of `f()`
/// the test will fail.
///
/// If there is no matching output file it will be created and filled with the
/// output of `f()`, but the test will fail.
fn dir_tests<F>(test_data_dir: &Path, paths: &[&str], outfile_extension: &str, f: F)
where
    F: Fn(&str, &Path) -> String,
{
    for (path, input_code) in collect_rust_files(test_data_dir, paths) {
        let actual = f(&input_code, &path);
        let path = path.with_extension(outfile_extension);
        expect_file![path].assert_eq(&actual)
    }
}

/// Collects all `.rs` files from `dir` subdirectories defined by `paths`.
fn collect_rust_files(root_dir: &Path, paths: &[&str]) -> Vec<(PathBuf, String)> {
    paths
        .iter()
        .flat_map(|path| {
            let path = root_dir.to_owned().join(path);
            rust_files_in_dir(&path).into_iter()
        })
        .map(|path| {
            let text = read_text(&path);
            (path, text)
        })
        .collect()
}

/// Collects paths to all `.rs` files from `dir` in a sorted `Vec<PathBuf>`.
fn rust_files_in_dir(dir: &Path) -> Vec<PathBuf> {
    let mut acc = Vec::new();
    for file in fs::read_dir(&dir).unwrap() {
        let file = file.unwrap();
        let path = file.path();
        if path.extension().unwrap_or_default() == "rs" {
            acc.push(path);
        }
    }
    acc.sort();
    acc
}

/// Read file and normalize newlines.
///
/// `rustc` seems to always normalize `\r\n` newlines to `\n`:
///
/// ```
/// let s = "
/// ";
/// assert_eq!(s.as_bytes(), &[10]);
/// ```
///
/// so this should always be correct.
fn read_text(path: &Path) -> String {
    fs::read_to_string(path)
        .unwrap_or_else(|_| panic!("File at {:?} should be valid", path))
        .replace("\r\n", "\n")
}
<commit_msg>internal: use single env var to controll all benchmarks<commit_after>mod sourcegen_tests;
mod sourcegen_ast;
mod ast_src;

use std::{
    fmt::Write,
    fs,
    path::{Path, PathBuf},
};

use ast::NameOwner;
use expect_test::expect_file;
use rayon::prelude::*;
use test_utils::{bench, bench_fixture, project_root};

use crate::{ast, fuzz, tokenize, AstNode, SourceFile, SyntaxError, TextRange, TextSize, Token};

#[test]
fn lexer_tests() {
    // FIXME:
    // * Add tests for unicode escapes in byte-character and [raw]-byte-string literals
    // * Add tests for unescape errors

    dir_tests(&test_data_dir(), &["lexer/ok"], "txt", |text, path| {
        let (tokens, errors) = tokenize(text);
        assert_errors_are_absent(&errors, path);
        dump_tokens_and_errors(&tokens, &errors, text)
    });
    dir_tests(&test_data_dir(), &["lexer/err"], "txt", |text, path| {
        let (tokens, errors) = tokenize(text);
        assert_errors_are_present(&errors, path);
        dump_tokens_and_errors(&tokens, &errors, text)
    });
}

#[test]
fn parse_smoke_test() {
    let code = r##"
fn main() {
    println!("Hello, world!")
}
    "##;

    let parse = SourceFile::parse(code);
    // eprintln!("{:#?}", parse.syntax_node());
    assert!(parse.ok().is_ok());
}

#[test]
fn benchmark_parser() {
    if std::env::var("RUN_SLOW_BENCHES").is_err() {
        return;
    }

    let data = bench_fixture::glorious_old_parser();
    let tree = {
        let _b = bench("parsing");
        let p = SourceFile::parse(&data);
        assert!(p.errors.is_empty());
        assert_eq!(p.tree().syntax.text_range().len(), 352474.into());
        p.tree()
    };

    {
        let _b = bench("tree traversal");
        let fn_names =
            tree.syntax().descendants().filter_map(ast::Fn::cast).filter_map(|f| f.name()).count();
        assert_eq!(fn_names, 268);
    }
}

#[test]
fn parser_tests() {
    dir_tests(&test_data_dir(), &["parser/inline/ok", "parser/ok"], "rast", |text, path| {
        let parse = SourceFile::parse(text);
        let errors = parse.errors();
        assert_errors_are_absent(errors, path);
        parse.debug_dump()
    });
    dir_tests(&test_data_dir(), &["parser/err", "parser/inline/err"], "rast", |text, path| {
        let parse = SourceFile::parse(text);
        let errors = parse.errors();
        assert_errors_are_present(errors, path);
        parse.debug_dump()
    });
}

#[test]
fn expr_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/expr/ok"],
        &["parser/fragments/expr/err"],
        crate::ast::Expr::parse,
    );
}

#[test]
fn path_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/path/ok"],
        &["parser/fragments/path/err"],
        crate::ast::Path::parse,
    );
}

#[test]
fn pattern_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/pattern/ok"],
        &["parser/fragments/pattern/err"],
        crate::ast::Pat::parse,
    );
}

#[test]
fn item_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/item/ok"],
        &["parser/fragments/item/err"],
        crate::ast::Item::parse,
    );
}

#[test]
fn type_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/type/ok"],
        &["parser/fragments/type/err"],
        crate::ast::Type::parse,
    );
}

#[test]
fn stmt_parser_tests() {
    fragment_parser_dir_test(
        &["parser/fragments/stmt/ok"],
        &["parser/fragments/stmt/err"],
        crate::ast::Stmt::parse,
    );
}

#[test]
fn parser_fuzz_tests() {
    for (_, text) in collect_rust_files(&test_data_dir(), &["parser/fuzz-failures"]) {
        fuzz::check_parser(&text)
    }
}

#[test]
fn reparse_fuzz_tests() {
    for (_, text) in collect_rust_files(&test_data_dir(), &["reparse/fuzz-failures"]) {
        let check = fuzz::CheckReparse::from_data(text.as_bytes()).unwrap();
        check.run();
    }
}

/// Test that Rust-analyzer can parse and validate the rust-analyzer
#[test]
fn self_hosting_parsing() {
    let crates_dir = project_root().join("crates");

    let mut files = ::sourcegen::list_rust_files(&crates_dir);
    files.retain(|path| {
        // Get all files which are not in the crates/syntax/test_data folder
        !path.components().any(|component| component.as_os_str() == "test_data")
    });

    assert!(
        files.len() > 100,
        "self_hosting_parsing found too few files - is it running in the right directory?"
    );

    let errors = files
        .into_par_iter()
        .filter_map(|file| {
            let text = read_text(&file);
            match SourceFile::parse(&text).ok() {
                Ok(_) => None,
                Err(err) => Some((file, err)),
            }
        })
        .collect::<Vec<_>>();

    if !errors.is_empty() {
        let errors = errors
            .into_iter()
            .map(|(path, err)| format!("{}: {:?}\n", path.display(), err[0]))
            .collect::<String>();
        panic!("Parsing errors:\n{}\n", errors);
    }
}

fn test_data_dir() -> PathBuf {
    project_root().join("crates/syntax/test_data")
}

fn assert_errors_are_present(errors: &[SyntaxError], path: &Path) {
    assert!(!errors.is_empty(), "There should be errors in the file {:?}", path.display());
}
fn assert_errors_are_absent(errors: &[SyntaxError], path: &Path) {
    assert_eq!(
        errors,
        &[] as &[SyntaxError],
        "There should be no errors in the file {:?}",
        path.display(),
    );
}

fn dump_tokens_and_errors(tokens: &[Token], errors: &[SyntaxError], text: &str) -> String {
    let mut acc = String::new();
    let mut offset: TextSize = 0.into();
    for token in tokens {
        let token_len = token.len;
        let token_text = &text[TextRange::at(offset, token.len)];
        offset += token.len;
        writeln!(acc, "{:?} {:?} {:?}", token.kind, token_len, token_text).unwrap();
    }
    for err in errors {
        writeln!(acc, "> error{:?} token({:?}) msg({})", err.range(), &text[err.range()], err)
            .unwrap();
    }
    acc
}

fn fragment_parser_dir_test<T, F>(ok_paths: &[&str], err_paths: &[&str], f: F)
where
    T: crate::AstNode,
    F: Fn(&str) -> Result<T, ()>,
{
    dir_tests(&test_data_dir(), ok_paths, "rast", |text, path| {
        if let Ok(node) = f(text) {
            format!("{:#?}", crate::ast::AstNode::syntax(&node))
        } else {
            panic!("Failed to parse '{:?}'", path);
        }
    });
    dir_tests(&test_data_dir(), err_paths, "rast", |text, path| {
        if f(text).is_ok() {
            panic!("'{:?}' successfully parsed when it should have errored", path);
        } else {
            "ERROR\n".to_owned()
        }
    });
}

/// Calls callback `f` with input code and file paths for each `.rs` file in `test_data_dir`
/// subdirectories defined by `paths`.
///
/// If the content of the matching output file differs from the output of `f()`
/// the test will fail.
///
/// If there is no matching output file it will be created and filled with the
/// output of `f()`, but the test will fail.
fn dir_tests<F>(test_data_dir: &Path, paths: &[&str], outfile_extension: &str, f: F)
where
    F: Fn(&str, &Path) -> String,
{
    for (path, input_code) in collect_rust_files(test_data_dir, paths) {
        let actual = f(&input_code, &path);
        let path = path.with_extension(outfile_extension);
        expect_file![path].assert_eq(&actual)
    }
}

/// Collects all `.rs` files from `dir` subdirectories defined by `paths`.
fn collect_rust_files(root_dir: &Path, paths: &[&str]) -> Vec<(PathBuf, String)> {
    paths
        .iter()
        .flat_map(|path| {
            let path = root_dir.to_owned().join(path);
            rust_files_in_dir(&path).into_iter()
        })
        .map(|path| {
            let text = read_text(&path);
            (path, text)
        })
        .collect()
}

/// Collects paths to all `.rs` files from `dir` in a sorted `Vec<PathBuf>`.
fn rust_files_in_dir(dir: &Path) -> Vec<PathBuf> {
    let mut acc = Vec::new();
    for file in fs::read_dir(&dir).unwrap() {
        let file = file.unwrap();
        let path = file.path();
        if path.extension().unwrap_or_default() == "rs" {
            acc.push(path);
        }
    }
    acc.sort();
    acc
}

/// Read file and normalize newlines.
///
/// `rustc` seems to always normalize `\r\n` newlines to `\n`:
///
/// ```
/// let s = "
/// ";
/// assert_eq!(s.as_bytes(), &[10]);
/// ```
///
/// so this should always be correct.
fn read_text(path: &Path) -> String {
    fs::read_to_string(path)
        .unwrap_or_else(|_| panic!("File at {:?} should be valid", path))
        .replace("\r\n", "\n")
}
<|endoftext|>


<commit_before>//! [GET /_matrix/client/r0/rooms/{roomId}/members](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-members)

use ruma_api::ruma_api;
use ruma_events::{room::member::MemberEvent, EventJson};
use ruma_identifiers::RoomId;
use serde::{Deserialize, Serialize};

ruma_api! {
    metadata {
        description: "Get membership events for a room.",
        method: GET,
        name: "get_member_events",
        path: "/_matrix/client/r0/rooms/:room_id/members",
        rate_limited: false,
        requires_authentication: true,
    }

    request {
        /// The room to get the member events for.
        #[ruma_api(path)]
        pub room_id: RoomId,

        /// The point in time (pagination token) to return members for in the room. This token can
        /// be obtained from a prev_batch token returned for each room by the sync API.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub at: Option<String>,

        /// The kind of memberships to filter for. Defaults to no filtering if unspecified. When
        /// specified alongside not_membership, the two parameters create an 'or' condition: either
        /// the membership is the same as membership or is not the same as not_membership.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub membership: Option<MembershipEventFilter>,

        /// The kind of memberships to *exclude* from the results. Defaults to no filtering if
        /// unspecified.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub not_membership: Option<MembershipEventFilter>,
    }

    response {
        /// A list of member events.
        pub chunk: Vec<EventJson<MemberEvent>>
    }

    error: crate::Error
}

/// The kind of membership events to filter for.
#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum MembershipEventFilter {
    /// The user has joined.
    Join,

    /// The user has been invited.
    Invite,

    /// The user has left.
    Leave,

    /// The user has been banned.
    Ban,
}
<commit_msg>Add a test to get_member_events<commit_after>//! [GET /_matrix/client/r0/rooms/{roomId}/members](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-members)

use ruma_api::ruma_api;
use ruma_events::{room::member::MemberEvent, EventJson};
use ruma_identifiers::RoomId;
use serde::{Deserialize, Serialize};

ruma_api! {
    metadata {
        description: "Get membership events for a room.",
        method: GET,
        name: "get_member_events",
        path: "/_matrix/client/r0/rooms/:room_id/members",
        rate_limited: false,
        requires_authentication: true,
    }

    request {
        /// The room to get the member events for.
        #[ruma_api(path)]
        pub room_id: RoomId,

        /// The point in time (pagination token) to return members for in the room. This token can
        /// be obtained from a prev_batch token returned for each room by the sync API.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub at: Option<String>,

        /// The kind of memberships to filter for. Defaults to no filtering if unspecified. When
        /// specified alongside not_membership, the two parameters create an 'or' condition: either
        /// the membership is the same as membership or is not the same as not_membership.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub membership: Option<MembershipEventFilter>,

        /// The kind of memberships to *exclude* from the results. Defaults to no filtering if
        /// unspecified.
        #[serde(skip_serializing_if = "Option::is_none")]
        #[ruma_api(query)]
        pub not_membership: Option<MembershipEventFilter>,
    }

    response {
        /// A list of member events.
        pub chunk: Vec<EventJson<MemberEvent>>
    }

    error: crate::Error
}

/// The kind of membership events to filter for.
#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum MembershipEventFilter {
    /// The user has joined.
    Join,

    /// The user has been invited.
    Invite,

    /// The user has left.
    Leave,

    /// The user has been banned.
    Ban,
}

#[cfg(test)]
mod tests {
    use std::convert::TryInto;

    use matches::assert_matches;

    use super::{MembershipEventFilter, Request};

    #[test]
    fn deserialization() {
        let uri = http::Uri::builder()
            .scheme("https")
            .authority("example.org")
            .path_and_query(
                "/_matrix/client/r0/rooms/!dummy%3Aexample.org/members\
                 ?not_membership=leave\
                 &at=1026",
            )
            .build()
            .unwrap();

        let req: Result<Request, _> =
            http::Request::builder().uri(uri).body(Vec::<u8>::new()).unwrap().try_into();

        assert_matches!(
            req,
            Ok(Request {
                room_id,
                at: Some(at),
                membership: None,
                not_membership: Some(MembershipEventFilter::Leave),
            }) if room_id == "!dummy:example.org" && at == "1026"
        );
    }
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

//! Execution plan for reading Parquet files

use std::fs::File;
use std::rc::Rc;
use std::sync::{Arc, Mutex};
use std::{fmt, thread};

use crate::error::{ExecutionError, Result};
use crate::execution::physical_plan::common;
use crate::execution::physical_plan::{ExecutionPlan, Partition};
use arrow::datatypes::{Schema, SchemaRef};
use arrow::error::{ArrowError, Result as ArrowResult};
use arrow::record_batch::{RecordBatch, RecordBatchReader};
use parquet::file::reader::SerializedFileReader;

use crossbeam::channel::{bounded, Receiver, RecvError, Sender};
use fmt::{Debug, Formatter};
use parquet::arrow::{ArrowReader, ParquetFileArrowReader};

/// Execution plan for scanning a Parquet file
#[derive(Debug)]
pub struct ParquetExec {
    /// Path to directory containing partitioned Parquet files with the same schema
    filenames: Vec<String>,
    /// Schema after projection is applied
    schema: SchemaRef,
    /// Projection for which columns to load
    projection: Vec<usize>,
    /// Batch size
    batch_size: usize,
}

impl ParquetExec {
    /// Create a new Parquet reader execution plan
    pub fn try_new(
        path: &str,
        projection: Option<Vec<usize>>,
        batch_size: usize,
    ) -> Result<Self> {
        let mut filenames: Vec<String> = vec![];
        common::build_file_list(path, &mut filenames, ".parquet")?;
        if filenames.is_empty() {
            Err(ExecutionError::General("No files found".to_string()))
        } else {
            let file = File::open(&filenames[0])?;
            let file_reader = Rc::new(SerializedFileReader::new(file)?);
            let mut arrow_reader = ParquetFileArrowReader::new(file_reader);
            let schema = arrow_reader.get_schema()?;

            let projection = match projection {
                Some(p) => p,
                None => (0..schema.fields().len()).collect(),
            };

            let projected_schema = Schema::new(
                projection
                    .iter()
                    .map(|i| schema.field(*i).clone())
                    .collect(),
            );

            Ok(Self {
                filenames,
                schema: Arc::new(projected_schema),
                projection,
                batch_size,
            })
        }
    }
}

impl ExecutionPlan for ParquetExec {
    fn schema(&self) -> SchemaRef {
        self.schema.clone()
    }

    fn partitions(&self) -> Result<Vec<Arc<dyn Partition>>> {
        let partitions = self
            .filenames
            .iter()
            .map(|filename| {
                Arc::new(ParquetPartition::new(
                    &filename,
                    self.projection.clone(),
                    self.schema.clone(),
                    self.batch_size,
                )) as Arc<dyn Partition>
            })
            .collect();
        Ok(partitions)
    }
}

struct ParquetPartition {
    iterator: Arc<Mutex<dyn RecordBatchReader + Send + Sync>>,
}

impl Debug for ParquetPartition {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        f.debug_struct("ParquetPartition").finish()
    }
}

impl ParquetPartition {
    /// Create a new Parquet partition
    pub fn new(
        filename: &str,
        projection: Vec<usize>,
        schema: SchemaRef,
        batch_size: usize,
    ) -> Self {
        // because the parquet implementation is not thread-safe, it is necessary to execute
        // on a thread and communicate with channels
        let (response_tx, response_rx): (
            Sender<ArrowResult<Option<RecordBatch>>>,
            Receiver<ArrowResult<Option<RecordBatch>>>,
        ) = bounded(2);

        let filename = filename.to_string();

        thread::spawn(move || {
            //TODO error handling, remove unwraps

            // open file
            let file = File::open(&filename).unwrap();
            match SerializedFileReader::new(file) {
                Ok(file_reader) => {
                    let file_reader = Rc::new(file_reader);

                    let mut arrow_reader = ParquetFileArrowReader::new(file_reader);

                    match arrow_reader
                        .get_record_reader_by_columns(projection, batch_size)
                    {
                        Ok(mut batch_reader) => loop {
                            match batch_reader.next_batch() {
                                Ok(Some(batch)) => {
                                    response_tx.send(Ok(Some(batch))).unwrap();
                                }
                                Ok(None) => {
                                    response_tx.send(Ok(None)).unwrap();
                                    break;
                                }
                                Err(e) => {
                                    response_tx
                                        .send(Err(ArrowError::ParquetError(format!(
                                            "{:?}",
                                            e
                                        ))))
                                        .unwrap();
                                    break;
                                }
                            }
                        },

                        Err(e) => {
                            response_tx
                                .send(Err(ArrowError::ParquetError(format!("{:?}", e))))
                                .unwrap();
                        }
                    }
                }

                Err(e) => {
                    response_tx
                        .send(Err(ArrowError::ParquetError(format!("{:?}", e))))
                        .unwrap();
                }
            }
        });

        let iterator = Arc::new(Mutex::new(ParquetIterator {
            schema,
            response_rx,
        }));

        Self { iterator }
    }
}

impl Partition for ParquetPartition {
    fn execute(&self) -> Result<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>> {
        Ok(self.iterator.clone())
    }
}

struct ParquetIterator {
    schema: SchemaRef,
    response_rx: Receiver<ArrowResult<Option<RecordBatch>>>,
}

impl RecordBatchReader for ParquetIterator {
    fn schema(&self) -> SchemaRef {
        self.schema.clone()
    }

    fn next_batch(&mut self) -> ArrowResult<Option<RecordBatch>> {
        match self.response_rx.recv() {
            Ok(batch) => batch,
            // RecvError means receiver has exited and closed the channel
            Err(RecvError) => Ok(None),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::env;

    #[test]
    fn test() -> Result<()> {
        let testdata =
            env::var("PARQUET_TEST_DATA").expect("PARQUET_TEST_DATA not defined");
        let filename = format!("{}/alltypes_plain.parquet", testdata);
        let parquet_exec = ParquetExec::try_new(&filename, Some(vec![0, 1, 2]), 1024)?;
        let partitions = parquet_exec.partitions()?;
        assert_eq!(partitions.len(), 1);

        let results = partitions[0].execute()?;
        let mut results = results.lock().unwrap();
        let batch = results.next_batch()?.unwrap();

        assert_eq!(8, batch.num_rows());
        assert_eq!(3, batch.num_columns());

        let schema = batch.schema();
        let field_names: Vec<&str> =
            schema.fields().iter().map(|f| f.name().as_str()).collect();
        assert_eq!(vec!["id", "bool_col", "tinyint_col"], field_names);

        let batch = results.next_batch()?;
        assert!(batch.is_none());

        let batch = results.next_batch()?;
        assert!(batch.is_none());

        let batch = results.next_batch()?;
        assert!(batch.is_none());

        Ok(())
    }
}
<commit_msg>ARROW-9712: [Rust] [DataFusion] Fix parquet error handling and general code improvements<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

//! Execution plan for reading Parquet files

use std::fs::File;
use std::rc::Rc;
use std::sync::{Arc, Mutex};
use std::{fmt, thread};

use crate::error::{ExecutionError, Result};
use crate::execution::physical_plan::common;
use crate::execution::physical_plan::{ExecutionPlan, Partition};
use arrow::datatypes::{Schema, SchemaRef};
use arrow::error::{ArrowError, Result as ArrowResult};
use arrow::record_batch::{RecordBatch, RecordBatchReader};
use parquet::file::reader::SerializedFileReader;

use crossbeam::channel::{bounded, Receiver, RecvError, Sender};
use fmt::{Debug, Formatter};
use parquet::arrow::{ArrowReader, ParquetFileArrowReader};

/// Execution plan for scanning a Parquet file
#[derive(Debug)]
pub struct ParquetExec {
    /// Path to directory containing partitioned Parquet files with the same schema
    filenames: Vec<String>,
    /// Schema after projection is applied
    schema: SchemaRef,
    /// Projection for which columns to load
    projection: Vec<usize>,
    /// Batch size
    batch_size: usize,
}

impl ParquetExec {
    /// Create a new Parquet reader execution plan
    pub fn try_new(
        path: &str,
        projection: Option<Vec<usize>>,
        batch_size: usize,
    ) -> Result<Self> {
        let mut filenames: Vec<String> = vec![];
        common::build_file_list(path, &mut filenames, ".parquet")?;
        if filenames.is_empty() {
            Err(ExecutionError::General("No files found".to_string()))
        } else {
            let file = File::open(&filenames[0])?;
            let file_reader = Rc::new(SerializedFileReader::new(file)?);
            let mut arrow_reader = ParquetFileArrowReader::new(file_reader);
            let schema = arrow_reader.get_schema()?;

            let projection = match projection {
                Some(p) => p,
                None => (0..schema.fields().len()).collect(),
            };

            let projected_schema = Schema::new(
                projection
                    .iter()
                    .map(|i| schema.field(*i).clone())
                    .collect(),
            );

            Ok(Self {
                filenames,
                schema: Arc::new(projected_schema),
                projection,
                batch_size,
            })
        }
    }
}

impl ExecutionPlan for ParquetExec {
    fn schema(&self) -> SchemaRef {
        self.schema.clone()
    }

    fn partitions(&self) -> Result<Vec<Arc<dyn Partition>>> {
        let partitions = self
            .filenames
            .iter()
            .map(|filename| {
                Arc::new(ParquetPartition::new(
                    &filename,
                    self.projection.clone(),
                    self.schema.clone(),
                    self.batch_size,
                )) as Arc<dyn Partition>
            })
            .collect();
        Ok(partitions)
    }
}

struct ParquetPartition {
    iterator: Arc<Mutex<dyn RecordBatchReader + Send + Sync>>,
}

impl Debug for ParquetPartition {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        f.debug_struct("ParquetPartition").finish()
    }
}

impl ParquetPartition {
    /// Create a new Parquet partition
    pub fn new(
        filename: &str,
        projection: Vec<usize>,
        schema: SchemaRef,
        batch_size: usize,
    ) -> Self {
        // because the parquet implementation is not thread-safe, it is necessary to execute
        // on a thread and communicate with channels
        let (response_tx, response_rx): (
            Sender<ArrowResult<Option<RecordBatch>>>,
            Receiver<ArrowResult<Option<RecordBatch>>>,
        ) = bounded(2);

        let filename = filename.to_string();

        thread::spawn(move || {
            if let Err(e) = read_file(&filename, projection, batch_size, response_tx) {
                println!("Parquet reader thread terminated due to error: {:?}", e);
            }
        });

        let iterator = Arc::new(Mutex::new(ParquetIterator {
            schema,
            response_rx,
        }));

        Self { iterator }
    }
}

fn send_result(
    response_tx: &Sender<ArrowResult<Option<RecordBatch>>>,
    result: ArrowResult<Option<RecordBatch>>,
) -> Result<()> {
    response_tx
        .send(result)
        .map_err(|e| ExecutionError::ExecutionError(format!("{:?}", e)))?;
    Ok(())
}

fn read_file(
    filename: &str,
    projection: Vec<usize>,
    batch_size: usize,
    response_tx: Sender<ArrowResult<Option<RecordBatch>>>,
) -> Result<()> {
    let file = File::open(&filename)?;
    let file_reader = Rc::new(SerializedFileReader::new(file)?);
    let mut arrow_reader = ParquetFileArrowReader::new(file_reader);
    let mut batch_reader =
        arrow_reader.get_record_reader_by_columns(projection.clone(), batch_size)?;
    loop {
        match batch_reader.next_batch() {
            Ok(Some(batch)) => send_result(&response_tx, Ok(Some(batch)))?,
            Ok(None) => {
                // finished reading file
                send_result(&response_tx, Ok(None))?;
                break;
            }
            Err(e) => {
                let err_msg = format!("Error reading batch from {}: {:?}", filename, e);
                // send error to operator
                send_result(
                    &response_tx,
                    Err(ArrowError::ParquetError(err_msg.clone())),
                )?;
                // terminate thread with error
                return Err(ExecutionError::ExecutionError(err_msg));
            }
        }
    }
    Ok(())
}

impl Partition for ParquetPartition {
    fn execute(&self) -> Result<Arc<Mutex<dyn RecordBatchReader + Send + Sync>>> {
        Ok(self.iterator.clone())
    }
}

struct ParquetIterator {
    schema: SchemaRef,
    response_rx: Receiver<ArrowResult<Option<RecordBatch>>>,
}

impl RecordBatchReader for ParquetIterator {
    fn schema(&self) -> SchemaRef {
        self.schema.clone()
    }

    fn next_batch(&mut self) -> ArrowResult<Option<RecordBatch>> {
        match self.response_rx.recv() {
            Ok(batch) => batch,
            // RecvError means receiver has exited and closed the channel
            Err(RecvError) => Ok(None),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::env;

    #[test]
    fn test() -> Result<()> {
        let testdata =
            env::var("PARQUET_TEST_DATA").expect("PARQUET_TEST_DATA not defined");
        let filename = format!("{}/alltypes_plain.parquet", testdata);
        let parquet_exec = ParquetExec::try_new(&filename, Some(vec![0, 1, 2]), 1024)?;
        let partitions = parquet_exec.partitions()?;
        assert_eq!(partitions.len(), 1);

        let results = partitions[0].execute()?;
        let mut results = results.lock().unwrap();
        let batch = results.next_batch()?.unwrap();

        assert_eq!(8, batch.num_rows());
        assert_eq!(3, batch.num_columns());

        let schema = batch.schema();
        let field_names: Vec<&str> =
            schema.fields().iter().map(|f| f.name().as_str()).collect();
        assert_eq!(vec!["id", "bool_col", "tinyint_col"], field_names);

        let batch = results.next_batch()?;
        assert!(batch.is_none());

        let batch = results.next_batch()?;
        assert!(batch.is_none());

        let batch = results.next_batch()?;
        assert!(batch.is_none());

        Ok(())
    }
}
<|endoftext|>


<commit_before>// Zinc, the bare metal stack for rust.
// Copyright 2014 Lionel Flandrin <lionel@svkt.org>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Driver for the Hitachi HD44780U LCD driver

use core::prelude::*;

use drivers::chario::CharIO;
use hal::pin::Gpio;
use hal::timer::Timer;

/// HD44780U driver context
pub struct Hd44780u<'a> {
  /// Timer used for protocol delays
  timer: &'a Timer + 'a,
  /// Register Selector wire
  rs:    &'a Gpio + 'a,
  /// Enable wire
  en:    &'a Gpio + 'a,
  /// The 4 data wires. Those must be wired to [d4, d5, d6, d7] in 4bit mode.
  data: [&'a Gpio + 'a, ..4],
}

/// The controller supports writing in either direction to accomodate various
/// languages.
pub enum MoveDir {
  /// Cursor moves right after write
  MoveRight,
  /// Cursor moves left after write
  MoveLeft,
}

/// The controller supports 5x8 and 5x10 dot fonts depending on the LCD used.
pub enum Font {
  /// Use 5x8 dot matrix font
  Font5x8,
  /// Use 5x10 dot matrix font
  Font5x10,
}

impl<'a> Hd44780u<'a> {
  /// Construct an Hd44780u instance
  pub fn new<'a>(timer: &'a Timer + 'a,
                 rs:    &'a Gpio  + 'a,
                 en:    &'a Gpio  + 'a,
                 data: [&'a Gpio  + 'a, ..4]) -> Hd44780u<'a> {
    Hd44780u { timer: timer, rs: rs, en: en, data: data }
  }

  /// Power up sequence for 4bit mode as detailed in page 46 of the datasheet.
  /// The two_lines and font parameters are used to set the LCD parameters. See
  /// `function_set` for more details.
  pub fn init(&self, two_lines: bool, font: Font) {
    // Start by pulling RS and EN low
    self.rs.set_low();
    self.en.set_low();

    // We need to wait at least 40ms after the chip is powered before we can
    // talk to it. Assume that the power just went on and wait.
    self.timer.wait_ms(50);

    // Set interface to 8bit
    self.write_nibble(0b0011);

    // Wait 4.1ms
    self.timer.wait_us(4100);

    // Set interface to 8bit
    self.write_nibble(0b0011);

    // Wait 100us
    self.timer.wait_us(100);

    // Set interface to 8bit (for the last time...)
    self.write_nibble(0b0011);
    self.timer.wait_us(100);

    // We can now finally switch to 4bit
    self.write_nibble(0b0010);
    self.timer.wait_us(100);

    // Set function
    self.function_set(two_lines, font);

    self.display_control(true, false, false);

    self.clear();

    // Set default mode
    self.mode_set(MoveRight, false);
  }

  /// Set cursor position to (`col`, `row`). (0, 0) is the top left.
  pub fn set_pos(&self, col: u8, row: u8) {
    // Rows are not continuous in DDRAM, there's a 0x40 offset between the
    // begining of each line.
    self.ddram_address_set(row * 0x40 + col);
  }

  /// Clear the entire display
  pub fn clear(&self) {
    self.instruction(0b0001);
    // I can't find in the datasheet how long this is supposed to take but it
    // seems to take at least a few ms.
    self.timer.wait_ms(5);
  }

  /// Return the cursor to the left of the first line of the display. Also
  /// returns the display to its original status if it was shifted (see
  /// `mode_set`).
  pub fn return_home(&self) {
    self.instruction(0b10);
    // This takes 1.52ms
    self.timer.wait_ms(2);
  }

  /// Set the writing direction and optional display shifting.
  ///
  /// If `shift_display` is true then when writing the cursor remains at the
  /// same place on the display and "pushes" existing data in the direction
  /// specified by dir.
  ///
  /// Calling `clear` resets the writing direction MoveRight but does not change
  /// the display shift setting.
  pub fn mode_set(&self, dir: MoveDir, shift_display: bool) {
    let mut cmd = 0b100;

    let d = match dir {
      MoveRight => 1,
      MoveLeft  => 0,
    };

    cmd |= d                     << 1;
    cmd |= (shift_display as u8) << 0;

    self.instruction(cmd);
  }

  /// Display control. If `on` is false nothing is displayed but the screen's
  /// contents remain in RAM and can be re-displayed at any moment. This
  /// function can also enable or disable the cursor and blinking.
  pub fn display_control(&self, on: bool, cursor: bool, blink: bool) {
    let mut cmd = 0b1000;

    cmd |= (on     as u8) << 2;
    cmd |= (cursor as u8) << 1;
    cmd |= (blink  as u8) << 0;

    self.instruction(cmd);
  }

  /// Shift cursor or display in a given direction
  pub fn shift(&self, dir: MoveDir, shift_display: bool) {
    let mut cmd = 0b10000;

    let d = match dir {
      MoveRight => 1,
      MoveLeft  => 0,
    };

    cmd |= (shift_display as u8) << 3;
    cmd |= d                     << 2;

    self.instruction(cmd);
  }

  /// Set the bus width, number of lines and character font size. This function
  /// can only be called once in the init sequence.
  ///
  /// Only 4bit wide interfaces are supported for now.
  ///
  /// `two_lines` should be
  /// `false` if the display only has one line, true otherwise.
  ///
  /// If each display character is a matrix of 5x10 dots `char_5x10` should be
  /// `true`. If it's 5x8 dots set it to `false`.
  fn function_set(&self, two_lines: bool, font: Font) {
    let mut cmd = 0b100000;

    // Only 4bit interface is supported at the moment
    cmd |= 0                 << 4;
    cmd |= (two_lines as u8) << 3;
    cmd |= match font {
      Font5x8  => 0,
      Font5x10 => 1,
    }                        << 2;

    self.instruction(cmd);
  }

  /// Set the curent read/write data address to the Character Generator RAM
  /// offset `addr`
  fn cgram_address_set(&self, addr: u8) {
    let mut cmd = 0b1000000;

    cmd |= addr & 0b111111;

    self.instruction(cmd);
  }

  /// Set the curent read/write data address to the Display Data RAM offset
  /// `addr`
  fn ddram_address_set(&self, addr: u8) {
    let mut cmd = 0b10000000;

    cmd |= addr & 0b1111111;

    self.instruction(cmd);
  }

  /// Create custom 5x8 char at index `pos` from `bitmap`. Only the 5LSBs of
  /// each bitmap lines are used. There can be only 8 custom 5x8 chars so pos
  /// must be in the range 0 ... 7.
  ///
  /// This resets the cursor positon to 0.
  pub fn custom_char_5x8(&self, pos: u8, bitmap: [ u8, ..8 ]) {
    if pos > 7 {
      panic!("Invalid character position");
    }

    self.cgram_address_set(pos << 3);

    for b in bitmap.iter() {
      self.data(b & 0b11111);
    }

    // Return the read/write pointer to the Display Data RAM
    self.set_pos(0, 0);
  }

  /// Create custom 5x10 char at index `pos` from `bitmap`. Only the 5LSBs of
  /// each bitmap lines are used. There can be only 8 custom 5x8 chars so pos
  /// must be in the range 0 ... 3.
  pub fn custom_char_5x10(&self, pos: u8, bitmap: [ u8, ..10 ]) {
    if pos > 3 {
      panic!("Invalid character position");
    }

    self.cgram_address_set(pos << 4);

    for b in bitmap.iter() {
      self.data(b & 0b11111);
    }

    // Return the read/write pointer to the Display Data RAM
    self.set_pos(0, 0);
  }

  /// Send a 8bit command code
  fn instruction(&self, cmd: u8) {
    self.rs.set_low();

    self.write_nibble(cmd >> 4);
    self.write_nibble(cmd);
    // Commands take *at least* 37us to execute, so default to that. If the
    // current instruction needs to delay for more than that the calling
    // function will wait some more.
    self.timer.wait_us(40);
  }

  /// Send 8bit of data
  fn data(&self, data: u8) {
    self.rs.set_high();

    self.write_nibble(data >> 4);
    self.write_nibble(data);
    // Data access take 37us
    self.timer.wait_us(40);
  }

  /// Write a 4bit value to the parallel interface.
  fn write_nibble(&self, v: u8) {
    // Set the 4 data lines in the parallel interface
    match v & 1 {
      0 => self.data[0].set_low(),
      _ => self.data[0].set_high(),
    }

    match v & 2 {
      0 => self.data[1].set_low(),
      _ => self.data[1].set_high(),
    }

    match v & 4 {
      0 => self.data[2].set_low(),
      _ => self.data[2].set_high(),
    }

    match v & 8 {
      0 => self.data[3].set_low(),
      _ => self.data[3].set_high(),
    }

    // Pulse the EN wire to notify the controller.
    self.en.set_high();
    // We need to assert EN for at least 450ns
    self.timer.wait_us(1);
    self.en.set_low();
  }
}

impl<'a> CharIO for Hd44780u<'a> {
  fn putc(&self, value: char) {
    self.data(value as u8);
  }
}
<commit_msg>hd44780u: Fix dereferences<commit_after>// Zinc, the bare metal stack for rust.
// Copyright 2014 Lionel Flandrin <lionel@svkt.org>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Driver for the Hitachi HD44780U LCD driver

use core::prelude::*;

use drivers::chario::CharIO;
use hal::pin::Gpio;
use hal::timer::Timer;

/// HD44780U driver context
pub struct Hd44780u<'a> {
  /// Timer used for protocol delays
  timer: &'a Timer + 'a,
  /// Register Selector wire
  rs:    &'a Gpio + 'a,
  /// Enable wire
  en:    &'a Gpio + 'a,
  /// The 4 data wires. Those must be wired to [d4, d5, d6, d7] in 4bit mode.
  data: [&'a Gpio + 'a, ..4],
}

/// The controller supports writing in either direction to accomodate various
/// languages.
pub enum MoveDir {
  /// Cursor moves right after write
  MoveRight,
  /// Cursor moves left after write
  MoveLeft,
}

/// The controller supports 5x8 and 5x10 dot fonts depending on the LCD used.
pub enum Font {
  /// Use 5x8 dot matrix font
  Font5x8,
  /// Use 5x10 dot matrix font
  Font5x10,
}

impl<'a> Hd44780u<'a> {
  /// Construct an Hd44780u instance
  pub fn new<'a>(timer: &'a Timer + 'a,
                 rs:    &'a Gpio  + 'a,
                 en:    &'a Gpio  + 'a,
                 data: [&'a Gpio  + 'a, ..4]) -> Hd44780u<'a> {
    Hd44780u { timer: timer, rs: rs, en: en, data: data }
  }

  /// Power up sequence for 4bit mode as detailed in page 46 of the datasheet.
  /// The two_lines and font parameters are used to set the LCD parameters. See
  /// `function_set` for more details.
  pub fn init(&self, two_lines: bool, font: Font) {
    // Start by pulling RS and EN low
    self.rs.set_low();
    self.en.set_low();

    // We need to wait at least 40ms after the chip is powered before we can
    // talk to it. Assume that the power just went on and wait.
    self.timer.wait_ms(50);

    // Set interface to 8bit
    self.write_nibble(0b0011);

    // Wait 4.1ms
    self.timer.wait_us(4100);

    // Set interface to 8bit
    self.write_nibble(0b0011);

    // Wait 100us
    self.timer.wait_us(100);

    // Set interface to 8bit (for the last time...)
    self.write_nibble(0b0011);
    self.timer.wait_us(100);

    // We can now finally switch to 4bit
    self.write_nibble(0b0010);
    self.timer.wait_us(100);

    // Set function
    self.function_set(two_lines, font);

    self.display_control(true, false, false);

    self.clear();

    // Set default mode
    self.mode_set(MoveRight, false);
  }

  /// Set cursor position to (`col`, `row`). (0, 0) is the top left.
  pub fn set_pos(&self, col: u8, row: u8) {
    // Rows are not continuous in DDRAM, there's a 0x40 offset between the
    // begining of each line.
    self.ddram_address_set(row * 0x40 + col);
  }

  /// Clear the entire display
  pub fn clear(&self) {
    self.instruction(0b0001);
    // I can't find in the datasheet how long this is supposed to take but it
    // seems to take at least a few ms.
    self.timer.wait_ms(5);
  }

  /// Return the cursor to the left of the first line of the display. Also
  /// returns the display to its original status if it was shifted (see
  /// `mode_set`).
  pub fn return_home(&self) {
    self.instruction(0b10);
    // This takes 1.52ms
    self.timer.wait_ms(2);
  }

  /// Set the writing direction and optional display shifting.
  ///
  /// If `shift_display` is true then when writing the cursor remains at the
  /// same place on the display and "pushes" existing data in the direction
  /// specified by dir.
  ///
  /// Calling `clear` resets the writing direction MoveRight but does not change
  /// the display shift setting.
  pub fn mode_set(&self, dir: MoveDir, shift_display: bool) {
    let mut cmd = 0b100;

    let d = match dir {
      MoveRight => 1,
      MoveLeft  => 0,
    };

    cmd |= d                     << 1;
    cmd |= (shift_display as u8) << 0;

    self.instruction(cmd);
  }

  /// Display control. If `on` is false nothing is displayed but the screen's
  /// contents remain in RAM and can be re-displayed at any moment. This
  /// function can also enable or disable the cursor and blinking.
  pub fn display_control(&self, on: bool, cursor: bool, blink: bool) {
    let mut cmd = 0b1000;

    cmd |= (on     as u8) << 2;
    cmd |= (cursor as u8) << 1;
    cmd |= (blink  as u8) << 0;

    self.instruction(cmd);
  }

  /// Shift cursor or display in a given direction
  pub fn shift(&self, dir: MoveDir, shift_display: bool) {
    let mut cmd = 0b10000;

    let d = match dir {
      MoveRight => 1,
      MoveLeft  => 0,
    };

    cmd |= (shift_display as u8) << 3;
    cmd |= d                     << 2;

    self.instruction(cmd);
  }

  /// Set the bus width, number of lines and character font size. This function
  /// can only be called once in the init sequence.
  ///
  /// Only 4bit wide interfaces are supported for now.
  ///
  /// `two_lines` should be
  /// `false` if the display only has one line, true otherwise.
  ///
  /// If each display character is a matrix of 5x10 dots `char_5x10` should be
  /// `true`. If it's 5x8 dots set it to `false`.
  fn function_set(&self, two_lines: bool, font: Font) {
    let mut cmd = 0b100000;

    // Only 4bit interface is supported at the moment
    cmd |= 0                 << 4;
    cmd |= (two_lines as u8) << 3;
    cmd |= match font {
      Font5x8  => 0,
      Font5x10 => 1,
    }                        << 2;

    self.instruction(cmd);
  }

  /// Set the curent read/write data address to the Character Generator RAM
  /// offset `addr`
  fn cgram_address_set(&self, addr: u8) {
    let mut cmd = 0b1000000;

    cmd |= addr & 0b111111;

    self.instruction(cmd);
  }

  /// Set the curent read/write data address to the Display Data RAM offset
  /// `addr`
  fn ddram_address_set(&self, addr: u8) {
    let mut cmd = 0b10000000;

    cmd |= addr & 0b1111111;

    self.instruction(cmd);
  }

  /// Create custom 5x8 char at index `pos` from `bitmap`. Only the 5LSBs of
  /// each bitmap lines are used. There can be only 8 custom 5x8 chars so pos
  /// must be in the range 0 ... 7.
  ///
  /// This resets the cursor positon to 0.
  pub fn custom_char_5x8(&self, pos: u8, bitmap: [ u8, ..8 ]) {
    if pos > 7 {
      panic!("Invalid character position");
    }

    self.cgram_address_set(pos << 3);

    for b in bitmap.iter() {
      self.data(*b & 0b11111);
    }

    // Return the read/write pointer to the Display Data RAM
    self.set_pos(0, 0);
  }

  /// Create custom 5x10 char at index `pos` from `bitmap`. Only the 5LSBs of
  /// each bitmap lines are used. There can be only 8 custom 5x8 chars so pos
  /// must be in the range 0 ... 3.
  pub fn custom_char_5x10(&self, pos: u8, bitmap: [ u8, ..10 ]) {
    if pos > 3 {
      panic!("Invalid character position");
    }

    self.cgram_address_set(pos << 4);

    for b in bitmap.iter() {
      self.data(*b & 0b11111);
    }

    // Return the read/write pointer to the Display Data RAM
    self.set_pos(0, 0);
  }

  /// Send a 8bit command code
  fn instruction(&self, cmd: u8) {
    self.rs.set_low();

    self.write_nibble(cmd >> 4);
    self.write_nibble(cmd);
    // Commands take *at least* 37us to execute, so default to that. If the
    // current instruction needs to delay for more than that the calling
    // function will wait some more.
    self.timer.wait_us(40);
  }

  /// Send 8bit of data
  fn data(&self, data: u8) {
    self.rs.set_high();

    self.write_nibble(data >> 4);
    self.write_nibble(data);
    // Data access take 37us
    self.timer.wait_us(40);
  }

  /// Write a 4bit value to the parallel interface.
  fn write_nibble(&self, v: u8) {
    // Set the 4 data lines in the parallel interface
    match v & 1 {
      0 => self.data[0].set_low(),
      _ => self.data[0].set_high(),
    }

    match v & 2 {
      0 => self.data[1].set_low(),
      _ => self.data[1].set_high(),
    }

    match v & 4 {
      0 => self.data[2].set_low(),
      _ => self.data[2].set_high(),
    }

    match v & 8 {
      0 => self.data[3].set_low(),
      _ => self.data[3].set_high(),
    }

    // Pulse the EN wire to notify the controller.
    self.en.set_high();
    // We need to assert EN for at least 450ns
    self.timer.wait_us(1);
    self.en.set_low();
  }
}

impl<'a> CharIO for Hd44780u<'a> {
  fn putc(&self, value: char) {
    self.data(value as u8);
  }
}
<|endoftext|>


<commit_before>#[derive(Serialize, Deserialize, Debug)]
pub struct Message {
    pub user: String,
    pub msg: String
}<commit_msg>added Command, CommandResponse<commit_after>#[derive(Serialize, Deserialize, Debug)]
pub struct Command {
    pub command: String,
    pub nickname: String
}

#[derive(Serialize, Deserialize, Debug)]
pub struct CommandResponse {
    pub status: String
}

#[derive(Serialize, Deserialize, Debug)]
pub struct Message {
    pub from: String,
    pub message: String,
    pub time: String
}
<|endoftext|>


<commit_before>extern crate cargo_tarpaulin;

use cargo_tarpaulin::launch_tarpaulin;
use cargo_tarpaulin::config::Config;
use std::env;
use std::time::Duration;

#[test]
fn incorrect_manifest_path() {
    let mut config = Config::default();
    config.manifest.push("__invalid_dir__");
    assert!(launch_tarpaulin(&config).is_err());
}

#[test]
fn simple_project_coverage() {
    let mut config = Config::default();
    config.test_timeout = Duration::from_secs(60);
    config.manifest = env::current_dir().unwrap();
    config.manifest.push("tests");
    config.manifest.push("data");
    config.manifest.push("simple_project");
    config.manifest.push("Cargo.toml");
    let res = launch_tarpaulin(&config).unwrap();

    let unused_hits = res.iter()
                         .filter(|x| x.path.file_name().unwrap() == "unused.rs")
                         .fold(0, |acc, ref x| acc + x.hits);
    assert_eq!(unused_hits, 0);

    let unused_hits = res.iter()
                         .filter(|x| x.path.file_name().unwrap() == "unused.rs")
                         .map(|x| x.line)
                         .collect::<Vec<_>>();

    assert_eq!(unused_hits.len(), 3);
    assert!(unused_hits.contains(&4));
    assert!(unused_hits.contains(&5));
    assert!(unused_hits.contains(&6));

    assert!(res.iter().any(|ref x| x.line == 6 &&
                                   x.hits == 0 &&
                                   x.path.file_name().unwrap() == "lib.rs"));

    assert!(res.iter().any(|ref x| x.line == 8 &&
                                   x.hits == 1 &&
                                   x.path.file_name().unwrap() == "lib.rs"));
}
<commit_msg>Updated integration test<commit_after>extern crate cargo_tarpaulin;

use cargo_tarpaulin::launch_tarpaulin;
use cargo_tarpaulin::config::Config;
use cargo_tarpaulin::traces::CoverageStat;
use std::env;
use std::time::Duration;

#[test]
fn incorrect_manifest_path() {
    let mut config = Config::default();
    config.manifest.push("__invalid_dir__");
    assert!(launch_tarpaulin(&config).is_err());
}

#[test]
fn simple_project_coverage() {
    let mut config = Config::default();
    config.test_timeout = Duration::from_secs(60);
    let mut test_dir = env::current_dir().unwrap();
    test_dir.push("tests");
    test_dir.push("data");
    test_dir.push("simple_project");
    config.manifest = test_dir.join("Cargo.toml");
    let res = launch_tarpaulin(&config).unwrap();
    let unused_file = test_dir.join("src/unused.rs");
    let unused_hits = res.covered_in_path(&unused_file);
    let unused_lines = res.coverable_in_path(&unused_file);
    assert_eq!(unused_hits, 0);
    assert_eq!(unused_lines, 3);
    let unused_hits = res.get_child_traces(&unused_file)
                         .iter()
                         .map(|x| x.line)
                         .collect::<Vec<_>>();

    assert_eq!(unused_hits.len(), 3);
    assert!(unused_hits.contains(&4));
    assert!(unused_hits.contains(&5));
    assert!(unused_hits.contains(&6));

    let lib_file = test_dir.join("src/lib.rs");
    let lib_traces = res.get_child_traces(&lib_file);
    for l in &lib_traces {
        if l.line == 6 {
            assert_eq!(CoverageStat::Line(0), l.stats);
        } else if l.line == 8 {
            assert_eq!(CoverageStat::Line(1), l.stats);
        }
    }
}
<|endoftext|>


<commit_before>use crate::ShardManagerContainer;
use serenity::prelude::*;
use serenity::model::prelude::*;
use serenity::framework::standard::{
    CommandResult,
    macros::command,
};

#[command]
#[owners_only]
async fn quit(ctx: &Context, msg: &Message) -> CommandResult {
    let data = ctx.data.read().await;

    if let Some(manager) = data.get::<ShardManagerContainer>() {
        manager.lock().await.shutdown_all().await;
    } else {
        msg.reply(ctx, "There was a problem getting the shard manager").await?;

        return Ok(());
    }

    msg.reply(ctx, "Shutting down!").await?;

    Ok(())
}
<commit_msg>Send the shutdown message earlier in the sample bot example (#984)<commit_after>use crate::ShardManagerContainer;
use serenity::prelude::*;
use serenity::model::prelude::*;
use serenity::framework::standard::{
    CommandResult,
    macros::command,
};

#[command]
#[owners_only]
async fn quit(ctx: &Context, msg: &Message) -> CommandResult {
    let data = ctx.data.read().await;

    if let Some(manager) = data.get::<ShardManagerContainer>() {
        msg.reply(ctx, "Shutting down!").await?;
        manager.lock().await.shutdown_all().await;
    } else {
        msg.reply(ctx, "There was a problem getting the shard manager").await?;

        return Ok(());
    }

    Ok(())
}
<|endoftext|>


<commit_before>// pest. The Elegant Parser
// Copyright (c) 2018 Dragoș Tiselice
//
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. All files in the project carrying such notice may not be copied,
// modified, or distributed except according to those terms.
#![no_std]
#![cfg_attr(feature = "const_prec_climber", feature(const_fn))]

//! # pest. The Elegant Parser
//!
//! pest is a general purpose parser written in Rust with a focus on accessibility, correctness,
//! and performance. It uses parsing expression grammars (or [PEG]) as input, which are similar in
//! spirit to regular expressions, but which offer the enhanced expressivity needed to parse
//! complex languages.
//!
//! [PEG]: https://en.wikipedia.org/wiki/Parsing_expression_grammar
//!
//! ## Getting started
//!
//! The recommended way to start parsing with pest is to read the official [book].
//!
//! Other helpful resources:
//!
//! * API reference on [docs.rs]
//! * play with grammars and share them on our [fiddle]
//! * leave feedback, ask questions, or greet us on [Gitter]
//!
//! [book]: https://pest-parser.github.io/book
//! [docs.rs]: https://docs.rs/pest
//! [fiddle]: https://pest-parser.github.io/#editor
//! [Gitter]: https://gitter.im/dragostis/pest
//!
//! ## Usage
//!
//! The core of pest is the trait [`Parser`], which provides an interface to the parsing
//! functionality.
//!
//! The accompanying crate `pest_derive` can automatically generate a [`Parser`] from a PEG
//! grammar. Using `pest_derive` is highly encouraged, but it is also possible to implement
//! [`Parser`] manually if required.
//!
//! ## `.pest` files
//!
//! Grammar definitions reside in custom `.pest` files located in the crate `src` directory.
//! Parsers are automatically generated from these files using `#[derive(Parser)]` and a special
//! `#[grammar = "..."]` attribute on a dummy struct.
//!
//! ```ignore
//! #[derive(Parser)]
//! #[grammar = "path/to/my_grammar.pest"] // relative to src
//! struct MyParser;
//! ```
//!
//! The syntax of `.pest` files is documented in the [`pest_derive` crate].
//!
//! ## Inline grammars
//!
//! Grammars can also be inlined by using the `#[grammar_inline = "..."]` attribute.
//!
//! [`Parser`]: trait.Parser.html
//! [`pest_derive` crate]: https://docs.rs/pest_derive/

#![doc(html_root_url = "https://docs.rs/pest")]

extern crate alloc;
#[cfg(feature = "std")]
extern crate std;
extern crate ucd_trie;

#[cfg(feature = "pretty-print")]
extern crate serde;
#[cfg(feature = "pretty-print")]
extern crate serde_json;

use core::fmt::Debug;
use core::hash::Hash;
pub use parser::Parser;
pub use parser_state::{state, Atomicity, Lookahead, MatchDir, ParseResult, ParserState};
pub use position::Position;
pub use span::{Lines, Span};
pub use token::Token;

pub mod error;
pub mod iterators;
mod macros;
mod parser;
mod parser_state;
mod position;
pub mod prec_climber;
mod span;
mod stack;
mod token;
#[doc(hidden)]
pub mod unicode;

/// A trait which parser rules must implement.
///
/// This trait is set up so that any struct that implements all of its required traits will
/// automatically implement this trait as well.
///
/// This is essentially a [trait alias](https://github.com/rust-lang/rfcs/pull/1733). When trait
/// aliases are implemented, this may be replaced by one.
pub trait RuleType: Copy + Debug + Eq + Hash + Ord {}

impl<T: Copy + Debug + Eq + Hash + Ord> RuleType for T {}
<commit_msg>pest: Adjust `const_prec_climber` feature gate<commit_after>// pest. The Elegant Parser
// Copyright (c) 2018 Dragoș Tiselice
//
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. All files in the project carrying such notice may not be copied,
// modified, or distributed except according to those terms.
#![no_std]
#![cfg_attr(feature = "const_prec_climber", feature(const_fn_trait_bound))]

//! # pest. The Elegant Parser
//!
//! pest is a general purpose parser written in Rust with a focus on accessibility, correctness,
//! and performance. It uses parsing expression grammars (or [PEG]) as input, which are similar in
//! spirit to regular expressions, but which offer the enhanced expressivity needed to parse
//! complex languages.
//!
//! [PEG]: https://en.wikipedia.org/wiki/Parsing_expression_grammar
//!
//! ## Getting started
//!
//! The recommended way to start parsing with pest is to read the official [book].
//!
//! Other helpful resources:
//!
//! * API reference on [docs.rs]
//! * play with grammars and share them on our [fiddle]
//! * leave feedback, ask questions, or greet us on [Gitter]
//!
//! [book]: https://pest-parser.github.io/book
//! [docs.rs]: https://docs.rs/pest
//! [fiddle]: https://pest-parser.github.io/#editor
//! [Gitter]: https://gitter.im/dragostis/pest
//!
//! ## Usage
//!
//! The core of pest is the trait [`Parser`], which provides an interface to the parsing
//! functionality.
//!
//! The accompanying crate `pest_derive` can automatically generate a [`Parser`] from a PEG
//! grammar. Using `pest_derive` is highly encouraged, but it is also possible to implement
//! [`Parser`] manually if required.
//!
//! ## `.pest` files
//!
//! Grammar definitions reside in custom `.pest` files located in the crate `src` directory.
//! Parsers are automatically generated from these files using `#[derive(Parser)]` and a special
//! `#[grammar = "..."]` attribute on a dummy struct.
//!
//! ```ignore
//! #[derive(Parser)]
//! #[grammar = "path/to/my_grammar.pest"] // relative to src
//! struct MyParser;
//! ```
//!
//! The syntax of `.pest` files is documented in the [`pest_derive` crate].
//!
//! ## Inline grammars
//!
//! Grammars can also be inlined by using the `#[grammar_inline = "..."]` attribute.
//!
//! [`Parser`]: trait.Parser.html
//! [`pest_derive` crate]: https://docs.rs/pest_derive/

#![doc(html_root_url = "https://docs.rs/pest")]

extern crate alloc;
#[cfg(feature = "std")]
extern crate std;
extern crate ucd_trie;

#[cfg(feature = "pretty-print")]
extern crate serde;
#[cfg(feature = "pretty-print")]
extern crate serde_json;

use core::fmt::Debug;
use core::hash::Hash;
pub use parser::Parser;
pub use parser_state::{state, Atomicity, Lookahead, MatchDir, ParseResult, ParserState};
pub use position::Position;
pub use span::{Lines, Span};
pub use token::Token;

pub mod error;
pub mod iterators;
mod macros;
mod parser;
mod parser_state;
mod position;
pub mod prec_climber;
mod span;
mod stack;
mod token;
#[doc(hidden)]
pub mod unicode;

/// A trait which parser rules must implement.
///
/// This trait is set up so that any struct that implements all of its required traits will
/// automatically implement this trait as well.
///
/// This is essentially a [trait alias](https://github.com/rust-lang/rfcs/pull/1733). When trait
/// aliases are implemented, this may be replaced by one.
pub trait RuleType: Copy + Debug + Eq + Hash + Ord {}

impl<T: Copy + Debug + Eq + Hash + Ord> RuleType for T {}
<|endoftext|>


<commit_before>/// Download from a URL to a path
#[cfg(feature = "request")]
pub async fn download<P: AsRef<std::path::Path>>(url: &str, path: P) -> eyre::Result<()> {
    use std::{fs::File, io};

    let response = reqwest::get(url).await?;
    let bytes = response.bytes().await?;
    let mut file = File::create(&path)?;
    io::copy(&mut bytes.as_ref(), &mut file)?;
    Ok(())
}
<commit_msg>fix(Downloads): Add user agent header to avoid 403s<commit_after>/// Download from a URL to a path
#[cfg(feature = "request")]
pub async fn download<P: AsRef<std::path::Path>>(url: &str, path: P) -> eyre::Result<()> {
    use std::{fs::File, io};

    // TODO: Allow an existing client to be passed into this function
    let client = reqwest::Client::new();
    let response = client
        .get(url)
        .header(reqwest::header::USER_AGENT, "Stencila")
        .send()
        .await?;
    let bytes = response.bytes().await?;
    let mut file = File::create(&path)?;
    io::copy(&mut bytes.as_ref(), &mut file)?;
    Ok(())
}
<|endoftext|>


<commit_before>use super::error::Result;
use crate::loader::Loader;
use crate::query_testing::{parse_position_comments, Assertion};
use ansi_term::Colour;
use std::fs;
use std::path::Path;
use tree_sitter::Point;
use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, Highlighter};

pub struct Failure {
    row: usize,
    column: usize,
    expected_highlight: String,
    actual_highlights: Vec<String>,
}

impl Failure {
    pub fn message(&self) -> String {
        let mut result = format!(
            "Failure - row: {}, column: {}, expected highlight '{}', actual highlights: ",
            self.row, self.column, self.expected_highlight
        );
        if self.actual_highlights.is_empty() {
            result += "none.";
        } else {
            for (i, actual_highlight) in self.actual_highlights.iter().enumerate() {
                if i > 0 {
                    result += ", ";
                }
                result += "'";
                result += actual_highlight;
                result += "'";
            }
        }
        result
    }
}

pub fn test_highlights(loader: &Loader, directory: &Path) -> Result<()> {
    let mut failed = false;
    let mut highlighter = Highlighter::new();

    println!("syntax highlighting:");
    for highlight_test_file in fs::read_dir(directory)? {
        let highlight_test_file = highlight_test_file?;
        let test_file_path = highlight_test_file.path();
        let test_file_name = highlight_test_file.file_name();
        let (language, language_config) = loader
            .language_configuration_for_file_name(&test_file_path)?
            .ok_or_else(|| format!("No language found for path {:?}", test_file_path))?;
        let highlight_config = language_config
            .highlight_config(language)?
            .ok_or_else(|| format!("No highlighting config found for {:?}", test_file_path))?;
        match test_highlight(
            &loader,
            &mut highlighter,
            highlight_config,
            fs::read(&test_file_path)?.as_slice(),
        ) {
            Ok(assertion_count) => {
                println!(
                    "  ✓ {} ({} assertions)",
                    Colour::Green.paint(test_file_name.to_string_lossy().as_ref()),
                    assertion_count
                );
            }
            Err(e) => {
                println!(
                    "  ✗ {}",
                    Colour::Red.paint(test_file_name.to_string_lossy().as_ref())
                );
                println!("    {}", e.message());
                failed = true;
            }
        }
    }

    if failed {
        Err(String::new().into())
    } else {
        Ok(())
    }
}

pub fn test_highlight(
    loader: &Loader,
    highlighter: &mut Highlighter,
    highlight_config: &HighlightConfiguration,
    source: &[u8],
) -> Result<usize> {
    // Highlight the file, and parse out all of the highlighting assertions.
    let highlight_names = loader.highlight_names();
    let highlights = get_highlight_positions(loader, highlighter, highlight_config, source)?;
    let assertions =
        parse_position_comments(highlighter.parser(), highlight_config.language, source)?;

    // Iterate through all of the highlighting assertions, checking each one against the
    // actual highlights.
    let mut i = 0;
    let mut actual_highlights = Vec::<&String>::new();
    for Assertion {
        position,
        expected_capture_name: expected_highlight,
    } in &assertions
    {
        let mut passed = false;
        actual_highlights.clear();

        'highlight_loop: loop {
            // The assertions are ordered by position, so skip past all of the highlights that
            // end at or before this assertion's position.
            if let Some(highlight) = highlights.get(i) {
                if highlight.1 <= *position {
                    i += 1;
                    continue;
                }

                // Iterate through all of the highlights that start at or before this assertion's,
                // position, looking for one that matches the assertion.
                let mut j = i;
                while let (false, Some(highlight)) = (passed, highlights.get(j)) {
                    if highlight.0 > *position {
                        break 'highlight_loop;
                    }

                    // If the highlight matches the assertion, this test passes. Otherwise,
                    // add this highlight to the list of actual highlights that span the
                    // assertion's position, in order to generate an error message in the event
                    // of a failure.
                    let highlight_name = &highlight_names[(highlight.2).0];
                    if *highlight_name == *expected_highlight {
                        passed = true;
                        break 'highlight_loop;
                    } else {
                        actual_highlights.push(highlight_name);
                    }

                    j += 1;
                }
            } else {
                break;
            }
        }

        if !passed {
            return Err(Failure {
                row: position.row,
                column: position.column,
                expected_highlight: expected_highlight.clone(),
                actual_highlights: actual_highlights.into_iter().cloned().collect(),
            }
            .into());
        }
    }

    Ok(assertions.len())
}

pub fn get_highlight_positions(
    loader: &Loader,
    highlighter: &mut Highlighter,
    highlight_config: &HighlightConfiguration,
    source: &[u8],
) -> Result<Vec<(Point, Point, Highlight)>> {
    let mut row = 0;
    let mut column = 0;
    let mut byte_offset = 0;
    let mut was_newline = false;
    let mut result = Vec::new();
    let mut highlight_stack = Vec::new();
    let source = String::from_utf8_lossy(source);
    let mut char_indices = source.char_indices();
    for event in highlighter.highlight(highlight_config, source.as_bytes(), None, |string| {
        loader.highlight_config_for_injection_string(string)
    })? {
        match event? {
            HighlightEvent::HighlightStart(h) => highlight_stack.push(h),
            HighlightEvent::HighlightEnd => {
                highlight_stack.pop();
            }
            HighlightEvent::Source { start, end } => {
                let mut start_position = Point::new(row, column);
                while byte_offset < end {
                    if byte_offset <= start {
                        start_position = Point::new(row, column);
                    }
                    if let Some((i, c)) = char_indices.next() {
                        if was_newline {
                            row += 1;
                            column = 0;
                        } else {
                            column += i - byte_offset;
                        }
                        was_newline = c == '\n';
                        byte_offset = i;
                    } else {
                        break;
                    }
                }
                if let Some(highlight) = highlight_stack.last() {
                    result.push((start_position, Point::new(row, column), *highlight))
                }
            }
        }
    }
    Ok(result)
}
<commit_msg>Fix tests.<commit_after>use super::error::Result;
use crate::loader::Loader;
use crate::query_testing::{parse_position_comments, Assertion};
use ansi_term::Colour;
use std::fs;
use std::path::Path;
use tree_sitter::Point;
use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, Highlighter};

pub struct Failure {
    row: usize,
    column: usize,
    expected_highlight: String,
    actual_highlights: Vec<String>,
}

impl Failure {
    pub fn message(&self) -> String {
        let mut result = format!(
            "Failure - row: {}, column: {}, expected highlight '{}', actual highlights: ",
            self.row, self.column, self.expected_highlight
        );
        if self.actual_highlights.is_empty() {
            result += "none.";
        } else {
            for (i, actual_highlight) in self.actual_highlights.iter().enumerate() {
                if i > 0 {
                    result += ", ";
                }
                result += "'";
                result += actual_highlight;
                result += "'";
            }
        }
        result
    }
}

pub fn test_highlights(loader: &Loader, directory: &Path) -> Result<()> {
    let mut failed = false;
    let mut highlighter = Highlighter::new();

    println!("syntax highlighting:");
    for highlight_test_file in fs::read_dir(directory)? {
        let highlight_test_file = highlight_test_file?;
        let test_file_path = highlight_test_file.path();
        let test_file_name = highlight_test_file.file_name();
        let (language, language_config) = loader
            .language_configuration_for_file_name(&test_file_path)?
            .ok_or_else(|| format!("No language found for path {:?}", test_file_path))?;
        let highlight_config = language_config
            .highlight_config(language)?
            .ok_or_else(|| format!("No highlighting config found for {:?}", test_file_path))?;
        match test_highlight(
            &loader,
            &mut highlighter,
            highlight_config,
            fs::read(&test_file_path)?.as_slice(),
        ) {
            Ok(assertion_count) => {
                println!(
                    "  ✓ {} ({} assertions)",
                    Colour::Green.paint(test_file_name.to_string_lossy().as_ref()),
                    assertion_count
                );
            }
            Err(e) => {
                println!(
                    "  ✗ {}",
                    Colour::Red.paint(test_file_name.to_string_lossy().as_ref())
                );
                println!("    {}", e.message());
                failed = true;
            }
        }
    }

    if failed {
        Err(String::new().into())
    } else {
        Ok(())
    }
}
pub fn iterate_assertions(
    assertions: &Vec<Assertion>,
    highlights: &Vec<(Point, Point, Highlight)>,
    highlight_names: &Vec<String>,
) -> Result<usize> {
    // Iterate through all of the highlighting assertions, checking each one against the
    // actual highlights.
    let mut i = 0;
    let mut actual_highlights = Vec::<&String>::new();
    for Assertion {
        position,
        expected_capture_name: expected_highlight,
    } in assertions
    {
        let mut passed = false;
        actual_highlights.clear();

        'highlight_loop: loop {
            // The assertions are ordered by position, so skip past all of the highlights that
            // end at or before this assertion's position.
            if let Some(highlight) = highlights.get(i) {
                if highlight.1 <= *position {
                    i += 1;
                    continue;
                }

                // Iterate through all of the highlights that start at or before this assertion's,
                // position, looking for one that matches the assertion.
                let mut j = i;
                while let (false, Some(highlight)) = (passed, highlights.get(j)) {
                    if highlight.0 > *position {
                        break 'highlight_loop;
                    }

                    // If the highlight matches the assertion, this test passes. Otherwise,
                    // add this highlight to the list of actual highlights that span the
                    // assertion's position, in order to generate an error message in the event
                    // of a failure.
                    let highlight_name = &highlight_names[(highlight.2).0];
                    if *highlight_name == *expected_highlight {
                        passed = true;
                        break 'highlight_loop;
                    } else {
                        actual_highlights.push(highlight_name);
                    }

                    j += 1;
                }
            } else {
                break;
            }
        }

        if !passed {
            return Err(Failure {
                row: position.row,
                column: position.column,
                expected_highlight: expected_highlight.clone(),
                actual_highlights: actual_highlights.into_iter().cloned().collect(),
            }
            .into());
        }
    }

    Ok(assertions.len())
}

pub fn test_highlight(
    loader: &Loader,
    highlighter: &mut Highlighter,
    highlight_config: &HighlightConfiguration,
    source: &[u8],
) -> Result<usize> {
    // Highlight the file, and parse out all of the highlighting assertions.
    let highlight_names = loader.highlight_names();
    let highlights = get_highlight_positions(loader, highlighter, highlight_config, source)?;
    let assertions =
        parse_position_comments(highlighter.parser(), highlight_config.language, source)?;

    iterate_assertions(&assertions, &highlights, &highlight_names)?;

    // Iterate through all of the highlighting assertions, checking each one against the
    // actual highlights.
    let mut i = 0;
    let mut actual_highlights = Vec::<&String>::new();
    for Assertion {
        position,
        expected_capture_name: expected_highlight,
    } in &assertions
    {
        let mut passed = false;
        actual_highlights.clear();

        'highlight_loop: loop {
            // The assertions are ordered by position, so skip past all of the highlights that
            // end at or before this assertion's position.
            if let Some(highlight) = highlights.get(i) {
                if highlight.1 <= *position {
                    i += 1;
                    continue;
                }

                // Iterate through all of the highlights that start at or before this assertion's,
                // position, looking for one that matches the assertion.
                let mut j = i;
                while let (false, Some(highlight)) = (passed, highlights.get(j)) {
                    if highlight.0 > *position {
                        break 'highlight_loop;
                    }

                    // If the highlight matches the assertion, this test passes. Otherwise,
                    // add this highlight to the list of actual highlights that span the
                    // assertion's position, in order to generate an error message in the event
                    // of a failure.
                    let highlight_name = &highlight_names[(highlight.2).0];
                    if *highlight_name == *expected_highlight {
                        passed = true;
                        break 'highlight_loop;
                    } else {
                        actual_highlights.push(highlight_name);
                    }

                    j += 1;
                }
            } else {
                break;
            }
        }

        if !passed {
            return Err(Failure {
                row: position.row,
                column: position.column,
                expected_highlight: expected_highlight.clone(),
                actual_highlights: actual_highlights.into_iter().cloned().collect(),
            }
            .into());
        }
    }

    Ok(assertions.len())
}

pub fn get_highlight_positions(
    loader: &Loader,
    highlighter: &mut Highlighter,
    highlight_config: &HighlightConfiguration,
    source: &[u8],
) -> Result<Vec<(Point, Point, Highlight)>> {
    let mut row = 0;
    let mut column = 0;
    let mut byte_offset = 0;
    let mut was_newline = false;
    let mut result = Vec::new();
    let mut highlight_stack = Vec::new();
    let source = String::from_utf8_lossy(source);
    let mut char_indices = source.char_indices();
    for event in highlighter.highlight(highlight_config, source.as_bytes(), None, |string| {
        loader.highlight_config_for_injection_string(string)
    })? {
        match event? {
            HighlightEvent::HighlightStart(h) => highlight_stack.push(h),
            HighlightEvent::HighlightEnd => {
                highlight_stack.pop();
            }
            HighlightEvent::Source { start, end } => {
                let mut start_position = Point::new(row, column);
                while byte_offset < end {
                    if byte_offset <= start {
                        start_position = Point::new(row, column);
                    }
                    if let Some((i, c)) = char_indices.next() {
                        if was_newline {
                            row += 1;
                            column = 0;
                        } else {
                            column += i - byte_offset;
                        }
                        was_newline = c == '\n';
                        byte_offset = i;
                    } else {
                        break;
                    }
                }
                if let Some(highlight) = highlight_stack.last() {
                    result.push((start_position, Point::new(row, column), *highlight))
                }
            }
        }
    }
    Ok(result)
}
<|endoftext|>