<commit_before>#![feature(if_let)]
#![feature(slicing_syntax)]
extern crate irc;

use std::default::Default;
use irc::data::config::Config;
use irc::server::{IrcServer, Server};
use irc::server::utils::Wrapper;

fn main() {
    let config = Config {
        nickname: Some(format!("pickles")),
        server: Some(format!("irc.fyrechat.net")),
        channels: Some(vec![format!("#vana")]),
        .. Default::default()
    };
    let irc_server = IrcServer::from_config(config).unwrap();
    // The wrapper provides us with methods like send_privmsg(...) and identify(...)
    let server = Wrapper::new(&irc_server);
    server.identify().unwrap();
    for message in server.iter() {
        print!("{}", message.into_string());
        if message.command[] == "PRIVMSG" {
            if let Some(msg) = message.suffix {
                if msg.contains("pickles") {
                    server.send_privmsg(message.args[0][], "Hi!").unwrap();
                }
            }
        }
    }
}
<commit_msg>Added alternate nicknames to the simple example.<commit_after>#![feature(if_let)]
#![feature(slicing_syntax)]
extern crate irc;

use std::default::Default;
use irc::data::config::Config;
use irc::server::{IrcServer, Server};
use irc::server::utils::Wrapper;

fn main() {
    let config = Config {
        nickname: Some(format!("pickles")),
        alt_nicks: Some(vec![format!("bananas"), format!("apples")]),
        server: Some(format!("irc.fyrechat.net")),
        channels: Some(vec![format!("#vana")]),
        .. Default::default()
    };
    let irc_server = IrcServer::from_config(config).unwrap();
    // The wrapper provides us with methods like send_privmsg(...) and identify(...)
    let server = Wrapper::new(&irc_server);
    server.identify().unwrap();
    for message in server.iter() {
        print!("{}", message.into_string());
        if message.command[] == "PRIVMSG" {
            if let Some(msg) = message.suffix {
                if msg.contains("pickles") {
                    server.send_privmsg(message.args[0][], "Hi!").unwrap();
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    backup_types::{
        epoch_ending::restore::EpochHistoryRestoreController,
        state_snapshot::restore::{StateSnapshotRestoreController, StateSnapshotRestoreOpt},
        transaction::restore::TransactionRestoreBatchController,
    },
    metadata,
    metadata::{cache::MetadataCacheOpt, TransactionBackupMeta},
    metrics::restore::{
        COORDINATOR_FAIL_TS, COORDINATOR_START_TS, COORDINATOR_SUCC_TS, COORDINATOR_TARGET_VERSION,
    },
    storage::BackupStorage,
    utils::{unix_timestamp_sec, GlobalRestoreOptions, RestoreRunMode},
};
use anyhow::{bail, Result};
use diem_logger::prelude::*;
use diem_types::transaction::Version;
use std::sync::Arc;
use structopt::StructOpt;

#[derive(StructOpt)]
pub struct RestoreCoordinatorOpt {
    #[structopt(flatten)]
    pub metadata_cache_opt: MetadataCacheOpt,
}

pub struct RestoreCoordinator {
    storage: Arc<dyn BackupStorage>,
    global_opt: GlobalRestoreOptions,
    metadata_cache_opt: MetadataCacheOpt,
}

impl RestoreCoordinator {
    pub fn new(
        opt: RestoreCoordinatorOpt,
        global_opt: GlobalRestoreOptions,
        storage: Arc<dyn BackupStorage>,
    ) -> Self {
        Self {
            storage,
            global_opt,
            metadata_cache_opt: opt.metadata_cache_opt,
        }
    }

    pub async fn run(self) -> Result<()> {
        info!("Restore coordinator started.");
        COORDINATOR_START_TS.set(unix_timestamp_sec());

        let ret = self.run_impl().await;

        if let Err(e) = &ret {
            error!(
                error = ?e,
                "Restore coordinator failed."
            );
            COORDINATOR_FAIL_TS.set(unix_timestamp_sec());
        } else {
            info!("Restore coordinator exiting with success.");
            COORDINATOR_SUCC_TS.set(unix_timestamp_sec());
        }

        ret
    }

    async fn run_impl(self) -> Result<()> {
        let metadata_view =
            metadata::cache::sync_and_load(&self.metadata_cache_opt, Arc::clone(&self.storage))
                .await?;

        let transactions = metadata_view.select_transaction_backups(self.target_version())?;
        let actual_target_version = self.get_actual_target_version(&transactions)?;
        let epoch_endings = metadata_view.select_epoch_ending_backups(actual_target_version)?;
        let state_snapshot = metadata_view.select_state_snapshot(actual_target_version)?;
        let replay_transactions_from_version = match &state_snapshot {
            Some(b) => b.version + 1,
            None => 0,
        };
        COORDINATOR_TARGET_VERSION.set(actual_target_version as i64);
        info!("Planned to restore to version {}.", actual_target_version);
        let txn_resume_point = match self.global_opt.run_mode.as_ref() {
            RestoreRunMode::Restore { restore_handler } => {
                restore_handler.get_next_expected_transaction_version()?
            }
            RestoreRunMode::Verify => {
                info!("This is a dry run.");
                0
            }
        };
        if txn_resume_point > 0 {
            warn!(
                "DB has existing transactions, will skip transaction backups before version {}",
                txn_resume_point
            );
        }

        let epoch_history = Arc::new(
            EpochHistoryRestoreController::new(
                epoch_endings
                    .into_iter()
                    .map(|backup| backup.manifest)
                    .collect(),
                self.global_opt.clone(),
                self.storage.clone(),
            )
            .run()
            .await?,
        );

        if let Some(backup) = state_snapshot {
            StateSnapshotRestoreController::new(
                StateSnapshotRestoreOpt {
                    manifest_handle: backup.manifest,
                    version: backup.version,
                },
                self.global_opt.clone(),
                Arc::clone(&self.storage),
                Some(Arc::clone(&epoch_history)),
            )
            .run()
            .await?;
        }

        let txn_manifests = transactions
            .into_iter()
            .skip_while(|b| b.last_version < txn_resume_point)
            .map(|b| b.manifest)
            .collect();
        TransactionRestoreBatchController::new(
            self.global_opt,
            self.storage,
            txn_manifests,
            Some(replay_transactions_from_version),
            Some(epoch_history),
        )
        .run()
        .await?;

        Ok(())
    }
}

impl RestoreCoordinator {
    fn target_version(&self) -> Version {
        self.global_opt.target_version
    }

    fn get_actual_target_version(
        &self,
        transaction_backups: &[TransactionBackupMeta],
    ) -> Result<Version> {
        if let Some(b) = transaction_backups.last() {
            if b.last_version > self.target_version() {
                Ok(self.target_version())
            } else {
                warn!(
                    "Can't find transaction backup containing the target version, \
                    will restore as much as possible"
                );
                Ok(b.last_version)
            }
        } else {
            bail!("No transaction backup found.")
        }
    }
}
<commit_msg>[backup] add --replay-all to db-restore<commit_after>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    backup_types::{
        epoch_ending::restore::EpochHistoryRestoreController,
        state_snapshot::restore::{StateSnapshotRestoreController, StateSnapshotRestoreOpt},
        transaction::restore::TransactionRestoreBatchController,
    },
    metadata,
    metadata::{cache::MetadataCacheOpt, TransactionBackupMeta},
    metrics::restore::{
        COORDINATOR_FAIL_TS, COORDINATOR_START_TS, COORDINATOR_SUCC_TS, COORDINATOR_TARGET_VERSION,
    },
    storage::BackupStorage,
    utils::{unix_timestamp_sec, GlobalRestoreOptions, RestoreRunMode},
};
use anyhow::{bail, Result};
use diem_logger::prelude::*;
use diem_types::transaction::Version;
use std::sync::Arc;
use structopt::StructOpt;

#[derive(StructOpt)]
pub struct RestoreCoordinatorOpt {
    #[structopt(flatten)]
    pub metadata_cache_opt: MetadataCacheOpt,
    #[structopt(
        long,
        help = "Replay all transactions, don't try to use a state snapshot."
    )]
    pub replay_all: bool,
}

pub struct RestoreCoordinator {
    storage: Arc<dyn BackupStorage>,
    global_opt: GlobalRestoreOptions,
    metadata_cache_opt: MetadataCacheOpt,
    replay_all: bool,
}

impl RestoreCoordinator {
    pub fn new(
        opt: RestoreCoordinatorOpt,
        global_opt: GlobalRestoreOptions,
        storage: Arc<dyn BackupStorage>,
    ) -> Self {
        Self {
            storage,
            global_opt,
            metadata_cache_opt: opt.metadata_cache_opt,
            replay_all: opt.replay_all,
        }
    }

    pub async fn run(self) -> Result<()> {
        info!("Restore coordinator started.");
        COORDINATOR_START_TS.set(unix_timestamp_sec());

        let ret = self.run_impl().await;

        if let Err(e) = &ret {
            error!(
                error = ?e,
                "Restore coordinator failed."
            );
            COORDINATOR_FAIL_TS.set(unix_timestamp_sec());
        } else {
            info!("Restore coordinator exiting with success.");
            COORDINATOR_SUCC_TS.set(unix_timestamp_sec());
        }

        ret
    }

    async fn run_impl(self) -> Result<()> {
        let metadata_view =
            metadata::cache::sync_and_load(&self.metadata_cache_opt, Arc::clone(&self.storage))
                .await?;

        let transactions = metadata_view.select_transaction_backups(self.target_version())?;
        let actual_target_version = self.get_actual_target_version(&transactions)?;
        let epoch_endings = metadata_view.select_epoch_ending_backups(actual_target_version)?;
        let state_snapshot = if self.replay_all {
            None
        } else {
            metadata_view.select_state_snapshot(actual_target_version)?
        };
        let replay_transactions_from_version = match &state_snapshot {
            Some(b) => b.version + 1,
            None => 0,
        };
        COORDINATOR_TARGET_VERSION.set(actual_target_version as i64);
        info!("Planned to restore to version {}.", actual_target_version);
        let txn_resume_point = match self.global_opt.run_mode.as_ref() {
            RestoreRunMode::Restore { restore_handler } => {
                restore_handler.get_next_expected_transaction_version()?
            }
            RestoreRunMode::Verify => {
                info!("This is a dry run.");
                0
            }
        };
        if txn_resume_point > 0 {
            warn!(
                "DB has existing transactions, will skip transaction backups before version {}",
                txn_resume_point
            );
        }

        let epoch_history = Arc::new(
            EpochHistoryRestoreController::new(
                epoch_endings
                    .into_iter()
                    .map(|backup| backup.manifest)
                    .collect(),
                self.global_opt.clone(),
                self.storage.clone(),
            )
            .run()
            .await?,
        );

        if let Some(backup) = state_snapshot {
            StateSnapshotRestoreController::new(
                StateSnapshotRestoreOpt {
                    manifest_handle: backup.manifest,
                    version: backup.version,
                },
                self.global_opt.clone(),
                Arc::clone(&self.storage),
                Some(Arc::clone(&epoch_history)),
            )
            .run()
            .await?;
        }

        let txn_manifests = transactions
            .into_iter()
            .skip_while(|b| b.last_version < txn_resume_point)
            .map(|b| b.manifest)
            .collect();
        TransactionRestoreBatchController::new(
            self.global_opt,
            self.storage,
            txn_manifests,
            Some(replay_transactions_from_version),
            Some(epoch_history),
        )
        .run()
        .await?;

        Ok(())
    }
}

impl RestoreCoordinator {
    fn target_version(&self) -> Version {
        self.global_opt.target_version
    }

    fn get_actual_target_version(
        &self,
        transaction_backups: &[TransactionBackupMeta],
    ) -> Result<Version> {
        if let Some(b) = transaction_backups.last() {
            if b.last_version > self.target_version() {
                Ok(self.target_version())
            } else {
                warn!(
                    "Can't find transaction backup containing the target version, \
                    will restore as much as possible"
                );
                Ok(b.last_version)
            }
        } else {
            bail!("No transaction backup found.")
        }
    }
}
<|endoftext|>


<commit_before>extern crate "pkg-config" as pkg_config;

use std::os;
use std::io::{mod, fs, Command};
use std::io::process::InheritFd;

fn main() {
    let mut opts = pkg_config::default_options("libgit2");
    opts.atleast_version = Some("0.21.0".to_string());
    match pkg_config::find_library_opts("libgit2", &opts) {
        Ok(()) => return,
        Err(..) => {}
    }

    match os::getenv("DEP_SSH2_ROOT") {
        Some(s) => {
            let prefix = os::getenv("CMAKE_PREFIX_PATH").unwrap_or(String::new());
            let mut v = os::split_paths(prefix.as_slice());
            v.push(Path::new(s));
            os::setenv("CMAKE_PREFIX_PATH", os::join_paths(v.as_slice()).unwrap());
        }
        None => {}
    }

    let mut cflags = os::getenv("CFLAGS").unwrap_or(String::new());
    let target = os::getenv("TARGET").unwrap();
    let mingw = target.contains("windows-gnu");
    cflags.push_str(" -ffunction-sections -fdata-sections");

    if target.contains("i686") {
        cflags.push_str(" -m32");
    } else if target.as_slice().contains("x86_64") {
        cflags.push_str(" -m64");
    }
    if !target.contains("i686") {
        cflags.push_str(" -fPIC");
    }

    let src = Path::new(os::getenv("CARGO_MANIFEST_DIR").unwrap());
    let dst = Path::new(os::getenv("OUT_DIR").unwrap());
    let _ = fs::mkdir(&dst.join("build"), io::USER_DIR);

    let mut cmd = Command::new("cmake");
    cmd.arg(src.join("libgit2"))
       .cwd(&dst.join("build"));
    if mingw {
        cmd.arg("-G").arg("Unix Makefiles");
    }
    run(cmd.arg("-DTHREADSAFE=ON")
           .arg("-DBUILD_SHARED_LIBS=OFF")
           .arg("-DBUILD_CLAR=OFF")
           .arg("-DCMAKE_BUILD_TYPE=RelWithDebInfo")
           .arg(format!("-DCMAKE_INSTALL_PREFIX={}", dst.display()))
           .arg("-DBUILD_EXAMPLES=OFF")
           .arg(format!("-DCMAKE_C_FLAGS={}", cflags)));
    run(Command::new("cmake")
                .arg("--build").arg(".")
                .arg("--target").arg("install")
                .cwd(&dst.join("build")));

    println!("cargo:rustc-flags=-L {} -lgit2:static",
             dst.join("lib").display());
    println!("cargo:root={}", dst.display());
    if mingw || target.contains("windows") {
        println!("cargo:rustc-flags=-l winhttp -l rpcrt4 -l ole32 \
                                    -l ws2_32 -l bcrypt -l crypt32");
    } else if target.contains("apple") {
        println!("cargo:rustc-flags=-l iconv");
    }
}

fn run(cmd: &mut Command) {
    println!("running: {}", cmd);
    assert!(cmd.stdout(InheritFd(1))
               .stderr(InheritFd(2))
               .status()
               .unwrap()
               .success());

}
<commit_msg>Space after -l<commit_after>extern crate "pkg-config" as pkg_config;

use std::os;
use std::io::{mod, fs, Command};
use std::io::process::InheritFd;

fn main() {
    let mut opts = pkg_config::default_options("libgit2");
    opts.atleast_version = Some("0.21.0".to_string());
    match pkg_config::find_library_opts("libgit2", &opts) {
        Ok(()) => return,
        Err(..) => {}
    }

    match os::getenv("DEP_SSH2_ROOT") {
        Some(s) => {
            let prefix = os::getenv("CMAKE_PREFIX_PATH").unwrap_or(String::new());
            let mut v = os::split_paths(prefix.as_slice());
            v.push(Path::new(s));
            os::setenv("CMAKE_PREFIX_PATH", os::join_paths(v.as_slice()).unwrap());
        }
        None => {}
    }

    let mut cflags = os::getenv("CFLAGS").unwrap_or(String::new());
    let target = os::getenv("TARGET").unwrap();
    let mingw = target.contains("windows-gnu");
    cflags.push_str(" -ffunction-sections -fdata-sections");

    if target.contains("i686") {
        cflags.push_str(" -m32");
    } else if target.as_slice().contains("x86_64") {
        cflags.push_str(" -m64");
    }
    if !target.contains("i686") {
        cflags.push_str(" -fPIC");
    }

    let src = Path::new(os::getenv("CARGO_MANIFEST_DIR").unwrap());
    let dst = Path::new(os::getenv("OUT_DIR").unwrap());
    let _ = fs::mkdir(&dst.join("build"), io::USER_DIR);

    let mut cmd = Command::new("cmake");
    cmd.arg(src.join("libgit2"))
       .cwd(&dst.join("build"));
    if mingw {
        cmd.arg("-G").arg("Unix Makefiles");
    }
    run(cmd.arg("-DTHREADSAFE=ON")
           .arg("-DBUILD_SHARED_LIBS=OFF")
           .arg("-DBUILD_CLAR=OFF")
           .arg("-DCMAKE_BUILD_TYPE=RelWithDebInfo")
           .arg(format!("-DCMAKE_INSTALL_PREFIX={}", dst.display()))
           .arg("-DBUILD_EXAMPLES=OFF")
           .arg(format!("-DCMAKE_C_FLAGS={}", cflags)));
    run(Command::new("cmake")
                .arg("--build").arg(".")
                .arg("--target").arg("install")
                .cwd(&dst.join("build")));

    println!("cargo:rustc-flags=-L {} -l git2:static",
             dst.join("lib").display());
    println!("cargo:root={}", dst.display());
    if mingw || target.contains("windows") {
        println!("cargo:rustc-flags=-l winhttp -l rpcrt4 -l ole32 \
                                    -l ws2_32 -l bcrypt -l crypt32");
    } else if target.contains("apple") {
        println!("cargo:rustc-flags=-l iconv");
    }
}

fn run(cmd: &mut Command) {
    println!("running: {}", cmd);
    assert!(cmd.stdout(InheritFd(1))
               .stderr(InheritFd(2))
               .status()
               .unwrap()
               .success());

}
<|endoftext|>


<commit_before>use syntax::ast;
use syntax::ext::base;
use syntax::ext::build::AstBuilder;
use syntax::ext::deriving::generic;
use syntax::codemap;
use syntax::parse::token;
use syntax::ptr::P;

/// Expand #[vertex_format]
pub fn expand(ecx: &mut base::ExtCtxt, span: codemap::Span,
              meta_item: &ast::MetaItem, item: &ast::Item,
              push: |P<ast::Item>|)
{
    generic::TraitDef {
        span: span,
        attributes: Vec::new(),
        path: generic::ty::Path {
            path: vec!["glium", "VertexFormat"],
            lifetime: None,
            params: Vec::new(),
            global: true,
        },
        additional_bounds: Vec::new(),
        generics: generic::ty::LifetimeBounds::empty(),
        methods: vec![
            generic::MethodDef {
                name: "build_bindings",
                generics: generic::ty::LifetimeBounds::empty(),
                explicit_self: None,
                args: vec![
                    generic::ty::Literal(generic::ty::Path {
                        path: vec!["Option"],
                        lifetime: None,
                        params: vec![box generic::ty::Self],
                        global: false,
                    })
                ],
                ret_ty: generic::ty::Literal(
                    generic::ty::Path::new(
                        vec!["glium", "VertexBindings"]
                    ),
                ),
                attributes: vec![
                    ecx.attribute(span.clone(), ecx.meta_list(span.clone(),
                        token::InternedString::new("allow"),
                        vec![ecx.meta_word(span.clone(),
                                token::InternedString::new("dead_assignment"))]
                    ))
                ],
                combine_substructure: generic::combine_substructure(body),
            },
        ],
    }.expand(ecx, meta_item, item, push);
}

fn body(ecx: &mut base::ExtCtxt, span: codemap::Span,
        substr: &generic::Substructure) -> P<ast::Expr>
{
    let ecx: &base::ExtCtxt = ecx;

    match substr.fields {
        &generic::StaticStruct(ref definition, generic::Named(ref fields)) => {
            let content = definition.fields.iter().zip(fields.iter())
                .map(|(def, &(ident, _))| {
                    let ref elem_type = def.node.ty;
                    let ident_str = token::get_ident(ident);
                    let ident_str = ident_str.get();

                    quote_expr!(ecx, {
                        bindings.push(($ident_str.to_string(), VertexAttrib {
                            offset: offset_sum,     // TODO: wrong, doesn't use alignment
                            data_type: GLDataTuple::get_gl_type(None::<$elem_type>),
                            elements_count: GLDataTuple::get_num_elems(None::<$elem_type>) as u32,
                        }));

                        offset_sum += mem::size_of::<$elem_type>();
                    })

                }).collect::<Vec<P<ast::Expr>>>();

            quote_expr!(ecx, {
                use glium::vertex_buffer::VertexAttrib;
                use glium::GLDataTuple;
                use std::mem;

                let mut bindings = Vec::new();
                let mut offset_sum = 0;
                $content;
                bindings
            })
        },

        _ => {
            ecx.span_err(span, "Unable to implement `glium::VertexFormat::build_bindings` \
                                on a non-structure");
            ecx.expr_lit(span, ast::LitNil)
        }
    }
}
<commit_msg>The #[vertex_format] macro now correctly calculates offsets<commit_after>use syntax::ast;
use syntax::ext::base;
use syntax::ext::build::AstBuilder;
use syntax::ext::deriving::generic;
use syntax::codemap;
use syntax::parse::token;
use syntax::ptr::P;

/// Expand #[vertex_format]
pub fn expand(ecx: &mut base::ExtCtxt, span: codemap::Span,
              meta_item: &ast::MetaItem, item: &ast::Item,
              push: |P<ast::Item>|)
{
    generic::TraitDef {
        span: span,
        attributes: Vec::new(),
        path: generic::ty::Path {
            path: vec!["glium", "VertexFormat"],
            lifetime: None,
            params: Vec::new(),
            global: true,
        },
        additional_bounds: Vec::new(),
        generics: generic::ty::LifetimeBounds::empty(),
        methods: vec![
            generic::MethodDef {
                name: "build_bindings",
                generics: generic::ty::LifetimeBounds::empty(),
                explicit_self: None,
                args: vec![
                    generic::ty::Literal(generic::ty::Path {
                        path: vec!["Option"],
                        lifetime: None,
                        params: vec![box generic::ty::Self],
                        global: false,
                    })
                ],
                ret_ty: generic::ty::Literal(
                    generic::ty::Path::new(
                        vec!["glium", "VertexBindings"]
                    ),
                ),
                attributes: vec![
                    ecx.attribute(span.clone(), ecx.meta_list(span.clone(),
                        token::InternedString::new("allow"),
                        vec![ecx.meta_word(span.clone(),
                                token::InternedString::new("dead_assignment"))]
                    ))
                ],
                combine_substructure: generic::combine_substructure(body),
            },
        ],
    }.expand(ecx, meta_item, item, push);
}

fn body(ecx: &mut base::ExtCtxt, span: codemap::Span,
        substr: &generic::Substructure) -> P<ast::Expr>
{
    let ecx: &base::ExtCtxt = ecx;
    let self_ty = &substr.type_ident;

    match substr.fields {
        &generic::StaticStruct(ref definition, generic::Named(ref fields)) => {
            let content = definition.fields.iter().zip(fields.iter())
                .map(|(def, &(ident, _))| {
                    let ref elem_type = def.node.ty;
                    let ident_str = token::get_ident(ident);
                    let ident_str = ident_str.get();

                    quote_expr!(ecx, {
                        let offset = {
                            let dummy: &$self_ty = unsafe { mem::transmute(0u) };
                            let dummy_field = &dummy.$ident;
                            let dummy_field: uint = unsafe { mem::transmute(dummy_field) };
                            dummy_field
                        };

                        bindings.push(($ident_str.to_string(), VertexAttrib {
                            offset: offset,
                            data_type: GLDataTuple::get_gl_type(None::<$elem_type>),
                            elements_count: GLDataTuple::get_num_elems(None::<$elem_type>) as u32,
                        }));
                    })

                }).collect::<Vec<P<ast::Expr>>>();

            quote_expr!(ecx, {
                use glium::vertex_buffer::VertexAttrib;
                use glium::GLDataTuple;
                use std::mem;

                let mut bindings = Vec::new();
                $content;
                bindings
            })
        },

        _ => {
            ecx.span_err(span, "Unable to implement `glium::VertexFormat::build_bindings` \
                                on a non-structure");
            ecx.expr_lit(span, ast::LitNil)
        }
    }
}
<|endoftext|>


<commit_before>use std::cmp::{max, min};

use cairo::{self, ImageSurface};
use cssparser::Parser;
use markup5ever::{expanded_name, local_name, namespace_url, ns};

use crate::drawing_ctx::DrawingCtx;
use crate::error::*;
use crate::node::{NodeResult, NodeTrait, RsvgNode};
use crate::parsers::{NumberOptionalNumber, Parse, ParseValue, ParseValueToParseError};
use crate::property_bag::PropertyBag;
use crate::rect::IRect;
use crate::surface_utils::{
    iterators::{PixelRectangle, Pixels},
    shared_surface::SharedImageSurface,
    EdgeMode, ImageSurfaceDataExt, Pixel,
};

use super::context::{FilterContext, FilterOutput, FilterResult};
use super::{FilterEffect, FilterError, PrimitiveWithInput};

/// Enumeration of the possible morphology operations.
enum Operator {
    Erode,
    Dilate,
}

/// The `feMorphology` filter primitive.
pub struct FeMorphology {
    base: PrimitiveWithInput,
    operator: Operator,
    radius: (f64, f64),
}

impl Default for FeMorphology {
    /// Constructs a new `Morphology` with empty properties.
    #[inline]
    fn default() -> FeMorphology {
        FeMorphology {
            base: PrimitiveWithInput::new::<Self>(),
            operator: Operator::Erode,
            radius: (0.0, 0.0),
        }
    }
}

impl NodeTrait for FeMorphology {
    impl_node_as_filter_effect!();

    fn set_atts(&mut self, parent: Option<&RsvgNode>, pbag: &PropertyBag<'_>) -> NodeResult {
        self.base.set_atts(parent, pbag)?;

        for (attr, value) in pbag.iter() {
            match attr.expanded() {
                expanded_name!(svg "operator") => self.operator = attr.parse(value)?,
                expanded_name!(svg "radius") => {
                    let NumberOptionalNumber(x, y) = attr.parse_to_parse_error_and_validate(
                        value,
                        |v: NumberOptionalNumber<f64>| {
                            if v.0 >= 0.0 && v.1 >= 0.0 {
                                Ok(v)
                            } else {
                                Err(ValueErrorKind::value_error("radius cannot be negative"))
                            }
                        }
                    )?;

                    self.radius = (x, y);
                }
                _ => (),
            }
        }

        Ok(())
    }
}

impl FilterEffect for FeMorphology {
    fn render(
        &self,
        _node: &RsvgNode,
        ctx: &FilterContext,
        draw_ctx: &mut DrawingCtx,
    ) -> Result<FilterResult, FilterError> {
        let input = self.base.get_input(ctx, draw_ctx)?;
        let bounds = self
            .base
            .get_bounds(ctx)
            .add_input(&input)
            .into_irect(draw_ctx);

        let (rx, ry) = self.radius;
        let (rx, ry) = ctx.paffine().transform_distance(rx, ry);

        // The radii can become negative here due to the transform.
        let rx = rx.abs();
        let ry = ry.abs();

        let mut output_surface = ImageSurface::create(
            cairo::Format::ARgb32,
            ctx.source_graphic().width(),
            ctx.source_graphic().height(),
        )?;

        let output_stride = output_surface.get_stride() as usize;
        {
            let mut output_data = output_surface.get_data().unwrap();

            for (x, y, _pixel) in Pixels::new(input.surface(), bounds) {
                // Compute the kernel rectangle bounds.
                let kernel_bounds = IRect::new(
                    (f64::from(x) - rx).floor() as i32,
                    (f64::from(y) - ry).floor() as i32,
                    (f64::from(x) + rx).ceil() as i32 + 1,
                    (f64::from(y) + ry).ceil() as i32 + 1,
                );

                // Compute the new pixel values.
                let initial = match self.operator {
                    Operator::Erode => u8::max_value(),
                    Operator::Dilate => u8::min_value(),
                };

                let mut output_pixel = Pixel {
                    r: initial,
                    g: initial,
                    b: initial,
                    a: initial,
                };

                for (_x, _y, pixel) in
                    PixelRectangle::new(&input.surface(), bounds, kernel_bounds, EdgeMode::None)
                {
                    let op = match self.operator {
                        Operator::Erode => min,
                        Operator::Dilate => max,
                    };

                    output_pixel.r = op(output_pixel.r, pixel.r);
                    output_pixel.g = op(output_pixel.g, pixel.g);
                    output_pixel.b = op(output_pixel.b, pixel.b);
                    output_pixel.a = op(output_pixel.a, pixel.a);
                }

                output_data.set_pixel(output_stride, output_pixel, x, y);
            }
        }

        Ok(FilterResult {
            name: self.base.result.clone(),
            output: FilterOutput {
                surface: SharedImageSurface::new(output_surface, input.surface().surface_type())?,
                bounds,
            },
        })
    }

    #[inline]
    fn is_affected_by_color_interpolation_filters(&self) -> bool {
        false
    }
}

impl Parse for Operator {
    fn parse<'i>(parser: &mut Parser<'i, '_>) -> Result<Operator, ValueErrorKind> {
        parse_identifiers!(
            parser,
            "erode" => Operator::Erode,
            "dilate" => Operator::Dilate,
        )
        .map_err(|_| ValueErrorKind::parse_error("parse error"))
    }
}
<commit_msg>Convert morphology::Operator to CssParseError<commit_after>use std::cmp::{max, min};

use cairo::{self, ImageSurface};
use cssparser::Parser;
use markup5ever::{expanded_name, local_name, namespace_url, ns};

use crate::drawing_ctx::DrawingCtx;
use crate::error::*;
use crate::node::{NodeResult, NodeTrait, RsvgNode};
use crate::parsers::{NumberOptionalNumber, ParseToParseError, ParseValueToParseError};
use crate::property_bag::PropertyBag;
use crate::rect::IRect;
use crate::surface_utils::{
    iterators::{PixelRectangle, Pixels},
    shared_surface::SharedImageSurface,
    EdgeMode, ImageSurfaceDataExt, Pixel,
};

use super::context::{FilterContext, FilterOutput, FilterResult};
use super::{FilterEffect, FilterError, PrimitiveWithInput};

/// Enumeration of the possible morphology operations.
enum Operator {
    Erode,
    Dilate,
}

/// The `feMorphology` filter primitive.
pub struct FeMorphology {
    base: PrimitiveWithInput,
    operator: Operator,
    radius: (f64, f64),
}

impl Default for FeMorphology {
    /// Constructs a new `Morphology` with empty properties.
    #[inline]
    fn default() -> FeMorphology {
        FeMorphology {
            base: PrimitiveWithInput::new::<Self>(),
            operator: Operator::Erode,
            radius: (0.0, 0.0),
        }
    }
}

impl NodeTrait for FeMorphology {
    impl_node_as_filter_effect!();

    fn set_atts(&mut self, parent: Option<&RsvgNode>, pbag: &PropertyBag<'_>) -> NodeResult {
        self.base.set_atts(parent, pbag)?;

        for (attr, value) in pbag.iter() {
            match attr.expanded() {
                expanded_name!(svg "operator") => self.operator = attr.parse_to_parse_error(value)?,
                expanded_name!(svg "radius") => {
                    let NumberOptionalNumber(x, y) = attr.parse_to_parse_error_and_validate(
                        value,
                        |v: NumberOptionalNumber<f64>| {
                            if v.0 >= 0.0 && v.1 >= 0.0 {
                                Ok(v)
                            } else {
                                Err(ValueErrorKind::value_error("radius cannot be negative"))
                            }
                        }
                    )?;

                    self.radius = (x, y);
                }
                _ => (),
            }
        }

        Ok(())
    }
}

impl FilterEffect for FeMorphology {
    fn render(
        &self,
        _node: &RsvgNode,
        ctx: &FilterContext,
        draw_ctx: &mut DrawingCtx,
    ) -> Result<FilterResult, FilterError> {
        let input = self.base.get_input(ctx, draw_ctx)?;
        let bounds = self
            .base
            .get_bounds(ctx)
            .add_input(&input)
            .into_irect(draw_ctx);

        let (rx, ry) = self.radius;
        let (rx, ry) = ctx.paffine().transform_distance(rx, ry);

        // The radii can become negative here due to the transform.
        let rx = rx.abs();
        let ry = ry.abs();

        let mut output_surface = ImageSurface::create(
            cairo::Format::ARgb32,
            ctx.source_graphic().width(),
            ctx.source_graphic().height(),
        )?;

        let output_stride = output_surface.get_stride() as usize;
        {
            let mut output_data = output_surface.get_data().unwrap();

            for (x, y, _pixel) in Pixels::new(input.surface(), bounds) {
                // Compute the kernel rectangle bounds.
                let kernel_bounds = IRect::new(
                    (f64::from(x) - rx).floor() as i32,
                    (f64::from(y) - ry).floor() as i32,
                    (f64::from(x) + rx).ceil() as i32 + 1,
                    (f64::from(y) + ry).ceil() as i32 + 1,
                );

                // Compute the new pixel values.
                let initial = match self.operator {
                    Operator::Erode => u8::max_value(),
                    Operator::Dilate => u8::min_value(),
                };

                let mut output_pixel = Pixel {
                    r: initial,
                    g: initial,
                    b: initial,
                    a: initial,
                };

                for (_x, _y, pixel) in
                    PixelRectangle::new(&input.surface(), bounds, kernel_bounds, EdgeMode::None)
                {
                    let op = match self.operator {
                        Operator::Erode => min,
                        Operator::Dilate => max,
                    };

                    output_pixel.r = op(output_pixel.r, pixel.r);
                    output_pixel.g = op(output_pixel.g, pixel.g);
                    output_pixel.b = op(output_pixel.b, pixel.b);
                    output_pixel.a = op(output_pixel.a, pixel.a);
                }

                output_data.set_pixel(output_stride, output_pixel, x, y);
            }
        }

        Ok(FilterResult {
            name: self.base.result.clone(),
            output: FilterOutput {
                surface: SharedImageSurface::new(output_surface, input.surface().surface_type())?,
                bounds,
            },
        })
    }

    #[inline]
    fn is_affected_by_color_interpolation_filters(&self) -> bool {
        false
    }
}

impl ParseToParseError for Operator {
    fn parse_to_parse_error<'i>(parser: &mut Parser<'i, '_>) -> Result<Self, CssParseError<'i>> {
        Ok(parse_identifiers!(
            parser,
            "erode" => Operator::Erode,
            "dilate" => Operator::Dilate,
        )?)
    }
}
<|endoftext|>


<commit_before>#![cfg(all(
    not(target_os = "fuchsia"),
    not(target_os = "android"),
    not(target_os = "windows"),
    not(target_os = "macos")
))]

extern crate libc;
extern crate pagecache;
extern crate rand;
extern crate sled;
extern crate tests;

use std::fs;
use std::mem::size_of;
use std::path::Path;
use std::thread;
use std::time::Duration;

use rand::Rng;

use pagecache::{Config, ConfigBuilder};

const CYCLE: usize = 256;

/// Verifies that the keys in the tree are correctly recovered.
/// Panics if they are incorrect.
/// Returns the key that should be resumed at, and the current cycle value.
fn verify(tree: &sled::Tree) -> (u32, u32) {
    // key 0 should always be the highest value, as that's where we increment
    // at some point, it might go down by one
    // it should never return, or go down again after that
    let mut iter = tree.iter();
    let highest = match iter.next() {
        Some(Ok((_k, v))) => slice_to_u32(&*v),
        Some(Err(e)) => panic!("{:?}", e),
        None => return (0, 0),
    };

    let highest_vec = u32_to_vec(highest);

    // find how far we got
    let mut contiguous: u32 = 0;
    let mut lowest = 0;
    for res in iter {
        let (_k, v) = res.unwrap();
        if &v[..4] == &highest_vec[..4] {
            contiguous += 1;
        } else {
            let expected = if highest == 0 {
                CYCLE as u32 - 1
            } else {
                (highest - 1) % CYCLE as u32
            };
            let actual = slice_to_u32(&*v);
            assert_eq!(expected, actual);
            lowest = actual;
            break;
        }
    }

    let lowest_vec = u32_to_vec(lowest);

    // ensure nothing changes after this point
    let low_beginning = u32_to_vec(contiguous + 1);

    for res in tree.range(&*low_beginning..) {
        let (k, v): (sled::IVec, _) = res.unwrap();
        assert_eq!(
            v,
            lowest_vec,
            "expected key {} to have value {}, instead it had value {}",
            slice_to_u32(&*k),
            lowest,
            slice_to_u32(&*v)
        );
    }

    (contiguous, highest)
}

fn u32_to_vec(u: u32) -> Vec<u8> {
    let buf: [u8; size_of::<u32>()] = u.to_be_bytes();
    buf.to_vec()
}

fn slice_to_u32(b: &[u8]) -> u32 {
    let mut buf = [0u8; size_of::<u32>()];
    buf.copy_from_slice(&b[..size_of::<u32>()]);

    u32::from_be_bytes(buf)
}

fn spawn_killah() {
    thread::spawn(|| {
        let runtime = rand::thread_rng().gen_range(0, 200);
        thread::sleep(Duration::from_millis(runtime));
        unsafe {
            libc::raise(9);
        }
    });
}

fn run(config: Config) {
    // tests::setup_logger();
    let crash_during_initialization = rand::thread_rng().gen_bool(0.1);

    if crash_during_initialization {
        spawn_killah();
    }

    let tree = sled::Db::start(config).unwrap();

    if !crash_during_initialization {
        spawn_killah();
    }

    let (key, highest) = verify(&tree);

    let mut hu = ((highest as usize) * CYCLE) + key as usize;
    assert_eq!(hu % CYCLE, key as usize);
    assert_eq!(hu / CYCLE, highest as usize);

    loop {
        hu += 1;

        if hu / CYCLE >= CYCLE {
            hu = 0;
        }

        let key = u32_to_vec((hu % CYCLE) as u32);

        let mut value = u32_to_vec((hu / CYCLE) as u32);
        let additional_len = rand::thread_rng().gen_range(0, 100_000);
        value.append(&mut vec![0u8; additional_len]);

        tree.set(&key, value).unwrap();
    }
}

fn run_without_snapshot() {
    let config = ConfigBuilder::new()
        .io_bufs(2)
        .blink_node_split_size(1024)
        .page_consolidation_threshold(10)
        .cache_bits(6)
        .cache_capacity(128 * 1024 * 1024)
        .flush_every_ms(Some(100))
        // drop io_buf_size to 1<<16, then 1<<17 to tease out
        // low hanging fruit more quickly
        .io_buf_size(100_000) // 1<<16 is 65k but might cause stalling
        .path("test_crashes".to_string())
        .snapshot_after_ops(1 << 56)
        .build();

    match thread::spawn(|| run(config)).join() {
        Err(e) => {
            println!("worker thread failed: {:?}", e);
            std::process::exit(15);
        }
        _ => {}
    }
}

fn run_with_snapshot() {
    let config = ConfigBuilder::new()
        .io_bufs(2)
        .blink_node_split_size(1024)
        .page_consolidation_threshold(10)
        .cache_bits(6)
        .cache_capacity(128 * 1024 * 1024)
        .flush_every_ms(Some(100))
        // drop io_buf_size to 1<<16, then 1<<17 to tease out
        // low hanging fruit more quickly
        .io_buf_size(100_000) // 1<<16 is 65k but might cause stalling
        .path("test_crashes_with_snapshot".to_string())
        .snapshot_after_ops(1 << 10)
        .build();

    match thread::spawn(|| run(config)).join() {
        Err(e) => {
            println!("worker thread failed: {:?}", e);
            std::process::exit(15);
        }
        _ => {}
    }
}

#[test]
#[ignore]
fn test_crash_recovery_with_runtime_snapshot() {
    cleanup_with_snapshots();
    for _ in 0..100 {
        let child = unsafe { libc::fork() };
        if child == 0 {
            run_with_snapshot()
        } else {
            let mut status = 0;
            unsafe {
                libc::waitpid(child, &mut status as *mut libc::c_int, 0);
            }
            if status != 9 {
                cleanup();
                panic!("child exited abnormally");
            }
        }
    }
    cleanup_with_snapshots();
}

#[test]
#[ignore]
fn test_crash_recovery_no_runtime_snapshot() {
    cleanup();
    for _ in 0..100 {
        let child = unsafe { libc::fork() };
        if child == 0 {
            run_without_snapshot()
        } else {
            let mut status = 0;
            unsafe {
                libc::waitpid(child, &mut status as *mut libc::c_int, 0);
            }
            if status != 9 {
                cleanup();
                panic!("child exited abnormally");
            }
        }
    }
    cleanup();
}

fn cleanup_with_snapshots() {
    let dir = Path::new("test_crashes_with_snapshot");
    if dir.exists() {
        fs::remove_dir_all(dir).unwrap();
    }
}

fn cleanup() {
    let dir = Path::new("test_crashes");
    if dir.exists() {
        fs::remove_dir_all(dir).unwrap();
    }
}
<commit_msg>Adjust crash detection test<commit_after>#![cfg(all(
    not(target_os = "fuchsia"),
    not(target_os = "android"),
    not(target_os = "windows"),
    not(target_os = "macos")
))]

extern crate libc;
extern crate pagecache;
extern crate rand;
extern crate sled;
extern crate tests;

use std::fs;
use std::mem::size_of;
use std::path::Path;
use std::thread;
use std::time::Duration;

use rand::Rng;

use pagecache::{Config, ConfigBuilder};

const CYCLE: usize = 256;

/// Verifies that the keys in the tree are correctly recovered.
/// Panics if they are incorrect.
/// Returns the key that should be resumed at, and the current cycle value.
fn verify(tree: &sled::Tree) -> (u32, u32) {
    // key 0 should always be the highest value, as that's where we increment
    // at some point, it might go down by one
    // it should never return, or go down again after that
    let mut iter = tree.iter();
    let highest = match iter.next() {
        Some(Ok((_k, v))) => slice_to_u32(&*v),
        Some(Err(e)) => panic!("{:?}", e),
        None => return (0, 0),
    };

    let highest_vec = u32_to_vec(highest);

    // find how far we got
    let mut contiguous: u32 = 0;
    let mut lowest = 0;
    for res in iter {
        let (_k, v) = res.unwrap();
        if &v[..4] == &highest_vec[..4] {
            contiguous += 1;
        } else {
            let expected = if highest == 0 {
                CYCLE as u32 - 1
            } else {
                (highest - 1) % CYCLE as u32
            };
            let actual = slice_to_u32(&*v);
            assert_eq!(expected, actual);
            lowest = actual;
            break;
        }
    }

    // ensure nothing changes after this point
    let low_beginning = u32_to_vec(contiguous + 1);

    for res in tree.range(&*low_beginning..) {
        let (k, v): (sled::IVec, _) = res.unwrap();
        assert_eq!(
            slice_to_u32(&*v),
            lowest,
            "expected key {} to have value {}, instead it had value {}",
            slice_to_u32(&*k),
            lowest,
            slice_to_u32(&*v)
        );
    }

    (contiguous, highest)
}

fn u32_to_vec(u: u32) -> Vec<u8> {
    let buf: [u8; size_of::<u32>()] = u.to_be_bytes();
    buf.to_vec()
}

fn slice_to_u32(b: &[u8]) -> u32 {
    let mut buf = [0u8; size_of::<u32>()];
    buf.copy_from_slice(&b[..size_of::<u32>()]);

    u32::from_be_bytes(buf)
}

fn spawn_killah() {
    thread::spawn(|| {
        let runtime = rand::thread_rng().gen_range(0, 200);
        thread::sleep(Duration::from_millis(runtime));
        unsafe {
            libc::raise(9);
        }
    });
}

fn run(config: Config) {
    // tests::setup_logger();
    let crash_during_initialization = rand::thread_rng().gen_bool(0.1);

    if crash_during_initialization {
        spawn_killah();
    }

    let tree = sled::Db::start(config).unwrap();

    if !crash_during_initialization {
        spawn_killah();
    }

    let (key, highest) = verify(&tree);

    let mut hu = ((highest as usize) * CYCLE) + key as usize;
    assert_eq!(hu % CYCLE, key as usize);
    assert_eq!(hu / CYCLE, highest as usize);

    loop {
        hu += 1;

        if hu / CYCLE >= CYCLE {
            hu = 0;
        }

        let key = u32_to_vec((hu % CYCLE) as u32);

        let mut value = u32_to_vec((hu / CYCLE) as u32);
        let additional_len = rand::thread_rng().gen_range(0, 100_000);
        value.append(&mut vec![0u8; additional_len]);

        tree.set(&key, value).unwrap();
    }
}

fn run_without_snapshot() {
    let config = ConfigBuilder::new()
        .io_bufs(2)
        .blink_node_split_size(1024)
        .page_consolidation_threshold(10)
        .cache_bits(6)
        .cache_capacity(128 * 1024 * 1024)
        .flush_every_ms(Some(100))
        // drop io_buf_size to 1<<16, then 1<<17 to tease out
        // low hanging fruit more quickly
        .io_buf_size(100_000) // 1<<16 is 65k but might cause stalling
        .path("test_crashes".to_string())
        .snapshot_after_ops(1 << 56)
        .build();

    match thread::spawn(|| run(config)).join() {
        Err(e) => {
            println!("worker thread failed: {:?}", e);
            std::process::exit(15);
        }
        _ => {}
    }
}

fn run_with_snapshot() {
    let config = ConfigBuilder::new()
        .io_bufs(2)
        .blink_node_split_size(1024)
        .page_consolidation_threshold(10)
        .cache_bits(6)
        .cache_capacity(128 * 1024 * 1024)
        .flush_every_ms(Some(100))
        // drop io_buf_size to 1<<16, then 1<<17 to tease out
        // low hanging fruit more quickly
        .io_buf_size(100_000) // 1<<16 is 65k but might cause stalling
        .path("test_crashes_with_snapshot".to_string())
        .snapshot_after_ops(1 << 10)
        .build();

    match thread::spawn(|| run(config)).join() {
        Err(e) => {
            println!("worker thread failed: {:?}", e);
            std::process::exit(15);
        }
        _ => {}
    }
}

#[test]
#[ignore]
fn test_crash_recovery_with_runtime_snapshot() {
    cleanup_with_snapshots();
    for _ in 0..100 {
        let child = unsafe { libc::fork() };
        if child == 0 {
            run_with_snapshot()
        } else {
            let mut status = 0;
            unsafe {
                libc::waitpid(child, &mut status as *mut libc::c_int, 0);
            }
            if status != 9 {
                cleanup();
                panic!("child exited abnormally");
            }
        }
    }
    cleanup_with_snapshots();
}

#[test]
#[ignore]
fn test_crash_recovery_no_runtime_snapshot() {
    cleanup();
    for _ in 0..100 {
        let child = unsafe { libc::fork() };
        if child == 0 {
            run_without_snapshot()
        } else {
            let mut status = 0;
            unsafe {
                libc::waitpid(child, &mut status as *mut libc::c_int, 0);
            }
            if status != 9 {
                cleanup();
                panic!("child exited abnormally");
            }
        }
    }
    cleanup();
}

fn cleanup_with_snapshots() {
    let dir = Path::new("test_crashes_with_snapshot");
    if dir.exists() {
        fs::remove_dir_all(dir).unwrap();
    }
}

fn cleanup() {
    let dir = Path::new("test_crashes");
    if dir.exists() {
        fs::remove_dir_all(dir).unwrap();
    }
}
<|endoftext|>


<commit_before>#![cfg(feature = "serde")]

extern crate serde;

use EnumMap;
use Internal;

use self::serde::ser::{Serialize, Serializer, SerializeMap};
use self::serde::de::{self, Deserialize, Deserializer, Error, MapAccess};

use core::fmt;
use core::marker::PhantomData;

/// Requires crate feature `"serde"`
impl<K: Internal<V> + Serialize, V: Serialize> Serialize for EnumMap<K, V> {
    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
        let mut map = serializer.serialize_map(Some(self.len()))?;
        for (key, value) in self {
            map.serialize_entry(&key, value)?;
        }
        map.end()
    }
}

/// Requires crate feature `"serde"`
impl<'de, K, V> Deserialize<'de> for EnumMap<K, V>
where
    K: Internal<V>
        + Internal<Option<V>>
        + Deserialize<'de>,
    V: Deserialize<'de>,
{
    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
        deserializer.deserialize_map(Visitor(PhantomData))
    }
}

struct Visitor<K, V>(PhantomData<(K, V)>);

impl<'de, K, V> de::Visitor<'de> for Visitor<K, V>
where
    K: Internal<V>
        + Internal<Option<V>>
        + Deserialize<'de>,
    V: Deserialize<'de>,
{
    type Value = EnumMap<K, V>;

    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(formatter, "a map")
    }

    fn visit_map<M: MapAccess<'de>>(self, mut access: M) -> Result<Self::Value, M::Error> {
        let mut entries = EnumMap::new();
        while let Some((key, value)) = access.next_entry()? {
            entries[key] = Some(value);
        }
        for value in entries.values() {
            value.as_ref().ok_or_else(|| M::Error::custom("key not specified"))?;
        }
        Ok(enum_map! { key => entries[key].take().unwrap() })
    }
}
<commit_msg>Reformat<commit_after>#![cfg(feature = "serde")]

extern crate serde;

use EnumMap;
use Internal;

use self::serde::ser::{Serialize, Serializer, SerializeMap};
use self::serde::de::{self, Deserialize, Deserializer, Error, MapAccess};

use core::fmt;
use core::marker::PhantomData;

/// Requires crate feature `"serde"`
impl<K: Internal<V> + Serialize, V: Serialize> Serialize for EnumMap<K, V> {
    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
        let mut map = serializer.serialize_map(Some(self.len()))?;
        for (key, value) in self {
            map.serialize_entry(&key, value)?;
        }
        map.end()
    }
}

/// Requires crate feature `"serde"`
impl<'de, K, V> Deserialize<'de> for EnumMap<K, V>
where
    K: Internal<V>
        + Internal<Option<V>>
        + Deserialize<'de>,
    V: Deserialize<'de>,
{
    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
        deserializer.deserialize_map(Visitor(PhantomData))
    }
}

struct Visitor<K, V>(PhantomData<(K, V)>);

impl<'de, K, V> de::Visitor<'de> for Visitor<K, V>
where
    K: Internal<V>
        + Internal<Option<V>>
        + Deserialize<'de>,
    V: Deserialize<'de>,
{
    type Value = EnumMap<K, V>;

    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(formatter, "a map")
    }

    fn visit_map<M: MapAccess<'de>>(self, mut access: M) -> Result<Self::Value, M::Error> {
        let mut entries = EnumMap::new();
        while let Some((key, value)) = access.next_entry()? {
            entries[key] = Some(value);
        }
        for value in entries.values() {
            value.as_ref().ok_or_else(
                || M::Error::custom("key not specified"),
            )?;
        }
        Ok(enum_map! { key => entries[key].take().unwrap() })
    }
}
<|endoftext|>


<commit_before>use super::helpers::fixtures::{get_language, get_property_sheet, get_property_sheet_json};
use lazy_static::lazy_static;
use std::ffi::CString;
use std::{ptr, slice, str};
use tree_sitter::{Language, PropertySheet};
use tree_sitter_highlight::{c, highlight, highlight_html, Highlight, HighlightEvent, Properties};

lazy_static! {
    static ref JS_SHEET: PropertySheet<Properties> =
        get_property_sheet("javascript", "highlights.json");
    static ref HTML_SHEET: PropertySheet<Properties> =
        get_property_sheet("html", "highlights.json");
    static ref EJS_SHEET: PropertySheet<Properties> =
        get_property_sheet("embedded-template", "highlights-ejs.json");
    static ref SCOPE_CLASS_STRINGS: Vec<String> = {
        let mut result = Vec::new();
        let mut i = 0;
        while let Some(highlight) = Highlight::from_usize(i) {
            result.push(format!("class={:?}", highlight));
            i += 1;
        }
        result
    };
}

#[test]
fn test_highlighting_injected_html_in_javascript() {
    let source = vec!["const s = html `<div>${a < b}</div>`;"].join("\n");

    assert_eq!(
        &to_token_vector(&source, get_language("javascript"), &JS_SHEET).unwrap(),
        &[vec![
            ("const", vec![Highlight::Keyword]),
            (" ", vec![]),
            ("s", vec![Highlight::Variable]),
            (" ", vec![]),
            ("=", vec![Highlight::Operator]),
            (" ", vec![]),
            ("html", vec![Highlight::Function]),
            (" ", vec![]),
            ("`<", vec![Highlight::String]),
            ("div", vec![Highlight::String, Highlight::Tag]),
            (">", vec![Highlight::String]),
            (
                "${",
                vec![
                    Highlight::String,
                    Highlight::Embedded,
                    Highlight::PunctuationSpecial
                ]
            ),
            (
                "a",
                vec![Highlight::String, Highlight::Embedded, Highlight::Variable]
            ),
            (" ", vec![Highlight::String, Highlight::Embedded]),
            (
                "<",
                vec![Highlight::String, Highlight::Embedded, Highlight::Operator]
            ),
            (" ", vec![Highlight::String, Highlight::Embedded]),
            (
                "b",
                vec![Highlight::String, Highlight::Embedded, Highlight::Variable]
            ),
            (
                "}",
                vec![
                    Highlight::String,
                    Highlight::Embedded,
                    Highlight::PunctuationSpecial
                ]
            ),
            ("</", vec![Highlight::String]),
            ("div", vec![Highlight::String, Highlight::Tag]),
            (">`", vec![Highlight::String]),
            (";", vec![Highlight::PunctuationDelimiter]),
        ]]
    );
}

#[test]
fn test_highlighting_injected_javascript_in_html() {
    let source = vec![
        "",
        "  ",
        "",
    ]
    .join("\n");

    assert_eq!(
        &to_token_vector(&source, get_language("html"), &HTML_SHEET).unwrap(),
        &[
            vec![("<", vec![]), ("body", vec![Highlight::Tag]), (">", vec![]),],
            vec![
                ("  <", vec![]),
                ("script", vec![Highlight::Tag]),
                (">", vec![]),
            ],
            vec![
                ("    ", vec![]),
                ("const", vec![Highlight::Keyword]),
                (" ", vec![]),
                ("x", vec![Highlight::Variable]),
                (" ", vec![]),
                ("=", vec![Highlight::Operator]),
                (" ", vec![]),
                ("new", vec![Highlight::Keyword]),
                (" ", vec![]),
                ("Thing", vec![Highlight::Constructor]),
                ("(", vec![Highlight::PunctuationBracket]),
                (")", vec![Highlight::PunctuationBracket]),
                (";", vec![Highlight::PunctuationDelimiter]),
            ],
            vec![
                ("  </", vec![]),
                ("script", vec![Highlight::Tag]),
                (">", vec![]),
            ],
            vec![
                ("</", vec![]),
                ("body", vec![Highlight::Tag]),
                (">", vec![]),
            ],
        ]
    );
}

#[test]
fn test_highlighting_multiline_nodes_to_html() {
    let source = vec![
        "const SOMETHING = `",
        "  one ${",
        "    two()",
        "  } three",
        "`",
        "",
    ]
    .join("\n");

    assert_eq!(
        &to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
        &[
            "const SOMETHING = `\n".to_string(),
            "  one ${\n".to_string(),
            "    two()\n".to_string(),
            "  } three\n".to_string(),
            "`\n".to_string(),
        ]
    );
}

#[test]
fn test_highlighting_empty_lines() {
    let source = vec![
        "class A {",
        "",
        "  b(c) {",
        "",
        "    d(e)",
        "",
        "  }",
        "",
        "}",
    ]
    .join("\n");

    assert_eq!(
        &to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
        &[
            "class A {\n".to_string(),
            "\n".to_string(),
            "  b(c) {\n".to_string(),
            "\n".to_string(),
            "    d(e)\n".to_string(),
            "\n".to_string(),
            "  }\n".to_string(),
            "\n".to_string(),
            "}\n".to_string(),
        ]
    );
}

#[test]
fn test_highlighting_ejs() {
    let source = vec!["
<% foo() %>
"].join("\n");

    assert_eq!(
        &to_token_vector(&source, get_language("embedded-template"), &EJS_SHEET).unwrap(),
        &[[
            ("<", vec![]),
            ("div", vec![Highlight::Tag]),
            (">", vec![]),
            ("<%", vec![Highlight::Keyword]),
            (" ", vec![]),
            ("foo", vec![Highlight::Function]),
            ("(", vec![Highlight::PunctuationBracket]),
            (")", vec![Highlight::PunctuationBracket]),
            (" ", vec![]),
            ("%>", vec![Highlight::Keyword]),
            ("</", vec![]),
            ("div", vec![Highlight::Tag]),
            (">", vec![])
        ]],
    );
}

#[test]
fn test_highlighting_via_c_api() {
    let js_lang = get_language("javascript");
    let html_lang = get_language("html");
    let js_sheet = get_property_sheet_json("javascript", "highlights.json");
    let js_sheet = c_string(&js_sheet);
    let html_sheet = get_property_sheet_json("html", "highlights.json");
    let html_sheet = c_string(&html_sheet);

    let class_tag = c_string("class=tag");
    let class_function = c_string("class=function");
    let class_string = c_string("class=string");
    let class_keyword = c_string("class=keyword");

    let js_scope_name = c_string("source.js");
    let html_scope_name = c_string("text.html.basic");
    let injection_regex = c_string("^(javascript|js)$");
    let source_code = c_string("");

    let attribute_strings = &mut [ptr::null(); Highlight::Unknown as usize + 1];
    attribute_strings[Highlight::Tag as usize] = class_tag.as_ptr();
    attribute_strings[Highlight::String as usize] = class_string.as_ptr();
    attribute_strings[Highlight::Keyword as usize] = class_keyword.as_ptr();
    attribute_strings[Highlight::Function as usize] = class_function.as_ptr();

    let highlighter = c::ts_highlighter_new(attribute_strings.as_ptr());
    let buffer = c::ts_highlight_buffer_new();

    c::ts_highlighter_add_language(
        highlighter,
        html_scope_name.as_ptr(),
        html_lang,
        html_sheet.as_ptr(),
        ptr::null_mut(),
    );
    c::ts_highlighter_add_language(
        highlighter,
        js_scope_name.as_ptr(),
        js_lang,
        js_sheet.as_ptr(),
        injection_regex.as_ptr(),
    );
    c::ts_highlighter_highlight(
        highlighter,
        html_scope_name.as_ptr(),
        source_code.as_ptr(),
        source_code.as_bytes().len() as u32,
        buffer,
        ptr::null_mut(),
    );

    let output_bytes = c::ts_highlight_buffer_content(buffer);
    let output_line_offsets = c::ts_highlight_buffer_line_offsets(buffer);
    let output_len = c::ts_highlight_buffer_len(buffer);
    let output_line_count = c::ts_highlight_buffer_line_count(buffer);

    let output_bytes = unsafe { slice::from_raw_parts(output_bytes, output_len as usize) };
    let output_line_offsets =
        unsafe { slice::from_raw_parts(output_line_offsets, output_line_count as usize) };

    let mut lines = Vec::new();
    for i in 0..(output_line_count as usize) {
        let line_start = output_line_offsets[i] as usize;
        let line_end = output_line_offsets
            .get(i + 1)
            .map(|x| *x as usize)
            .unwrap_or(output_bytes.len());
        lines.push(str::from_utf8(&output_bytes[line_start..line_end]).unwrap());
    }

    assert_eq!(
        lines,
        vec![
            "<script>",
            "const a = b('c');",
            "c.d();",
            "</script>",
        ]
    );

    c::ts_highlighter_delete(highlighter);
    c::ts_highlight_buffer_delete(buffer);
}

fn c_string(s: &str) -> CString {
    CString::new(s.as_bytes().to_vec()).unwrap()
}

fn test_language_for_injection_string<'a>(
    string: &str,
) -> Option<(Language, &'a PropertySheet<Properties>)> {
    match string {
        "javascript" => Some((get_language("javascript"), &JS_SHEET)),
        "html" => Some((get_language("html"), &HTML_SHEET)),
        _ => None,
    }
}

fn to_html<'a>(
    src: &'a str,
    language: Language,
    property_sheet: &'a PropertySheet<Properties>,
) -> Result<Vec<String>, String> {
    highlight_html(
        src.as_bytes(),
        language,
        property_sheet,
        &test_language_for_injection_string,
        &|highlight| SCOPE_CLASS_STRINGS[highlight as usize].as_str(),
    )
}

fn to_token_vector<'a>(
    src: &'a str,
    language: Language,
    property_sheet: &'a PropertySheet<Properties>,
) -> Result<Vec<Vec<(&'a str, Vec<Highlight>)>>, String> {
    let mut lines = Vec::new();
    let mut highlights = Vec::new();
    let mut line = Vec::new();
    for event in highlight(
        src.as_bytes(),
        language,
        property_sheet,
        &test_language_for_injection_string,
    )? {
        match event {
            HighlightEvent::HighlightStart(s) => highlights.push(s),
            HighlightEvent::HighlightEnd => {
                highlights.pop();
            }
            HighlightEvent::Source(s) => {
                for (i, l) in s.lines().enumerate() {
                    if i > 0 {
                        lines.push(line);
                        line = Vec::new();
                    }
                    if l.len() > 0 {
                        line.push((l, highlights.clone()));
                    }
                }
            }
        }
    }
    lines.push(line);
    Ok(lines)
}
<commit_msg>Add a unit test for local variable tracking in syntax highlighting<commit_after>use super::helpers::fixtures::{get_language, get_property_sheet, get_property_sheet_json};
use lazy_static::lazy_static;
use std::ffi::CString;
use std::{ptr, slice, str};
use tree_sitter::{Language, PropertySheet};
use tree_sitter_highlight::{c, highlight, highlight_html, Highlight, HighlightEvent, Properties};

lazy_static! {
    static ref JS_SHEET: PropertySheet<Properties> =
        get_property_sheet("javascript", "highlights.json");
    static ref HTML_SHEET: PropertySheet<Properties> =
        get_property_sheet("html", "highlights.json");
    static ref EJS_SHEET: PropertySheet<Properties> =
        get_property_sheet("embedded-template", "highlights-ejs.json");
    static ref SCOPE_CLASS_STRINGS: Vec<String> = {
        let mut result = Vec::new();
        let mut i = 0;
        while let Some(highlight) = Highlight::from_usize(i) {
            result.push(format!("class={:?}", highlight));
            i += 1;
        }
        result
    };
}

#[test]
fn test_highlighting_injected_html_in_javascript() {
    let source = vec!["const s = html `<div>${a < b}</div>`;"].join("\n");

    assert_eq!(
        &to_token_vector(&source, get_language("javascript"), &JS_SHEET).unwrap(),
        &[vec![
            ("const", vec![Highlight::Keyword]),
            (" ", vec![]),
            ("s", vec![Highlight::Variable]),
            (" ", vec![]),
            ("=", vec![Highlight::Operator]),
            (" ", vec![]),
            ("html", vec![Highlight::Function]),
            (" ", vec![]),
            ("`<", vec![Highlight::String]),
            ("div", vec![Highlight::String, Highlight::Tag]),
            (">", vec![Highlight::String]),
            (
                "${",
                vec![
                    Highlight::String,
                    Highlight::Embedded,
                    Highlight::PunctuationSpecial
                ]
            ),
            (
                "a",
                vec![Highlight::String, Highlight::Embedded, Highlight::Variable]
            ),
            (" ", vec![Highlight::String, Highlight::Embedded]),
            (
                "<",
                vec![Highlight::String, Highlight::Embedded, Highlight::Operator]
            ),
            (" ", vec![Highlight::String, Highlight::Embedded]),
            (
                "b",
                vec![Highlight::String, Highlight::Embedded, Highlight::Variable]
            ),
            (
                "}",
                vec![
                    Highlight::String,
                    Highlight::Embedded,
                    Highlight::PunctuationSpecial
                ]
            ),
            ("</", vec![Highlight::String]),
            ("div", vec![Highlight::String, Highlight::Tag]),
            (">`", vec![Highlight::String]),
            (";", vec![Highlight::PunctuationDelimiter]),
        ]]
    );
}

#[test]
fn test_highlighting_injected_javascript_in_html() {
    let source = vec![
        "",
        "  ",
        "",
    ]
    .join("\n");

    assert_eq!(
        &to_token_vector(&source, get_language("html"), &HTML_SHEET).unwrap(),
        &[
            vec![("<", vec![]), ("body", vec![Highlight::Tag]), (">", vec![]),],
            vec![
                ("  <", vec![]),
                ("script", vec![Highlight::Tag]),
                (">", vec![]),
            ],
            vec![
                ("    ", vec![]),
                ("const", vec![Highlight::Keyword]),
                (" ", vec![]),
                ("x", vec![Highlight::Variable]),
                (" ", vec![]),
                ("=", vec![Highlight::Operator]),
                (" ", vec![]),
                ("new", vec![Highlight::Keyword]),
                (" ", vec![]),
                ("Thing", vec![Highlight::Constructor]),
                ("(", vec![Highlight::PunctuationBracket]),
                (")", vec![Highlight::PunctuationBracket]),
                (";", vec![Highlight::PunctuationDelimiter]),
            ],
            vec![
                ("  </", vec![]),
                ("script", vec![Highlight::Tag]),
                (">", vec![]),
            ],
            vec![
                ("</", vec![]),
                ("body", vec![Highlight::Tag]),
                (">", vec![]),
            ],
        ]
    );
}

#[test]
fn test_highlighting_multiline_nodes_to_html() {
    let source = vec![
        "const SOMETHING = `",
        "  one ${",
        "    two()",
        "  } three",
        "`",
        "",
    ]
    .join("\n");

    assert_eq!(
        &to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
        &[
            "const SOMETHING = `\n".to_string(),
            "  one ${\n".to_string(),
            "    two()\n".to_string(),
            "  } three\n".to_string(),
            "`\n".to_string(),
        ]
    );
}

#[test]
fn test_highlighting_with_local_variable_tracking() {
    let source = vec![
        "module.exports = function a(b) {",
        "  const module = c;",
        "  console.log(module, b);",
        "}",
        "",
    ]
    .join("\n");

    assert_eq!(
        &to_token_vector(&source, get_language("javascript"), &JS_SHEET).unwrap(),
        &[
            vec![
                ("module", vec![Highlight::VariableBuiltin]),
                (".", vec![Highlight::PunctuationDelimiter]),
                ("exports", vec![Highlight::Property]),
                (" ", vec![]),
                ("=", vec![Highlight::Operator]),
                (" ", vec![]),
                ("function", vec![Highlight::Keyword]),
                (" ", vec![]),
                ("a", vec![Highlight::Function]),
                ("(", vec![Highlight::PunctuationBracket]),
                ("b", vec![Highlight::VariableParameter]),
                (")", vec![Highlight::PunctuationBracket]),
                (" ", vec![]),
                ("{", vec![Highlight::PunctuationBracket])
            ],
            vec![
                ("  ", vec![]),
                ("const", vec![Highlight::Keyword]),
                (" ", vec![]),
                ("module", vec![Highlight::Variable]),
                (" ", vec![]),
                ("=", vec![Highlight::Operator]),
                (" ", vec![]),
                ("c", vec![Highlight::Variable]),
                (";", vec![Highlight::PunctuationDelimiter])
            ],
            vec![
                ("  ", vec![]),
                ("console", vec![Highlight::VariableBuiltin]),
                (".", vec![Highlight::PunctuationDelimiter]),
                ("log", vec![Highlight::Function]),
                ("(", vec![Highlight::PunctuationBracket]),
                // Not a builtin, because `module` was defined as a variable above.
                ("module", vec![Highlight::Variable]),
                (",", vec![Highlight::PunctuationDelimiter]),
                (" ", vec![]),
                // A parameter, because `b` was defined as a parameter above.
                ("b", vec![Highlight::VariableParameter]),
                (")", vec![Highlight::PunctuationBracket]),
                (";", vec![Highlight::PunctuationDelimiter]),
                ("}", vec![Highlight::PunctuationBracket])
            ]
        ],
    );
}

#[test]
fn test_highlighting_empty_lines() {
    let source = vec![
        "class A {",
        "",
        "  b(c) {",
        "",
        "    d(e)",
        "",
        "  }",
        "",
        "}",
    ]
    .join("\n");

    assert_eq!(
        &to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
        &[
            "class A {\n".to_string(),
            "\n".to_string(),
            "  b(c) {\n".to_string(),
            "\n".to_string(),
            "    d(e)\n".to_string(),
            "\n".to_string(),
            "  }\n".to_string(),
            "\n".to_string(),
            "}\n".to_string(),
        ]
    );
}

#[test]
fn test_highlighting_ejs() {
    let source = vec!["
<% foo() %>
"].join("\n");

    assert_eq!(
        &to_token_vector(&source, get_language("embedded-template"), &EJS_SHEET).unwrap(),
        &[[
            ("<", vec![]),
            ("div", vec![Highlight::Tag]),
            (">", vec![]),
            ("<%", vec![Highlight::Keyword]),
            (" ", vec![]),
            ("foo", vec![Highlight::Function]),
            ("(", vec![Highlight::PunctuationBracket]),
            (")", vec![Highlight::PunctuationBracket]),
            (" ", vec![]),
            ("%>", vec![Highlight::Keyword]),
            ("</", vec![]),
            ("div", vec![Highlight::Tag]),
            (">", vec![])
        ]],
    );
}

#[test]
fn test_highlighting_via_c_api() {
    let js_lang = get_language("javascript");
    let html_lang = get_language("html");
    let js_sheet = get_property_sheet_json("javascript", "highlights.json");
    let js_sheet = c_string(&js_sheet);
    let html_sheet = get_property_sheet_json("html", "highlights.json");
    let html_sheet = c_string(&html_sheet);

    let class_tag = c_string("class=tag");
    let class_function = c_string("class=function");
    let class_string = c_string("class=string");
    let class_keyword = c_string("class=keyword");

    let js_scope_name = c_string("source.js");
    let html_scope_name = c_string("text.html.basic");
    let injection_regex = c_string("^(javascript|js)$");
    let source_code = c_string("");

    let attribute_strings = &mut [ptr::null(); Highlight::Unknown as usize + 1];
    attribute_strings[Highlight::Tag as usize] = class_tag.as_ptr();
    attribute_strings[Highlight::String as usize] = class_string.as_ptr();
    attribute_strings[Highlight::Keyword as usize] = class_keyword.as_ptr();
    attribute_strings[Highlight::Function as usize] = class_function.as_ptr();

    let highlighter = c::ts_highlighter_new(attribute_strings.as_ptr());
    let buffer = c::ts_highlight_buffer_new();

    c::ts_highlighter_add_language(
        highlighter,
        html_scope_name.as_ptr(),
        html_lang,
        html_sheet.as_ptr(),
        ptr::null_mut(),
    );
    c::ts_highlighter_add_language(
        highlighter,
        js_scope_name.as_ptr(),
        js_lang,
        js_sheet.as_ptr(),
        injection_regex.as_ptr(),
    );
    c::ts_highlighter_highlight(
        highlighter,
        html_scope_name.as_ptr(),
        source_code.as_ptr(),
        source_code.as_bytes().len() as u32,
        buffer,
        ptr::null_mut(),
    );

    let output_bytes = c::ts_highlight_buffer_content(buffer);
    let output_line_offsets = c::ts_highlight_buffer_line_offsets(buffer);
    let output_len = c::ts_highlight_buffer_len(buffer);
    let output_line_count = c::ts_highlight_buffer_line_count(buffer);

    let output_bytes = unsafe { slice::from_raw_parts(output_bytes, output_len as usize) };
    let output_line_offsets =
        unsafe { slice::from_raw_parts(output_line_offsets, output_line_count as usize) };

    let mut lines = Vec::new();
    for i in 0..(output_line_count as usize) {
        let line_start = output_line_offsets[i] as usize;
        let line_end = output_line_offsets
            .get(i + 1)
            .map(|x| *x as usize)
            .unwrap_or(output_bytes.len());
        lines.push(str::from_utf8(&output_bytes[line_start..line_end]).unwrap());
    }

    assert_eq!(
        lines,
        vec![
            "<script>",
            "const a = b('c');",
            "c.d();",
            "</script>",
        ]
    );

    c::ts_highlighter_delete(highlighter);
    c::ts_highlight_buffer_delete(buffer);
}

fn c_string(s: &str) -> CString {
    CString::new(s.as_bytes().to_vec()).unwrap()
}

fn test_language_for_injection_string<'a>(
    string: &str,
) -> Option<(Language, &'a PropertySheet<Properties>)> {
    match string {
        "javascript" => Some((get_language("javascript"), &JS_SHEET)),
        "html" => Some((get_language("html"), &HTML_SHEET)),
        _ => None,
    }
}

fn to_html<'a>(
    src: &'a str,
    language: Language,
    property_sheet: &'a PropertySheet<Properties>,
) -> Result<Vec<String>, String> {
    highlight_html(
        src.as_bytes(),
        language,
        property_sheet,
        &test_language_for_injection_string,
        &|highlight| SCOPE_CLASS_STRINGS[highlight as usize].as_str(),
    )
}

fn to_token_vector<'a>(
    src: &'a str,
    language: Language,
    property_sheet: &'a PropertySheet<Properties>,
) -> Result<Vec<Vec<(&'a str, Vec<Highlight>)>>, String> {
    let mut lines = Vec::new();
    let mut highlights = Vec::new();
    let mut line = Vec::new();
    for event in highlight(
        src.as_bytes(),
        language,
        property_sheet,
        &test_language_for_injection_string,
    )? {
        match event {
            HighlightEvent::HighlightStart(s) => highlights.push(s),
            HighlightEvent::HighlightEnd => {
                highlights.pop();
            }
            HighlightEvent::Source(s) => {
                for (i, l) in s.lines().enumerate() {
                    if i > 0 {
                        lines.push(line);
                        line = Vec::new();
                    }
                    if l.len() > 0 {
                        line.push((l, highlights.clone()));
                    }
                }
            }
        }
    }
    lines.push(line);
    Ok(lines)
}
<|endoftext|>