<commit_before>use super::{ImplTraitContext, LoweringContext, ParamMode};

use rustc_ast::ptr::P;
use rustc_ast::*;
use rustc_data_structures::stack::ensure_sufficient_stack;
use rustc_errors::Applicability;
use rustc_hir as hir;
use rustc_hir::def::Res;
use rustc_span::symbol::Ident;
use rustc_span::{source_map::Spanned, Span};

impl<'a, 'hir> LoweringContext<'a, 'hir> {
    crate fn lower_pat(&mut self, p: &Pat) -> &'hir hir::Pat<'hir> {
        ensure_sufficient_stack(|| {
            let node = match p.kind {
                PatKind::Wild => hir::PatKind::Wild,
                PatKind::Ident(ref binding_mode, ident, ref sub) => {
                    let lower_sub = |this: &mut Self| sub.as_ref().map(|s| this.lower_pat(&*s));
                    let node = self.lower_pat_ident(p, binding_mode, ident, lower_sub);
                    node
                }
                PatKind::Lit(ref e) => hir::PatKind::Lit(self.lower_expr(e)),
                PatKind::TupleStruct(ref path, ref pats) => {
                    let qpath = self.lower_qpath(
                        p.id,
                        &None,
                        path,
                        ParamMode::Optional,
                        ImplTraitContext::disallowed(),
                    );
                    let (pats, ddpos) = self.lower_pat_tuple(pats, "tuple struct");
                    hir::PatKind::TupleStruct(qpath, pats, ddpos)
                }
                PatKind::Or(ref pats) => hir::PatKind::Or(
                    self.arena.alloc_from_iter(pats.iter().map(|x| self.lower_pat(x))),
                ),
                PatKind::Path(ref qself, ref path) => {
                    let qpath = self.lower_qpath(
                        p.id,
                        qself,
                        path,
                        ParamMode::Optional,
                        ImplTraitContext::disallowed(),
                    );
                    hir::PatKind::Path(qpath)
                }
                PatKind::Struct(ref path, ref fields, etc) => {
                    let qpath = self.lower_qpath(
                        p.id,
                        &None,
                        path,
                        ParamMode::Optional,
                        ImplTraitContext::disallowed(),
                    );

                    let fs = self.arena.alloc_from_iter(fields.iter().map(|f| hir::FieldPat {
                        hir_id: self.next_id(),
                        ident: f.ident,
                        pat: self.lower_pat(&f.pat),
                        is_shorthand: f.is_shorthand,
                        span: f.span,
                    }));
                    hir::PatKind::Struct(qpath, fs, etc)
                }
                PatKind::Tuple(ref pats) => {
                    let (pats, ddpos) = self.lower_pat_tuple(pats, "tuple");
                    hir::PatKind::Tuple(pats, ddpos)
                }
                PatKind::Box(ref inner) => hir::PatKind::Box(self.lower_pat(inner)),
                PatKind::Ref(ref inner, mutbl) => hir::PatKind::Ref(self.lower_pat(inner), mutbl),
                PatKind::Range(ref e1, ref e2, Spanned { node: ref end, .. }) => {
                    hir::PatKind::Range(
                        e1.as_deref().map(|e| self.lower_expr(e)),
                        e2.as_deref().map(|e| self.lower_expr(e)),
                        self.lower_range_end(end, e2.is_some()),
                    )
                }
                PatKind::Slice(ref pats) => self.lower_pat_slice(pats),
                PatKind::Rest => {
                    // If we reach here the `..` pattern is not semantically allowed.
                    self.ban_illegal_rest_pat(p.span)
                }
                // FIXME: consider not using recursion to lower this.
                PatKind::Paren(ref inner) => return self.lower_pat(inner),
                PatKind::MacCall(_) => panic!("{:?} shouldn't exist here", p.span),
            };

            self.pat_with_node_id_of(p, node)
        })
    }

    fn lower_pat_tuple(
        &mut self,
        pats: &[P<Pat>],
        ctx: &str,
    ) -> (&'hir [&'hir hir::Pat<'hir>], Option<usize>) {
        let mut elems = Vec::with_capacity(pats.len());
        let mut rest = None;

        let mut iter = pats.iter().enumerate();
        for (idx, pat) in iter.by_ref() {
            // Interpret the first `..` pattern as a sub-tuple pattern.
            // Note that unlike for slice patterns,
            // where `xs @ ..` is a legal sub-slice pattern,
            // it is not a legal sub-tuple pattern.
            match pat.kind {
                // Found a sub-tuple rest pattern
                PatKind::Rest => {
                    rest = Some((idx, pat.span));
                    break;
                }
                // Found a sub-tuple pattern `$binding_mode $ident @ ..`.
                // This is not allowed as a sub-tuple pattern
                PatKind::Ident(ref _bm, ident, Some(ref sub)) if sub.is_rest() => {
                    let sp = pat.span;
                    self.diagnostic()
                        .struct_span_err(
                            sp,
                            &format!("`{} @` is not allowed in a {}", ident.name, ctx),
                        )
                        .span_label(sp, "this is only allowed in slice patterns")
                        .help("remove this and bind each tuple field independently")
                        .span_suggestion_verbose(
                            sp,
                            &format!("if you don't need to use the contents of {}, discard the tuple's remaining fields", ident),
                            "..".to_string(),
                            Applicability::MaybeIncorrect,
                        )
                        .emit();
                }
                _ => {}
            }

            // It was not a sub-tuple pattern so lower it normally.
            elems.push(self.lower_pat(pat));
        }

        for (_, pat) in iter {
            // There was a previous sub-tuple pattern; make sure we don't allow more...
            if pat.is_rest() {
                // ...but there was one again, so error.
                self.ban_extra_rest_pat(pat.span, rest.unwrap().1, ctx);
            } else {
                elems.push(self.lower_pat(pat));
            }
        }

        (self.arena.alloc_from_iter(elems), rest.map(|(ddpos, _)| ddpos))
    }

    /// Lower a slice pattern of form `[pat_0, ..., pat_n]` into
    /// `hir::PatKind::Slice(before, slice, after)`.
    ///
    /// When encountering `($binding_mode $ident @)? ..` (`slice`),
    /// this is interpreted as a sub-slice pattern semantically.
    /// Patterns that follow, which are not like `slice` -- or an error occurs, are in `after`.
    fn lower_pat_slice(&mut self, pats: &[P<Pat>]) -> hir::PatKind<'hir> {
        let mut before = Vec::new();
        let mut after = Vec::new();
        let mut slice = None;
        let mut prev_rest_span = None;

        // Lowers `$bm $ident @ ..` to `$bm $ident @ _`.
        let lower_rest_sub = |this: &mut Self, pat, bm, ident, sub| {
            let lower_sub = |this: &mut Self| Some(this.pat_wild_with_node_id_of(sub));
            let node = this.lower_pat_ident(pat, bm, ident, lower_sub);
            this.pat_with_node_id_of(pat, node)
        };

        let mut iter = pats.iter();
        // Lower all the patterns until the first occurrence of a sub-slice pattern.
        for pat in iter.by_ref() {
            match pat.kind {
                // Found a sub-slice pattern `..`. Record, lower it to `_`, and stop here.
                PatKind::Rest => {
                    prev_rest_span = Some(pat.span);
                    slice = Some(self.pat_wild_with_node_id_of(pat));
                    break;
                }
                // Found a sub-slice pattern `$binding_mode $ident @ ..`.
                // Record, lower it to `$binding_mode $ident @ _`, and stop here.
                PatKind::Ident(ref bm, ident, Some(ref sub)) if sub.is_rest() => {
                    prev_rest_span = Some(sub.span);
                    slice = Some(lower_rest_sub(self, pat, bm, ident, sub));
                    break;
                }
                // It was not a subslice pattern so lower it normally.
                _ => before.push(self.lower_pat(pat)),
            }
        }

        // Lower all the patterns after the first sub-slice pattern.
        for pat in iter {
            // There was a previous subslice pattern; make sure we don't allow more.
            let rest_span = match pat.kind {
                PatKind::Rest => Some(pat.span),
                PatKind::Ident(ref bm, ident, Some(ref sub)) if sub.is_rest() => {
                    // #69103: Lower into `binding @ _` as above to avoid ICEs.
                    after.push(lower_rest_sub(self, pat, bm, ident, sub));
                    Some(sub.span)
                }
                _ => None,
            };
            if let Some(rest_span) = rest_span {
                // We have e.g., `[a, .., b, ..]`. That's no good, error!
                self.ban_extra_rest_pat(rest_span, prev_rest_span.unwrap(), "slice");
            } else {
                // Lower the pattern normally.
                after.push(self.lower_pat(pat));
            }
        }

        hir::PatKind::Slice(
            self.arena.alloc_from_iter(before),
            slice,
            self.arena.alloc_from_iter(after),
        )
    }

    fn lower_pat_ident(
        &mut self,
        p: &Pat,
        binding_mode: &BindingMode,
        ident: Ident,
        lower_sub: impl FnOnce(&mut Self) -> Option<&'hir hir::Pat<'hir>>,
    ) -> hir::PatKind<'hir> {
        match self.resolver.get_partial_res(p.id).map(|d| d.base_res()) {
            // `None` can occur in body-less function signatures
            res @ (None | Some(Res::Local(_))) => {
                let canonical_id = match res {
                    Some(Res::Local(id)) => id,
                    _ => p.id,
                };

                hir::PatKind::Binding(
                    self.lower_binding_mode(binding_mode),
                    self.lower_node_id(canonical_id),
                    ident,
                    lower_sub(self),
                )
            }
            Some(res) => hir::PatKind::Path(hir::QPath::Resolved(
                None,
                self.arena.alloc(hir::Path {
                    span: ident.span,
                    res: self.lower_res(res),
                    segments: arena_vec![self; hir::PathSegment::from_ident(ident)],
                }),
            )),
        }
    }

    fn lower_binding_mode(&mut self, b: &BindingMode) -> hir::BindingAnnotation {
        match *b {
            BindingMode::ByValue(Mutability::Not) => hir::BindingAnnotation::Unannotated,
            BindingMode::ByRef(Mutability::Not) => hir::BindingAnnotation::Ref,
            BindingMode::ByValue(Mutability::Mut) => hir::BindingAnnotation::Mutable,
            BindingMode::ByRef(Mutability::Mut) => hir::BindingAnnotation::RefMut,
        }
    }

    fn pat_wild_with_node_id_of(&mut self, p: &Pat) -> &'hir hir::Pat<'hir> {
        self.pat_with_node_id_of(p, hir::PatKind::Wild)
    }

    /// Construct a `Pat` with the `HirId` of `p.id` lowered.
    fn pat_with_node_id_of(&mut self, p: &Pat, kind: hir::PatKind<'hir>) -> &'hir hir::Pat<'hir> {
        self.arena.alloc(hir::Pat { hir_id: self.lower_node_id(p.id), kind, span: p.span })
    }

    /// Emit a friendly error for extra `..` patterns in a tuple/tuple struct/slice pattern.
    fn ban_extra_rest_pat(&self, sp: Span, prev_sp: Span, ctx: &str) {
        self.diagnostic()
            .struct_span_err(sp, &format!("`..` can only be used once per {} pattern", ctx))
            .span_label(sp, &format!("can only be used once per {} pattern", ctx))
            .span_label(prev_sp, "previously used here")
            .emit();
    }

    /// Used to ban the `..` pattern in places it shouldn't be semantically.
    fn ban_illegal_rest_pat(&self, sp: Span) -> hir::PatKind<'hir> {
        self.diagnostic()
            .struct_span_err(sp, "`..` patterns are not allowed here")
            .note("only allowed in tuple, tuple struct, and slice patterns")
            .emit();

        // We're not in a list context so `..` can be reasonably treated
        // as `_` because it should always be valid and roughly matches the
        // intent of `..` (notice that the rest of a single slot is that slot).
        hir::PatKind::Wild
    }

    fn lower_range_end(&mut self, e: &RangeEnd, has_end: bool) -> hir::RangeEnd {
        match *e {
            RangeEnd::Excluded if has_end => hir::RangeEnd::Excluded,
            // No end; so `X..` behaves like `RangeFrom`.
            RangeEnd::Excluded | RangeEnd::Included(_) => hir::RangeEnd::Included,
        }
    }
}
<commit_msg>Rollup merge of #78292 - bugadani:recursion, r=nagisa<commit_after>use super::{ImplTraitContext, LoweringContext, ParamMode};

use rustc_ast::ptr::P;
use rustc_ast::*;
use rustc_data_structures::stack::ensure_sufficient_stack;
use rustc_errors::Applicability;
use rustc_hir as hir;
use rustc_hir::def::Res;
use rustc_span::symbol::Ident;
use rustc_span::{source_map::Spanned, Span};

impl<'a, 'hir> LoweringContext<'a, 'hir> {
    crate fn lower_pat(&mut self, mut pattern: &Pat) -> &'hir hir::Pat<'hir> {
        ensure_sufficient_stack(|| {
            // loop here to avoid recursion
            let node = loop {
                match pattern.kind {
                    PatKind::Wild => break hir::PatKind::Wild,
                    PatKind::Ident(ref binding_mode, ident, ref sub) => {
                        let lower_sub = |this: &mut Self| sub.as_ref().map(|s| this.lower_pat(&*s));
                        break self.lower_pat_ident(pattern, binding_mode, ident, lower_sub);
                    }
                    PatKind::Lit(ref e) => break hir::PatKind::Lit(self.lower_expr(e)),
                    PatKind::TupleStruct(ref path, ref pats) => {
                        let qpath = self.lower_qpath(
                            pattern.id,
                            &None,
                            path,
                            ParamMode::Optional,
                            ImplTraitContext::disallowed(),
                        );
                        let (pats, ddpos) = self.lower_pat_tuple(pats, "tuple struct");
                        break hir::PatKind::TupleStruct(qpath, pats, ddpos);
                    }
                    PatKind::Or(ref pats) => {
                        break hir::PatKind::Or(
                            self.arena.alloc_from_iter(pats.iter().map(|x| self.lower_pat(x))),
                        );
                    }
                    PatKind::Path(ref qself, ref path) => {
                        let qpath = self.lower_qpath(
                            pattern.id,
                            qself,
                            path,
                            ParamMode::Optional,
                            ImplTraitContext::disallowed(),
                        );
                        break hir::PatKind::Path(qpath);
                    }
                    PatKind::Struct(ref path, ref fields, etc) => {
                        let qpath = self.lower_qpath(
                            pattern.id,
                            &None,
                            path,
                            ParamMode::Optional,
                            ImplTraitContext::disallowed(),
                        );

                        let fs = self.arena.alloc_from_iter(fields.iter().map(|f| hir::FieldPat {
                            hir_id: self.next_id(),
                            ident: f.ident,
                            pat: self.lower_pat(&f.pat),
                            is_shorthand: f.is_shorthand,
                            span: f.span,
                        }));
                        break hir::PatKind::Struct(qpath, fs, etc);
                    }
                    PatKind::Tuple(ref pats) => {
                        let (pats, ddpos) = self.lower_pat_tuple(pats, "tuple");
                        break hir::PatKind::Tuple(pats, ddpos);
                    }
                    PatKind::Box(ref inner) => {
                        break hir::PatKind::Box(self.lower_pat(inner));
                    }
                    PatKind::Ref(ref inner, mutbl) => {
                        break hir::PatKind::Ref(self.lower_pat(inner), mutbl);
                    }
                    PatKind::Range(ref e1, ref e2, Spanned { node: ref end, .. }) => {
                        break hir::PatKind::Range(
                            e1.as_deref().map(|e| self.lower_expr(e)),
                            e2.as_deref().map(|e| self.lower_expr(e)),
                            self.lower_range_end(end, e2.is_some()),
                        );
                    }
                    PatKind::Slice(ref pats) => break self.lower_pat_slice(pats),
                    PatKind::Rest => {
                        // If we reach here the `..` pattern is not semantically allowed.
                        break self.ban_illegal_rest_pat(pattern.span);
                    }
                    // return inner to be processed in next loop
                    PatKind::Paren(ref inner) => pattern = inner,
                    PatKind::MacCall(_) => panic!("{:?} shouldn't exist here", pattern.span),
                }
            };

            self.pat_with_node_id_of(pattern, node)
        })
    }

    fn lower_pat_tuple(
        &mut self,
        pats: &[P<Pat>],
        ctx: &str,
    ) -> (&'hir [&'hir hir::Pat<'hir>], Option<usize>) {
        let mut elems = Vec::with_capacity(pats.len());
        let mut rest = None;

        let mut iter = pats.iter().enumerate();
        for (idx, pat) in iter.by_ref() {
            // Interpret the first `..` pattern as a sub-tuple pattern.
            // Note that unlike for slice patterns,
            // where `xs @ ..` is a legal sub-slice pattern,
            // it is not a legal sub-tuple pattern.
            match pat.kind {
                // Found a sub-tuple rest pattern
                PatKind::Rest => {
                    rest = Some((idx, pat.span));
                    break;
                }
                // Found a sub-tuple pattern `$binding_mode $ident @ ..`.
                // This is not allowed as a sub-tuple pattern
                PatKind::Ident(ref _bm, ident, Some(ref sub)) if sub.is_rest() => {
                    let sp = pat.span;
                    self.diagnostic()
                        .struct_span_err(
                            sp,
                            &format!("`{} @` is not allowed in a {}", ident.name, ctx),
                        )
                        .span_label(sp, "this is only allowed in slice patterns")
                        .help("remove this and bind each tuple field independently")
                        .span_suggestion_verbose(
                            sp,
                            &format!("if you don't need to use the contents of {}, discard the tuple's remaining fields", ident),
                            "..".to_string(),
                            Applicability::MaybeIncorrect,
                        )
                        .emit();
                }
                _ => {}
            }

            // It was not a sub-tuple pattern so lower it normally.
            elems.push(self.lower_pat(pat));
        }

        for (_, pat) in iter {
            // There was a previous sub-tuple pattern; make sure we don't allow more...
            if pat.is_rest() {
                // ...but there was one again, so error.
                self.ban_extra_rest_pat(pat.span, rest.unwrap().1, ctx);
            } else {
                elems.push(self.lower_pat(pat));
            }
        }

        (self.arena.alloc_from_iter(elems), rest.map(|(ddpos, _)| ddpos))
    }

    /// Lower a slice pattern of form `[pat_0, ..., pat_n]` into
    /// `hir::PatKind::Slice(before, slice, after)`.
    ///
    /// When encountering `($binding_mode $ident @)? ..` (`slice`),
    /// this is interpreted as a sub-slice pattern semantically.
    /// Patterns that follow, which are not like `slice` -- or an error occurs, are in `after`.
    fn lower_pat_slice(&mut self, pats: &[P<Pat>]) -> hir::PatKind<'hir> {
        let mut before = Vec::new();
        let mut after = Vec::new();
        let mut slice = None;
        let mut prev_rest_span = None;

        // Lowers `$bm $ident @ ..` to `$bm $ident @ _`.
        let lower_rest_sub = |this: &mut Self, pat, bm, ident, sub| {
            let lower_sub = |this: &mut Self| Some(this.pat_wild_with_node_id_of(sub));
            let node = this.lower_pat_ident(pat, bm, ident, lower_sub);
            this.pat_with_node_id_of(pat, node)
        };

        let mut iter = pats.iter();
        // Lower all the patterns until the first occurrence of a sub-slice pattern.
        for pat in iter.by_ref() {
            match pat.kind {
                // Found a sub-slice pattern `..`. Record, lower it to `_`, and stop here.
                PatKind::Rest => {
                    prev_rest_span = Some(pat.span);
                    slice = Some(self.pat_wild_with_node_id_of(pat));
                    break;
                }
                // Found a sub-slice pattern `$binding_mode $ident @ ..`.
                // Record, lower it to `$binding_mode $ident @ _`, and stop here.
                PatKind::Ident(ref bm, ident, Some(ref sub)) if sub.is_rest() => {
                    prev_rest_span = Some(sub.span);
                    slice = Some(lower_rest_sub(self, pat, bm, ident, sub));
                    break;
                }
                // It was not a subslice pattern so lower it normally.
                _ => before.push(self.lower_pat(pat)),
            }
        }

        // Lower all the patterns after the first sub-slice pattern.
        for pat in iter {
            // There was a previous subslice pattern; make sure we don't allow more.
            let rest_span = match pat.kind {
                PatKind::Rest => Some(pat.span),
                PatKind::Ident(ref bm, ident, Some(ref sub)) if sub.is_rest() => {
                    // #69103: Lower into `binding @ _` as above to avoid ICEs.
                    after.push(lower_rest_sub(self, pat, bm, ident, sub));
                    Some(sub.span)
                }
                _ => None,
            };
            if let Some(rest_span) = rest_span {
                // We have e.g., `[a, .., b, ..]`. That's no good, error!
                self.ban_extra_rest_pat(rest_span, prev_rest_span.unwrap(), "slice");
            } else {
                // Lower the pattern normally.
                after.push(self.lower_pat(pat));
            }
        }

        hir::PatKind::Slice(
            self.arena.alloc_from_iter(before),
            slice,
            self.arena.alloc_from_iter(after),
        )
    }

    fn lower_pat_ident(
        &mut self,
        p: &Pat,
        binding_mode: &BindingMode,
        ident: Ident,
        lower_sub: impl FnOnce(&mut Self) -> Option<&'hir hir::Pat<'hir>>,
    ) -> hir::PatKind<'hir> {
        match self.resolver.get_partial_res(p.id).map(|d| d.base_res()) {
            // `None` can occur in body-less function signatures
            res @ (None | Some(Res::Local(_))) => {
                let canonical_id = match res {
                    Some(Res::Local(id)) => id,
                    _ => p.id,
                };

                hir::PatKind::Binding(
                    self.lower_binding_mode(binding_mode),
                    self.lower_node_id(canonical_id),
                    ident,
                    lower_sub(self),
                )
            }
            Some(res) => hir::PatKind::Path(hir::QPath::Resolved(
                None,
                self.arena.alloc(hir::Path {
                    span: ident.span,
                    res: self.lower_res(res),
                    segments: arena_vec![self; hir::PathSegment::from_ident(ident)],
                }),
            )),
        }
    }

    fn lower_binding_mode(&mut self, b: &BindingMode) -> hir::BindingAnnotation {
        match *b {
            BindingMode::ByValue(Mutability::Not) => hir::BindingAnnotation::Unannotated,
            BindingMode::ByRef(Mutability::Not) => hir::BindingAnnotation::Ref,
            BindingMode::ByValue(Mutability::Mut) => hir::BindingAnnotation::Mutable,
            BindingMode::ByRef(Mutability::Mut) => hir::BindingAnnotation::RefMut,
        }
    }

    fn pat_wild_with_node_id_of(&mut self, p: &Pat) -> &'hir hir::Pat<'hir> {
        self.pat_with_node_id_of(p, hir::PatKind::Wild)
    }

    /// Construct a `Pat` with the `HirId` of `p.id` lowered.
    fn pat_with_node_id_of(&mut self, p: &Pat, kind: hir::PatKind<'hir>) -> &'hir hir::Pat<'hir> {
        self.arena.alloc(hir::Pat { hir_id: self.lower_node_id(p.id), kind, span: p.span })
    }

    /// Emit a friendly error for extra `..` patterns in a tuple/tuple struct/slice pattern.
    fn ban_extra_rest_pat(&self, sp: Span, prev_sp: Span, ctx: &str) {
        self.diagnostic()
            .struct_span_err(sp, &format!("`..` can only be used once per {} pattern", ctx))
            .span_label(sp, &format!("can only be used once per {} pattern", ctx))
            .span_label(prev_sp, "previously used here")
            .emit();
    }

    /// Used to ban the `..` pattern in places it shouldn't be semantically.
    fn ban_illegal_rest_pat(&self, sp: Span) -> hir::PatKind<'hir> {
        self.diagnostic()
            .struct_span_err(sp, "`..` patterns are not allowed here")
            .note("only allowed in tuple, tuple struct, and slice patterns")
            .emit();

        // We're not in a list context so `..` can be reasonably treated
        // as `_` because it should always be valid and roughly matches the
        // intent of `..` (notice that the rest of a single slot is that slot).
        hir::PatKind::Wild
    }

    fn lower_range_end(&mut self, e: &RangeEnd, has_end: bool) -> hir::RangeEnd {
        match *e {
            RangeEnd::Excluded if has_end => hir::RangeEnd::Excluded,
            // No end; so `X..` behaves like `RangeFrom`.
            RangeEnd::Excluded | RangeEnd::Included(_) => hir::RangeEnd::Included,
        }
    }
}
<|endoftext|>


<commit_before>// twobit module containing a native implementation of the twobit query library
#![crate_id = "twobit#0.2"]
#![crate_type = "dylib"]
#![feature(macro_rules)]

extern crate native;
extern crate rustrt;

use std::os::MemoryMap;
use std::collections::hashmap::HashMap;
use std::io::{ IoResult, IoError };

use rustrt::rtio::RtioFileStream;

struct Block { start: u32, length: u32 }

struct Sequence {
	n_dna_bases: u32,
	unk_blocks: Vec<Block>,
	mask_blocks: Vec<Block>,
	dna_start: *mut u8
}

pub struct TwoBit {
	seqs: HashMap<String, Sequence>,

	#[allow(dead_code)]
	data: MemoryMap // this is needed to keep the memory map alive
}

macro_rules! try_rt(
    ($e:expr) => (match $e { Ok(e) => e, Err(rustrt::rtio::IoError{code: code, extra: _, detail: _}) => return Err(IoError::from_errno(code, true)) })
)

fn mmap_read_u32(ptr: * mut u8, offset: uint) -> u32 {
	return unsafe {
		let tmp : *u32 = std::mem::transmute(ptr.offset(offset as int) as *[u8, ..4]);
		*tmp };
}

fn mmap_read_u8(ptr: * mut u8, offset: uint) -> u8 {
	return unsafe {
		*ptr.offset(offset as int)
	};
}

fn read_blocks(data: * mut u8, offset: u32) -> (Vec<Block>, u32) {
	let len = mmap_read_u32(data, offset as uint);
	let mut vec = Vec::<Block>::new();

	if len > 0 {
		let off1 = offset + 4;
		let off2 = offset + 4 + len*4;

		for i in range(0, len) {
			let start = mmap_read_u32(data, (off1 + i*4) as uint);
			let size = mmap_read_u32(data, (off2 + i*4) as uint);

			vec.push(Block{ start: start, length: size });
		}
	}

	return (vec, offset + 4 + 2*len*4);
}

fn mmap_read_index(data: *mut u8, count: u32) -> HashMap<String, Sequence> {
		let mut index = HashMap::with_capacity(count as uint);

		let mut header_start = 16u;

		for _ in range(0, count) {
			let name_size = mmap_read_u8(data, header_start);
			let name = unsafe { String::from_raw_parts(name_size as uint, name_size as uint, data.offset((header_start + 1) as int)) };
			let offset = mmap_read_u32(data, header_start + 1 + name_size as uint);

			// get actual info
			let dna_size = mmap_read_u32(data, offset as uint);

			// unknown blocks
			let (unk_blocks, offset) = read_blocks(data, offset + 4);

			// masked blocks
			let (mask_blocks, offset) = read_blocks(data, offset);

			// actual pointer to DNA data
			let dna_ptr = unsafe { data.offset((offset + 4) as int) }; // + reserved

			index.insert(name, Sequence { n_dna_bases: dna_size, unk_blocks: unk_blocks, mask_blocks: mask_blocks, dna_start: dna_ptr });

			header_start = header_start + 1 + name_size as uint + 4;
		}

		return index;
}

fn byte_to_base(value: u8, offset: uint) -> char {
	let bases = ['T', 'C', 'A', 'G'];
	let rev_offset = 3 - offset;
	let mask = 3 << (rev_offset * 2);
	let idx = (value & mask) >> (rev_offset * 2);

	return bases[idx as uint];
}

impl TwoBit {

	pub fn new(filename: &str) -> IoResult<TwoBit> {
		// open file
		let mut fh = try_rt!(native::io::file::open(&filename.to_c_str(), rustrt::rtio::Open, rustrt::rtio::Read));
		let fs = try_rt!(fh.fstat());

		// build memory map
		let mmap = match MemoryMap::new(fs.size as uint, [ std::os::MapReadable, std::os::MapFd(fh.fd())]) {
			Ok(val) => val,
			Err(_) => return Err(IoError{kind: std::io::OtherIoError, desc: "Memory map failed!", detail: None})
		};

		// TODO: add madvise call

		// validate header
		let val = mmap_read_u32(mmap.data, 0);

		if val != 0x1A412743 {
			return Err(IoError { kind: std::io::OtherIoError, desc: "Invalid signature or wrong architecture.", detail: None });
		}

		let val = mmap_read_u32(mmap.data, 4);
		if val != 0 {
			return Err(IoError { kind: std::io::OtherIoError, desc: "Unknown file version.", detail: None });
		} // TODO: actually report version found

		let n_sequences = mmap_read_u32(mmap.data, 8);
		if n_sequences == 0 {
			return Err(IoError { kind: std::io::OtherIoError, desc: "Zero sequence count.", detail: None });
		}

		let val = mmap_read_u32(mmap.data, 12);
		if val != 0 {
			return Err(IoError { kind: std::io::OtherIoError, desc: "Reserved bytes not zero.", detail: None });
		} // TODO: actually report value found

		// parse index
		let index = mmap_read_index(mmap.data, n_sequences);

		return Ok(TwoBit { seqs: index, data: mmap });
	}

	pub fn sequence(&self, chrom: &str, start: u32, end: u32) -> Option<String> {
		match self.seqs.find(&String::from_str(chrom)) {
			Some(ref seq) => {
				let mut rsize = (end - start + 1) as uint;
				let mut end = end;
				let mut result = String::from_char(rsize, 'N');

				if end >= seq.n_dna_bases {
					end = seq.n_dna_bases - 1;
					rsize = (end - start + 1) as uint;
				}

				unsafe {
					let bvec = result.as_mut_bytes();

					/* fill sequence */
					let mut block = seq.dna_start.offset( (start / 4) as int);
					let mut offset = (start % 4) as uint;

					for i in range(0, rsize) {
						bvec[i] = byte_to_base(*block, offset) as u8;

						offset = offset + 1;
						if offset == 4 {
							offset = 0;
							block = block.offset(1);
						}
					}

					/* fill in Ns */
					for item in seq.unk_blocks.iter() {
						let mut bstart = item.start;
						let mut bsize = item.length;
						let bend = bstart + bsize - 1;

						if bstart <= end && bend >= start {
							if bstart < start {
								bsize = bsize - (start - bstart);
								bstart = start;
							}

							let mut j = 0;
							let mut k = bstart;
							while j < bsize && k <= end {
								bvec[(k - start) as uint] = 'N' as u8;

								j = j + 1;
								k = k + 1;
							}
						}
					}
				}

				Some(result)
			},
			None => None
		}
	}

	pub fn sequence_len(&self, chrom: &str) -> Option<u32> {
		match self.seqs.find(&String::from_str(chrom)) {
			Some(&Sequence{ n_dna_bases: n, ..}) => Some(n),
			None => None
		}
	}
}
<commit_msg>Function to get all sequence names<commit_after>// twobit module containing a native implementation of the twobit query library
#![crate_id = "twobit#0.2"]
#![crate_type = "dylib"]
#![feature(macro_rules)]

extern crate native;
extern crate rustrt;

use std::os::MemoryMap;
use std::collections::hashmap::HashMap;
use std::io::{ IoResult, IoError };

use rustrt::rtio::RtioFileStream;

struct Block { start: u32, length: u32 }

struct Sequence {
	n_dna_bases: u32,
	unk_blocks: Vec<Block>,
	mask_blocks: Vec<Block>,
	dna_start: *mut u8
}

pub struct TwoBit {
	seqs: HashMap<String, Sequence>,

	#[allow(dead_code)]
	data: MemoryMap // this is needed to keep the memory map alive
}

macro_rules! try_rt(
    ($e:expr) => (match $e { Ok(e) => e, Err(rustrt::rtio::IoError{code: code, extra: _, detail: _}) => return Err(IoError::from_errno(code, true)) })
)

fn mmap_read_u32(ptr: * mut u8, offset: uint) -> u32 {
	return unsafe {
		let tmp : *u32 = std::mem::transmute(ptr.offset(offset as int) as *[u8, ..4]);
		*tmp };
}

fn mmap_read_u8(ptr: * mut u8, offset: uint) -> u8 {
	return unsafe {
		*ptr.offset(offset as int)
	};
}

fn read_blocks(data: * mut u8, offset: u32) -> (Vec<Block>, u32) {
	let len = mmap_read_u32(data, offset as uint);
	let mut vec = Vec::<Block>::new();

	if len > 0 {
		let off1 = offset + 4;
		let off2 = offset + 4 + len*4;

		for i in range(0, len) {
			let start = mmap_read_u32(data, (off1 + i*4) as uint);
			let size = mmap_read_u32(data, (off2 + i*4) as uint);

			vec.push(Block{ start: start, length: size });
		}
	}

	return (vec, offset + 4 + 2*len*4);
}

fn mmap_read_index(data: *mut u8, count: u32) -> HashMap<String, Sequence> {
		let mut index = HashMap::with_capacity(count as uint);

		let mut header_start = 16u;

		for _ in range(0, count) {
			let name_size = mmap_read_u8(data, header_start);
			let name = unsafe { String::from_raw_parts(name_size as uint, name_size as uint, data.offset((header_start + 1) as int)) };
			let offset = mmap_read_u32(data, header_start + 1 + name_size as uint);

			// get actual info
			let dna_size = mmap_read_u32(data, offset as uint);

			// unknown blocks
			let (unk_blocks, offset) = read_blocks(data, offset + 4);

			// masked blocks
			let (mask_blocks, offset) = read_blocks(data, offset);

			// actual pointer to DNA data
			let dna_ptr = unsafe { data.offset((offset + 4) as int) }; // + reserved

			index.insert(name, Sequence { n_dna_bases: dna_size, unk_blocks: unk_blocks, mask_blocks: mask_blocks, dna_start: dna_ptr });

			header_start = header_start + 1 + name_size as uint + 4;
		}

		return index;
}

fn byte_to_base(value: u8, offset: uint) -> char {
	let bases = ['T', 'C', 'A', 'G'];
	let rev_offset = 3 - offset;
	let mask = 3 << (rev_offset * 2);
	let idx = (value & mask) >> (rev_offset * 2);

	return bases[idx as uint];
}

impl TwoBit {

	pub fn new(filename: &str) -> IoResult<TwoBit> {
		// open file
		let mut fh = try_rt!(native::io::file::open(&filename.to_c_str(), rustrt::rtio::Open, rustrt::rtio::Read));
		let fs = try_rt!(fh.fstat());

		// build memory map
		let mmap = match MemoryMap::new(fs.size as uint, [ std::os::MapReadable, std::os::MapFd(fh.fd())]) {
			Ok(val) => val,
			Err(_) => return Err(IoError{kind: std::io::OtherIoError, desc: "Memory map failed!", detail: None})
		};

		// TODO: add madvise call

		// validate header
		let val = mmap_read_u32(mmap.data, 0);

		if val != 0x1A412743 {
			return Err(IoError { kind: std::io::OtherIoError, desc: "Invalid signature or wrong architecture.", detail: None });
		}

		let val = mmap_read_u32(mmap.data, 4);
		if val != 0 {
			return Err(IoError { kind: std::io::OtherIoError, desc: "Unknown file version.", detail: None });
		} // TODO: actually report version found

		let n_sequences = mmap_read_u32(mmap.data, 8);
		if n_sequences == 0 {
			return Err(IoError { kind: std::io::OtherIoError, desc: "Zero sequence count.", detail: None });
		}

		let val = mmap_read_u32(mmap.data, 12);
		if val != 0 {
			return Err(IoError { kind: std::io::OtherIoError, desc: "Reserved bytes not zero.", detail: None });
		} // TODO: actually report value found

		// parse index
		let index = mmap_read_index(mmap.data, n_sequences);

		return Ok(TwoBit { seqs: index, data: mmap });
	}

	pub fn sequence(&self, chrom: &str, start: u32, end: u32) -> Option<String> {
		match self.seqs.find(&String::from_str(chrom)) {
			Some(ref seq) => {
				let mut rsize = (end - start + 1) as uint;
				let mut end = end;
				let mut result = String::from_char(rsize, 'N');

				if end >= seq.n_dna_bases {
					end = seq.n_dna_bases - 1;
					rsize = (end - start + 1) as uint;
				}

				unsafe {
					let bvec = result.as_mut_bytes();

					/* fill sequence */
					let mut block = seq.dna_start.offset( (start / 4) as int);
					let mut offset = (start % 4) as uint;

					for i in range(0, rsize) {
						bvec[i] = byte_to_base(*block, offset) as u8;

						offset = offset + 1;
						if offset == 4 {
							offset = 0;
							block = block.offset(1);
						}
					}

					/* fill in Ns */
					for item in seq.unk_blocks.iter() {
						let mut bstart = item.start;
						let mut bsize = item.length;
						let bend = bstart + bsize - 1;

						if bstart <= end && bend >= start {
							if bstart < start {
								bsize = bsize - (start - bstart);
								bstart = start;
							}

							let mut j = 0;
							let mut k = bstart;
							while j < bsize && k <= end {
								bvec[(k - start) as uint] = 'N' as u8;

								j = j + 1;
								k = k + 1;
							}
						}
					}
				}

				Some(result)
			},
			None => None
		}
	}

	pub fn sequence_len(&self, chrom: &str) -> Option<u32> {
		match self.seqs.find(&String::from_str(chrom)) {
			Some(&Sequence{ n_dna_bases: n, ..}) => Some(n),
			None => None
		}
	}

	pub fn sequence_names<'a>(&'a self) -> Vec<&'a String> {
		self.seqs.keys().collect()
	}
}
<|endoftext|>


<commit_before>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use serde::{Deserialize, Serialize};

#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StateSyncConfig {
    // Size of chunk to request for state synchronization
    pub chunk_limit: u64,
    // The timeout of the state sync client to process a commit notification (in milliseconds)
    pub client_commit_timeout_ms: u64,
    // default timeout used for long polling to remote peer
    pub long_poll_timeout_ms: u64,
    // valid maximum chunk limit for sanity check
    pub max_chunk_limit: u64,
    // valid maximum timeout limit for sanity check
    pub max_timeout_ms: u64,
    // The timeout of the state sync coordinator to receive a commit ack from mempool (in milliseconds)
    pub mempool_commit_timeout_ms: u64,
    // default timeout to make state sync progress by sending chunk requests to a certain number of networks
    // if no progress is made by sending chunk requests to a number of networks,
    // the next sync request will be multicasted, i.e. sent to more networks
    pub multicast_timeout_ms: u64,
    // The timeout for ensuring sync requests are making progress (i.e., the maximum time between
    // commits when processing a sync request).
    pub sync_request_timeout_ms: u64,
    // interval used for checking state synchronization progress
    pub tick_interval_ms: u64,

    // Everything above belongs to state sync v1 and will be removed in the future.
    pub data_streaming_service: DataStreamingServiceConfig,
    pub aptos_data_client: AptosDataClientConfig,
    pub state_sync_driver: StateSyncDriverConfig,
    pub storage_service: StorageServiceConfig,
}

impl Default for StateSyncConfig {
    fn default() -> Self {
        Self {
            chunk_limit: 1000,
            client_commit_timeout_ms: 5_000,
            long_poll_timeout_ms: 10_000,
            max_chunk_limit: 1000,
            max_timeout_ms: 120_000,
            mempool_commit_timeout_ms: 5_000,
            multicast_timeout_ms: 30_000,
            sync_request_timeout_ms: 60_000,
            tick_interval_ms: 100,
            data_streaming_service: DataStreamingServiceConfig::default(),
            aptos_data_client: AptosDataClientConfig::default(),
            state_sync_driver: StateSyncDriverConfig::default(),
            storage_service: StorageServiceConfig::default(),
        }
    }
}

/// The bootstrapping mode determines how the node will bootstrap to the latest
/// blockchain state, e.g., directly download the latest account states.
#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
pub enum BootstrappingMode {
    ApplyTransactionOutputsFromGenesis, // Applies transaction outputs (starting at genesis)
    DownloadLatestAccountStates,        // Downloads the account states (at the latest version)
    ExecuteTransactionsFromGenesis,     // Executes transactions (starting at genesis)
}

/// The continuous syncing mode determines how the node will stay up-to-date
/// once it has bootstrapped and the blockchain continues to grow, e.g.,
/// continuously executing all transactions.
#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
pub enum ContinuousSyncingMode {
    ExecuteTransactions,     // Executes transactions to stay up-to-date
    ApplyTransactionOutputs, // Applies transaction outputs to stay up-to-date
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StateSyncDriverConfig {
    pub bootstrapping_mode: BootstrappingMode, // The mode by which to bootstrap
    pub enable_state_sync_v2: bool,            // If the node should sync with state sync v2
    pub continuous_syncing_mode: ContinuousSyncingMode, // The mode by which to sync after bootstrapping
    pub progress_check_interval_ms: u64, // The interval (ms) at which to check state sync progress
    pub max_connection_deadline_secs: u64, // The max time (secs) to wait for connections from peers
    pub max_stream_wait_time_ms: u64,    // The max time (ms) to wait for a data stream notification
}

/// The default state sync driver config will be the one that gets (and keeps)
/// the node up-to-date as quickly and cheaply as possible.
impl Default for StateSyncDriverConfig {
    fn default() -> Self {
        Self {
            bootstrapping_mode: BootstrappingMode::DownloadLatestAccountStates,
            enable_state_sync_v2: false,
            continuous_syncing_mode: ContinuousSyncingMode::ApplyTransactionOutputs,
            progress_check_interval_ms: 100,
            max_connection_deadline_secs: 10,
            max_stream_wait_time_ms: 2000,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StorageServiceConfig {
    pub max_account_states_chunk_sizes: u64, // Max num of accounts per chunk
    pub max_concurrent_requests: u64,        // Max num of concurrent storage server tasks
    pub max_epoch_chunk_size: u64,           // Max num of epoch ending ledger infos per chunk
    pub max_transaction_chunk_size: u64,     // Max num of transactions per chunk
    pub max_transaction_output_chunk_size: u64, // Max num of transaction outputs per chunk
}

impl Default for StorageServiceConfig {
    fn default() -> Self {
        Self {
            max_account_states_chunk_sizes: 3000,
            max_concurrent_requests: 50,
            max_epoch_chunk_size: 100,
            max_transaction_chunk_size: 3000,
            max_transaction_output_chunk_size: 3000,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct DataStreamingServiceConfig {
    // The interval (milliseconds) at which to refresh the global data summary.
    pub global_summary_refresh_interval_ms: u64,

    // Maximum number of concurrent data client requests (per stream).
    pub max_concurrent_requests: u64,

    // Maximum channel sizes for each data stream listener. If messages are not
    // consumed, they will be dropped (oldest messages first). The remaining
    // messages will be retrieved using FIFO ordering.
    pub max_data_stream_channel_sizes: u64,

    // Maximum number of retries for a single client request before a data
    // stream will terminate.
    pub max_request_retry: u64,

    // Maximum number of notification ID to response context mappings held in
    // memory. Once the number grows beyond this value, garbage collection occurs.
    pub max_notification_id_mappings: u64,

    // The interval (milliseconds) at which to check the progress of each stream.
    pub progress_check_interval_ms: u64,
}

impl Default for DataStreamingServiceConfig {
    fn default() -> Self {
        Self {
            global_summary_refresh_interval_ms: 300,
            max_concurrent_requests: 5,
            max_data_stream_channel_sizes: 1000,
            max_request_retry: 5,
            max_notification_id_mappings: 2000,
            progress_check_interval_ms: 100,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct AptosDataClientConfig {
    pub response_timeout_ms: u64, // Timeout (in milliseconds) when waiting for a response
    pub summary_poll_interval_ms: u64, // Interval (in milliseconds) between data summary polls
}

impl Default for AptosDataClientConfig {
    fn default() -> Self {
        Self {
            response_timeout_ms: 3_000,
            summary_poll_interval_ms: 300,
        }
    }
}
<commit_msg>[State Sync] Tweaks to default configs for DevNet.<commit_after>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use serde::{Deserialize, Serialize};

#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StateSyncConfig {
    // Size of chunk to request for state synchronization
    pub chunk_limit: u64,
    // The timeout of the state sync client to process a commit notification (in milliseconds)
    pub client_commit_timeout_ms: u64,
    // default timeout used for long polling to remote peer
    pub long_poll_timeout_ms: u64,
    // valid maximum chunk limit for sanity check
    pub max_chunk_limit: u64,
    // valid maximum timeout limit for sanity check
    pub max_timeout_ms: u64,
    // The timeout of the state sync coordinator to receive a commit ack from mempool (in milliseconds)
    pub mempool_commit_timeout_ms: u64,
    // default timeout to make state sync progress by sending chunk requests to a certain number of networks
    // if no progress is made by sending chunk requests to a number of networks,
    // the next sync request will be multicasted, i.e. sent to more networks
    pub multicast_timeout_ms: u64,
    // The timeout for ensuring sync requests are making progress (i.e., the maximum time between
    // commits when processing a sync request).
    pub sync_request_timeout_ms: u64,
    // interval used for checking state synchronization progress
    pub tick_interval_ms: u64,

    // Everything above belongs to state sync v1 and will be removed in the future.
    pub data_streaming_service: DataStreamingServiceConfig,
    pub aptos_data_client: AptosDataClientConfig,
    pub state_sync_driver: StateSyncDriverConfig,
    pub storage_service: StorageServiceConfig,
}

impl Default for StateSyncConfig {
    fn default() -> Self {
        Self {
            chunk_limit: 1000,
            client_commit_timeout_ms: 5_000,
            long_poll_timeout_ms: 10_000,
            max_chunk_limit: 1000,
            max_timeout_ms: 120_000,
            mempool_commit_timeout_ms: 5_000,
            multicast_timeout_ms: 30_000,
            sync_request_timeout_ms: 60_000,
            tick_interval_ms: 100,
            data_streaming_service: DataStreamingServiceConfig::default(),
            aptos_data_client: AptosDataClientConfig::default(),
            state_sync_driver: StateSyncDriverConfig::default(),
            storage_service: StorageServiceConfig::default(),
        }
    }
}

/// The bootstrapping mode determines how the node will bootstrap to the latest
/// blockchain state, e.g., directly download the latest account states.
#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
pub enum BootstrappingMode {
    ApplyTransactionOutputsFromGenesis, // Applies transaction outputs (starting at genesis)
    DownloadLatestAccountStates,        // Downloads the account states (at the latest version)
    ExecuteTransactionsFromGenesis,     // Executes transactions (starting at genesis)
}

/// The continuous syncing mode determines how the node will stay up-to-date
/// once it has bootstrapped and the blockchain continues to grow, e.g.,
/// continuously executing all transactions.
#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
pub enum ContinuousSyncingMode {
    ExecuteTransactions,     // Executes transactions to stay up-to-date
    ApplyTransactionOutputs, // Applies transaction outputs to stay up-to-date
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StateSyncDriverConfig {
    pub bootstrapping_mode: BootstrappingMode, // The mode by which to bootstrap
    pub enable_state_sync_v2: bool,            // If the node should sync with state sync v2
    pub continuous_syncing_mode: ContinuousSyncingMode, // The mode by which to sync after bootstrapping
    pub progress_check_interval_ms: u64, // The interval (ms) at which to check state sync progress
    pub max_connection_deadline_secs: u64, // The max time (secs) to wait for connections from peers
    pub max_stream_wait_time_ms: u64,    // The max time (ms) to wait for a data stream notification
}

/// The default state sync driver config will be the one that gets (and keeps)
/// the node up-to-date as quickly and cheaply as possible.
impl Default for StateSyncDriverConfig {
    fn default() -> Self {
        Self {
            bootstrapping_mode: BootstrappingMode::DownloadLatestAccountStates,
            enable_state_sync_v2: false,
            continuous_syncing_mode: ContinuousSyncingMode::ApplyTransactionOutputs,
            progress_check_interval_ms: 100,
            max_connection_deadline_secs: 10,
            max_stream_wait_time_ms: 10_000,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct StorageServiceConfig {
    pub max_account_states_chunk_sizes: u64, // Max num of accounts per chunk
    pub max_concurrent_requests: u64,        // Max num of concurrent storage server tasks
    pub max_epoch_chunk_size: u64,           // Max num of epoch ending ledger infos per chunk
    pub max_transaction_chunk_size: u64,     // Max num of transactions per chunk
    pub max_transaction_output_chunk_size: u64, // Max num of transaction outputs per chunk
}

impl Default for StorageServiceConfig {
    fn default() -> Self {
        Self {
            max_account_states_chunk_sizes: 3000,
            max_concurrent_requests: 50,
            max_epoch_chunk_size: 100,
            max_transaction_chunk_size: 3000,
            max_transaction_output_chunk_size: 3000,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct DataStreamingServiceConfig {
    // The interval (milliseconds) at which to refresh the global data summary.
    pub global_summary_refresh_interval_ms: u64,

    // Maximum number of concurrent data client requests (per stream).
    pub max_concurrent_requests: u64,

    // Maximum channel sizes for each data stream listener. If messages are not
    // consumed, they will be dropped (oldest messages first). The remaining
    // messages will be retrieved using FIFO ordering.
    pub max_data_stream_channel_sizes: u64,

    // Maximum number of retries for a single client request before a data
    // stream will terminate.
    pub max_request_retry: u64,

    // Maximum number of notification ID to response context mappings held in
    // memory. Once the number grows beyond this value, garbage collection occurs.
    pub max_notification_id_mappings: u64,

    // The interval (milliseconds) at which to check the progress of each stream.
    pub progress_check_interval_ms: u64,
}

impl Default for DataStreamingServiceConfig {
    fn default() -> Self {
        Self {
            global_summary_refresh_interval_ms: 300,
            max_concurrent_requests: 5,
            max_data_stream_channel_sizes: 1000,
            max_request_retry: 5,
            max_notification_id_mappings: 2000,
            progress_check_interval_ms: 100,
        }
    }
}

#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct AptosDataClientConfig {
    pub response_timeout_ms: u64, // Timeout (in milliseconds) when waiting for a response
    pub summary_poll_interval_ms: u64, // Interval (in milliseconds) between data summary polls
}

impl Default for AptosDataClientConfig {
    fn default() -> Self {
        Self {
            response_timeout_ms: 10_000,
            summary_poll_interval_ms: 1_000,
        }
    }
}
<|endoftext|>


<commit_before>//! Types for the *m.ignored_user_list* event.

use ruma_events_macros::EventContent;
use ruma_identifiers::UserId;
use serde::{Deserialize, Serialize};

use crate::GlobalAccountDataEvent;

/// A list of users to ignore.
pub type IgnoredUserListEvent = GlobalAccountDataEvent<IgnoredUserListEventContent>;

/// The payload for `IgnoredUserListEvent`.
#[derive(Clone, Debug, Deserialize, Serialize, EventContent)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
#[ruma_event(type = "m.ignored_user_list")]
pub struct IgnoredUserListEventContent {
    /// A list of users to ignore.
    #[serde(with = "ruma_serde::vec_as_map_of_empty")]
    pub ignored_users: Vec<UserId>,
}

impl IgnoredUserListEventContent {
    /// Creates a new `IgnoredUserListEventContent` from the given user IDs.
    pub fn new(ignored_users: Vec<UserId>) -> Self {
        Self { ignored_users }
    }
}

#[cfg(test)]
mod tests {
    use matches::assert_matches;
    use ruma_identifiers::user_id;
    use ruma_serde::Raw;
    use serde_json::{from_value as from_json_value, json, to_value as to_json_value};

    use super::IgnoredUserListEventContent;
    use crate::{AnyGlobalAccountDataEventContent, GlobalAccountDataEvent};

    #[test]
    fn serialization() {
        let ignored_user_list_event = GlobalAccountDataEvent {
            content: IgnoredUserListEventContent {
                ignored_users: vec![user_id!("@carl:example.com")],
            },
        };

        let json = json!({
            "content": {
                "ignored_users": {
                    "@carl:example.com": {}
                }
            },
            "type": "m.ignored_user_list"
        });

        assert_eq!(to_json_value(ignored_user_list_event).unwrap(), json);
    }

    #[test]
    fn deserialization() {
        let json = json!({
            "content": {
                "ignored_users": {
                    "@carl:example.com": {}
                }
            },
            "type": "m.ignored_user_list"
        });

        assert_matches!(
            from_json_value::<Raw<GlobalAccountDataEvent<AnyGlobalAccountDataEventContent>>>(json)
                .unwrap()
                .deserialize()
                .unwrap(),
            GlobalAccountDataEvent {
                content: AnyGlobalAccountDataEventContent::IgnoredUserList(IgnoredUserListEventContent {
                    ignored_users,
                }),
            } if ignored_users == vec![user_id!("@carl:example.com")]
        );
    }
}
<commit_msg>events: Update IgnoredUserListEvent test<commit_after>//! Types for the *m.ignored_user_list* event.

use ruma_events_macros::EventContent;
use ruma_identifiers::UserId;
use serde::{Deserialize, Serialize};

use crate::GlobalAccountDataEvent;

/// A list of users to ignore.
pub type IgnoredUserListEvent = GlobalAccountDataEvent<IgnoredUserListEventContent>;

/// The payload for `IgnoredUserListEvent`.
#[derive(Clone, Debug, Deserialize, Serialize, EventContent)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
#[ruma_event(type = "m.ignored_user_list")]
pub struct IgnoredUserListEventContent {
    /// A list of users to ignore.
    #[serde(with = "ruma_serde::vec_as_map_of_empty")]
    pub ignored_users: Vec<UserId>,
}

impl IgnoredUserListEventContent {
    /// Creates a new `IgnoredUserListEventContent` from the given user IDs.
    pub fn new(ignored_users: Vec<UserId>) -> Self {
        Self { ignored_users }
    }
}

#[cfg(test)]
mod tests {
    use matches::assert_matches;
    use ruma_identifiers::user_id;
    use ruma_serde::Raw;
    use serde_json::{from_value as from_json_value, json, to_value as to_json_value};

    use super::{IgnoredUserListEvent, IgnoredUserListEventContent};
    use crate::{AnyGlobalAccountDataEvent, GlobalAccountDataEvent};

    #[test]
    fn serialization() {
        let ignored_user_list_event = GlobalAccountDataEvent {
            content: IgnoredUserListEventContent {
                ignored_users: vec![user_id!("@carl:example.com")],
            },
        };

        let json = json!({
            "content": {
                "ignored_users": {
                    "@carl:example.com": {}
                }
            },
            "type": "m.ignored_user_list"
        });

        assert_eq!(to_json_value(ignored_user_list_event).unwrap(), json);
    }

    #[test]
    fn deserialization() {
        let json = json!({
            "content": {
                "ignored_users": {
                    "@carl:example.com": {}
                }
            },
            "type": "m.ignored_user_list"
        });

        assert_matches!(
            from_json_value::<Raw<AnyGlobalAccountDataEvent>>(json)
                .unwrap()
                .deserialize()
                .unwrap(),
            AnyGlobalAccountDataEvent::IgnoredUserList(
                IgnoredUserListEvent {
                    content: IgnoredUserListEventContent {
                        ignored_users
                    },
                })
         if ignored_users == vec![user_id!("@carl:example.com")]
        );
    }
}
<|endoftext|>


<commit_before>//! Semantic Tokens helpers

use std::ops;

use lsp_types::{
    Range, SemanticToken, SemanticTokenModifier, SemanticTokenType, SemanticTokens,
    SemanticTokensEdit,
};

macro_rules! define_semantic_token_types {
    ($(($ident:ident, $string:literal)),*$(,)?) => {
        $(pub(crate) const $ident: SemanticTokenType = SemanticTokenType::new($string);)*

        pub(crate) const SUPPORTED_TYPES: &[SemanticTokenType] = &[
            SemanticTokenType::COMMENT,
            SemanticTokenType::KEYWORD,
            SemanticTokenType::STRING,
            SemanticTokenType::NUMBER,
            SemanticTokenType::REGEXP,
            SemanticTokenType::OPERATOR,
            SemanticTokenType::NAMESPACE,
            SemanticTokenType::TYPE,
            SemanticTokenType::STRUCT,
            SemanticTokenType::CLASS,
            SemanticTokenType::INTERFACE,
            SemanticTokenType::ENUM,
            SemanticTokenType::ENUM_MEMBER,
            SemanticTokenType::TYPE_PARAMETER,
            SemanticTokenType::FUNCTION,
            SemanticTokenType::METHOD,
            SemanticTokenType::PROPERTY,
            SemanticTokenType::MACRO,
            SemanticTokenType::VARIABLE,
            SemanticTokenType::PARAMETER,
            $($ident),*
        ];
    };
}

define_semantic_token_types![
    (ANGLE, "angle"),
    (ARITHMETIC, "arithmetic"),
    (ATTRIBUTE, "attribute"),
    (BITWISE, "bitwise"),
    (BOOLEAN, "boolean"),
    (BRACE, "brace"),
    (BRACKET, "bracket"),
    (BUILTIN_ATTRIBUTE, "builtinAttribute"),
    (BUILTIN_TYPE, "builtinType"),
    (CHAR, "character"),
    (COLON, "colon"),
    (COMMA, "comma"),
    (COMPARISON, "comparison"),
    (CONST_PARAMETER, "constParameter"),
    (DOT, "dot"),
    (ESCAPE_SEQUENCE, "escapeSequence"),
    (FORMAT_SPECIFIER, "formatSpecifier"),
    (GENERIC, "generic"),
    (LABEL, "label"),
    (LIFETIME, "lifetime"),
    (LOGICAL, "logical"),
    (OPERATOR, "operator"),
    (PARENTHESIS, "parenthesis"),
    (PUNCTUATION, "punctuation"),
    (SELF_KEYWORD, "selfKeyword"),
    (SEMICOLON, "semicolon"),
    (TYPE_ALIAS, "typeAlias"),
    (UNION, "union"),
    (UNRESOLVED_REFERENCE, "unresolvedReference"),
];

macro_rules! define_semantic_token_modifiers {
    ($(($ident:ident, $string:literal)),*$(,)?) => {
        $(pub(crate) const $ident: SemanticTokenModifier = SemanticTokenModifier::new($string);)*

        pub(crate) const SUPPORTED_MODIFIERS: &[SemanticTokenModifier] = &[
            SemanticTokenModifier::DOCUMENTATION,
            SemanticTokenModifier::DECLARATION,
            SemanticTokenModifier::DEFINITION,
            SemanticTokenModifier::STATIC,
            SemanticTokenModifier::ABSTRACT,
            SemanticTokenModifier::DEPRECATED,
            SemanticTokenModifier::READONLY,
            $($ident),*
        ];
    };
}

define_semantic_token_modifiers![
    (ASYNC, "async"),
    (ATTRIBUTE_MODIFIER, "attribute"),
    (CALLABLE, "callable"),
    (CONSTANT, "constant"),
    (CONSUMING, "consuming"),
    (CONTROL_FLOW, "controlFlow"),
    (INJECTED, "injected"),
    (INTRA_DOC_LINK, "intraDocLink"),
    (LIBRARY, "library"),
    (MUTABLE, "mutable"),
    (PUBLIC, "public"),
    (REFERENCE, "reference"),
    (TRAIT_MODIFIER, "trait"),
    (UNSAFE, "unsafe"),
];

#[derive(Default)]
pub(crate) struct ModifierSet(pub(crate) u32);

impl ops::BitOrAssign<SemanticTokenModifier> for ModifierSet {
    fn bitor_assign(&mut self, rhs: SemanticTokenModifier) {
        let idx = SUPPORTED_MODIFIERS.iter().position(|it| it == &rhs).unwrap();
        self.0 |= 1 << idx;
    }
}

/// Tokens are encoded relative to each other.
///
/// This is a direct port of <https://github.com/microsoft/vscode-languageserver-node/blob/f425af9de46a0187adb78ec8a46b9b2ce80c5412/server/src/sematicTokens.proposed.ts#L45>
pub(crate) struct SemanticTokensBuilder {
    id: String,
    prev_line: u32,
    prev_char: u32,
    data: Vec<SemanticToken>,
}

impl SemanticTokensBuilder {
    pub(crate) fn new(id: String) -> Self {
        SemanticTokensBuilder { id, prev_line: 0, prev_char: 0, data: Default::default() }
    }

    /// Push a new token onto the builder
    pub(crate) fn push(&mut self, range: Range, token_index: u32, modifier_bitset: u32) {
        let mut push_line = range.start.line as u32;
        let mut push_char = range.start.character as u32;

        if !self.data.is_empty() {
            push_line -= self.prev_line;
            if push_line == 0 {
                push_char -= self.prev_char;
            }
        }

        // A token cannot be multiline
        let token_len = range.end.character - range.start.character;

        let token = SemanticToken {
            delta_line: push_line,
            delta_start: push_char,
            length: token_len as u32,
            token_type: token_index,
            token_modifiers_bitset: modifier_bitset,
        };

        self.data.push(token);

        self.prev_line = range.start.line as u32;
        self.prev_char = range.start.character as u32;
    }

    pub(crate) fn build(self) -> SemanticTokens {
        SemanticTokens { result_id: Some(self.id), data: self.data }
    }
}

pub(crate) fn diff_tokens(old: &[SemanticToken], new: &[SemanticToken]) -> Vec<SemanticTokensEdit> {
    let offset = new.iter().zip(old.iter()).take_while(|&(n, p)| n == p).count();

    let (_, old) = old.split_at(offset);
    let (_, new) = new.split_at(offset);

    let offset_from_end =
        new.iter().rev().zip(old.iter().rev()).take_while(|&(n, p)| n == p).count();

    let (old, _) = old.split_at(old.len() - offset_from_end);
    let (new, _) = new.split_at(new.len() - offset_from_end);

    if old.is_empty() && new.is_empty() {
        vec![]
    } else {
        // The lsp data field is actually a byte-diff but we
        // travel in tokens so `start` and `delete_count` are in multiples of the
        // serialized size of `SemanticToken`.
        vec![SemanticTokensEdit {
            start: 5 * offset as u32,
            delete_count: 5 * old.len() as u32,
            data: Some(new.into()),
        }]
    }
}

pub(crate) fn type_index(ty: SemanticTokenType) -> u32 {
    SUPPORTED_TYPES.iter().position(|it| *it == ty).unwrap() as u32
}

#[cfg(test)]
mod tests {
    use super::*;

    fn from(t: (u32, u32, u32, u32, u32)) -> SemanticToken {
        SemanticToken {
            delta_line: t.0,
            delta_start: t.1,
            length: t.2,
            token_type: t.3,
            token_modifiers_bitset: t.4,
        }
    }

    #[test]
    fn test_diff_insert_at_end() {
        let before = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
        let after = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10)), from((11, 12, 13, 14, 15))];

        let edits = diff_tokens(&before, &after);
        assert_eq!(
            edits[0],
            SemanticTokensEdit {
                start: 10,
                delete_count: 0,
                data: Some(vec![from((11, 12, 13, 14, 15))])
            }
        );
    }

    #[test]
    fn test_diff_insert_at_beginning() {
        let before = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
        let after = [from((11, 12, 13, 14, 15)), from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];

        let edits = diff_tokens(&before, &after);
        assert_eq!(
            edits[0],
            SemanticTokensEdit {
                start: 0,
                delete_count: 0,
                data: Some(vec![from((11, 12, 13, 14, 15))])
            }
        );
    }

    #[test]
    fn test_diff_insert_in_middle() {
        let before = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
        let after = [
            from((1, 2, 3, 4, 5)),
            from((10, 20, 30, 40, 50)),
            from((60, 70, 80, 90, 100)),
            from((6, 7, 8, 9, 10)),
        ];

        let edits = diff_tokens(&before, &after);
        assert_eq!(
            edits[0],
            SemanticTokensEdit {
                start: 5,
                delete_count: 0,
                data: Some(vec![from((10, 20, 30, 40, 50)), from((60, 70, 80, 90, 100))])
            }
        );
    }

    #[test]
    fn test_diff_remove_from_end() {
        let before = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10)), from((11, 12, 13, 14, 15))];
        let after = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];

        let edits = diff_tokens(&before, &after);
        assert_eq!(edits[0], SemanticTokensEdit { start: 10, delete_count: 5, data: Some(vec![]) });
    }

    #[test]
    fn test_diff_remove_from_beginning() {
        let before = [from((11, 12, 13, 14, 15)), from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
        let after = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];

        let edits = diff_tokens(&before, &after);
        assert_eq!(edits[0], SemanticTokensEdit { start: 0, delete_count: 5, data: Some(vec![]) });
    }

    #[test]
    fn test_diff_remove_from_middle() {
        let before = [
            from((1, 2, 3, 4, 5)),
            from((10, 20, 30, 40, 50)),
            from((60, 70, 80, 90, 100)),
            from((6, 7, 8, 9, 10)),
        ];
        let after = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];

        let edits = diff_tokens(&before, &after);
        assert_eq!(edits[0], SemanticTokensEdit { start: 5, delete_count: 10, data: Some(vec![]) });
    }
}
<commit_msg>Fix add missing semantic token modifier in SUPPORTED_MODIFIERS<commit_after>//! Semantic Tokens helpers

use std::ops;

use lsp_types::{
    Range, SemanticToken, SemanticTokenModifier, SemanticTokenType, SemanticTokens,
    SemanticTokensEdit,
};

macro_rules! define_semantic_token_types {
    ($(($ident:ident, $string:literal)),*$(,)?) => {
        $(pub(crate) const $ident: SemanticTokenType = SemanticTokenType::new($string);)*

        pub(crate) const SUPPORTED_TYPES: &[SemanticTokenType] = &[
            SemanticTokenType::COMMENT,
            SemanticTokenType::KEYWORD,
            SemanticTokenType::STRING,
            SemanticTokenType::NUMBER,
            SemanticTokenType::REGEXP,
            SemanticTokenType::OPERATOR,
            SemanticTokenType::NAMESPACE,
            SemanticTokenType::TYPE,
            SemanticTokenType::STRUCT,
            SemanticTokenType::CLASS,
            SemanticTokenType::INTERFACE,
            SemanticTokenType::ENUM,
            SemanticTokenType::ENUM_MEMBER,
            SemanticTokenType::TYPE_PARAMETER,
            SemanticTokenType::FUNCTION,
            SemanticTokenType::METHOD,
            SemanticTokenType::PROPERTY,
            SemanticTokenType::MACRO,
            SemanticTokenType::VARIABLE,
            SemanticTokenType::PARAMETER,
            $($ident),*
        ];
    };
}

define_semantic_token_types![
    (ANGLE, "angle"),
    (ARITHMETIC, "arithmetic"),
    (ATTRIBUTE, "attribute"),
    (BITWISE, "bitwise"),
    (BOOLEAN, "boolean"),
    (BRACE, "brace"),
    (BRACKET, "bracket"),
    (BUILTIN_ATTRIBUTE, "builtinAttribute"),
    (BUILTIN_TYPE, "builtinType"),
    (CHAR, "character"),
    (COLON, "colon"),
    (COMMA, "comma"),
    (COMPARISON, "comparison"),
    (CONST_PARAMETER, "constParameter"),
    (DOT, "dot"),
    (ESCAPE_SEQUENCE, "escapeSequence"),
    (FORMAT_SPECIFIER, "formatSpecifier"),
    (GENERIC, "generic"),
    (LABEL, "label"),
    (LIFETIME, "lifetime"),
    (LOGICAL, "logical"),
    (OPERATOR, "operator"),
    (PARENTHESIS, "parenthesis"),
    (PUNCTUATION, "punctuation"),
    (SELF_KEYWORD, "selfKeyword"),
    (SEMICOLON, "semicolon"),
    (TYPE_ALIAS, "typeAlias"),
    (UNION, "union"),
    (UNRESOLVED_REFERENCE, "unresolvedReference"),
];

macro_rules! define_semantic_token_modifiers {
    ($(($ident:ident, $string:literal)),*$(,)?) => {
        $(pub(crate) const $ident: SemanticTokenModifier = SemanticTokenModifier::new($string);)*

        pub(crate) const SUPPORTED_MODIFIERS: &[SemanticTokenModifier] = &[
            SemanticTokenModifier::DOCUMENTATION,
            SemanticTokenModifier::DECLARATION,
            SemanticTokenModifier::DEFINITION,
            SemanticTokenModifier::STATIC,
            SemanticTokenModifier::ABSTRACT,
            SemanticTokenModifier::DEPRECATED,
            SemanticTokenModifier::READONLY,
            SemanticTokenModifier::DEFAULT_LIBRARY,
            $($ident),*
        ];
    };
}

define_semantic_token_modifiers![
    (ASYNC, "async"),
    (ATTRIBUTE_MODIFIER, "attribute"),
    (CALLABLE, "callable"),
    (CONSTANT, "constant"),
    (CONSUMING, "consuming"),
    (CONTROL_FLOW, "controlFlow"),
    (INJECTED, "injected"),
    (INTRA_DOC_LINK, "intraDocLink"),
    (LIBRARY, "library"),
    (MUTABLE, "mutable"),
    (PUBLIC, "public"),
    (REFERENCE, "reference"),
    (TRAIT_MODIFIER, "trait"),
    (UNSAFE, "unsafe"),
];

#[derive(Default)]
pub(crate) struct ModifierSet(pub(crate) u32);

impl ops::BitOrAssign<SemanticTokenModifier> for ModifierSet {
    fn bitor_assign(&mut self, rhs: SemanticTokenModifier) {
        let idx = SUPPORTED_MODIFIERS.iter().position(|it| it == &rhs).unwrap();
        self.0 |= 1 << idx;
    }
}

/// Tokens are encoded relative to each other.
///
/// This is a direct port of <https://github.com/microsoft/vscode-languageserver-node/blob/f425af9de46a0187adb78ec8a46b9b2ce80c5412/server/src/sematicTokens.proposed.ts#L45>
pub(crate) struct SemanticTokensBuilder {
    id: String,
    prev_line: u32,
    prev_char: u32,
    data: Vec<SemanticToken>,
}

impl SemanticTokensBuilder {
    pub(crate) fn new(id: String) -> Self {
        SemanticTokensBuilder { id, prev_line: 0, prev_char: 0, data: Default::default() }
    }

    /// Push a new token onto the builder
    pub(crate) fn push(&mut self, range: Range, token_index: u32, modifier_bitset: u32) {
        let mut push_line = range.start.line as u32;
        let mut push_char = range.start.character as u32;

        if !self.data.is_empty() {
            push_line -= self.prev_line;
            if push_line == 0 {
                push_char -= self.prev_char;
            }
        }

        // A token cannot be multiline
        let token_len = range.end.character - range.start.character;

        let token = SemanticToken {
            delta_line: push_line,
            delta_start: push_char,
            length: token_len as u32,
            token_type: token_index,
            token_modifiers_bitset: modifier_bitset,
        };

        self.data.push(token);

        self.prev_line = range.start.line as u32;
        self.prev_char = range.start.character as u32;
    }

    pub(crate) fn build(self) -> SemanticTokens {
        SemanticTokens { result_id: Some(self.id), data: self.data }
    }
}

pub(crate) fn diff_tokens(old: &[SemanticToken], new: &[SemanticToken]) -> Vec<SemanticTokensEdit> {
    let offset = new.iter().zip(old.iter()).take_while(|&(n, p)| n == p).count();

    let (_, old) = old.split_at(offset);
    let (_, new) = new.split_at(offset);

    let offset_from_end =
        new.iter().rev().zip(old.iter().rev()).take_while(|&(n, p)| n == p).count();

    let (old, _) = old.split_at(old.len() - offset_from_end);
    let (new, _) = new.split_at(new.len() - offset_from_end);

    if old.is_empty() && new.is_empty() {
        vec![]
    } else {
        // The lsp data field is actually a byte-diff but we
        // travel in tokens so `start` and `delete_count` are in multiples of the
        // serialized size of `SemanticToken`.
        vec![SemanticTokensEdit {
            start: 5 * offset as u32,
            delete_count: 5 * old.len() as u32,
            data: Some(new.into()),
        }]
    }
}

pub(crate) fn type_index(ty: SemanticTokenType) -> u32 {
    SUPPORTED_TYPES.iter().position(|it| *it == ty).unwrap() as u32
}

#[cfg(test)]
mod tests {
    use super::*;

    fn from(t: (u32, u32, u32, u32, u32)) -> SemanticToken {
        SemanticToken {
            delta_line: t.0,
            delta_start: t.1,
            length: t.2,
            token_type: t.3,
            token_modifiers_bitset: t.4,
        }
    }

    #[test]
    fn test_diff_insert_at_end() {
        let before = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
        let after = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10)), from((11, 12, 13, 14, 15))];

        let edits = diff_tokens(&before, &after);
        assert_eq!(
            edits[0],
            SemanticTokensEdit {
                start: 10,
                delete_count: 0,
                data: Some(vec![from((11, 12, 13, 14, 15))])
            }
        );
    }

    #[test]
    fn test_diff_insert_at_beginning() {
        let before = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
        let after = [from((11, 12, 13, 14, 15)), from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];

        let edits = diff_tokens(&before, &after);
        assert_eq!(
            edits[0],
            SemanticTokensEdit {
                start: 0,
                delete_count: 0,
                data: Some(vec![from((11, 12, 13, 14, 15))])
            }
        );
    }

    #[test]
    fn test_diff_insert_in_middle() {
        let before = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
        let after = [
            from((1, 2, 3, 4, 5)),
            from((10, 20, 30, 40, 50)),
            from((60, 70, 80, 90, 100)),
            from((6, 7, 8, 9, 10)),
        ];

        let edits = diff_tokens(&before, &after);
        assert_eq!(
            edits[0],
            SemanticTokensEdit {
                start: 5,
                delete_count: 0,
                data: Some(vec![from((10, 20, 30, 40, 50)), from((60, 70, 80, 90, 100))])
            }
        );
    }

    #[test]
    fn test_diff_remove_from_end() {
        let before = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10)), from((11, 12, 13, 14, 15))];
        let after = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];

        let edits = diff_tokens(&before, &after);
        assert_eq!(edits[0], SemanticTokensEdit { start: 10, delete_count: 5, data: Some(vec![]) });
    }

    #[test]
    fn test_diff_remove_from_beginning() {
        let before = [from((11, 12, 13, 14, 15)), from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
        let after = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];

        let edits = diff_tokens(&before, &after);
        assert_eq!(edits[0], SemanticTokensEdit { start: 0, delete_count: 5, data: Some(vec![]) });
    }

    #[test]
    fn test_diff_remove_from_middle() {
        let before = [
            from((1, 2, 3, 4, 5)),
            from((10, 20, 30, 40, 50)),
            from((60, 70, 80, 90, 100)),
            from((6, 7, 8, 9, 10)),
        ];
        let after = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];

        let edits = diff_tokens(&before, &after);
        assert_eq!(edits[0], SemanticTokensEdit { start: 5, delete_count: 10, data: Some(vec![]) });
    }
}
<|endoftext|>


<commit_before>use std::fs;

use kite_rocksdb::RocksDBIndexStore;

use analysis::AnalyzerSpec;
use analysis::tokenizers::TokenizerSpec;
use analysis::filters::FilterSpec;
use analysis::ngram_generator::Edge;
use index::Index;

use api::persistent;
use api::iron::prelude::*;
use api::iron::status;
use api::router::Router;
use api::utils::json_response;

pub fn view_get_index(req: &mut Request) -> IronResult<Response> {
    let ref system = get_system!(req);
    let ref index_name = read_path_parameter!(req, "index").unwrap_or("");

    // Lock index array
    let indices = system.indices.read().unwrap();

    // Check index exists
    get_index_or_404!(indices, *index_name);

    // TODO
    return Ok(json_response(status::Ok, "{}"));
}

pub fn view_put_index(req: &mut Request) -> IronResult<Response> {
    let ref system = get_system!(req);
    let ref index_name = read_path_parameter!(req, "index").unwrap_or("");

    // Lock index array
    let mut indices = system.indices.write().unwrap();

    // Load data from body
    // let data = json_from_request_body!(req);

    // Find index
    let index_ref = indices.names.find_one(&index_name);

    match index_ref {
        Some(_) => {
            // Update existing index
            // TODO

            system.log.info("[api] updated index", b!("index" => *index_name));
        }
        None => {
            // Create index
            let mut indices_dir = system.get_indices_dir();
            indices_dir.push(index_name);
            indices_dir.set_extension("rsi");
            let mut index = Index::new(index_name.clone().to_owned(), RocksDBIndexStore::create(indices_dir).unwrap());

            // Insert standard and edgengram analyzers
            // TODO: Load these from index settings
            index.analyzers.insert("standard".to_string(), AnalyzerSpec {
                tokenizer: TokenizerSpec::Standard,
                filters: vec![
                    FilterSpec::Lowercase,
                    FilterSpec::ASCIIFolding,
                ]
            });
            index.analyzers.insert("edgengram_analyzer".to_string(), AnalyzerSpec {
                tokenizer: TokenizerSpec::Standard,
                filters: vec![
                    FilterSpec::Lowercase,
                    FilterSpec::ASCIIFolding,
                    FilterSpec::NGram {
                        min_size: 2,
                        max_size: 15,
                        edge: Edge::Left
                    },
                ]
            });

            // TODO: load settings

            let index_ref = indices.insert(index);
            indices.names.insert_canonical(index_name.clone().to_owned(), index_ref).unwrap();

            system.log.info("[api] created index", b!("index" => *index_name));
        }
    }

    return Ok(json_response(status::Ok, "{\"acknowledged\": true}"));
}

pub fn view_delete_index(req: &mut Request) -> IronResult<Response> {
    let ref system = get_system!(req);
    let ref index_name = read_path_parameter!(req, "index").unwrap_or("");

    // Lock index array
    let mut indices = system.indices.write().unwrap();

    // Make sure the index exists
    get_index_or_404!(indices, *index_name);

    // Remove index from array
    let index_ref = indices.names.find_one(*index_name);

    if let Some(index_ref) = index_ref {
        indices.remove(&index_ref);

        // Delete canonical name
        indices.names.delete_canonical(&index_name, index_ref).unwrap();
    }

    // Delete file
    let mut indices_dir = system.get_indices_dir();
    indices_dir.push(index_name);
    indices_dir.set_extension("rsi");
    match fs::remove_dir_all(&indices_dir) {
        Ok(()) => {},
        Err(e) => {
            system.log.warn("[api] failed to delete index data", b!("index" => *index_name, "error" => format!("{}", e)));
        }
    }

    system.log.info("[api] deleted index", b!("index" => *index_name));

    return Ok(json_response(status::Ok, "{\"acknowledged\": true}"));
}

pub fn view_post_refresh_index(_req: &mut Request) -> IronResult<Response> {
    // let ref system = get_system!(req);
    // let ref index_name = read_path_parameter!(req, "index").unwrap_or("");

    // Lock index array
    // TODO
    // let mut indices = system.indices.write().unwrap();

    // TODO: {"_shards":{"total":10,"successful":5,"failed":0}}
    return Ok(json_response(status::Ok, "{\"acknowledged\": true}"));
}
<commit_msg>Make delete by alias work properly<commit_after>use std::fs;

use kite_rocksdb::RocksDBIndexStore;

use analysis::AnalyzerSpec;
use analysis::tokenizers::TokenizerSpec;
use analysis::filters::FilterSpec;
use analysis::ngram_generator::Edge;
use index::Index;

use api::persistent;
use api::iron::prelude::*;
use api::iron::status;
use api::router::Router;
use api::utils::json_response;

pub fn view_get_index(req: &mut Request) -> IronResult<Response> {
    let ref system = get_system!(req);
    let ref index_name = read_path_parameter!(req, "index").unwrap_or("");

    // Lock index array
    let indices = system.indices.read().unwrap();

    // Check index exists
    get_index_or_404!(indices, *index_name);

    // TODO
    return Ok(json_response(status::Ok, "{}"));
}

pub fn view_put_index(req: &mut Request) -> IronResult<Response> {
    let ref system = get_system!(req);
    let ref index_name = read_path_parameter!(req, "index").unwrap_or("");

    // Lock index array
    let mut indices = system.indices.write().unwrap();

    // Load data from body
    // let data = json_from_request_body!(req);

    // Find index
    let index_ref = indices.names.find_one(&index_name);

    match index_ref {
        Some(_) => {
            // Update existing index
            // TODO

            system.log.info("[api] updated index", b!("index" => *index_name));
        }
        None => {
            // Create index
            let mut indices_dir = system.get_indices_dir();
            indices_dir.push(index_name);
            indices_dir.set_extension("rsi");
            let mut index = Index::new(index_name.clone().to_owned(), RocksDBIndexStore::create(indices_dir).unwrap());

            // Insert standard and edgengram analyzers
            // TODO: Load these from index settings
            index.analyzers.insert("standard".to_string(), AnalyzerSpec {
                tokenizer: TokenizerSpec::Standard,
                filters: vec![
                    FilterSpec::Lowercase,
                    FilterSpec::ASCIIFolding,
                ]
            });
            index.analyzers.insert("edgengram_analyzer".to_string(), AnalyzerSpec {
                tokenizer: TokenizerSpec::Standard,
                filters: vec![
                    FilterSpec::Lowercase,
                    FilterSpec::ASCIIFolding,
                    FilterSpec::NGram {
                        min_size: 2,
                        max_size: 15,
                        edge: Edge::Left
                    },
                ]
            });

            // TODO: load settings

            let index_ref = indices.insert(index);
            indices.names.insert_canonical(index_name.clone().to_owned(), index_ref).unwrap();

            system.log.info("[api] created index", b!("index" => *index_name));
        }
    }

    return Ok(json_response(status::Ok, "{\"acknowledged\": true}"));
}

pub fn view_delete_index(req: &mut Request) -> IronResult<Response> {
    let ref system = get_system!(req);
    let ref index_selector = read_path_parameter!(req, "index").unwrap_or("");

    // Lock index array
    let mut indices = system.indices.write().unwrap();

    // Make sure the index exists
    get_index_or_404!(indices, *index_selector);

    // Remove indices
    for index_ref in indices.names.find(*index_selector) {
        // Get the index name
        let index_name = {
            if let Some(index) = indices.get(&index_ref) {
                index.name().to_string()
            } else {
                // Index doesn't exist
                continue;
            }
        };

        // Remove index from array
        indices.remove(&index_ref);

        // Delete canonical name
        indices.names.delete_canonical(&index_name, index_ref).unwrap();

        // Delete file
        let mut indices_dir = system.get_indices_dir();
        indices_dir.push(&index_name);
        indices_dir.set_extension("rsi");
        match fs::remove_dir_all(&indices_dir) {
            Ok(()) => {},
            Err(e) => {
                system.log.warn("[api] failed to delete index data", b!("index" => format!("{}", index_name), "error" => format!("{}", e)));
            }
        }

        system.log.info("[api] deleted index", b!("index" => format!("{}", index_name)));
    }

    return Ok(json_response(status::Ok, "{\"acknowledged\": true}"));
}

pub fn view_post_refresh_index(_req: &mut Request) -> IronResult<Response> {
    // let ref system = get_system!(req);
    // let ref index_name = read_path_parameter!(req, "index").unwrap_or("");

    // Lock index array
    // TODO
    // let mut indices = system.indices.write().unwrap();

    // TODO: {"_shards":{"total":10,"successful":5,"failed":0}}
    return Ok(json_response(status::Ok, "{\"acknowledged\": true}"));
}
<|endoftext|>