<commit_before>use numbers::*;
use std::mem;
use std::ops::*;
use std;
#[cfg(any(feature="use_sse", feature="use_avx"))]
use stdsimd::simd;

/// SIMD methods which have `f32` or `f64` specific implementation.
pub trait Simd<T>: Sized
    where T: Sized + Sync + Send
{
    /// The type of real valued array which matches a SIMD register.
    type Array;

    /// SIMD register to array.
    fn to_array(self) -> Self::Array;

    /// The type of complex valued array which matches a SIMD register.
    type ComplexArray;

    /// Number of elements in a SIMD register.
    fn len() -> usize;

    /// Loads a SIMD register from an array. If the end of the array is approached
    /// then the load code wraps around to the beginning and starts to load the first
    /// elements again.
    fn load_wrap_unchecked(array: &[T], idx: usize) -> Self;

    /// Creates a SIMD register loaded with a complex value.
    fn from_complex(value: Complex<T>) -> Self;

    /// Add a real number to the register.
    fn add_real(self, value: T) -> Self;

    /// Add a complex number to the register.
    fn add_complex(self, value: Complex<T>) -> Self;

    /// Scale the register by a real number.
    fn scale_real(self, value: T) -> Self;

    /// Scale the register by a complex number.
    fn scale_complex(self, value: Complex<T>) -> Self;

    /// Store the complex norm squared in the first half of the vector.
    fn complex_abs_squared(self) -> Self;

    /// Store the complex norm in the first half of the vector.
    fn complex_abs(self) -> Self;

    /// Same as `complex_abs_squared` but stores the result
    /// as complex number
    fn complex_abs_squared2(self) -> Self;

    /// Same as `complex_abs` but stores the result
    /// as complex number where the imaginary part is 0.
    fn complex_abs2(self) -> Self;

    /// Calculates the square root of the register.
    fn sqrt(self) -> Self;

    /// Stores the first half of the vector in an array.
    /// Useful e.g. in combination with `complex_abs_squared`.
    fn store_half_unchecked(self, target: &mut [T], index: usize);

    /// Multiplies the register with a complex value.
    fn mul_complex(self, value: Self) -> Self;

    /// Divides the register by a complex value.
    fn div_complex(self, value: Self) -> Self;

    /// Calculates the sum of all register elements, assuming that they
    /// are real valued.
    fn sum_real(&self) -> T;

    /// Calculates the sum of all register elements, assuming that they
    /// are complex valued.
    fn sum_complex(&self) -> Complex<T>;

    fn max(self, other: Self) -> Self;

    fn min(self, other: Self) -> Self;
}

/// Dirty workaround since the stdsimd doesn't implement conversion traits (yet?).
pub trait SimdFrom<T> {
    fn regfrom(T) -> Self;
}

/// SIMD methods which share their implementation independent if it's a `f32` or `f64` register.
pub trait SimdGeneric<T>: Simd<T> + SimdApproximations<T>
    + Add<Self, Output=Self> + Sub<Self, Output=Self> + Mul<Self, Output=Self> + Div<Self, Output=Self>
    + Copy + Clone + Sync + Send + Sized + Zero
    where T: Sized + Sync + Send
{
    /// On some CPU architectures memory access needs to be aligned or otherwise
    /// the process will crash. This method takes a vector an divides it in three ranges:
    /// beginning, center, end. Beginning and end may not be loaded directly as SIMD registers.
    /// Center will contain most of the data.
    fn calc_data_alignment_reqs(array: &[T]) -> (usize, usize, usize);

    /// Converts a real valued array which has exactly the size of a SIMD register
    /// into a SIMD register.
    fn from_array(array: Self::Array) -> Self;

    /// Converts the SIMD register into a complex valued array.
    fn to_complex_array(self) -> Self::ComplexArray;

    /// Converts a complex valued array which has exactly the size of a SIMD register
    /// into a SIMD register.
    fn from_complex_array(array: Self::ComplexArray) -> Self;

    /// Executed the given function on each element of the register.
    /// Register elements are assumed to be real valued.
    fn iter_over_vector<F>(self, op: F) -> Self where F: FnMut(T) -> T;

    /// Executed the given function on each element of the register.
    /// Register elements are assumed to be complex valued.
    fn iter_over_complex_vector<F>(self, op: F) -> Self where F: FnMut(Complex<T>) -> Complex<T>;

    /// Converts an array slice into a slice of SIMD registers.
    fn array_to_regs(array: &[T]) -> &[Self];

    /// Converts a mutable array slice into a slice of mutable SIMD registers.
    fn array_to_regs_mut(array: &mut [T]) -> &mut [Self];

    /// Loads a SIMD register from an array without any bound checks.
    fn load_unchecked(array: &[T], idx: usize) -> Self;

    /// Stores a SIMD register into an array without any bound checks.
    fn store_unchecked(self, target: &mut [T], index: usize);

    /// Returns one element from the register.
    fn extract(self, idx: u32) -> T;

    /// Creates a new SIMD register where every element equals `value`.
    fn splat(value: T) -> Self;

    /// Stores a SIMD register into an array.
    fn store(self, array: &mut [T], index: usize);
}

/// Approximated and faster implementation of some numeric standard function.
/// The approximations are implemented based on SIMD registers.
/// Refer to the documentation of the `ApproximatedOps` trait (which is part of
/// the public API of this lib) for some information about accuracy and speed.
pub trait SimdApproximations<T>
{
    /// Returns the natural logarithm of the number.
    fn ln_approx(self) -> Self;

    /// Returns `e^(self)`, (the exponential function).
    fn exp_approx(self) -> Self;

    /// Computes the sine of a number (in radians).
    fn sin_approx(self) -> Self;

    /// Computes the cosine of a number (in radians).
    fn cos_approx(self) -> Self;

    /// An implementation detail which leaked into the trait defintion
    /// for convenience. Use `sin_approx` or `cos_approx` instead of this
    /// function.
    ///
    /// Since the implementation of sine and cosine is almost identical
    /// the implementation is easier with a boolean `is_sin` flag which
    /// determines if the sine or cosine is requried.
    fn sin_cos_approx(self, is_sin: bool) -> Self;
}

/// Private struct copied over from the `simd` crate to implement the `load_unchecked`
/// and `store_unchecked` methods for the `SimdGeneric` trait.
#[repr(packed)]
#[derive(Debug, Copy, Clone)]
struct Unalign<T>(T);

macro_rules! simd_generic_impl {
    ($data_type:ident, $mod: ident::$reg:ident)
    =>
    {
        impl Zero for $mod::$reg {
            fn zero() -> Self {
                Self::splat(0.0)
            }
        }

        impl SimdGeneric<$data_type> for $mod::$reg {
            #[inline]
            fn calc_data_alignment_reqs(array: &[$data_type]) -> (usize, usize, usize) {
                let data_length = array.len();
                let addr = array.as_ptr();
                let scalar_left = (addr as usize % mem::size_of::<Self>()) / mem::size_of::<f32>();
                if scalar_left + Self::len() > data_length {
                    // Result order: scalar_left, scalar_right, vectorization_length
                    (data_length, data_length, 0)
                } else {
                    let right = (data_length - scalar_left) % Self::len();
                    (scalar_left, data_length - right, data_length - right)
                }
            }

            #[inline]
            fn from_array(array: Self::Array) -> Self {
                Self::load(&array, 0)
            }

            #[inline]
            fn to_complex_array(self) -> Self::ComplexArray {
                unsafe { mem::transmute(self.to_array()) }
            }

            #[inline]
            fn from_complex_array(array: Self::ComplexArray) -> Self {
                Self::from_array(unsafe { mem::transmute(array) })
            }

            #[inline]
            fn iter_over_vector<F>(self, mut op: F) -> Self
                where F: FnMut($data_type) -> $data_type {
                let mut array = self.to_array();
                for n in &mut array {
                    *n = op(*n);
                }
                Self::from_array(array)
            }

            #[inline]
            fn iter_over_complex_vector<F>(self,  mut op: F) -> Self
                where F: FnMut(Complex<$data_type>) -> Complex<$data_type> {
                let mut array = self.to_complex_array();
                for n in &mut array {
                    *n = op(*n);
                }
                Self::from_complex_array(array)
            }

            #[inline]
            fn array_to_regs(array: &[$data_type]) -> &[Self] {
                unsafe {
                    let len = array.len();
                    let reg_len = Self::len();
                    if len % reg_len != 0 {
                        panic!("Argument must be dividable by {}", reg_len);
                    }
                    let trans: &[Self] = mem::transmute(array);
                    &trans[0 .. len / reg_len]
                }
            }

            #[inline]
            fn array_to_regs_mut(array: &mut [$data_type]) -> &mut [Self] {
                unsafe {
                    let len = array.len();
                    let reg_len = Self::len();
                    if len % reg_len != 0 {
                        panic!("Argument must be dividable by {}", reg_len);
                    }
                    let trans: &mut [Self] = mem::transmute(array);
                    &mut trans[0 .. len / reg_len]
                }
            }

            #[inline]
            fn load_unchecked(array: &[$data_type], idx: usize) -> Self {
                let loaded = unsafe {
                    let data = array.as_ptr();
                    *(data.offset(idx as isize) as *const Unalign<Self>)
                };
                loaded.0
            }

            #[inline]
            fn store_unchecked(self, array: &mut [$data_type], idx: usize) {
                unsafe {
                    let place = array.as_mut_ptr();
                    *(place.offset(idx as isize) as *mut Unalign<Self>) = Unalign(self)
                }
            }

            #[inline]
            fn extract(self, idx: u32) -> $data_type {
                Self::extract(self, idx)
            }

            #[inline]
            fn splat(value: $data_type) -> Self {
                Self::splat(value)
            }

            #[inline]
            fn store(self, array: &mut [$data_type], index: usize) {
                Self::store(self, array, index);
            }
        }
    }
}

#[cfg(feature="use_avx")]
mod avx;
#[cfg(feature="use_avx")]
simd_generic_impl!(f32, simd::f32x8);
#[cfg(feature="use_avx")]
simd_generic_impl!(f64, simd::f64x4);

#[cfg(feature="use_sse")]
mod sse;
#[cfg(feature="use_sse")]
simd_generic_impl!(f32, simd::f32x4);
#[cfg(feature="use_sse")]
simd_generic_impl!(f64, simd::f64x2);

#[cfg(any(feature="use_sse", feature="use_avx"))]
mod approximations;

pub mod fallback;
mod approx_fallback;

simd_generic_impl!(f32, fallback::f32x4);
simd_generic_impl!(f64, fallback::f64x2);

pub struct RegType<Reg> {
    _type: std::marker::PhantomData<Reg>
}

impl<Reg> RegType<Reg> {
    pub fn new() -> Self {
        RegType {
            _type: std::marker::PhantomData
        }
    }
}

/// Selects a SIMD register type and passes it as 2nd argument to a function.
/// The macro tries to mimic the Rust syntax of a method call.
macro_rules! sel_reg(
    ($self_:ident.$method: ident::<$type: ident>($($args: expr),*)) => {
        if cfg_feature_enabled!("avx512") && cfg!(feature="use_avx512") {
            $self_.$method(RegType::<<$type as ToSimd>::RegAvx>::new(), $($args),*)
        } else if cfg_feature_enabled!("avx2") && cfg!(feature="use_avx") {
            $self_.$method(RegType::<<$type as ToSimd>::RegAvx>::new(), $($args),*)
        } else if cfg_feature_enabled!("sse2") && cfg!(feature="use_sse") {
            $self_.$method(RegType::<<$type as ToSimd>::RegSse>::new(), $($args),*)
        } else {
            $self_.$method(RegType::<<$type as ToSimd>::RegFallback>::new(), $($args),*)
        }
    };
    ($method: ident::<$type: ident>($($args: expr),*)) => {
        if cfg_feature_enabled!("avx512") && cfg!(feature="use_avx512") {
            $method(RegType::<<$type as ToSimd>::RegAvx>::new(), $($args),*)
        } else if cfg_feature_enabled!("avx2") && cfg!(feature="use_avx") {
            $method(RegType::<<$type as ToSimd>::RegAvx>::new(), $($args),*)
        } else if cfg_feature_enabled!("sse2") && cfg!(feature="use_sse") {
            $method(RegType::<<$type as ToSimd>::RegSse>::new(), $($args),*)
        } else {
            $method(RegType::<<$type as ToSimd>::RegFallback>::new(), $($args),*)
        }
    };
);<commit_msg>avx512 target needs to be double checked<commit_after>use numbers::*;
use std::mem;
use std::ops::*;
use std;
#[cfg(any(feature="use_sse", feature="use_avx"))]
use stdsimd::simd;

/// SIMD methods which have `f32` or `f64` specific implementation.
pub trait Simd<T>: Sized
    where T: Sized + Sync + Send
{
    /// The type of real valued array which matches a SIMD register.
    type Array;

    /// SIMD register to array.
    fn to_array(self) -> Self::Array;

    /// The type of complex valued array which matches a SIMD register.
    type ComplexArray;

    /// Number of elements in a SIMD register.
    fn len() -> usize;

    /// Loads a SIMD register from an array. If the end of the array is approached
    /// then the load code wraps around to the beginning and starts to load the first
    /// elements again.
    fn load_wrap_unchecked(array: &[T], idx: usize) -> Self;

    /// Creates a SIMD register loaded with a complex value.
    fn from_complex(value: Complex<T>) -> Self;

    /// Add a real number to the register.
    fn add_real(self, value: T) -> Self;

    /// Add a complex number to the register.
    fn add_complex(self, value: Complex<T>) -> Self;

    /// Scale the register by a real number.
    fn scale_real(self, value: T) -> Self;

    /// Scale the register by a complex number.
    fn scale_complex(self, value: Complex<T>) -> Self;

    /// Store the complex norm squared in the first half of the vector.
    fn complex_abs_squared(self) -> Self;

    /// Store the complex norm in the first half of the vector.
    fn complex_abs(self) -> Self;

    /// Same as `complex_abs_squared` but stores the result
    /// as complex number
    fn complex_abs_squared2(self) -> Self;

    /// Same as `complex_abs` but stores the result
    /// as complex number where the imaginary part is 0.
    fn complex_abs2(self) -> Self;

    /// Calculates the square root of the register.
    fn sqrt(self) -> Self;

    /// Stores the first half of the vector in an array.
    /// Useful e.g. in combination with `complex_abs_squared`.
    fn store_half_unchecked(self, target: &mut [T], index: usize);

    /// Multiplies the register with a complex value.
    fn mul_complex(self, value: Self) -> Self;

    /// Divides the register by a complex value.
    fn div_complex(self, value: Self) -> Self;

    /// Calculates the sum of all register elements, assuming that they
    /// are real valued.
    fn sum_real(&self) -> T;

    /// Calculates the sum of all register elements, assuming that they
    /// are complex valued.
    fn sum_complex(&self) -> Complex<T>;

    fn max(self, other: Self) -> Self;

    fn min(self, other: Self) -> Self;
}

/// Dirty workaround since the stdsimd doesn't implement conversion traits (yet?).
pub trait SimdFrom<T> {
    fn regfrom(T) -> Self;
}

/// SIMD methods which share their implementation independent if it's a `f32` or `f64` register.
pub trait SimdGeneric<T>: Simd<T> + SimdApproximations<T>
    + Add<Self, Output=Self> + Sub<Self, Output=Self> + Mul<Self, Output=Self> + Div<Self, Output=Self>
    + Copy + Clone + Sync + Send + Sized + Zero
    where T: Sized + Sync + Send
{
    /// On some CPU architectures memory access needs to be aligned or otherwise
    /// the process will crash. This method takes a vector an divides it in three ranges:
    /// beginning, center, end. Beginning and end may not be loaded directly as SIMD registers.
    /// Center will contain most of the data.
    fn calc_data_alignment_reqs(array: &[T]) -> (usize, usize, usize);

    /// Converts a real valued array which has exactly the size of a SIMD register
    /// into a SIMD register.
    fn from_array(array: Self::Array) -> Self;

    /// Converts the SIMD register into a complex valued array.
    fn to_complex_array(self) -> Self::ComplexArray;

    /// Converts a complex valued array which has exactly the size of a SIMD register
    /// into a SIMD register.
    fn from_complex_array(array: Self::ComplexArray) -> Self;

    /// Executed the given function on each element of the register.
    /// Register elements are assumed to be real valued.
    fn iter_over_vector<F>(self, op: F) -> Self where F: FnMut(T) -> T;

    /// Executed the given function on each element of the register.
    /// Register elements are assumed to be complex valued.
    fn iter_over_complex_vector<F>(self, op: F) -> Self where F: FnMut(Complex<T>) -> Complex<T>;

    /// Converts an array slice into a slice of SIMD registers.
    fn array_to_regs(array: &[T]) -> &[Self];

    /// Converts a mutable array slice into a slice of mutable SIMD registers.
    fn array_to_regs_mut(array: &mut [T]) -> &mut [Self];

    /// Loads a SIMD register from an array without any bound checks.
    fn load_unchecked(array: &[T], idx: usize) -> Self;

    /// Stores a SIMD register into an array without any bound checks.
    fn store_unchecked(self, target: &mut [T], index: usize);

    /// Returns one element from the register.
    fn extract(self, idx: u32) -> T;

    /// Creates a new SIMD register where every element equals `value`.
    fn splat(value: T) -> Self;

    /// Stores a SIMD register into an array.
    fn store(self, array: &mut [T], index: usize);
}

/// Approximated and faster implementation of some numeric standard function.
/// The approximations are implemented based on SIMD registers.
/// Refer to the documentation of the `ApproximatedOps` trait (which is part of
/// the public API of this lib) for some information about accuracy and speed.
pub trait SimdApproximations<T>
{
    /// Returns the natural logarithm of the number.
    fn ln_approx(self) -> Self;

    /// Returns `e^(self)`, (the exponential function).
    fn exp_approx(self) -> Self;

    /// Computes the sine of a number (in radians).
    fn sin_approx(self) -> Self;

    /// Computes the cosine of a number (in radians).
    fn cos_approx(self) -> Self;

    /// An implementation detail which leaked into the trait defintion
    /// for convenience. Use `sin_approx` or `cos_approx` instead of this
    /// function.
    ///
    /// Since the implementation of sine and cosine is almost identical
    /// the implementation is easier with a boolean `is_sin` flag which
    /// determines if the sine or cosine is requried.
    fn sin_cos_approx(self, is_sin: bool) -> Self;
}

/// Private struct copied over from the `simd` crate to implement the `load_unchecked`
/// and `store_unchecked` methods for the `SimdGeneric` trait.
#[repr(packed)]
#[derive(Debug, Copy, Clone)]
struct Unalign<T>(T);

macro_rules! simd_generic_impl {
    ($data_type:ident, $mod: ident::$reg:ident)
    =>
    {
        impl Zero for $mod::$reg {
            fn zero() -> Self {
                Self::splat(0.0)
            }
        }

        impl SimdGeneric<$data_type> for $mod::$reg {
            #[inline]
            fn calc_data_alignment_reqs(array: &[$data_type]) -> (usize, usize, usize) {
                let data_length = array.len();
                let addr = array.as_ptr();
                let scalar_left = (addr as usize % mem::size_of::<Self>()) / mem::size_of::<f32>();
                if scalar_left + Self::len() > data_length {
                    // Result order: scalar_left, scalar_right, vectorization_length
                    (data_length, data_length, 0)
                } else {
                    let right = (data_length - scalar_left) % Self::len();
                    (scalar_left, data_length - right, data_length - right)
                }
            }

            #[inline]
            fn from_array(array: Self::Array) -> Self {
                Self::load(&array, 0)
            }

            #[inline]
            fn to_complex_array(self) -> Self::ComplexArray {
                unsafe { mem::transmute(self.to_array()) }
            }

            #[inline]
            fn from_complex_array(array: Self::ComplexArray) -> Self {
                Self::from_array(unsafe { mem::transmute(array) })
            }

            #[inline]
            fn iter_over_vector<F>(self, mut op: F) -> Self
                where F: FnMut($data_type) -> $data_type {
                let mut array = self.to_array();
                for n in &mut array {
                    *n = op(*n);
                }
                Self::from_array(array)
            }

            #[inline]
            fn iter_over_complex_vector<F>(self,  mut op: F) -> Self
                where F: FnMut(Complex<$data_type>) -> Complex<$data_type> {
                let mut array = self.to_complex_array();
                for n in &mut array {
                    *n = op(*n);
                }
                Self::from_complex_array(array)
            }

            #[inline]
            fn array_to_regs(array: &[$data_type]) -> &[Self] {
                unsafe {
                    let len = array.len();
                    let reg_len = Self::len();
                    if len % reg_len != 0 {
                        panic!("Argument must be dividable by {}", reg_len);
                    }
                    let trans: &[Self] = mem::transmute(array);
                    &trans[0 .. len / reg_len]
                }
            }

            #[inline]
            fn array_to_regs_mut(array: &mut [$data_type]) -> &mut [Self] {
                unsafe {
                    let len = array.len();
                    let reg_len = Self::len();
                    if len % reg_len != 0 {
                        panic!("Argument must be dividable by {}", reg_len);
                    }
                    let trans: &mut [Self] = mem::transmute(array);
                    &mut trans[0 .. len / reg_len]
                }
            }

            #[inline]
            fn load_unchecked(array: &[$data_type], idx: usize) -> Self {
                let loaded = unsafe {
                    let data = array.as_ptr();
                    *(data.offset(idx as isize) as *const Unalign<Self>)
                };
                loaded.0
            }

            #[inline]
            fn store_unchecked(self, array: &mut [$data_type], idx: usize) {
                unsafe {
                    let place = array.as_mut_ptr();
                    *(place.offset(idx as isize) as *mut Unalign<Self>) = Unalign(self)
                }
            }

            #[inline]
            fn extract(self, idx: u32) -> $data_type {
                Self::extract(self, idx)
            }

            #[inline]
            fn splat(value: $data_type) -> Self {
                Self::splat(value)
            }

            #[inline]
            fn store(self, array: &mut [$data_type], index: usize) {
                Self::store(self, array, index);
            }
        }
    }
}

#[cfg(feature="use_avx")]
mod avx;
#[cfg(feature="use_avx")]
simd_generic_impl!(f32, simd::f32x8);
#[cfg(feature="use_avx")]
simd_generic_impl!(f64, simd::f64x4);

#[cfg(feature="use_sse")]
mod sse;
#[cfg(feature="use_sse")]
simd_generic_impl!(f32, simd::f32x4);
#[cfg(feature="use_sse")]
simd_generic_impl!(f64, simd::f64x2);

#[cfg(any(feature="use_sse", feature="use_avx"))]
mod approximations;

pub mod fallback;
mod approx_fallback;

simd_generic_impl!(f32, fallback::f32x4);
simd_generic_impl!(f64, fallback::f64x2);

pub struct RegType<Reg> {
    _type: std::marker::PhantomData<Reg>
}

impl<Reg> RegType<Reg> {
    pub fn new() -> Self {
        RegType {
            _type: std::marker::PhantomData
        }
    }
}

/// Selects a SIMD register type and passes it as 2nd argument to a function.
/// The macro tries to mimic the Rust syntax of a method call.
macro_rules! sel_reg(
    ($self_:ident.$method: ident::<$type: ident>($($args: expr),*)) => {
        /*if cfg_feature_enabled!("avx512") && cfg!(feature="use_avx512") {
            $self_.$method(RegType::<<$type as ToSimd>::RegAvx>::new(), $($args),*)
        } else */if cfg_feature_enabled!("avx2") && cfg!(feature="use_avx") {
            $self_.$method(RegType::<<$type as ToSimd>::RegAvx>::new(), $($args),*)
        } else if cfg_feature_enabled!("sse2") && cfg!(feature="use_sse") {
            $self_.$method(RegType::<<$type as ToSimd>::RegSse>::new(), $($args),*)
        } else {
            $self_.$method(RegType::<<$type as ToSimd>::RegFallback>::new(), $($args),*)
        }
    };
    ($method: ident::<$type: ident>($($args: expr),*)) => {
        /*if cfg_feature_enabled!("avx512") && cfg!(feature="use_avx512") {
            $method(RegType::<<$type as ToSimd>::RegAvx>::new(), $($args),*)
        } else */if cfg_feature_enabled!("avx2") && cfg!(feature="use_avx") {
            $method(RegType::<<$type as ToSimd>::RegAvx>::new(), $($args),*)
        } else if cfg_feature_enabled!("sse2") && cfg!(feature="use_sse") {
            $method(RegType::<<$type as ToSimd>::RegSse>::new(), $($args),*)
        } else {
            $method(RegType::<<$type as ToSimd>::RegFallback>::new(), $($args),*)
        }
    };
);<|endoftext|>


<commit_before>use stdio;
use utils::outb;
use pic::PIC_remap;

mod pic;
/* Defines an IDT entry */
#[packed]
struct IDTEntry {
    base_lo: u16,
    sel: u16,        /* Our kernel segment goes here! */
    zero: u8,        /* This will ALWAYS be set to 0! */
    flags: u8,       /* Set using the above table! */
    base_hi: u16
}

/* Defines an IDT pointer */
#[packed]
struct IDTPointer {
    limit: u16,
    base: u32
}

/* Declare an IDT of 256 entries. Although we will only use the
 *  first 32 entries in this tutorial, the rest exists as a bit
 *  of a trap. If any undefined IDT entry is hit, it normally
 *  will cause an "Unhandled Interrupt" exception. Any descriptor
 *  for which the 'presence' bit is cleared (0) will generate an
 *  "Unhandled Interrupt" exception */

#[no_mangle]
static mut idt: [IDTEntry, ..256] = [IDTEntry {base_lo: 0, sel: 0, zero: 0, flags: 0, base_hi: 0}, ..256];

#[no_mangle]
pub static mut idtp: IDTPointer = IDTPointer {limit: 0, base: 0};

/* Use this function to set an entry in the IDT. A lot simpler
*  than twiddling with the GDT ;) */
#[no_mangle]
unsafe fn idt_set_gate(num: u8, f: extern "C" unsafe fn(), sel: u16, flags: u8)
{

    let base = f as u32;
    idt[num].sel = sel;
    idt[num].flags = flags;
    idt[num].base_hi = (base >> 16) as u16;
    idt[num].base_lo = (base & (1 << 16 - 1)) as u16;
}

/* Installs the IDT */
extern {
    fn idt_load(x: *IDTPointer);
    fn _interrupt_handler_kbd_wrapper ();
    fn test1 ();
}

#[no_mangle]
pub unsafe fn idt_install() {
    /* Sets the special IDT pointer up, just like in 'gdt.c' */
    //idtp.limit = ((super::core::mem::size_of::<IDTEntry>() * 256) - 1) as u16;
    //idtp.base = &idt as *[IDTEntry, ..256] as u32;

    /* Add any new ISRs to the IDT here using idt_set_gate */
    //let on_flags: u8 = 0b10001110; // on, ring = 0
    //let flags = 0x8E;
    //let mut i: uint = 0;
    //while i < 256 {
    //    idt_set_gate(i as u8, _interrupt_handler_kbd_wrapper, 0x08, flags);
    //    i += 1;
    //}
    PIC_remap();
    outb(0x21,0xfd); // Keyboard interrupts only
    outb(0xa1,0xff);
    asm!("sti");
    //asm!("lidt ($0)" :: "r" (idtp));

    /* Points the processor's internal register to the new IDT */
}

static mut char: u8 = 0;
#[no_mangle]
pub unsafe fn _interrupt_handler_kbd() {
    let mut i: uint = 1;
    let N: uint = 80000000;
    stdio::write(0, 6, "Int: ");
    stdio::putchar(5 + (char as u16), 6, char + 65);
    while i % N != 0 {
        i += 1;
    }
    let x = (((i % N + 1) * 0) as u8);
    char += x + 1;
}
<commit_msg>I can press more than one key!<commit_after>use stdio;
use utils::{outb, inb};
use pic::PIC_remap;

mod pic;
/* Defines an IDT entry */
#[packed]
struct IDTEntry {
    base_lo: u16,
    sel: u16,        /* Our kernel segment goes here! */
    zero: u8,        /* This will ALWAYS be set to 0! */
    flags: u8,       /* Set using the above table! */
    base_hi: u16
}

/* Defines an IDT pointer */
#[packed]
struct IDTPointer {
    limit: u16,
    base: u32
}

/* Declare an IDT of 256 entries. Although we will only use the
 *  first 32 entries in this tutorial, the rest exists as a bit
 *  of a trap. If any undefined IDT entry is hit, it normally
 *  will cause an "Unhandled Interrupt" exception. Any descriptor
 *  for which the 'presence' bit is cleared (0) will generate an
 *  "Unhandled Interrupt" exception */

#[no_mangle]
static mut idt: [IDTEntry, ..256] = [IDTEntry {base_lo: 0, sel: 0, zero: 0, flags: 0, base_hi: 0}, ..256];

#[no_mangle]
pub static mut idtp: IDTPointer = IDTPointer {limit: 0, base: 0};

/* Use this function to set an entry in the IDT. A lot simpler
*  than twiddling with the GDT ;) */
#[no_mangle]
unsafe fn idt_set_gate(num: u8, f: extern "C" unsafe fn(), sel: u16, flags: u8)
{

    let base = f as u32;
    idt[num].sel = sel;
    idt[num].flags = flags;
    idt[num].base_hi = (base >> 16) as u16;
    idt[num].base_lo = (base & (1 << 16 - 1)) as u16;
}

/* Installs the IDT */
extern {
    fn idt_load(x: *IDTPointer);
    fn _interrupt_handler_kbd_wrapper ();
    fn test1 ();
}

#[no_mangle]
pub unsafe fn idt_install() {
    /* Sets the special IDT pointer up, just like in 'gdt.c' */
    //idtp.limit = ((super::core::mem::size_of::<IDTEntry>() * 256) - 1) as u16;
    //idtp.base = &idt as *[IDTEntry, ..256] as u32;

    /* Add any new ISRs to the IDT here using idt_set_gate */
    //let on_flags: u8 = 0b10001110; // on, ring = 0
    //let flags = 0x8E;
    //let mut i: uint = 0;
    //while i < 256 {
    //    idt_set_gate(i as u8, _interrupt_handler_kbd_wrapper, 0x08, flags);
    //    i += 1;
    //}
    PIC_remap();
    outb(0x21,0xfd); // Keyboard interrupts only
    outb(0xa1,0xff);
    asm!("sti");
    //asm!("lidt ($0)" :: "r" (idtp));

    /* Points the processor's internal register to the new IDT */
}

static mut pos: u8 = 0;
#[no_mangle]
pub unsafe fn _interrupt_handler_kbd() {
    let mut i: uint = 1;
    let N: uint = 80000000;
    stdio::write(0, 6, "Int: ");
    while i % N != 0 {
        i += 1;
    }
    let x = (((i % N + 1) * 0) as u8);
    pos += x + 1;
    outb(0x20, 0x20);
    let key = inb(0x60);
    stdio::putchar(5 + (pos as u16), 6, key);
}
<|endoftext|>


<commit_before>//! Intel's Restricted Transactional Memory (RTM).
//!
//! This CPU feature is available on Intel Broadwell or later CPUs (and some Haswell).
//!
//! [Wikipedia][wikipedia_rtm] provides a quick overview of the assembly instructions, and
//! Intel's [programming considerations][intel_consid] details what sorts of instructions within a
//! transaction are likely to cause an abort.
//!
//! [wikipedia_rtm]: https://en.wikipedia.org/wiki/Transactional_Synchronization_Extensions#Restricted_Transactional_Memory
//! [intel_consid]: https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-intel-transactional-synchronization-extensions-intel-tsx-programming-considerations

#[cfg(test)]
use stdsimd_test::assert_instr;

extern "C" {
    #[link_name = "llvm.x86.xbegin"]
    fn x86_xbegin() -> i32;
    #[link_name = "llvm.x86.xend"]
    fn x86_xend() -> ();
    #[link_name = "llvm.x86.xabort"]
    fn x86_xabort(imm8: i8) -> ();
    #[link_name = "llvm.x86.xtest"]
    fn x86_xtest() -> i32;
}

/// Transaction successfully started.
pub const _XBEGIN_STARTED: u32 = !0;

/// Transaction explicitly aborted with xabort. The parameter passed to xabort is available with
/// _xabort_code(status).
pub const _XABORT_EXPLICIT: u32 = 1 << 0;

/// Transaction retry is possible.
pub const _XABORT_RETRY: u32 = 1 << 1;

/// Transaction abort due to a memory conflict with another thread.
pub const _XABORT_CONFLICT: u32 = 1 << 2;

/// Transaction abort due to the transaction using too much memory.
pub const _XABORT_CAPACITY: u32 = 1 << 3;

/// Transaction abort due to a debug trap.
pub const _XABORT_DEBUG: u32 = 1 << 4;

/// Transaction abort in a inner nested transaction.
pub const _XABORT_NESTED: u32 = 1 << 5;

/// Specifies the start of a restricted transactional memory (RTM) code region and returns a value
/// indicating status.
///
/// [Intel's documentation](https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-xbegin).
#[inline]
#[target_feature(enable = "rtm")]
#[cfg_attr(test, assert_instr(xbegin))]
pub unsafe fn _xbegin() -> u32 {
    x86_xbegin() as _
}

/// Specifies the end of a restricted transactional memory (RTM) code region.
///
/// [Intel's documentation](https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-xend).
#[inline]
#[target_feature(enable = "rtm")]
#[cfg_attr(test, assert_instr(xend))]
pub unsafe fn _xend() {
    x86_xend()
}

/// Forces a restricted transactional memory (RTM) region to abort.
///
/// [Intel's documentation](https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-xabort).
#[inline]
#[target_feature(enable = "rtm")]
#[cfg_attr(test, assert_instr(xabort))]
pub unsafe fn _xabort(imm8: u32) {
    macro_rules! call {
        ($imm8:expr) => {
            x86_xabort($imm8)
        };
    }
    constify_imm8!(imm8, call)
}

/// Queries whether the processor is executing in a transactional region identified by restricted
/// transactional memory (RTM) or hardware lock elision (HLE).
///
/// [Intel's documentation](https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-xtest).
#[inline]
#[target_feature(enable = "rtm")]
#[cfg_attr(test, assert_instr(xtest))]
pub unsafe fn _xtest() -> bool {
    x86_xtest() != 0
}

/// Retrieves the parameter passed to [`_xabort`] when [`_xbegin`]'s status has the
/// `_XABORT_EXPLICIT` flag set.
#[inline]
pub const fn _xabort_code(status: u32) -> u32 {
    (status >> 24) & 0xFF
}

#[cfg(test)]
mod tests {
    use stdsimd_test::simd_test;

    use crate::core_arch::x86::*;

    #[simd_test(enable = "rtm")]
    unsafe fn test_xbegin_xend() {
        let mut x = 0;
        for _ in 0..10 {
            let code = rtm::_xbegin();
            if code == _XBEGIN_STARTED {
                x += 1;
                rtm::_xend();
                assert_eq!(x, 1);
                break
            }
            assert_eq!(x, 0);
        }
    }

    #[simd_test(enable = "rtm")]
    unsafe fn test_xabort() {
        // aborting outside a transactional region does nothing
        _xabort(0);

        for abort_code in 0..10 {
            let mut x = 0;
            let code = rtm::_xbegin();
            if code == _XBEGIN_STARTED {
                x += 1;
                rtm::_xabort(abort_code);
            } else if code & _XABORT_EXPLICIT != 0 {
                let test_abort_code = rtm::_xabort_code(code);
                assert_eq!(test_abort_code, abort_code);
            }
            assert_eq!(x, 0);
        }
    }

    #[simd_test(enable = "rtm")]
    unsafe fn test_xtest() {
        assert_eq!(_xtest(), false);

        for _ in 0..10 {
            let code = rtm::_xbegin();
            if code == _XBEGIN_STARTED {
                let in_tx = _xtest();
                rtm::_xend();

                // putting the assert inside the transaction would abort the transaction on fail
                // without any output/panic/etc
                assert_eq!(in_tx, true);
                break
            }
        }
    }
}<commit_msg>Add link to intel instruction set reference<commit_after>//! Intel's Restricted Transactional Memory (RTM).
//!
//! This CPU feature is available on Intel Broadwell or later CPUs (and some Haswell).
//!
//! The reference is [Intel 64 and IA-32 Architectures Software Developer's
//! Manual Volume 2: Instruction Set Reference, A-Z][intel64_ref].
//!
//! [Wikipedia][wikipedia_rtm] provides a quick overview of the assembly instructions, and
//! Intel's [programming considerations][intel_consid] details what sorts of instructions within a
//! transaction are likely to cause an abort.
//!
//! [intel64_ref]: http://www.intel.de/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
//! [wikipedia_rtm]: https://en.wikipedia.org/wiki/Transactional_Synchronization_Extensions#Restricted_Transactional_Memory
//! [intel_consid]: https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-intel-transactional-synchronization-extensions-intel-tsx-programming-considerations

#[cfg(test)]
use stdsimd_test::assert_instr;

extern "C" {
    #[link_name = "llvm.x86.xbegin"]
    fn x86_xbegin() -> i32;
    #[link_name = "llvm.x86.xend"]
    fn x86_xend() -> ();
    #[link_name = "llvm.x86.xabort"]
    fn x86_xabort(imm8: i8) -> ();
    #[link_name = "llvm.x86.xtest"]
    fn x86_xtest() -> i32;
}

/// Transaction successfully started.
pub const _XBEGIN_STARTED: u32 = !0;

/// Transaction explicitly aborted with xabort. The parameter passed to xabort is available with
/// _xabort_code(status).
pub const _XABORT_EXPLICIT: u32 = 1 << 0;

/// Transaction retry is possible.
pub const _XABORT_RETRY: u32 = 1 << 1;

/// Transaction abort due to a memory conflict with another thread.
pub const _XABORT_CONFLICT: u32 = 1 << 2;

/// Transaction abort due to the transaction using too much memory.
pub const _XABORT_CAPACITY: u32 = 1 << 3;

/// Transaction abort due to a debug trap.
pub const _XABORT_DEBUG: u32 = 1 << 4;

/// Transaction abort in a inner nested transaction.
pub const _XABORT_NESTED: u32 = 1 << 5;

/// Specifies the start of a restricted transactional memory (RTM) code region and returns a value
/// indicating status.
///
/// [Intel's documentation](https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-xbegin).
#[inline]
#[target_feature(enable = "rtm")]
#[cfg_attr(test, assert_instr(xbegin))]
pub unsafe fn _xbegin() -> u32 {
    x86_xbegin() as _
}

/// Specifies the end of a restricted transactional memory (RTM) code region.
///
/// [Intel's documentation](https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-xend).
#[inline]
#[target_feature(enable = "rtm")]
#[cfg_attr(test, assert_instr(xend))]
pub unsafe fn _xend() {
    x86_xend()
}

/// Forces a restricted transactional memory (RTM) region to abort.
///
/// [Intel's documentation](https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-xabort).
#[inline]
#[target_feature(enable = "rtm")]
#[cfg_attr(test, assert_instr(xabort))]
pub unsafe fn _xabort(imm8: u32) {
    macro_rules! call {
        ($imm8:expr) => {
            x86_xabort($imm8)
        };
    }
    constify_imm8!(imm8, call)
}

/// Queries whether the processor is executing in a transactional region identified by restricted
/// transactional memory (RTM) or hardware lock elision (HLE).
///
/// [Intel's documentation](https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-xtest).
#[inline]
#[target_feature(enable = "rtm")]
#[cfg_attr(test, assert_instr(xtest))]
pub unsafe fn _xtest() -> bool {
    x86_xtest() != 0
}

/// Retrieves the parameter passed to [`_xabort`] when [`_xbegin`]'s status has the
/// `_XABORT_EXPLICIT` flag set.
#[inline]
pub const fn _xabort_code(status: u32) -> u32 {
    (status >> 24) & 0xFF
}

#[cfg(test)]
mod tests {
    use stdsimd_test::simd_test;

    use crate::core_arch::x86::*;

    #[simd_test(enable = "rtm")]
    unsafe fn test_xbegin_xend() {
        let mut x = 0;
        for _ in 0..10 {
            let code = rtm::_xbegin();
            if code == _XBEGIN_STARTED {
                x += 1;
                rtm::_xend();
                assert_eq!(x, 1);
                break
            }
            assert_eq!(x, 0);
        }
    }

    #[simd_test(enable = "rtm")]
    unsafe fn test_xabort() {
        // aborting outside a transactional region does nothing
        _xabort(0);

        for abort_code in 0..10 {
            let mut x = 0;
            let code = rtm::_xbegin();
            if code == _XBEGIN_STARTED {
                x += 1;
                rtm::_xabort(abort_code);
            } else if code & _XABORT_EXPLICIT != 0 {
                let test_abort_code = rtm::_xabort_code(code);
                assert_eq!(test_abort_code, abort_code);
            }
            assert_eq!(x, 0);
        }
    }

    #[simd_test(enable = "rtm")]
    unsafe fn test_xtest() {
        assert_eq!(_xtest(), false);

        for _ in 0..10 {
            let code = rtm::_xbegin();
            if code == _XBEGIN_STARTED {
                let in_tx = _xtest();
                rtm::_xend();

                // putting the assert inside the transaction would abort the transaction on fail
                // without any output/panic/etc
                assert_eq!(in_tx, true);
                break
            }
        }
    }
}<|endoftext|>


<commit_before>use std::any::Any;
use std::marker::PhantomData;
use std::sync::atomic::{AtomicUsize, Ordering};

use once_cell::sync::OnceCell;

use crate::context::Context;
use crate::lifecycle::InstanceData;

static COUNTER: AtomicUsize = AtomicUsize::new(0);

fn next_id() -> usize {
    COUNTER.fetch_add(1, Ordering::SeqCst)
}

/// A cell that can be used to allocate data that is global to an instance
/// of a Neon addon.
#[derive(Default)]
pub struct Global<T> {
    _type: PhantomData<T>,
    id: OnceCell<usize>,
}

impl<T> Global<T> {
    pub const fn new() -> Self {
        Self {
            _type: PhantomData,
            id: OnceCell::new(),
        }
    }

    fn id(&self) -> usize {
        *self.id.get_or_init(next_id)
    }
}

impl<T: Any + Send + 'static> Global<T> {
    pub fn get<'cx, 'a, C>(&self, cx: &'a mut C) -> Option<&'cx T>
    where
        C: Context<'cx>,
    {
        let r: Option<&T> = InstanceData::globals(cx)
            .get(self.id())
            .as_ref()
            .map(|boxed| boxed.downcast_ref().unwrap());

        unsafe {
            std::mem::transmute(r)
        }
    }

    pub fn get_or_init<'cx, 'a, C, F>(&self, cx: &'a mut C, f: F) -> &'cx T
    where
        C: Context<'cx>,
        F: FnOnce() -> T,
    {
        let r: &T = InstanceData::globals(cx)
            .get(self.id())
            .get_or_insert_with(|| Box::new(f()))
            .downcast_ref()
            .unwrap();

        unsafe {
            std::mem::transmute(r)
        }
    }
}
<commit_msg>Explicitly name the types in the transmute<commit_after>use std::any::Any;
use std::marker::PhantomData;
use std::sync::atomic::{AtomicUsize, Ordering};

use once_cell::sync::OnceCell;

use crate::context::Context;
use crate::lifecycle::InstanceData;

static COUNTER: AtomicUsize = AtomicUsize::new(0);

fn next_id() -> usize {
    COUNTER.fetch_add(1, Ordering::SeqCst)
}

/// A cell that can be used to allocate data that is global to an instance
/// of a Neon addon.
#[derive(Default)]
pub struct Global<T> {
    _type: PhantomData<T>,
    id: OnceCell<usize>,
}

impl<T> Global<T> {
    pub const fn new() -> Self {
        Self {
            _type: PhantomData,
            id: OnceCell::new(),
        }
    }

    fn id(&self) -> usize {
        *self.id.get_or_init(next_id)
    }
}

impl<T: Any + Send + 'static> Global<T> {
    pub fn get<'cx, 'a, C>(&self, cx: &'a mut C) -> Option<&'cx T>
    where
        C: Context<'cx>,
    {
        let r: Option<&T> = InstanceData::globals(cx)
            .get(self.id())
            .as_ref()
            .map(|boxed| boxed.downcast_ref().unwrap());

        unsafe {
            std::mem::transmute(r)
        }
    }

    pub fn get_or_init<'cx, 'a, C, F>(&self, cx: &'a mut C, f: F) -> &'cx T
    where
        C: Context<'cx>,
        F: FnOnce() -> T,
    {
        let r: &T = InstanceData::globals(cx)
            .get(self.id())
            .get_or_insert_with(|| Box::new(f()))
            .downcast_ref()
            .unwrap();

        unsafe {
            std::mem::transmute::<&'a T, &'cx T>(r)
        }
    }
}
<|endoftext|>


<commit_before>pub(crate) mod tags;

mod highlights;
mod injector;

mod highlight;
mod format;
mod macro_;
mod inject;

mod html;
#[cfg(test)]
mod tests;

use hir::{InFile, Name, Semantics};
use ide_db::{RootDatabase, SymbolKind};
use rustc_hash::FxHashMap;
use syntax::{
    ast::{self, HasFormatSpecifier},
    match_ast, AstNode, AstToken, Direction, NodeOrToken,
    SyntaxKind::*,
    SyntaxNode, TextRange, WalkEvent, T,
};

use crate::{
    syntax_highlighting::{
        format::highlight_format_string, highlights::Highlights, macro_::MacroHighlighter,
        tags::Highlight,
    },
    FileId, HlMod, HlTag,
};

pub(crate) use html::highlight_as_html;

#[derive(Debug, Clone, Copy)]
pub struct HlRange {
    pub range: TextRange,
    pub highlight: Highlight,
    pub binding_hash: Option<u64>,
}

// Feature: Semantic Syntax Highlighting
//
// rust-analyzer highlights the code semantically.
// For example, `Bar` in `foo::Bar` might be colored differently depending on whether `Bar` is an enum or a trait.
// rust-analyzer does not specify colors directly, instead it assigns a tag (like `struct`) and a set of modifiers (like `declaration`) to each token.
// It's up to the client to map those to specific colors.
//
// The general rule is that a reference to an entity gets colored the same way as the entity itself.
// We also give special modifier for `mut` and `&mut` local variables.
//
//
// .Token Tags
//
// Rust-analyzer currently emits the following token tags:
//
// - For items:
// +
// [horizontal]
// enum:: Emitted for enums.
// function:: Emitted for free-standing functions.
// macro:: Emitted for macros.
// method:: Emitted for associated functions, also knowns as methods.
// namespace:: Emitted for modules.
// struct:: Emitted for structs.
// trait:: Emitted for traits.
// typeAlias:: Emitted for type aliases and `Self` in `impl`s.
// union:: Emitted for unions.
//
// - For literals:
// +
// [horizontal]
// boolean:: Emitted for the boolean literals `true` and `false`.
// character:: Emitted for character literals.
// number:: Emitted for numeric literals.
// string:: Emitted for string literals.
// escapeSequence:: Emitted for escaped sequences inside strings like `\n`.
// formatSpecifier:: Emitted for format specifiers `{:?}` in `format!`-like macros.
//
// - For operators:
// +
// [horizontal]
// operator:: Emitted for general operators.
// arithmetic:: Emitted for the arithmetic operators `+`, `-`, `*`, `/`, `+=`, `-=`, `*=`, `/=`.
// bitwise:: Emitted for the bitwise operators `|`, `&`, `!`, `^`, `|=`, `&=`, `^=`.
// comparison:: Emitted for the comparison operators `>`, `<`, `==`, `>=`, `<=`, `!=`.
// logical:: Emitted for the logical operators `||`, `&&`, `!`.
//
// - For punctuation:
// +
// [horizontal]
// punctuation:: Emitted for general punctuation.
// angle:: Emitted for `<>` angle brackets.
// brace:: Emitted for `{}` braces.
// bracket:: Emitted for `[]` brackets.
// parenthesis:: Emitted for `()` parentheses.
// colon:: Emitted for the `:` token.
// comma:: Emitted for the `,` token.
// dot:: Emitted for the `.` token.
// Semi:: Emitted for the `;` token.
//
// //-
//
// [horizontal]
// attribute:: Emitted for the `#[` `]` tokens.
// builtinAttribute:: Emitted for names to builtin attributes in attribute path, the `repr` in `#[repr(u8)]` for example.
// builtinType:: Emitted for builtin types like `u32`, `str` and `f32`.
// comment:: Emitted for comments.
// constParameter:: Emitted for const parameters.
// enumMember:: Emitted for enum variants.
// generic:: Emitted for generic tokens that have no mapping.
// keyword:: Emitted for keywords.
// label:: Emitted for labels.
// lifetime:: Emitted for lifetimes.
// parameter:: Emitted for non-self function parameters.
// property:: Emitted for struct and union fields.
// selfKeyword:: Emitted for the self function parameter and self path-specifier.
// typeParameter:: Emitted for type parameters.
// unresolvedReference:: Emitted for unresolved references, names that rust-analyzer can't find the definition of.
// variable:: Emitted for locals, constants and statics.
//
//
// .Token Modifiers
//
// Token modifiers allow to style some elements in the source code more precisely.
//
// Rust-analyzer currently emits the following token modifiers:
//
// [horizontal]
// async:: Emitted for async functions and the `async` and `await` keywords.
// attribute:: Emitted for tokens inside attributes.
// callable:: Emitted for locals whose types implements one of the `Fn*` traits.
// constant:: Emitted for consts.
// consuming:: Emitted for locals that are being consumed when use in a function call.
// controlFlow:: Emitted for control-flow related tokens, this includes the `?` operator.
// declaration:: Emitted for names of definitions, like `foo` in `fn foo() {}`.
// documentation:: Emitted for documentation comments.
// injected:: Emitted for doc-string injected highlighting like rust source blocks in documentation.
// intraDocLink:: Emitted for intra doc links in doc-strings.
// library:: Emitted for items that are defined outside of the current crate.
// mutable:: Emitted for mutable locals and statics as well as functions taking `&mut self`.
// public:: Emitted for items that are from the current crate and are `pub`.
// reference:: Emitted for locals behind a reference and functions taking `self` by reference.
// static:: Emitted for "static" functions, also known as functions that do not take a `self` param, as well as statics and consts.
// trait:: Emitted for associated trait items.
// unsafe:: Emitted for unsafe operations, like unsafe function calls, as well as the `unsafe` token.
//
//
// image::https://user-images.githubusercontent.com/48062697/113164457-06cfb980-9239-11eb-819b-0f93e646acf8.png[]
// image::https://user-images.githubusercontent.com/48062697/113187625-f7f50100-9250-11eb-825e-91c58f236071.png[]
pub(crate) fn highlight(
    db: &RootDatabase,
    file_id: FileId,
    range_to_highlight: Option<TextRange>,
    syntactic_name_ref_highlighting: bool,
) -> Vec<HlRange> {
    let _p = profile::span("highlight");
    let sema = Semantics::new(db);

    // Determine the root based on the given range.
    let (root, range_to_highlight) = {
        let source_file = sema.parse(file_id);
        let source_file = source_file.syntax();
        match range_to_highlight {
            Some(range) => {
                let node = match source_file.covering_element(range) {
                    NodeOrToken::Node(it) => it,
                    NodeOrToken::Token(it) => it.parent().unwrap_or_else(|| source_file.clone()),
                };
                (node, range)
            }
            None => (source_file.clone(), source_file.text_range()),
        }
    };

    let mut hl = highlights::Highlights::new(root.text_range());
    traverse(
        &mut hl,
        &sema,
        InFile::new(file_id.into(), &root),
        sema.scope(&root).krate(),
        range_to_highlight,
        syntactic_name_ref_highlighting,
    );
    hl.to_vec()
}

fn traverse(
    hl: &mut Highlights,
    sema: &Semantics<RootDatabase>,
    root: InFile<&SyntaxNode>,
    krate: Option<hir::Crate>,
    range_to_highlight: TextRange,
    syntactic_name_ref_highlighting: bool,
) {
    let mut bindings_shadow_count: FxHashMap<Name, u32> = FxHashMap::default();

    let mut current_macro_call: Option<ast::MacroCall> = None;
    let mut current_attr_macro_call = None;
    let mut current_macro: Option<ast::Macro> = None;
    let mut macro_highlighter = MacroHighlighter::default();
    let mut inside_attribute = false;

    // Walk all nodes, keeping track of whether we are inside a macro or not.
    // If in macro, expand it first and highlight the expanded code.
    for event in root.value.preorder_with_tokens() {
        let event_range = match &event {
            WalkEvent::Enter(it) | WalkEvent::Leave(it) => it.text_range(),
        };

        // Element outside of the viewport, no need to highlight
        if range_to_highlight.intersect(event_range).is_none() {
            continue;
        }

        match event.clone() {
            WalkEvent::Enter(NodeOrToken::Node(node)) => {
                match_ast! {
                    match node {
                        ast::MacroCall(mcall) => {
                            if let Some(range) = macro_call_range(&mcall) {
                                hl.add(HlRange {
                                    range,
                                    highlight: HlTag::Symbol(SymbolKind::Macro).into(),
                                    binding_hash: None,
                                });
                            }
                            current_macro_call = Some(mcall);
                            continue;
                        },
                        ast::Macro(mac) => {
                            macro_highlighter.init();
                            current_macro = Some(mac);
                            continue;
                        },
                        ast::Item(item) => {
                            if sema.is_attr_macro_call(&item) {
                                current_attr_macro_call = Some(item);
                            }
                        },
                        ast::Attr(__) => inside_attribute = true,
                        _ => ()
                    }
                }
            }
            WalkEvent::Leave(NodeOrToken::Node(node)) => {
                match_ast! {
                    match node {
                        ast::MacroCall(mcall) => {
                            assert_eq!(current_macro_call, Some(mcall));
                            current_macro_call = None;
                        },
                        ast::Macro(mac) => {
                            assert_eq!(current_macro, Some(mac));
                            current_macro = None;
                            macro_highlighter = MacroHighlighter::default();
                        },
                        ast::Item(item) => {
                            if current_attr_macro_call == Some(item) {
                                current_attr_macro_call = None;
                            }
                        },
                        ast::Attr(__) => inside_attribute = false,
                        _ => ()
                    }
                }
            }
            _ => (),
        }

        let element = match event {
            WalkEvent::Enter(it) => it,
            WalkEvent::Leave(it) => {
                if let Some(node) = it.as_node() {
                    inject::doc_comment(hl, sema, root.with_value(node));
                }
                continue;
            }
        };

        let range = element.text_range();

        if current_macro.is_some() {
            if let Some(tok) = element.as_token() {
                macro_highlighter.advance(tok);
            }
        }

        let element_to_highlight = if current_macro_call.is_some() && element.kind() != COMMENT {
            // Inside a macro -- expand it first
            let token = match element.clone().into_token() {
                Some(it) if it.parent().map_or(false, |it| it.kind() == TOKEN_TREE) => it,
                _ => continue,
            };
            let token = sema.descend_into_macros(token.clone());
            match token.parent() {
                Some(parent) => {
                    // We only care Name and Name_ref
                    match (token.kind(), parent.kind()) {
                        (IDENT, NAME | NAME_REF) => parent.into(),
                        _ => token.into(),
                    }
                }
                None => token.into(),
            }
        } else if current_attr_macro_call.is_some() {
            let token = match element.clone().into_token() {
                Some(it) => it,
                _ => continue,
            };
            let token = sema.descend_into_macros(token.clone());
            match token.parent() {
                Some(parent) => {
                    // We only care Name and Name_ref
                    match (token.kind(), parent.kind()) {
                        (IDENT, NAME | NAME_REF) => parent.into(),
                        _ => token.into(),
                    }
                }
                None => token.into(),
            }
        } else {
            element.clone()
        };

        if let Some(token) = element.as_token().cloned().and_then(ast::String::cast) {
            if token.is_raw() {
                let expanded = element_to_highlight.as_token().unwrap().clone();
                if inject::ra_fixture(hl, sema, token, expanded).is_some() {
                    continue;
                }
            }
        }

        if let Some(_) = macro_highlighter.highlight(element_to_highlight.clone()) {
            continue;
        }

        if let Some((mut highlight, binding_hash)) = highlight::element(
            sema,
            krate,
            &mut bindings_shadow_count,
            syntactic_name_ref_highlighting,
            element_to_highlight.clone(),
        ) {
            if inside_attribute {
                highlight = highlight | HlMod::Attribute;
            }

            hl.add(HlRange { range, highlight, binding_hash });
        }

        if let Some(string) = element_to_highlight.as_token().cloned().and_then(ast::String::cast) {
            highlight_format_string(hl, &string, range);
            // Highlight escape sequences
            if let Some(char_ranges) = string.char_ranges() {
                for (piece_range, _) in char_ranges.iter().filter(|(_, char)| char.is_ok()) {
                    if string.text()[piece_range.start().into()..].starts_with('\\') {
                        hl.add(HlRange {
                            range: piece_range + range.start(),
                            highlight: HlTag::EscapeSequence.into(),
                            binding_hash: None,
                        });
                    }
                }
            }
        }
    }
}

fn macro_call_range(macro_call: &ast::MacroCall) -> Option<TextRange> {
    let path = macro_call.path()?;
    let name_ref = path.segment()?.name_ref()?;

    let range_start = name_ref.syntax().text_range().start();
    let mut range_end = name_ref.syntax().text_range().end();
    for sibling in path.syntax().siblings_with_tokens(Direction::Next) {
        match sibling.kind() {
            T![!] | IDENT => range_end = sibling.text_range().end(),
            _ => (),
        }
    }

    Some(TextRange::new(range_start, range_end))
}
<commit_msg>Add comment<commit_after>pub(crate) mod tags;

mod highlights;
mod injector;

mod highlight;
mod format;
mod macro_;
mod inject;

mod html;
#[cfg(test)]
mod tests;

use hir::{InFile, Name, Semantics};
use ide_db::{RootDatabase, SymbolKind};
use rustc_hash::FxHashMap;
use syntax::{
    ast::{self, HasFormatSpecifier},
    match_ast, AstNode, AstToken, Direction, NodeOrToken,
    SyntaxKind::*,
    SyntaxNode, TextRange, WalkEvent, T,
};

use crate::{
    syntax_highlighting::{
        format::highlight_format_string, highlights::Highlights, macro_::MacroHighlighter,
        tags::Highlight,
    },
    FileId, HlMod, HlTag,
};

pub(crate) use html::highlight_as_html;

#[derive(Debug, Clone, Copy)]
pub struct HlRange {
    pub range: TextRange,
    pub highlight: Highlight,
    pub binding_hash: Option<u64>,
}

// Feature: Semantic Syntax Highlighting
//
// rust-analyzer highlights the code semantically.
// For example, `Bar` in `foo::Bar` might be colored differently depending on whether `Bar` is an enum or a trait.
// rust-analyzer does not specify colors directly, instead it assigns a tag (like `struct`) and a set of modifiers (like `declaration`) to each token.
// It's up to the client to map those to specific colors.
//
// The general rule is that a reference to an entity gets colored the same way as the entity itself.
// We also give special modifier for `mut` and `&mut` local variables.
//
//
// .Token Tags
//
// Rust-analyzer currently emits the following token tags:
//
// - For items:
// +
// [horizontal]
// enum:: Emitted for enums.
// function:: Emitted for free-standing functions.
// macro:: Emitted for macros.
// method:: Emitted for associated functions, also knowns as methods.
// namespace:: Emitted for modules.
// struct:: Emitted for structs.
// trait:: Emitted for traits.
// typeAlias:: Emitted for type aliases and `Self` in `impl`s.
// union:: Emitted for unions.
//
// - For literals:
// +
// [horizontal]
// boolean:: Emitted for the boolean literals `true` and `false`.
// character:: Emitted for character literals.
// number:: Emitted for numeric literals.
// string:: Emitted for string literals.
// escapeSequence:: Emitted for escaped sequences inside strings like `\n`.
// formatSpecifier:: Emitted for format specifiers `{:?}` in `format!`-like macros.
//
// - For operators:
// +
// [horizontal]
// operator:: Emitted for general operators.
// arithmetic:: Emitted for the arithmetic operators `+`, `-`, `*`, `/`, `+=`, `-=`, `*=`, `/=`.
// bitwise:: Emitted for the bitwise operators `|`, `&`, `!`, `^`, `|=`, `&=`, `^=`.
// comparison:: Emitted for the comparison operators `>`, `<`, `==`, `>=`, `<=`, `!=`.
// logical:: Emitted for the logical operators `||`, `&&`, `!`.
//
// - For punctuation:
// +
// [horizontal]
// punctuation:: Emitted for general punctuation.
// angle:: Emitted for `<>` angle brackets.
// brace:: Emitted for `{}` braces.
// bracket:: Emitted for `[]` brackets.
// parenthesis:: Emitted for `()` parentheses.
// colon:: Emitted for the `:` token.
// comma:: Emitted for the `,` token.
// dot:: Emitted for the `.` token.
// Semi:: Emitted for the `;` token.
//
// //-
//
// [horizontal]
// attribute:: Emitted for the `#[` `]` tokens.
// builtinAttribute:: Emitted for names to builtin attributes in attribute path, the `repr` in `#[repr(u8)]` for example.
// builtinType:: Emitted for builtin types like `u32`, `str` and `f32`.
// comment:: Emitted for comments.
// constParameter:: Emitted for const parameters.
// enumMember:: Emitted for enum variants.
// generic:: Emitted for generic tokens that have no mapping.
// keyword:: Emitted for keywords.
// label:: Emitted for labels.
// lifetime:: Emitted for lifetimes.
// parameter:: Emitted for non-self function parameters.
// property:: Emitted for struct and union fields.
// selfKeyword:: Emitted for the self function parameter and self path-specifier.
// typeParameter:: Emitted for type parameters.
// unresolvedReference:: Emitted for unresolved references, names that rust-analyzer can't find the definition of.
// variable:: Emitted for locals, constants and statics.
//
//
// .Token Modifiers
//
// Token modifiers allow to style some elements in the source code more precisely.
//
// Rust-analyzer currently emits the following token modifiers:
//
// [horizontal]
// async:: Emitted for async functions and the `async` and `await` keywords.
// attribute:: Emitted for tokens inside attributes.
// callable:: Emitted for locals whose types implements one of the `Fn*` traits.
// constant:: Emitted for consts.
// consuming:: Emitted for locals that are being consumed when use in a function call.
// controlFlow:: Emitted for control-flow related tokens, this includes the `?` operator.
// declaration:: Emitted for names of definitions, like `foo` in `fn foo() {}`.
// defaultLibrary:: Emitted for items from built-in crates (std, core, alloc, test and proc_macro).
// documentation:: Emitted for documentation comments.
// injected:: Emitted for doc-string injected highlighting like rust source blocks in documentation.
// intraDocLink:: Emitted for intra doc links in doc-strings.
// library:: Emitted for items that are defined outside of the current crate.
// mutable:: Emitted for mutable locals and statics as well as functions taking `&mut self`.
// public:: Emitted for items that are from the current crate and are `pub`.
// reference:: Emitted for locals behind a reference and functions taking `self` by reference.
// static:: Emitted for "static" functions, also known as functions that do not take a `self` param, as well as statics and consts.
// trait:: Emitted for associated trait items.
// unsafe:: Emitted for unsafe operations, like unsafe function calls, as well as the `unsafe` token.
//
//
// image::https://user-images.githubusercontent.com/48062697/113164457-06cfb980-9239-11eb-819b-0f93e646acf8.png[]
// image::https://user-images.githubusercontent.com/48062697/113187625-f7f50100-9250-11eb-825e-91c58f236071.png[]
pub(crate) fn highlight(
    db: &RootDatabase,
    file_id: FileId,
    range_to_highlight: Option<TextRange>,
    syntactic_name_ref_highlighting: bool,
) -> Vec<HlRange> {
    let _p = profile::span("highlight");
    let sema = Semantics::new(db);

    // Determine the root based on the given range.
    let (root, range_to_highlight) = {
        let source_file = sema.parse(file_id);
        let source_file = source_file.syntax();
        match range_to_highlight {
            Some(range) => {
                let node = match source_file.covering_element(range) {
                    NodeOrToken::Node(it) => it,
                    NodeOrToken::Token(it) => it.parent().unwrap_or_else(|| source_file.clone()),
                };
                (node, range)
            }
            None => (source_file.clone(), source_file.text_range()),
        }
    };

    let mut hl = highlights::Highlights::new(root.text_range());
    traverse(
        &mut hl,
        &sema,
        InFile::new(file_id.into(), &root),
        sema.scope(&root).krate(),
        range_to_highlight,
        syntactic_name_ref_highlighting,
    );
    hl.to_vec()
}

fn traverse(
    hl: &mut Highlights,
    sema: &Semantics<RootDatabase>,
    root: InFile<&SyntaxNode>,
    krate: Option<hir::Crate>,
    range_to_highlight: TextRange,
    syntactic_name_ref_highlighting: bool,
) {
    let mut bindings_shadow_count: FxHashMap<Name, u32> = FxHashMap::default();

    let mut current_macro_call: Option<ast::MacroCall> = None;
    let mut current_attr_macro_call = None;
    let mut current_macro: Option<ast::Macro> = None;
    let mut macro_highlighter = MacroHighlighter::default();
    let mut inside_attribute = false;

    // Walk all nodes, keeping track of whether we are inside a macro or not.
    // If in macro, expand it first and highlight the expanded code.
    for event in root.value.preorder_with_tokens() {
        let event_range = match &event {
            WalkEvent::Enter(it) | WalkEvent::Leave(it) => it.text_range(),
        };

        // Element outside of the viewport, no need to highlight
        if range_to_highlight.intersect(event_range).is_none() {
            continue;
        }

        match event.clone() {
            WalkEvent::Enter(NodeOrToken::Node(node)) => {
                match_ast! {
                    match node {
                        ast::MacroCall(mcall) => {
                            if let Some(range) = macro_call_range(&mcall) {
                                hl.add(HlRange {
                                    range,
                                    highlight: HlTag::Symbol(SymbolKind::Macro).into(),
                                    binding_hash: None,
                                });
                            }
                            current_macro_call = Some(mcall);
                            continue;
                        },
                        ast::Macro(mac) => {
                            macro_highlighter.init();
                            current_macro = Some(mac);
                            continue;
                        },
                        ast::Item(item) => {
                            if sema.is_attr_macro_call(&item) {
                                current_attr_macro_call = Some(item);
                            }
                        },
                        ast::Attr(__) => inside_attribute = true,
                        _ => ()
                    }
                }
            }
            WalkEvent::Leave(NodeOrToken::Node(node)) => {
                match_ast! {
                    match node {
                        ast::MacroCall(mcall) => {
                            assert_eq!(current_macro_call, Some(mcall));
                            current_macro_call = None;
                        },
                        ast::Macro(mac) => {
                            assert_eq!(current_macro, Some(mac));
                            current_macro = None;
                            macro_highlighter = MacroHighlighter::default();
                        },
                        ast::Item(item) => {
                            if current_attr_macro_call == Some(item) {
                                current_attr_macro_call = None;
                            }
                        },
                        ast::Attr(__) => inside_attribute = false,
                        _ => ()
                    }
                }
            }
            _ => (),
        }

        let element = match event {
            WalkEvent::Enter(it) => it,
            WalkEvent::Leave(it) => {
                if let Some(node) = it.as_node() {
                    inject::doc_comment(hl, sema, root.with_value(node));
                }
                continue;
            }
        };

        let range = element.text_range();

        if current_macro.is_some() {
            if let Some(tok) = element.as_token() {
                macro_highlighter.advance(tok);
            }
        }

        let element_to_highlight = if current_macro_call.is_some() && element.kind() != COMMENT {
            // Inside a macro -- expand it first
            let token = match element.clone().into_token() {
                Some(it) if it.parent().map_or(false, |it| it.kind() == TOKEN_TREE) => it,
                _ => continue,
            };
            let token = sema.descend_into_macros(token.clone());
            match token.parent() {
                Some(parent) => {
                    // We only care Name and Name_ref
                    match (token.kind(), parent.kind()) {
                        (IDENT, NAME | NAME_REF) => parent.into(),
                        _ => token.into(),
                    }
                }
                None => token.into(),
            }
        } else if current_attr_macro_call.is_some() {
            let token = match element.clone().into_token() {
                Some(it) => it,
                _ => continue,
            };
            let token = sema.descend_into_macros(token.clone());
            match token.parent() {
                Some(parent) => {
                    // We only care Name and Name_ref
                    match (token.kind(), parent.kind()) {
                        (IDENT, NAME | NAME_REF) => parent.into(),
                        _ => token.into(),
                    }
                }
                None => token.into(),
            }
        } else {
            element.clone()
        };

        if let Some(token) = element.as_token().cloned().and_then(ast::String::cast) {
            if token.is_raw() {
                let expanded = element_to_highlight.as_token().unwrap().clone();
                if inject::ra_fixture(hl, sema, token, expanded).is_some() {
                    continue;
                }
            }
        }

        if let Some(_) = macro_highlighter.highlight(element_to_highlight.clone()) {
            continue;
        }

        if let Some((mut highlight, binding_hash)) = highlight::element(
            sema,
            krate,
            &mut bindings_shadow_count,
            syntactic_name_ref_highlighting,
            element_to_highlight.clone(),
        ) {
            if inside_attribute {
                highlight = highlight | HlMod::Attribute;
            }

            hl.add(HlRange { range, highlight, binding_hash });
        }

        if let Some(string) = element_to_highlight.as_token().cloned().and_then(ast::String::cast) {
            highlight_format_string(hl, &string, range);
            // Highlight escape sequences
            if let Some(char_ranges) = string.char_ranges() {
                for (piece_range, _) in char_ranges.iter().filter(|(_, char)| char.is_ok()) {
                    if string.text()[piece_range.start().into()..].starts_with('\\') {
                        hl.add(HlRange {
                            range: piece_range + range.start(),
                            highlight: HlTag::EscapeSequence.into(),
                            binding_hash: None,
                        });
                    }
                }
            }
        }
    }
}

fn macro_call_range(macro_call: &ast::MacroCall) -> Option<TextRange> {
    let path = macro_call.path()?;
    let name_ref = path.segment()?.name_ref()?;

    let range_start = name_ref.syntax().text_range().start();
    let mut range_end = name_ref.syntax().text_range().end();
    for sibling in path.syntax().siblings_with_tokens(Direction::Next) {
        match sibling.kind() {
            T![!] | IDENT => range_end = sibling.text_range().end(),
            _ => (),
        }
    }

    Some(TextRange::new(range_start, range_end))
}
<|endoftext|>


<commit_before>mod consts;
mod nominal;
mod traits;
mod use_item;

pub(crate) use self::{
    expressions::{match_arm_list, named_field_list},
    nominal::{enum_variant_list, named_field_def_list},
    traits::{impl_item_list, trait_item_list},
    use_item::use_tree_list,
};
use super::*;

// test mod_contents
// fn foo() {}
// macro_rules! foo {}
// foo::bar!();
// super::baz! {}
// struct S;
pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) {
    attributes::inner_attributes(p);
    while !p.at(EOF) && !(stop_on_r_curly && p.at(R_CURLY)) {
        item_or_macro(p, stop_on_r_curly, ItemFlavor::Mod)
    }
}

pub(super) enum ItemFlavor {
    Mod,
    Trait,
}

pub(super) const ITEM_RECOVERY_SET: TokenSet = token_set![
    FN_KW, STRUCT_KW, ENUM_KW, IMPL_KW, TRAIT_KW, CONST_KW, STATIC_KW, LET_KW, MOD_KW, PUB_KW,
    CRATE_KW
];

pub(super) fn item_or_macro(p: &mut Parser, stop_on_r_curly: bool, flavor: ItemFlavor) {
    let m = p.start();
    match maybe_item(p, flavor) {
        MaybeItem::Item(kind) => {
            m.complete(p, kind);
        }
        MaybeItem::None => {
            if paths::is_path_start(p) {
                match macro_call(p) {
                    BlockLike::Block => (),
                    BlockLike::NotBlock => {
                        p.expect(SEMI);
                    }
                }
                m.complete(p, MACRO_CALL);
            } else {
                m.abandon(p);
                if p.at(L_CURLY) {
                    error_block(p, "expected an item");
                } else if p.at(R_CURLY) && !stop_on_r_curly {
                    let e = p.start();
                    p.error("unmatched `}`");
                    p.bump();
                    e.complete(p, ERROR);
                } else if !p.at(EOF) && !p.at(R_CURLY) {
                    p.err_and_bump("expected an item");
                } else {
                    p.error("expected an item");
                }
            }
        }
        MaybeItem::Modifiers => {
            p.error("expected fn, trait or impl");
            m.complete(p, ERROR);
        }
    }
}

pub(super) enum MaybeItem {
    None,
    Item(SyntaxKind),
    Modifiers,
}

pub(super) fn maybe_item(p: &mut Parser, flavor: ItemFlavor) -> MaybeItem {
    attributes::outer_attributes(p);
    opt_visibility(p);
    if let Some(kind) = items_without_modifiers(p) {
        return MaybeItem::Item(kind);
    }

    let mut has_mods = false;
    // modifiers
    has_mods |= p.eat(CONST_KW);

    // test_err unsafe_block_in_mod
    // fn foo(){} unsafe { } fn bar(){}
    if p.at(UNSAFE_KW) && p.nth(1) != L_CURLY {
        p.eat(UNSAFE_KW);
        has_mods = true;
    }
    if p.at(EXTERN_KW) {
        has_mods = true;
        abi(p);
    }
    if p.at(IDENT) && p.at_contextual_kw("auto") && p.nth(1) == TRAIT_KW {
        p.bump_remap(AUTO_KW);
        has_mods = true;
    }
    if p.at(IDENT) && p.at_contextual_kw("default") && p.nth(1) == IMPL_KW {
        p.bump_remap(DEFAULT_KW);
        has_mods = true;
    }

    // items
    let kind = match p.current() {
        // test extern_fn
        // extern fn foo() {}

        // test const_fn
        // const fn foo() {}

        // test const_unsafe_fn
        // const unsafe fn foo() {}

        // test unsafe_extern_fn
        // unsafe extern "C" fn foo() {}

        // test unsafe_fn
        // unsafe fn foo() {}
        FN_KW => {
            fn_def(p, flavor);
            FN_DEF
        }

        // test unsafe_trait
        // unsafe trait T {}

        // test auto_trait
        // auto trait T {}

        // test unsafe_auto_trait
        // unsafe auto trait T {}
        TRAIT_KW => {
            traits::trait_def(p);
            TRAIT_DEF
        }

        // test unsafe_impl
        // unsafe impl Foo {}

        // test default_impl
        // default impl Foo {}

        // test unsafe_default_impl
        // unsafe default impl Foo {}
        IMPL_KW => {
            traits::impl_item(p);
            IMPL_ITEM
        }
        _ => {
            return if has_mods {
                MaybeItem::Modifiers
            } else {
                MaybeItem::None
            };
        }
    };

    MaybeItem::Item(kind)
}

fn items_without_modifiers(p: &mut Parser) -> Option<SyntaxKind> {
    let la = p.nth(1);
    let kind = match p.current() {
        // test extern_crate
        // extern crate foo;
        EXTERN_KW if la == CRATE_KW => {
            extern_crate_item(p);
            EXTERN_CRATE_ITEM
        }
        TYPE_KW => {
            type_def(p);
            TYPE_DEF
        }
        MOD_KW => {
            mod_item(p);
            MODULE
        }
        STRUCT_KW => {
            // test struct_items
            // struct Foo;
            // struct Foo {}
            // struct Foo();
            // struct Foo(String, usize);
            // struct Foo {
            //     a: i32,
            //     b: f32,
            // }
            nominal::struct_def(p, STRUCT_KW);
            if p.at(SEMI) {
                p.err_and_bump(
                    "expected item, found `;`\n\
                     consider removing this semicolon",
                );
            }
            STRUCT_DEF
        }
        IDENT if p.at_contextual_kw("union") => {
            // test union_items
            // union Foo {}
            // union Foo {
            //     a: i32,
            //     b: f32,
            // }
            nominal::struct_def(p, UNION_KW);
            STRUCT_DEF
        }
        ENUM_KW => {
            nominal::enum_def(p);
            ENUM_DEF
        }
        USE_KW => {
            use_item::use_item(p);
            USE_ITEM
        }
        CONST_KW if (la == IDENT || la == MUT_KW) => {
            consts::const_def(p);
            CONST_DEF
        }
        STATIC_KW => {
            consts::static_def(p);
            STATIC_DEF
        }
        // test extern_block
        // extern {}
        EXTERN_KW
            if la == L_CURLY || ((la == STRING || la == RAW_STRING) && p.nth(2) == L_CURLY) =>
        {
            abi(p);
            extern_item_list(p);
            EXTERN_BLOCK
        }
        _ => return None,
    };
    Some(kind)
}

fn extern_crate_item(p: &mut Parser) {
    assert!(p.at(EXTERN_KW));
    p.bump();
    assert!(p.at(CRATE_KW));
    p.bump();
    name(p);
    opt_alias(p);
    p.expect(SEMI);
}

pub(crate) fn extern_item_list(p: &mut Parser) {
    assert!(p.at(L_CURLY));
    let m = p.start();
    p.bump();
    mod_contents(p, true);
    p.expect(R_CURLY);
    m.complete(p, EXTERN_ITEM_LIST);
}

fn fn_def(p: &mut Parser, flavor: ItemFlavor) {
    assert!(p.at(FN_KW));
    p.bump();

    name_r(p, ITEM_RECOVERY_SET);
    // test function_type_params
    // fn foo<T: Clone + Copy>(){}
    type_params::opt_type_param_list(p);

    if p.at(L_PAREN) {
        match flavor {
            ItemFlavor::Mod => params::param_list(p),
            ItemFlavor::Trait => params::param_list_opt_patterns(p),
        }
    } else {
        p.error("expected function arguments");
    }
    // test function_ret_type
    // fn foo() {}
    // fn bar() -> () {}
    opt_fn_ret_type(p);

    // test function_where_clause
    // fn foo<T>() where T: Copy {}
    type_params::opt_where_clause(p);

    // test fn_decl
    // trait T { fn foo(); }
    if p.at(SEMI) {
        p.bump();
    } else {
        expressions::block(p)
    }
}

// test type_item
// type Foo = Bar;
fn type_def(p: &mut Parser) {
    assert!(p.at(TYPE_KW));
    p.bump();

    name(p);

    // test type_item_type_params
    // type Result<T> = ();
    type_params::opt_type_param_list(p);

    if p.at(COLON) {
        type_params::bounds(p);
    }

    // test type_item_where_clause
    // type Foo where Foo: Copy = ();
    type_params::opt_where_clause(p);

    if p.eat(EQ) {
        types::type_(p);
    }
    p.expect(SEMI);
}

pub(crate) fn mod_item(p: &mut Parser) {
    assert!(p.at(MOD_KW));
    p.bump();

    name(p);
    if p.at(L_CURLY) {
        mod_item_list(p);
    } else if !p.eat(SEMI) {
        p.error("expected `;` or `{`");
    }
}

pub(crate) fn mod_item_list(p: &mut Parser) {
    assert!(p.at(L_CURLY));
    let m = p.start();
    p.bump();
    mod_contents(p, true);
    p.expect(R_CURLY);
    m.complete(p, ITEM_LIST);
}

fn macro_call(p: &mut Parser) -> BlockLike {
    assert!(paths::is_path_start(p));
    paths::use_path(p);
    macro_call_after_excl(p)
}

pub(super) fn macro_call_after_excl(p: &mut Parser) -> BlockLike {
    p.expect(EXCL);
    p.eat(IDENT);
    match p.current() {
        L_CURLY => {
            token_tree(p);
            BlockLike::Block
        }
        L_PAREN | L_BRACK => {
            token_tree(p);
            BlockLike::NotBlock
        }
        _ => {
            p.error("expected `{`, `[`, `(`");
            BlockLike::NotBlock
        }
    }
}

pub(crate) fn token_tree(p: &mut Parser) {
    let closing_paren_kind = match p.current() {
        L_CURLY => R_CURLY,
        L_PAREN => R_PAREN,
        L_BRACK => R_BRACK,
        _ => unreachable!(),
    };
    let m = p.start();
    p.bump();
    while !p.at(EOF) && !p.at(closing_paren_kind) {
        match p.current() {
            L_CURLY | L_PAREN | L_BRACK => token_tree(p),
            R_CURLY => {
                p.error("unmatched `}`");
                m.complete(p, TOKEN_TREE);
                return;
            }
            R_PAREN | R_BRACK => p.err_and_bump("unmatched brace"),
            _ => p.bump(),
        }
    }
    p.expect(closing_paren_kind);
    m.complete(p, TOKEN_TREE);
}
<commit_msg>Fix parsing of contextual_keyword union<commit_after>mod consts;
mod nominal;
mod traits;
mod use_item;

pub(crate) use self::{
    expressions::{match_arm_list, named_field_list},
    nominal::{enum_variant_list, named_field_def_list},
    traits::{impl_item_list, trait_item_list},
    use_item::use_tree_list,
};
use super::*;

// test mod_contents
// fn foo() {}
// macro_rules! foo {}
// foo::bar!();
// super::baz! {}
// struct S;
pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) {
    attributes::inner_attributes(p);
    while !p.at(EOF) && !(stop_on_r_curly && p.at(R_CURLY)) {
        item_or_macro(p, stop_on_r_curly, ItemFlavor::Mod)
    }
}

pub(super) enum ItemFlavor {
    Mod,
    Trait,
}

pub(super) const ITEM_RECOVERY_SET: TokenSet = token_set![
    FN_KW, STRUCT_KW, ENUM_KW, IMPL_KW, TRAIT_KW, CONST_KW, STATIC_KW, LET_KW, MOD_KW, PUB_KW,
    CRATE_KW
];

pub(super) fn item_or_macro(p: &mut Parser, stop_on_r_curly: bool, flavor: ItemFlavor) {
    let m = p.start();
    match maybe_item(p, flavor) {
        MaybeItem::Item(kind) => {
            m.complete(p, kind);
        }
        MaybeItem::None => {
            if paths::is_path_start(p) {
                match macro_call(p) {
                    BlockLike::Block => (),
                    BlockLike::NotBlock => {
                        p.expect(SEMI);
                    }
                }
                m.complete(p, MACRO_CALL);
            } else {
                m.abandon(p);
                if p.at(L_CURLY) {
                    error_block(p, "expected an item");
                } else if p.at(R_CURLY) && !stop_on_r_curly {
                    let e = p.start();
                    p.error("unmatched `}`");
                    p.bump();
                    e.complete(p, ERROR);
                } else if !p.at(EOF) && !p.at(R_CURLY) {
                    p.err_and_bump("expected an item");
                } else {
                    p.error("expected an item");
                }
            }
        }
        MaybeItem::Modifiers => {
            p.error("expected fn, trait or impl");
            m.complete(p, ERROR);
        }
    }
}

pub(super) enum MaybeItem {
    None,
    Item(SyntaxKind),
    Modifiers,
}

pub(super) fn maybe_item(p: &mut Parser, flavor: ItemFlavor) -> MaybeItem {
    attributes::outer_attributes(p);
    opt_visibility(p);
    if let Some(kind) = items_without_modifiers(p) {
        return MaybeItem::Item(kind);
    }

    let mut has_mods = false;
    // modifiers
    has_mods |= p.eat(CONST_KW);

    // test_err unsafe_block_in_mod
    // fn foo(){} unsafe { } fn bar(){}
    if p.at(UNSAFE_KW) && p.nth(1) != L_CURLY {
        p.eat(UNSAFE_KW);
        has_mods = true;
    }
    if p.at(EXTERN_KW) {
        has_mods = true;
        abi(p);
    }
    if p.at(IDENT) && p.at_contextual_kw("auto") && p.nth(1) == TRAIT_KW {
        p.bump_remap(AUTO_KW);
        has_mods = true;
    }
    if p.at(IDENT) && p.at_contextual_kw("default") && p.nth(1) == IMPL_KW {
        p.bump_remap(DEFAULT_KW);
        has_mods = true;
    }

    // items
    let kind = match p.current() {
        // test extern_fn
        // extern fn foo() {}

        // test const_fn
        // const fn foo() {}

        // test const_unsafe_fn
        // const unsafe fn foo() {}

        // test unsafe_extern_fn
        // unsafe extern "C" fn foo() {}

        // test unsafe_fn
        // unsafe fn foo() {}
        FN_KW => {
            fn_def(p, flavor);
            FN_DEF
        }

        // test unsafe_trait
        // unsafe trait T {}

        // test auto_trait
        // auto trait T {}

        // test unsafe_auto_trait
        // unsafe auto trait T {}
        TRAIT_KW => {
            traits::trait_def(p);
            TRAIT_DEF
        }

        // test unsafe_impl
        // unsafe impl Foo {}

        // test default_impl
        // default impl Foo {}

        // test unsafe_default_impl
        // unsafe default impl Foo {}
        IMPL_KW => {
            traits::impl_item(p);
            IMPL_ITEM
        }
        _ => {
            return if has_mods {
                MaybeItem::Modifiers
            } else {
                MaybeItem::None
            };
        }
    };

    MaybeItem::Item(kind)
}

fn items_without_modifiers(p: &mut Parser) -> Option<SyntaxKind> {
    let la = p.nth(1);
    let kind = match p.current() {
        // test extern_crate
        // extern crate foo;
        EXTERN_KW if la == CRATE_KW => {
            extern_crate_item(p);
            EXTERN_CRATE_ITEM
        }
        TYPE_KW => {
            type_def(p);
            TYPE_DEF
        }
        MOD_KW => {
            mod_item(p);
            MODULE
        }
        STRUCT_KW => {
            // test struct_items
            // struct Foo;
            // struct Foo {}
            // struct Foo();
            // struct Foo(String, usize);
            // struct Foo {
            //     a: i32,
            //     b: f32,
            // }
            nominal::struct_def(p, STRUCT_KW);
            if p.at(SEMI) {
                p.err_and_bump(
                    "expected item, found `;`\n\
                     consider removing this semicolon",
                );
            }
            STRUCT_DEF
        }
        IDENT if p.at_contextual_kw("union") && p.nth(1) == IDENT => {
            // test union_items
            // union Foo {}
            // union Foo {
            //     a: i32,
            //     b: f32,
            // }
            nominal::struct_def(p, UNION_KW);
            STRUCT_DEF
        }
        ENUM_KW => {
            nominal::enum_def(p);
            ENUM_DEF
        }
        USE_KW => {
            use_item::use_item(p);
            USE_ITEM
        }
        CONST_KW if (la == IDENT || la == MUT_KW) => {
            consts::const_def(p);
            CONST_DEF
        }
        STATIC_KW => {
            consts::static_def(p);
            STATIC_DEF
        }
        // test extern_block
        // extern {}
        EXTERN_KW
            if la == L_CURLY || ((la == STRING || la == RAW_STRING) && p.nth(2) == L_CURLY) =>
        {
            abi(p);
            extern_item_list(p);
            EXTERN_BLOCK
        }
        _ => return None,
    };
    Some(kind)
}

fn extern_crate_item(p: &mut Parser) {
    assert!(p.at(EXTERN_KW));
    p.bump();
    assert!(p.at(CRATE_KW));
    p.bump();
    name(p);
    opt_alias(p);
    p.expect(SEMI);
}

pub(crate) fn extern_item_list(p: &mut Parser) {
    assert!(p.at(L_CURLY));
    let m = p.start();
    p.bump();
    mod_contents(p, true);
    p.expect(R_CURLY);
    m.complete(p, EXTERN_ITEM_LIST);
}

fn fn_def(p: &mut Parser, flavor: ItemFlavor) {
    assert!(p.at(FN_KW));
    p.bump();

    name_r(p, ITEM_RECOVERY_SET);
    // test function_type_params
    // fn foo<T: Clone + Copy>(){}
    type_params::opt_type_param_list(p);

    if p.at(L_PAREN) {
        match flavor {
            ItemFlavor::Mod => params::param_list(p),
            ItemFlavor::Trait => params::param_list_opt_patterns(p),
        }
    } else {
        p.error("expected function arguments");
    }
    // test function_ret_type
    // fn foo() {}
    // fn bar() -> () {}
    opt_fn_ret_type(p);

    // test function_where_clause
    // fn foo<T>() where T: Copy {}
    type_params::opt_where_clause(p);

    // test fn_decl
    // trait T { fn foo(); }
    if p.at(SEMI) {
        p.bump();
    } else {
        expressions::block(p)
    }
}

// test type_item
// type Foo = Bar;
fn type_def(p: &mut Parser) {
    assert!(p.at(TYPE_KW));
    p.bump();

    name(p);

    // test type_item_type_params
    // type Result<T> = ();
    type_params::opt_type_param_list(p);

    if p.at(COLON) {
        type_params::bounds(p);
    }

    // test type_item_where_clause
    // type Foo where Foo: Copy = ();
    type_params::opt_where_clause(p);

    if p.eat(EQ) {
        types::type_(p);
    }
    p.expect(SEMI);
}

pub(crate) fn mod_item(p: &mut Parser) {
    assert!(p.at(MOD_KW));
    p.bump();

    name(p);
    if p.at(L_CURLY) {
        mod_item_list(p);
    } else if !p.eat(SEMI) {
        p.error("expected `;` or `{`");
    }
}

pub(crate) fn mod_item_list(p: &mut Parser) {
    assert!(p.at(L_CURLY));
    let m = p.start();
    p.bump();
    mod_contents(p, true);
    p.expect(R_CURLY);
    m.complete(p, ITEM_LIST);
}

fn macro_call(p: &mut Parser) -> BlockLike {
    assert!(paths::is_path_start(p));
    paths::use_path(p);
    macro_call_after_excl(p)
}

pub(super) fn macro_call_after_excl(p: &mut Parser) -> BlockLike {
    p.expect(EXCL);
    p.eat(IDENT);
    match p.current() {
        L_CURLY => {
            token_tree(p);
            BlockLike::Block
        }
        L_PAREN | L_BRACK => {
            token_tree(p);
            BlockLike::NotBlock
        }
        _ => {
            p.error("expected `{`, `[`, `(`");
            BlockLike::NotBlock
        }
    }
}

pub(crate) fn token_tree(p: &mut Parser) {
    let closing_paren_kind = match p.current() {
        L_CURLY => R_CURLY,
        L_PAREN => R_PAREN,
        L_BRACK => R_BRACK,
        _ => unreachable!(),
    };
    let m = p.start();
    p.bump();
    while !p.at(EOF) && !p.at(closing_paren_kind) {
        match p.current() {
            L_CURLY | L_PAREN | L_BRACK => token_tree(p),
            R_CURLY => {
                p.error("unmatched `}`");
                m.complete(p, TOKEN_TREE);
                return;
            }
            R_PAREN | R_BRACK => p.err_and_bump("unmatched brace"),
            _ => p.bump(),
        }
    }
    p.expect(closing_paren_kind);
    m.complete(p, TOKEN_TREE);
}
<|endoftext|>


<commit_before>use Term;
use query::Weight;
use core::SegmentReader;
use query::Scorer;
use DocSet;
use postings::SegmentPostings;
use schema::IndexRecordOption;
use super::term_scorer::TermScorer;
use Result;

pub struct TermWeight {
    pub(crate) num_docs: u32,
    pub(crate) doc_freq: u32,
    pub(crate) term: Term,
    pub(crate) index_record_option: IndexRecordOption,
}

impl Weight for TermWeight {
    fn scorer(&self, reader: &SegmentReader) -> Result<Box<Scorer>> {
        let specialized_scorer = self.specialized_scorer(reader)?;
        Ok(box specialized_scorer)
    }

    fn count(&self, reader: &SegmentReader) -> Result<u32> {
        if reader.num_deleted_docs() == 0 {
            Ok(self.doc_freq)
        } else {
            Ok(self.specialized_scorer(reader)?.count())
        }
    }
}

impl TermWeight {
    fn idf(&self) -> f32 {
        1.0 + (self.num_docs as f32 / (self.doc_freq as f32 + 1.0)).ln()
    }

    /// If the field is not found, returns an empty `DocSet`.
    pub fn specialized_scorer(
        &self,
        reader: &SegmentReader,
    ) -> Result<TermScorer> {
        let field = self.term.field();
        let inverted_index = reader.inverted_index(field);
        let fieldnorm_reader_opt = reader.get_fieldnorms_reader(field);
        let postings_opt: Option<SegmentPostings> =
            inverted_index.read_postings(&self.term, self.index_record_option);
        if let Some(segment_postings) = postings_opt {
            Ok(TermScorer {
                idf: self.idf(),
                fieldnorm_reader_opt,
                postings: segment_postings,
            })
        } else {
            Ok(TermScorer {
                idf: 1f32,
                fieldnorm_reader_opt: None,
                postings: SegmentPostings::empty(),
            })
        }
    }
}
<commit_msg>Bugfix on TermWeight::count()<commit_after>use Term;
use query::Weight;
use core::SegmentReader;
use query::Scorer;
use DocSet;
use postings::SegmentPostings;
use schema::IndexRecordOption;
use super::term_scorer::TermScorer;
use Result;

pub struct TermWeight {
    pub(crate) num_docs: u32,
    pub(crate) doc_freq: u32,
    pub(crate) term: Term,
    pub(crate) index_record_option: IndexRecordOption,
}

impl Weight for TermWeight {
    fn scorer(&self, reader: &SegmentReader) -> Result<Box<Scorer>> {
        let specialized_scorer = self.specialized_scorer(reader)?;
        Ok(box specialized_scorer)
    }

    fn count(&self, reader: &SegmentReader) -> Result<u32> {
        if reader.num_deleted_docs() == 0 {
            let field = self.term.field();
            Ok(reader
                .inverted_index(field)
                .get_term_info(&self.term)
                .map(|term_info| term_info.doc_freq)
                .unwrap_or(0))
        } else {
            Ok(self.specialized_scorer(reader)?.count())
        }
    }
}

impl TermWeight {
    fn idf(&self) -> f32 {
        1.0 + (self.num_docs as f32 / (self.doc_freq as f32 + 1.0)).ln()
    }

    /// If the field is not found, returns an empty `DocSet`.
    pub fn specialized_scorer(
        &self,
        reader: &SegmentReader,
    ) -> Result<TermScorer> {
        let field = self.term.field();
        let inverted_index = reader.inverted_index(field);
        let fieldnorm_reader_opt = reader.get_fieldnorms_reader(field);
        let postings_opt: Option<SegmentPostings> =
            inverted_index.read_postings(&self.term, self.index_record_option);
        if let Some(segment_postings) = postings_opt {
            Ok(TermScorer {
                idf: self.idf(),
                fieldnorm_reader_opt,
                postings: segment_postings,
            })
        } else {
            Ok(TermScorer {
                idf: 1f32,
                fieldnorm_reader_opt: None,
                postings: SegmentPostings::empty(),
            })
        }
    }
}
<|endoftext|>