<commit_before>#[doc = "The main parser interface"];
import dvec::extensions;

export parse_sess;
export next_node_id;
export new_parser_from_file;
export new_parser_from_source_str;
export parse_crate_from_file;
export parse_crate_from_crate_file;
export parse_crate_from_source_str;
export parse_expr_from_source_str;
export parse_item_from_source_str;
export parse_from_source_str;

import parser::parser;
import attr::parser_attr;
import common::parser_common;
import ast::node_id;
import util::interner;
import lexer::reader;

type parse_sess = @{
    cm: codemap::codemap,
    mut next_id: node_id,
    span_diagnostic: diagnostic::span_handler,
    // these two must be kept up to date
    mut chpos: uint,
    mut byte_pos: uint
};

fn parse_crate_from_file(input: str, cfg: ast::crate_cfg, sess: parse_sess) ->
   @ast::crate {
    if str::ends_with(input, ".rc") {
        parse_crate_from_crate_file(input, cfg, sess)
    } else if str::ends_with(input, ".rs") {
        parse_crate_from_source_file(input, cfg, sess)
    } else {
        sess.span_diagnostic.handler().fatal("unknown input file type: " +
                                             input)
    }
}

fn parse_crate_from_crate_file(input: str, cfg: ast::crate_cfg,
                               sess: parse_sess) -> @ast::crate {
    let p = new_parser_from_file(sess, cfg, input, parser::CRATE_FILE);
    let lo = p.span.lo;
    let prefix = path::dirname(p.reader.filemap.name);
    let leading_attrs = p.parse_inner_attrs_and_next();
    let { inner: crate_attrs, next: first_cdir_attr } = leading_attrs;
    let cdirs = p.parse_crate_directives(token::EOF, first_cdir_attr);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    let cx = @{sess: sess, cfg: /* FIXME: bad */ copy p.cfg};
    let (companionmod, _) = path::splitext(path::basename(input));
    let (m, attrs) = eval::eval_crate_directives_to_mod(
        cx, cdirs, prefix, option::some(companionmod));
    let mut hi = p.span.hi;
    p.expect(token::EOF);
    ret @ast_util::respan(ast_util::mk_sp(lo, hi),
                          {directives: cdirs,
                           module: m,
                           attrs: crate_attrs + attrs,
                           config: /* FIXME: bad */ copy p.cfg});
}

fn parse_crate_from_source_file(input: str, cfg: ast::crate_cfg,
                                sess: parse_sess) -> @ast::crate {
    let p = new_parser_from_file(sess, cfg, input, parser::SOURCE_FILE);
    let r = p.parse_crate_mod(cfg);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn parse_crate_from_source_str(name: str, source: @str, cfg: ast::crate_cfg,
                               sess: parse_sess) -> @ast::crate {
    let p = new_parser_from_source_str(
        sess, cfg, name, codemap::fss_none, source);
    let r = p.parse_crate_mod(cfg);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn parse_expr_from_source_str(name: str, source: @str, cfg: ast::crate_cfg,
                              sess: parse_sess) -> @ast::expr {
    let p = new_parser_from_source_str(
        sess, cfg, name, codemap::fss_none, source);
    let r = p.parse_expr();
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn parse_item_from_source_str(name: str, source: @str, cfg: ast::crate_cfg,
                              +attrs: [ast::attribute], vis: ast::visibility,
                              sess: parse_sess) -> option<@ast::item> {
    let p = new_parser_from_source_str(
        sess, cfg, name, codemap::fss_none, source);
    let r = p.parse_item(attrs, vis);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn parse_from_source_str<T>(f: fn (p: parser) -> T,
                            name: str, ss: codemap::file_substr,
                            source: @str, cfg: ast::crate_cfg,
                            sess: parse_sess)
    -> T
{
    let p = new_parser_from_source_str(sess, cfg, name, ss, source);
    let r = f(p);
    if !p.reader.is_eof() {
        p.reader.fatal("expected end-of-string");
    }
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn next_node_id(sess: parse_sess) -> node_id {
    let rv = sess.next_id;
    sess.next_id += 1;
    // ID 0 is reserved for the crate and doesn't actually exist in the AST
    assert rv != 0;
    ret rv;
}

fn new_parser_from_source_str(sess: parse_sess, cfg: ast::crate_cfg,
                              +name: str, +ss: codemap::file_substr,
                              source: @str) -> parser {
    let ftype = parser::SOURCE_FILE;
    let filemap = codemap::new_filemap_w_substr
        (name, ss, source, sess.chpos, sess.byte_pos);
    sess.cm.files.push(filemap);
    let itr = @interner::mk::<@str>({|x|str::hash(*x)}, {|x,y|str::eq(*x, *y)});
    let rdr = lexer::new_reader(sess.span_diagnostic,
                                filemap, itr);
    ret parser(sess, cfg, rdr, ftype);
}

fn new_parser_from_file(sess: parse_sess, cfg: ast::crate_cfg, +path: str,
                        ftype: parser::file_type) ->
   parser {
    let res = io::read_whole_file_str(path);
    alt res {
      result::ok(_) { /* Continue. */ }
      result::err(e) { sess.span_diagnostic.handler().fatal(e); }
    }
    // FIXME: This copy is unfortunate (#2319).
    let src = @copy result::unwrap(res);
    let filemap = codemap::new_filemap(path, src, sess.chpos, sess.byte_pos);
    sess.cm.files.push(filemap);
       let itr = @interner::mk::<@str>({|x|str::hash(*x)}, {|x,y|str::eq(*x, *y)});
    let rdr = lexer::new_reader(sess.span_diagnostic, filemap, itr);
    ret parser(sess, cfg, rdr, ftype);
}
<commit_msg>syntax: Don't copy source string<commit_after>#[doc = "The main parser interface"];
import dvec::extensions;

export parse_sess;
export next_node_id;
export new_parser_from_file;
export new_parser_from_source_str;
export parse_crate_from_file;
export parse_crate_from_crate_file;
export parse_crate_from_source_str;
export parse_expr_from_source_str;
export parse_item_from_source_str;
export parse_from_source_str;

import parser::parser;
import attr::parser_attr;
import common::parser_common;
import ast::node_id;
import util::interner;
import lexer::reader;

type parse_sess = @{
    cm: codemap::codemap,
    mut next_id: node_id,
    span_diagnostic: diagnostic::span_handler,
    // these two must be kept up to date
    mut chpos: uint,
    mut byte_pos: uint
};

fn parse_crate_from_file(input: str, cfg: ast::crate_cfg, sess: parse_sess) ->
   @ast::crate {
    if str::ends_with(input, ".rc") {
        parse_crate_from_crate_file(input, cfg, sess)
    } else if str::ends_with(input, ".rs") {
        parse_crate_from_source_file(input, cfg, sess)
    } else {
        sess.span_diagnostic.handler().fatal("unknown input file type: " +
                                             input)
    }
}

fn parse_crate_from_crate_file(input: str, cfg: ast::crate_cfg,
                               sess: parse_sess) -> @ast::crate {
    let p = new_parser_from_file(sess, cfg, input, parser::CRATE_FILE);
    let lo = p.span.lo;
    let prefix = path::dirname(p.reader.filemap.name);
    let leading_attrs = p.parse_inner_attrs_and_next();
    let { inner: crate_attrs, next: first_cdir_attr } = leading_attrs;
    let cdirs = p.parse_crate_directives(token::EOF, first_cdir_attr);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    let cx = @{sess: sess, cfg: /* FIXME: bad */ copy p.cfg};
    let (companionmod, _) = path::splitext(path::basename(input));
    let (m, attrs) = eval::eval_crate_directives_to_mod(
        cx, cdirs, prefix, option::some(companionmod));
    let mut hi = p.span.hi;
    p.expect(token::EOF);
    ret @ast_util::respan(ast_util::mk_sp(lo, hi),
                          {directives: cdirs,
                           module: m,
                           attrs: crate_attrs + attrs,
                           config: /* FIXME: bad */ copy p.cfg});
}

fn parse_crate_from_source_file(input: str, cfg: ast::crate_cfg,
                                sess: parse_sess) -> @ast::crate {
    let p = new_parser_from_file(sess, cfg, input, parser::SOURCE_FILE);
    let r = p.parse_crate_mod(cfg);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn parse_crate_from_source_str(name: str, source: @str, cfg: ast::crate_cfg,
                               sess: parse_sess) -> @ast::crate {
    let p = new_parser_from_source_str(
        sess, cfg, name, codemap::fss_none, source);
    let r = p.parse_crate_mod(cfg);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn parse_expr_from_source_str(name: str, source: @str, cfg: ast::crate_cfg,
                              sess: parse_sess) -> @ast::expr {
    let p = new_parser_from_source_str(
        sess, cfg, name, codemap::fss_none, source);
    let r = p.parse_expr();
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn parse_item_from_source_str(name: str, source: @str, cfg: ast::crate_cfg,
                              +attrs: [ast::attribute], vis: ast::visibility,
                              sess: parse_sess) -> option<@ast::item> {
    let p = new_parser_from_source_str(
        sess, cfg, name, codemap::fss_none, source);
    let r = p.parse_item(attrs, vis);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn parse_from_source_str<T>(f: fn (p: parser) -> T,
                            name: str, ss: codemap::file_substr,
                            source: @str, cfg: ast::crate_cfg,
                            sess: parse_sess)
    -> T
{
    let p = new_parser_from_source_str(sess, cfg, name, ss, source);
    let r = f(p);
    if !p.reader.is_eof() {
        p.reader.fatal("expected end-of-string");
    }
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn next_node_id(sess: parse_sess) -> node_id {
    let rv = sess.next_id;
    sess.next_id += 1;
    // ID 0 is reserved for the crate and doesn't actually exist in the AST
    assert rv != 0;
    ret rv;
}

fn new_parser_from_source_str(sess: parse_sess, cfg: ast::crate_cfg,
                              +name: str, +ss: codemap::file_substr,
                              source: @str) -> parser {
    let ftype = parser::SOURCE_FILE;
    let filemap = codemap::new_filemap_w_substr
        (name, ss, source, sess.chpos, sess.byte_pos);
    sess.cm.files.push(filemap);
    let itr = @interner::mk::<@str>({|x|str::hash(*x)}, {|x,y|str::eq(*x, *y)});
    let rdr = lexer::new_reader(sess.span_diagnostic,
                                filemap, itr);
    ret parser(sess, cfg, rdr, ftype);
}

fn new_parser_from_file(sess: parse_sess, cfg: ast::crate_cfg, +path: str,
                        ftype: parser::file_type) ->
   parser {
    let res = io::read_whole_file_str(path);
    alt res {
      result::ok(_) { /* Continue. */ }
      result::err(e) { sess.span_diagnostic.handler().fatal(e); }
    }
    let src = @result::unwrap(res);
    let filemap = codemap::new_filemap(path, src, sess.chpos, sess.byte_pos);
    sess.cm.files.push(filemap);
       let itr = @interner::mk::<@str>({|x|str::hash(*x)}, {|x,y|str::eq(*x, *y)});
    let rdr = lexer::new_reader(sess.span_diagnostic, filemap, itr);
    ret parser(sess, cfg, rdr, ftype);
}
<|endoftext|>


<commit_before>//! Provides a keyframe transformation which is a transform associated
//! with a specific point in time

use bspline;
use la;

use linalg::{self, quaternion, Vector, Matrix4, Quaternion, Transform};

/// A transformation associated with a specific point in time. Note
/// that this transform is now more implicit since they keyframe's times
/// are stored as knots in the b-spline animation path
#[derive(Debug, Copy, Clone)]
pub struct Keyframe {
    pub translation: Vector,
    pub rotation: Quaternion,
    pub scaling: Matrix4,
}

impl Keyframe {
    /// Construct a new keyframe transformation, The transform will
    /// be stored in a decomposed form, M = TRS.
    pub fn new(transform: &Transform) -> Keyframe {
        let (t, r, s) = Keyframe::decompose(transform);
        Keyframe { translation: t, rotation: r, scaling: s }
    }
    /// Construct the keyframe from the decomposed transformation
    pub fn from_parts(translation: &Vector, rotation: &Quaternion, scaling: &Matrix4) -> Keyframe {
        Keyframe { translation: *translation, rotation: *rotation, scaling: *scaling }
    }
    /// Decompose the transformation into its component translation, rotation and
    /// scaling operations.
    fn decompose(transform: &Transform) -> (Vector, Quaternion, Matrix4) {
        let m = transform.mat;
        let translation = Vector::new(*m.at(0, 3), *m.at(1, 3), *m.at(2, 3));
        // Robust matrix decomposition, based on Mitsuba:
        // We use SVD to extract rotation and scaling matrices that properly account for flip
        let la_mat = la::Matrix::<f64>::new(3, 3, vec![*m.at(0, 0) as f64, *m.at(0, 1) as f64, *m.at(0, 2) as f64,
                                                       *m.at(1, 0) as f64, *m.at(1, 1) as f64, *m.at(1, 2) as f64,
                                                       *m.at(2, 0) as f64, *m.at(2, 1) as f64, *m.at(2, 2) as f64]);
        // TODO: More explanation of the math going on here, why do we choose these matrices fo
        // q and p. q is the basis transform of the matrix without scaling, while p is the scaling
        // translated from the basis into R^3. Is this explanation correct?
        let svd = la::SVD::<f64>::new(&la_mat);
        let mut q = svd.get_u() * svd.get_v().t();
        let mut p = svd.get_v() * svd.get_s() * svd.get_v().t();
        if q.det() < 0.0 {
            q = -q;
            p = -p;
        }
        let rotation = Quaternion::from_matrix(
                            &Matrix4::new([q.get(0, 0) as f32, q.get(0, 1) as f32, q.get(0, 2) as f32, 0.0,
                                           q.get(1, 0) as f32, q.get(1, 1) as f32, q.get(1, 2) as f32, 0.0,
                                           q.get(2, 0) as f32, q.get(2, 1) as f32, q.get(2, 2) as f32, 0.0,
                                           0.0, 0.0, 0.0, 1.0]));
        let mut scaling = Matrix4::identity();
        *scaling.at_mut(0, 0) = p.get(0, 0) as f32;
        *scaling.at_mut(1, 1) = p.get(1, 1) as f32;
        *scaling.at_mut(2, 2) = p.get(2, 2) as f32;
        (translation, rotation, scaling)
    }
    /// Return the transformation stored for this keyframe
    pub fn transform(&self) -> Transform {
        let m = self.rotation.to_matrix() * self.scaling;
        Transform::translate(&self.translation) * Transform::from_mat(&m)
    }
}

impl bspline::Interpolate for Keyframe {
    fn interpolate(&self, other: &Keyframe, t: f32) -> Keyframe {
        let translation = (1.0 - t) * self.translation + t * other.translation;
        let rotation = quaternion::slerp(t, &self.rotation, &other.rotation);
        let scaling: Matrix4 = self.scaling.iter().zip(other.scaling.iter())
            .map(|(a, b)| linalg::lerp(t, a, b)).collect();
        Keyframe::from_parts(&translation, &rotation, &scaling)
    }
}

<commit_msg>We don't need to store the full scaling matrix now, just the x,y,z vals<commit_after>//! Provides a keyframe transformation which is a transform associated
//! with a specific point in time

use bspline;
use la;

use linalg::{quaternion, Vector, Matrix4, Quaternion, Transform};

/// A transformation associated with a specific point in time. Note
/// that this transform is now more implicit since they keyframe's times
/// are stored as knots in the b-spline animation path
#[derive(Debug, Copy, Clone)]
pub struct Keyframe {
    pub translation: Vector,
    pub rotation: Quaternion,
    pub scaling: Vector,
}

impl Keyframe {
    /// Construct a new keyframe transformation, The transform will
    /// be stored in a decomposed form, M = TRS.
    pub fn new(transform: &Transform) -> Keyframe {
        let (t, r, s) = Keyframe::decompose(transform);
        Keyframe { translation: t, rotation: r, scaling: s }
    }
    /// Construct the keyframe from the decomposed transformation
    pub fn from_parts(translation: &Vector, rotation: &Quaternion, scaling: &Vector) -> Keyframe {
        Keyframe { translation: *translation, rotation: *rotation, scaling: *scaling }
    }
    /// Decompose the transformation into its component translation, rotation and
    /// scaling operations.
    fn decompose(transform: &Transform) -> (Vector, Quaternion, Vector) {
        let m = transform.mat;
        let translation = Vector::new(*m.at(0, 3), *m.at(1, 3), *m.at(2, 3));
        // Robust matrix decomposition, based on Mitsuba:
        // We use SVD to extract rotation and scaling matrices that properly account for flip
        let la_mat = la::Matrix::<f64>::new(3, 3, vec![*m.at(0, 0) as f64, *m.at(0, 1) as f64, *m.at(0, 2) as f64,
                                                       *m.at(1, 0) as f64, *m.at(1, 1) as f64, *m.at(1, 2) as f64,
                                                       *m.at(2, 0) as f64, *m.at(2, 1) as f64, *m.at(2, 2) as f64]);
        // TODO: More explanation of the math going on here, why do we choose these matrices for
        // q and p. q is the basis transform of the matrix without scaling so it represents the
        // rotation, while p is the scaling transformed from the basis into the canonical basis
        // for R^3, giving scaling in the canonical basis. Is this intuition correct?
        let svd = la::SVD::<f64>::new(&la_mat);
        let mut q = svd.get_u() * svd.get_v().t();
        let mut p = svd.get_v() * svd.get_s() * svd.get_v().t();
        if q.det() < 0.0 {
            q = -q;
            p = -p;
        }
        let rotation = Quaternion::from_matrix(
                            &Matrix4::new([q.get(0, 0) as f32, q.get(0, 1) as f32, q.get(0, 2) as f32, 0.0,
                                           q.get(1, 0) as f32, q.get(1, 1) as f32, q.get(1, 2) as f32, 0.0,
                                           q.get(2, 0) as f32, q.get(2, 1) as f32, q.get(2, 2) as f32, 0.0,
                                           0.0, 0.0, 0.0, 1.0]));
        let scaling = Vector::new(p.get(0, 0) as f32, p.get(1, 1) as f32, p.get(2, 2) as f32);
        (translation, rotation, scaling)
    }
    /// Return the transformation stored for this keyframe
    pub fn transform(&self) -> Transform {
        let m = self.rotation.to_matrix();
        Transform::translate(&self.translation) * Transform::from_mat(&m) * Transform::scale(&self.scaling)
    }
}

impl bspline::Interpolate for Keyframe {
    fn interpolate(&self, other: &Keyframe, t: f32) -> Keyframe {
        let translation = (1.0 - t) * self.translation + t * other.translation;
        let rotation = quaternion::slerp(t, &self.rotation, &other.rotation);
        let scaling = (1.0 - t) * self.scaling + t * other.scaling;
        Keyframe::from_parts(&translation, &rotation, &scaling)
    }
}

<|endoftext|>


<commit_before>use num::Integer;
use euclid::Point2D;
use hex::Dir6;

/// User data for field of view cells.
pub trait FovValue: Eq + Clone {
    /// Construct a new FovValue for a position based on the previous one along the line of sight.
    fn advance(&self, offset: Point2D<i32>) -> Option<Self>;

    /// Return whether the given offset contains an isometric wall tile.
    ///
    /// Optional method for showing acute corners of fake isometric rooms in fov.
    fn is_fake_isometric_wall(&self, offset: Point2D<i32>) -> bool {
        let _ = offset;
        false
    }
}

/// Field of view iterator for a hexagonal map.
pub struct HexFov<T> {
    stack: Vec<Arc<T>>,
    /// Extra values generated by special cases.
    side_channel: Vec<(Point2D<i32>, T)>,
}

impl<T: FovValue> HexFov<T> {
    /// Create a new field of view iterator with a seed userdata for the origin position.
    pub fn new(init: T) -> HexFov<T> {
        // We could run f for (0, 0) here, but the traditional way for the FOV to work is to only
        // consider your surroundings, not the origin site itself.
        HexFov {
            stack: vec![Arc::new(PolarPoint::new(0.0, 1),
                                 PolarPoint::new(6.0, 1),
                                 init.clone())],
            // The FOV algorithm will not generate the origin point, so we use
            // the side channel to explicitly add it in the beginning.
            side_channel: vec![(Point2D::new(0, 0), init)],
        }
    }

    /// Add visible horizontal corners to fake-isometric rooms.
    fn make_corners_visible(&mut self, current: &Arc<T>) {
        // We're moving along a vertical line on the hex circle, so there are side
        // points to check.
        if let Some(side_pos) = current.pt.side_point() {
            let next = current.pt.next();
            // If the next cell is within the current span and the current cell is
            // wallform,
            if next.is_below(current.end) &&
               current.prev_value.is_fake_isometric_wall(current.pt.to_v2()) {
                // and if the next cell is visible,
                if let Some(next_value) = current.prev_value.advance(next.to_v2()) {
                    // and if the current and the next cell are in the same value group,
                    // both the next cell and the third corner point cell are
                    // wallforms, and the side point would not be otherwise
                    // visible:
                    if next_value == current.prev_value &&
                       next_value.is_fake_isometric_wall(next.to_v2()) &&
                       current.prev_value.advance(side_pos).is_none() &&
                       current.prev_value.is_fake_isometric_wall(side_pos) {
                        // Add the side point to the side channel.
                        self.side_channel.push((side_pos, current.prev_value.clone()));
                    }
                }
            }
        }
    }
}

impl<T: FovValue> Iterator for HexFov<T> {
    type Item = (Point2D<i32>, T);
    fn next(&mut self) -> Option<(Point2D<i32>, T)> {
        // Empty the side channel before proceeding to the algorithm proper.
        if let Some(ret) = self.side_channel.pop() {
            return Some(ret);
        }

        // Start processing the next arc in the stack.
        if let Some(current) = self.stack.pop() {
            if current.arc_has_split(&mut self.stack) {
                return self.next();
            }

            assert!(current.group_value == current.prev_value.advance(current.pt.to_v2()));

            self.make_corners_visible(&current);

            let pos = current.pt.to_v2();
            let ret = current.group_value.clone();

            current.advance(&mut self.stack);

            if let Some(ret) = ret {
                return Some((pos, ret));
            } else {
                return self.next();
            }
        } else {
            None
        }
    }
}

struct Arc<T> {
    /// Start point of current arc.
    begin: PolarPoint,
    /// Point currently being processed.
    pt: PolarPoint,
    /// End point of current arc.
    end: PolarPoint,
    /// The user value from previous iteration.
    prev_value: T,
    /// The user value for this group.
    group_value: Option<T>,
}

impl<T: FovValue> Arc<T> {
    pub fn new(begin: PolarPoint, end: PolarPoint, prev_value: T) -> Arc<T> {
        let group_value = prev_value.advance(begin.to_v2());
        Arc {
            begin: begin,
            pt: begin,
            end: end,
            prev_value: prev_value,
            group_value: group_value,
        }
    }

    /// Consume the given arc and add its descendent, if any, to the stack.
    pub fn advance(mut self, stack: &mut Vec<Arc<T>>) {
        self.pt = self.pt.next();
        if self.pt.is_below(self.end) {
            stack.push(self);
        } else {
            if let Some(group_value) = self.group_value {
                stack.push(Arc::new(self.begin.further(), self.end.further(), group_value));
            }
        }
    }

    /// If the arc has advanced into a different value group, split it into the given stack.
    ///
    /// Return true if arc was split, false otherwise.
    pub fn arc_has_split(&self, stack: &mut Vec<Arc<T>>) -> bool {
        let next_value = self.prev_value.advance(self.pt.to_v2());
        if next_value != self.group_value {
            // Using the literal instead of the constructor to avoid recomputing next_value.
            stack.push(Arc {
                begin: self.pt,
                pt: self.pt,
                end: self.end,
                prev_value: self.prev_value.clone(),
                group_value: next_value,
            });

            // Extend current arc if it has a group value.
            if let Some(ref group_value) = self.group_value {
                stack.push(Arc::new(self.begin.further(), self.pt.further(), group_value.clone()));
            }

            true
        } else {
            false
        }
    }
}

/// Points on a hex circle expressed in polar coordinates.
#[derive(Copy, Clone, PartialEq)]
struct PolarPoint {
    pos: f32,
    radius: u32,
}

impl PolarPoint {
    pub fn new(pos: f32, radius: u32) -> PolarPoint {
        PolarPoint {
            pos: pos,
            radius: radius,
        }
    }
    /// Index of the discrete hex cell along the circle that corresponds to this point.
    fn winding_index(self) -> i32 { (self.pos + 0.5).floor() as i32 }

    pub fn is_below(self, other: PolarPoint) -> bool { self.winding_index() < other.end_index() }

    fn end_index(self) -> i32 { (self.pos + 0.5).ceil() as i32 }

    pub fn to_v2(self) -> Point2D<i32> {
        if self.radius == 0 {
            return Point2D::new(0, 0);
        }
        let index = self.winding_index();
        let sector = index.mod_floor(&(self.radius as i32 * 6)) / self.radius as i32;
        let offset = index.mod_floor(&(self.radius as i32));

        let rod = Dir6::from_int(sector).to_v2();
        let tangent = Dir6::from_int((sector + 2) % 6).to_v2();

        rod * (self.radius as i32) + tangent * offset
    }

    /// If this point and the next point are adjacent vertically (along the xy
    /// axis), return the point outside of the circle between the two points.
    ///
    /// This is a helper function for the FOV special case where acute corners
    /// of fake isometric rooms are marked visible even though strict hex FOV
    /// logic would keep them unseen.
    pub fn side_point(self) -> Option<Point2D<i32>> {
        let next = self.next();
        let a = self.to_v2();
        let b = next.to_v2();

        if b.x == a.x + 1 && b.y == a.y + 1 {
            // Going down the right rim.
            Some(Point2D::new(a.x + 1, a.y))
        } else if b.x == a.x - 1 && b.y == a.y - 1 {
            // Going up the left rim.
            Some(Point2D::new(a.x - 1, a.y))
        } else {
            None
        }
    }

    /// The point corresponding to this one on the hex circle with radius +1.
    pub fn further(self) -> PolarPoint {
        PolarPoint::new(self.pos * (self.radius + 1) as f32 / self.radius as f32,
                        self.radius + 1)
    }

    /// The point next to this one along the hex circle.
    pub fn next(self) -> PolarPoint { PolarPoint::new((self.pos + 0.5).floor() + 0.5, self.radius) }
}

#[cfg(test)]
mod test {
    use std::collections::HashMap;
    use std::iter::FromIterator;
    use euclid::Point2D;
    use super::{FovValue, HexFov};
    use hex::HexGeom;

    #[derive(PartialEq, Eq, Clone)]
    struct Cell1 {
        range: i32,
    }

    impl FovValue for Cell1 {
        fn advance(&self, offset: Point2D<i32>) -> Option<Self> {
            if offset.hex_dist() < self.range { Some(self.clone()) } else { None }
        }
    }

    #[derive(PartialEq, Eq, Clone)]
    struct Cell2 {
        range: i32,
    }

    impl FovValue for Cell2 {
        fn advance(&self, offset: Point2D<i32>) -> Option<Self> {
            if offset.hex_dist() < self.range { Some(self.clone()) } else { None }
        }

        fn is_fake_isometric_wall(&self, offset: Point2D<i32>) -> bool {
            let _ = offset;
            true
        }
    }

    #[test]
    fn trivial_fov() {
        // Just draw a small circle.
        let field: HashMap<Point2D<i32>, Cell1> = HashMap::from_iter(HexFov::new(Cell1 {
            range: 2,
        }));
        assert!(field.contains_key(&Point2D::new(1, 0)));
        assert!(!field.contains_key(&Point2D::new(1, -1)));

        // Now test out the fake-isometric corners.
        let field: HashMap<Point2D<i32>, Cell2> = HashMap::from_iter(HexFov::new(Cell2 {
            range: 2,
        }));
        assert!(field.contains_key(&Point2D::new(1, 0)));
        assert!(field.contains_key(&Point2D::new(1, -1)));
    }
}
<commit_msg>Allow floats in FovValue<commit_after>use num::Integer;
use euclid::Point2D;
use hex::Dir6;

/// User data for field of view cells.
pub trait FovValue: PartialEq + Clone {
    /// Construct a new FovValue for a position based on the previous one along the line of sight.
    fn advance(&self, offset: Point2D<i32>) -> Option<Self>;

    /// Return whether the given offset contains an isometric wall tile.
    ///
    /// Optional method for showing acute corners of fake isometric rooms in fov.
    fn is_fake_isometric_wall(&self, offset: Point2D<i32>) -> bool {
        let _ = offset;
        false
    }
}

/// Field of view iterator for a hexagonal map.
pub struct HexFov<T> {
    stack: Vec<Arc<T>>,
    /// Extra values generated by special cases.
    side_channel: Vec<(Point2D<i32>, T)>,
}

impl<T: FovValue> HexFov<T> {
    /// Create a new field of view iterator with a seed userdata for the origin position.
    pub fn new(init: T) -> HexFov<T> {
        // We could run f for (0, 0) here, but the traditional way for the FOV to work is to only
        // consider your surroundings, not the origin site itself.
        HexFov {
            stack: vec![Arc::new(PolarPoint::new(0.0, 1),
                                 PolarPoint::new(6.0, 1),
                                 init.clone())],
            // The FOV algorithm will not generate the origin point, so we use
            // the side channel to explicitly add it in the beginning.
            side_channel: vec![(Point2D::new(0, 0), init)],
        }
    }

    /// Add visible horizontal corners to fake-isometric rooms.
    fn make_corners_visible(&mut self, current: &Arc<T>) {
        // We're moving along a vertical line on the hex circle, so there are side
        // points to check.
        if let Some(side_pos) = current.pt.side_point() {
            let next = current.pt.next();
            // If the next cell is within the current span and the current cell is
            // wallform,
            if next.is_below(current.end) &&
               current.prev_value.is_fake_isometric_wall(current.pt.to_v2()) {
                // and if the next cell is visible,
                if let Some(next_value) = current.prev_value.advance(next.to_v2()) {
                    // and if the current and the next cell are in the same value group,
                    // both the next cell and the third corner point cell are
                    // wallforms, and the side point would not be otherwise
                    // visible:
                    if next_value == current.prev_value &&
                       next_value.is_fake_isometric_wall(next.to_v2()) &&
                       current.prev_value.advance(side_pos).is_none() &&
                       current.prev_value.is_fake_isometric_wall(side_pos) {
                        // Add the side point to the side channel.
                        self.side_channel.push((side_pos, current.prev_value.clone()));
                    }
                }
            }
        }
    }
}

impl<T: FovValue> Iterator for HexFov<T> {
    type Item = (Point2D<i32>, T);
    fn next(&mut self) -> Option<(Point2D<i32>, T)> {
        // Empty the side channel before proceeding to the algorithm proper.
        if let Some(ret) = self.side_channel.pop() {
            return Some(ret);
        }

        // Start processing the next arc in the stack.
        if let Some(current) = self.stack.pop() {
            if current.arc_has_split(&mut self.stack) {
                return self.next();
            }

            assert!(current.group_value == current.prev_value.advance(current.pt.to_v2()));

            self.make_corners_visible(&current);

            let pos = current.pt.to_v2();
            let ret = current.group_value.clone();

            current.advance(&mut self.stack);

            if let Some(ret) = ret {
                return Some((pos, ret));
            } else {
                return self.next();
            }
        } else {
            None
        }
    }
}

struct Arc<T> {
    /// Start point of current arc.
    begin: PolarPoint,
    /// Point currently being processed.
    pt: PolarPoint,
    /// End point of current arc.
    end: PolarPoint,
    /// The user value from previous iteration.
    prev_value: T,
    /// The user value for this group.
    group_value: Option<T>,
}

impl<T: FovValue> Arc<T> {
    pub fn new(begin: PolarPoint, end: PolarPoint, prev_value: T) -> Arc<T> {
        let group_value = prev_value.advance(begin.to_v2());
        Arc {
            begin: begin,
            pt: begin,
            end: end,
            prev_value: prev_value,
            group_value: group_value,
        }
    }

    /// Consume the given arc and add its descendent, if any, to the stack.
    pub fn advance(mut self, stack: &mut Vec<Arc<T>>) {
        self.pt = self.pt.next();
        if self.pt.is_below(self.end) {
            stack.push(self);
        } else {
            if let Some(group_value) = self.group_value {
                stack.push(Arc::new(self.begin.further(), self.end.further(), group_value));
            }
        }
    }

    /// If the arc has advanced into a different value group, split it into the given stack.
    ///
    /// Return true if arc was split, false otherwise.
    pub fn arc_has_split(&self, stack: &mut Vec<Arc<T>>) -> bool {
        let next_value = self.prev_value.advance(self.pt.to_v2());
        if next_value != self.group_value {
            // Using the literal instead of the constructor to avoid recomputing next_value.
            stack.push(Arc {
                begin: self.pt,
                pt: self.pt,
                end: self.end,
                prev_value: self.prev_value.clone(),
                group_value: next_value,
            });

            // Extend current arc if it has a group value.
            if let Some(ref group_value) = self.group_value {
                stack.push(Arc::new(self.begin.further(), self.pt.further(), group_value.clone()));
            }

            true
        } else {
            false
        }
    }
}

/// Points on a hex circle expressed in polar coordinates.
#[derive(Copy, Clone, PartialEq)]
struct PolarPoint {
    pos: f32,
    radius: u32,
}

impl PolarPoint {
    pub fn new(pos: f32, radius: u32) -> PolarPoint {
        PolarPoint {
            pos: pos,
            radius: radius,
        }
    }
    /// Index of the discrete hex cell along the circle that corresponds to this point.
    fn winding_index(self) -> i32 { (self.pos + 0.5).floor() as i32 }

    pub fn is_below(self, other: PolarPoint) -> bool { self.winding_index() < other.end_index() }

    fn end_index(self) -> i32 { (self.pos + 0.5).ceil() as i32 }

    pub fn to_v2(self) -> Point2D<i32> {
        if self.radius == 0 {
            return Point2D::new(0, 0);
        }
        let index = self.winding_index();
        let sector = index.mod_floor(&(self.radius as i32 * 6)) / self.radius as i32;
        let offset = index.mod_floor(&(self.radius as i32));

        let rod = Dir6::from_int(sector).to_v2();
        let tangent = Dir6::from_int((sector + 2) % 6).to_v2();

        rod * (self.radius as i32) + tangent * offset
    }

    /// If this point and the next point are adjacent vertically (along the xy
    /// axis), return the point outside of the circle between the two points.
    ///
    /// This is a helper function for the FOV special case where acute corners
    /// of fake isometric rooms are marked visible even though strict hex FOV
    /// logic would keep them unseen.
    pub fn side_point(self) -> Option<Point2D<i32>> {
        let next = self.next();
        let a = self.to_v2();
        let b = next.to_v2();

        if b.x == a.x + 1 && b.y == a.y + 1 {
            // Going down the right rim.
            Some(Point2D::new(a.x + 1, a.y))
        } else if b.x == a.x - 1 && b.y == a.y - 1 {
            // Going up the left rim.
            Some(Point2D::new(a.x - 1, a.y))
        } else {
            None
        }
    }

    /// The point corresponding to this one on the hex circle with radius +1.
    pub fn further(self) -> PolarPoint {
        PolarPoint::new(self.pos * (self.radius + 1) as f32 / self.radius as f32,
                        self.radius + 1)
    }

    /// The point next to this one along the hex circle.
    pub fn next(self) -> PolarPoint { PolarPoint::new((self.pos + 0.5).floor() + 0.5, self.radius) }
}

#[cfg(test)]
mod test {
    use std::collections::HashMap;
    use std::iter::FromIterator;
    use euclid::Point2D;
    use super::{FovValue, HexFov};
    use hex::HexGeom;

    #[derive(PartialEq, Eq, Clone)]
    struct Cell1 {
        range: i32,
    }

    impl FovValue for Cell1 {
        fn advance(&self, offset: Point2D<i32>) -> Option<Self> {
            if offset.hex_dist() < self.range { Some(self.clone()) } else { None }
        }
    }

    #[derive(PartialEq, Eq, Clone)]
    struct Cell2 {
        range: i32,
    }

    impl FovValue for Cell2 {
        fn advance(&self, offset: Point2D<i32>) -> Option<Self> {
            if offset.hex_dist() < self.range { Some(self.clone()) } else { None }
        }

        fn is_fake_isometric_wall(&self, offset: Point2D<i32>) -> bool {
            let _ = offset;
            true
        }
    }

    #[test]
    fn trivial_fov() {
        // Just draw a small circle.
        let field: HashMap<Point2D<i32>, Cell1> = HashMap::from_iter(HexFov::new(Cell1 {
            range: 2,
        }));
        assert!(field.contains_key(&Point2D::new(1, 0)));
        assert!(!field.contains_key(&Point2D::new(1, -1)));

        // Now test out the fake-isometric corners.
        let field: HashMap<Point2D<i32>, Cell2> = HashMap::from_iter(HexFov::new(Cell2 {
            range: 2,
        }));
        assert!(field.contains_key(&Point2D::new(1, 0)));
        assert!(field.contains_key(&Point2D::new(1, -1)));
    }
}
<|endoftext|>


<commit_before>fn get_words(sec: &str, w: usize) -> Vec<String> {
    let lim = sec.len() - w + 1;
    let mut words = Vec::new();

    for i in 0..lim {
        let mut word = String::new();
        word.push(sec.as_bytes()[i] as char);
        word.push(sec.as_bytes()[i + 1] as char);
        word.push(sec.as_bytes()[i + 2] as char);

        words.push(word);
    }

    words
}

fn get_hsp(w1: &String, w2: &String, match_scr: i32, mism_scr: i32) -> i32 {
    let mut hsp = 0;

    let mut chars2 = w2.chars();
    for c in w1.chars() {
        hsp += if c == chars2.next().unwrap() {
            match_scr
        } else {
            mism_scr
        }
    }

    hsp
}

fn expand_words(align1: &mut String,
                align2: &mut String,
                sec_1: &str,
                sec_2: &str,
                indx1_i: &mut usize,
                indx1_f: &mut usize,
                indx2_i: &mut usize,
                indx2_f: &mut usize)
                -> (bool, bool) {
    if *indx1_i != 0 && *indx2_i != 0 {
        let mut ps_align1 = align1.chars().rev().collect::<String>();
        let mut ps_align2 = align2.chars().rev().collect::<String>();
        ps_align1.push(sec_1.as_bytes()[*indx1_i - 1] as char);
        ps_align2.push(sec_2.as_bytes()[*indx2_i - 1] as char);
        *align1 = ps_align1.chars().rev().collect::<String>();
        *align2 = ps_align2.chars().rev().collect::<String>();
        *indx1_i -= 1;
        *indx2_i -= 1;
    }

    if *indx1_f < sec_1.len() && *indx2_f < sec_2.len() {
        align1.push(sec_1.as_bytes()[*indx1_f] as char);
        align2.push(sec_2.as_bytes()[*indx2_f] as char);
        *indx1_f += 1;
        *indx2_f += 1;
    }

    (!(*indx1_i != 0 && *indx2_i != 0), !(*indx1_f < sec_1.len() && *indx2_f < sec_2.len()))
}

fn print_alignment(align1: String, align2: String) {
    let mut chars2 = align2.chars();
    let mut end_first_it = false;
    let mut algn1 = String::new();
    let mut algn2 = String::new();

    for c1 in align1.chars() {
        let c2 = chars2.next().unwrap();
        if c1 == c2 {
            end_first_it = true;
        }
        if end_first_it {
            algn1.push(c1);
            algn2.push(c2);
        }
    }

    println!(">{}\n>{}", algn1, algn2);
}

fn align_secuence(sec_1: &str,
                  sec_2: &str,
                  w: usize,
                  match_scr: i32,
                  mism_scr: i32,
                  hps: i32,
                  u: i32) {
    let words_s1 = get_words(sec_1, w);
    let words_s2 = get_words(sec_2, w);

    let mut indx1_i: usize = 0;
    let mut indx2_i: usize = 0;
    let mut align1 = String::new();
    let mut align2 = String::new();

    'outer: for word1 in words_s1.iter() {
        indx2_i = 0;
        for word2 in words_s2.iter() {
            if get_hsp(word1, word2, match_scr, mism_scr) >= hps {
                align1 = word1.clone();
                align2 = word2.clone();
                break 'outer;
            }
            indx2_i += 1;
        }
        indx1_i += 1;
    }

    let mut indx1_f = indx1_i + w;
    let mut indx2_f = indx2_i + w;

    let mut algn1 = align1.clone();
    let mut algn2 = align2.clone();

    while {
        let hsp = get_hsp(&align1, &align2, match_scr, mism_scr);
        // println!("{}, {}\n{} -> {}\n", align1, align2, hsp, hsp > u);
        hsp > u
    } {
        algn1 = align1.clone();
        algn2 = align2.clone();

        let (a, b) = expand_words(&mut align1, &mut align2, &sec_1, &sec_2, &mut indx1_i, &mut indx1_f, &mut indx2_i, &mut indx2_f);

        if a && b {
            algn1 = align1.clone();
            algn2 = align2.clone();
            break;
        }
    }

    print_alignment(algn1, algn2);
}

fn main() {
    align_secuence("TCAGATCACGG", "GTATCGCTC", 3, 1, -2, 3, -3);
}
<commit_msg>Prettified output<commit_after>fn get_words(sec: &str, w: usize) -> Vec<String> {
    let lim = sec.len() - w + 1;
    let mut words = Vec::new();

    for i in 0..lim {
        let mut word = String::new();
        word.push(sec.as_bytes()[i] as char);
        word.push(sec.as_bytes()[i + 1] as char);
        word.push(sec.as_bytes()[i + 2] as char);

        words.push(word);
    }

    words
}

fn get_hsp(w1: &String, w2: &String, match_scr: i32, mism_scr: i32) -> i32 {
    let mut hsp = 0;

    let mut chars2 = w2.chars();
    for c in w1.chars() {
        hsp += if c == chars2.next().unwrap() {
            match_scr
        } else {
            mism_scr
        }
    }

    hsp
}

fn expand_words(align1: &mut String,
                align2: &mut String,
                sec_1: &str,
                sec_2: &str,
                indx1_i: &mut usize,
                indx1_f: &mut usize,
                indx2_i: &mut usize,
                indx2_f: &mut usize)
                -> (bool, bool) {
    if *indx1_i != 0 && *indx2_i != 0 {
        let mut ps_align1 = align1.chars().rev().collect::<String>();
        let mut ps_align2 = align2.chars().rev().collect::<String>();
        ps_align1.push(sec_1.as_bytes()[*indx1_i - 1] as char);
        ps_align2.push(sec_2.as_bytes()[*indx2_i - 1] as char);
        *align1 = ps_align1.chars().rev().collect::<String>();
        *align2 = ps_align2.chars().rev().collect::<String>();
        *indx1_i -= 1;
        *indx2_i -= 1;
    }

    if *indx1_f < sec_1.len() && *indx2_f < sec_2.len() {
        align1.push(sec_1.as_bytes()[*indx1_f] as char);
        align2.push(sec_2.as_bytes()[*indx2_f] as char);
        *indx1_f += 1;
        *indx2_f += 1;
    }

    (!(*indx1_i != 0 && *indx2_i != 0), !(*indx1_f < sec_1.len() && *indx2_f < sec_2.len()))
}

fn print_alignment(align1: String, align2: String) {
    let mut chars2 = align2.chars();
    let mut end_first_it = false;
    let mut algn1 = String::new();
    let mut algn2 = String::new();

    for c1 in align1.chars() {
        let c2 = chars2.next().unwrap();
        if c1 == c2 {
            end_first_it = true;
        }
        if end_first_it {
            algn1.push(c1);
            algn2.push(c2);
        }
    }

    println!("S1> {}\nS2> {}", algn1, algn2);
}

fn align_secuence(sec_1: &str,
                  sec_2: &str,
                  w: usize,
                  match_scr: i32,
                  mism_scr: i32,
                  hps: i32,
                  u: i32) {
    let words_s1 = get_words(sec_1, w);
    let words_s2 = get_words(sec_2, w);

    let mut indx1_i: usize = 0;
    let mut indx2_i: usize = 0;
    let mut align1 = String::new();
    let mut align2 = String::new();

    'outer: for word1 in words_s1.iter() {
        indx2_i = 0;
        for word2 in words_s2.iter() {
            if get_hsp(word1, word2, match_scr, mism_scr) >= hps {
                align1 = word1.clone();
                align2 = word2.clone();
                break 'outer;
            }
            indx2_i += 1;
        }
        indx1_i += 1;
    }

    let mut indx1_f = indx1_i + w;
    let mut indx2_f = indx2_i + w;

    let mut algn1 = align1.clone();
    let mut algn2 = align2.clone();

    while {
        let hsp = get_hsp(&align1, &align2, match_scr, mism_scr);
        // println!("{}, {}\n{} -> {}\n", align1, align2, hsp, hsp > u);
        hsp > u
    } {
        algn1 = align1.clone();
        algn2 = align2.clone();

        let (a, b) = expand_words(&mut align1, &mut align2, &sec_1, &sec_2, &mut indx1_i, &mut indx1_f, &mut indx2_i, &mut indx2_f);

        if a && b {
            algn1 = align1.clone();
            algn2 = align2.clone();
            break;
        }
    }

    print_alignment(algn1, algn2);
}

fn main() {
    align_secuence("TCAGATCACGG", "GTATCGCTC", 3, 1, -2, 3, -3);
}
<|endoftext|>


<commit_before>use amq_protocol::frame::{GenError, Offset, gen_frame, parse_frame};
use log::{error, trace};
use mio::{Evented, Events, Poll, PollOpt, Ready, Registration, SetReadiness, Token};
use parking_lot::Mutex;

use std::{
  io::{self, Read, Write},
  sync::{
    Arc,
    atomic::{AtomicBool, Ordering},
  },
  thread::{self, Builder as ThreadBuilder, JoinHandle},
  time::{Duration, Instant},
};

use crate::{
  buffer::Buffer,
  connection::Connection,
  connection_status::ConnectionState,
  error::{Error, ErrorKind},
};

const SOCKET:   Token = Token(1);
const DATA:     Token = Token(2);
const CONTINUE: Token = Token(3);

const FRAMES_STORAGE: usize = 32;

pub(crate) struct IoLoop<T> {
  inner: Arc<Mutex<Inner<T>>>,
}

impl<T: Evented + Read + Write + Send + 'static> IoLoop<T> {
  pub(crate) fn new(connection: Connection, socket: T) -> Result<Self, Error> {
    let inner = Arc::new(Mutex::new(Inner::new(connection, socket)?));
    Ok(Self { inner })
  }

  pub(crate) fn run(&self) -> Result<(), Error> {
    let inner  = self.inner.clone();
    self.inner.lock().connection.set_io_loop(ThreadBuilder::new().name("io_loop".to_owned()).spawn(move || {
      let mut events = Events::with_capacity(1024);
      loop {
        inner.lock().run(&mut events)?;
      }
    }).map_err(ErrorKind::IOError)?);
    Ok(())
  }
}

#[derive(Clone, Debug)]
pub(crate) struct IoLoopHandle {
  handle: Arc<Mutex<Option<JoinHandle<Result<(), Error>>>>>,
}

impl Default for IoLoopHandle {
  fn default() -> Self {
    Self { handle: Arc::new(Mutex::new(None)) }
  }
}

impl IoLoopHandle {
  pub(crate) fn register(&self, handle: JoinHandle<Result<(), Error>>) {
    *self.handle.lock() = Some(handle);
  }

  pub(crate) fn wait(&self) -> Result<(), Error> {
    if let Some(handle) = self.handle.lock().take() {
      handle.join().expect("io loop")?
    }
    Ok(())
  }
}

#[derive(PartialEq)]
enum Status {
  Initial,
  Setup,
}

struct Inner<T> {
  connection:     Connection,
  socket:         T,
  status:         Status,
  poll:           Poll,
  registration:   Registration,
  set_readiness:  SetReadiness,
  hb_handle:      Option<JoinHandle<()>>,
  frame_size:     usize,
  receive_buffer: Buffer,
  send_buffer:    Buffer,
  can_write:      bool,
  can_read:       bool,
  has_data:       bool,
  send_heartbeat: Arc<AtomicBool>,
}

impl<T> Drop for Inner<T> {
  fn drop(&mut self) {
    self.connection.run().expect("io loop failed");
    if let Some(hb_handle) = self.hb_handle.take() {
      hb_handle.join().expect("heartbeat loop failed");
    }
  }
}

impl<T: Evented + Read + Write + Send + 'static> Inner<T> {
  fn new(connection: Connection, socket: T) -> Result<Self, Error> {
    let frame_size = std::cmp::max(8192, connection.configuration().frame_max() as usize);
    let (registration, set_readiness) = Registration::new2();
    let inner = Self {
      connection,
      socket,
      status:         Status::Initial,
      poll:           Poll::new().map_err(ErrorKind::IOError)?,
      registration,
      set_readiness,
      hb_handle:      None,
      frame_size,
      receive_buffer: Buffer::with_capacity(FRAMES_STORAGE * frame_size),
      send_buffer:    Buffer::with_capacity(FRAMES_STORAGE * frame_size),
      can_write:      false,
      can_read:       false,
      has_data:       false,
      send_heartbeat: Arc::new(AtomicBool::new(false)),
    };
    inner.poll.register(&inner.socket, SOCKET, Ready::readable() | Ready::writable(), PollOpt::edge()).map_err(ErrorKind::IOError)?;
    inner.poll.register(&inner.connection, DATA, Ready::readable(), PollOpt::edge()).map_err(ErrorKind::IOError)?;
    inner.poll.register(&inner.registration, CONTINUE, Ready::readable(), PollOpt::edge()).map_err(ErrorKind::IOError)?;
    Ok(inner)
  }

  fn start_heartbeat(&mut self, interval: Duration) -> Result<(), Error> {
    let send_hartbeat = self.send_heartbeat.clone();
    let hb_handle = ThreadBuilder::new().name("heartbeat".to_owned()).spawn(move || {
      loop {
        let start         = Instant::now();
        let mut remaining = interval;

        loop {
          thread::park_timeout(remaining);
          let elapsed = start.elapsed();
          if elapsed >= remaining {
            break
          }
          remaining -= interval - elapsed;
        }

        send_hartbeat.store(true, Ordering::Relaxed);
      }
    }).map_err(ErrorKind::IOError)?;
    self.hb_handle = Some(hb_handle);
    Ok(())
  }

  fn heartbeat(&mut self) -> Result<(), Error> {
    self.connection.send_heartbeat()?;
    self.send_heartbeat.store(false, Ordering::Relaxed);
    Ok(())
  }

  fn ensure_setup(&mut self) -> Result<(), Error> {
    if self.status != Status::Setup && self.connection.status().connected() {
      let frame_max = self.connection.configuration().frame_max() as usize;
      self.frame_size = std::cmp::max(self.frame_size, frame_max);
      self.receive_buffer.grow(FRAMES_STORAGE * self.frame_size);
      self.send_buffer.grow(FRAMES_STORAGE * self.frame_size);
      let heartbeat = self.connection.configuration().heartbeat();
      if heartbeat != 0 {
        trace!("io_loop: start heartbeat");
        self.start_heartbeat(Duration::from_secs(heartbeat as u64))?;
        trace!("io_loop: heartbeat started");
      }
      self.status = Status::Setup;
    }
    Ok(())
  }

  fn wants_to_write(&self) -> bool {
    self.can_write && self.has_data
  }

  fn wants_to_read(&self) -> bool {
    self.can_read
  }

  fn do_run(&mut self) -> Result<(), Error> {
    trace!("io_loop do_run; can_read={}, can_write={}, has_data={}", self.can_read, self.can_write, self.has_data);
    loop {
      if self.send_heartbeat.load(Ordering::Relaxed) {
        self.heartbeat()?;
      }
      if self.wants_to_write() && !self.connection.status().blocked() {
        if let Err(e) = self.write_to_stream() {
          match e.kind() {
            ErrorKind::IOError(e) if e.kind() == io::ErrorKind::WouldBlock => self.can_write = false,
            _ => {
              error!("error writing: {:?}", e);
              self.connection.set_error()?;
              return Err(e);
            }
          }
        }
        self.send_buffer.shift_unless_available(self.frame_size);
      }
      if self.wants_to_read() {
        if let Err(e) = self.read_from_stream() {
          match e.kind() {
            ErrorKind::IOError(e) if e.kind() == io::ErrorKind::WouldBlock => self.can_read = false,
            _ => {
              error!("error reading: {:?}", e);
              self.connection.set_error()?;
              return Err(e);
            }
          }
        }
        self.receive_buffer.shift_unless_available(self.frame_size);
      }
      if self.can_parse() {
        self.parse()?;
      }
      if !self.wants_to_read() || !self.wants_to_write() {
        if self.wants_to_read() || self.can_parse() || self.has_data {
          trace!("io_loop send continue; can_read={}, can_write={}, has_data={}", self.can_read, self.can_write, self.has_data);
          self.set_readiness.set_readiness(Ready::readable()).map_err(ErrorKind::IOError)?;
        }
        break;
      }
    }
    trace!("io_loop do_run done; can_read={}, can_write={}, has_data={}", self.can_read, self.can_write, self.has_data);
    Ok(())
  }

  fn run(&mut self, events: &mut Events) -> Result<(), Error> {
    trace!("io_loop run");
    self.ensure_setup()?;
    trace!("io_loop poll");
    self.poll.poll(events, None).map_err(ErrorKind::IOError)?;
    trace!("io_loop poll done");
    for event in events.iter() {
      match event.token() {
        SOCKET    => {
          if event.readiness().is_readable() {
            self.can_read = true;
          }
          if event.readiness().is_writable() {
            self.can_write = true;
          }
        },
        DATA      => self.has_data = true,
        _         => {},
      }
    }
    self.do_run()
  }

  fn can_parse(&self) -> bool {
    self.receive_buffer.available_data() > 0
  }

  fn write_to_stream(&mut self) -> Result<(), Error> {
    self.serialize()?;

    self.socket.write(&self.send_buffer.data()).map(|sz| {
      trace!("wrote {} bytes", sz);
      self.send_buffer.consume(sz);
    }).map_err(|e| ErrorKind::IOError(e).into())
  }

  fn read_from_stream(&mut self) -> Result<(), Error> {
    let state = self.connection.status().state();
    if state == ConnectionState::Initial || state == ConnectionState::Closed || state == ConnectionState::Error {
      self.connection.set_error()?;
      Err(ErrorKind::InvalidConnectionState(state).into())
    } else {
      self.socket.read(&mut self.receive_buffer.space()).map(|sz| {
        trace!("read {} bytes", sz);
        self.receive_buffer.fill(sz);
      }).map_err(|e| ErrorKind::IOError(e).into())
    }
  }

  fn serialize(&mut self) -> Result<(), Error> {
    if let Some((send_id, next_msg)) = self.connection.next_frame() {
      trace!("will write to buffer: {:?}", next_msg);
      match gen_frame(self.send_buffer.space(), &next_msg).map(|tup| tup.0) {
        Ok(sz) => {
          self.send_buffer.fill(sz);
          self.connection.mark_sent(send_id);
          Ok(())
        },
        Err(e) => {
          match e {
            GenError::BufferTooSmall(_) => {
              // Requeue msg
              self.connection.requeue_frame(send_id, next_msg)?;
              self.send_buffer.shift();
              Ok(())
            },
            GenError::InvalidOffset | GenError::CustomError(_) | GenError::NotYetImplemented => {
              error!("error generating frame: {:?}", e);
              self.connection.set_error()?;
              Err(ErrorKind::SerialisationError(e).into())
            }
          }
        }
      }
    } else {
      self.has_data = false;
      Ok(())
    }
  }

  fn parse(&mut self) -> Result<(), Error> {
    match parse_frame(self.receive_buffer.data()) {
      Ok((i, f)) => {

        let consumed = self.receive_buffer.data().offset(i);
        self.receive_buffer.consume(consumed);

        if let Err(e) = self.connection.handle_frame(f) {
          self.connection.set_error()?;
          Err(e)
        } else {
          Ok(())
        }
      },
      Err(e) => {
        if e.is_incomplete() {
          self.receive_buffer.shift();
          Ok(())
        } else {
          error!("parse error: {:?}", e);
          self.connection.set_error()?;
          Err(ErrorKind::ParsingError(format!("{:?}", e)).into())
        }
      }
    }
  }
}
<commit_msg>async: io_loop: merge do_run in run<commit_after>use amq_protocol::frame::{GenError, Offset, gen_frame, parse_frame};
use log::{error, trace};
use mio::{Evented, Events, Poll, PollOpt, Ready, Registration, SetReadiness, Token};
use parking_lot::Mutex;

use std::{
  io::{self, Read, Write},
  sync::{
    Arc,
    atomic::{AtomicBool, Ordering},
  },
  thread::{self, Builder as ThreadBuilder, JoinHandle},
  time::{Duration, Instant},
};

use crate::{
  buffer::Buffer,
  connection::Connection,
  connection_status::ConnectionState,
  error::{Error, ErrorKind},
};

const SOCKET:   Token = Token(1);
const DATA:     Token = Token(2);
const CONTINUE: Token = Token(3);

const FRAMES_STORAGE: usize = 32;

pub(crate) struct IoLoop<T> {
  inner: Arc<Mutex<Inner<T>>>,
}

impl<T: Evented + Read + Write + Send + 'static> IoLoop<T> {
  pub(crate) fn new(connection: Connection, socket: T) -> Result<Self, Error> {
    let inner = Arc::new(Mutex::new(Inner::new(connection, socket)?));
    Ok(Self { inner })
  }

  pub(crate) fn run(&self) -> Result<(), Error> {
    let inner  = self.inner.clone();
    self.inner.lock().connection.set_io_loop(ThreadBuilder::new().name("io_loop".to_owned()).spawn(move || {
      let mut events = Events::with_capacity(1024);
      loop {
        inner.lock().run(&mut events)?;
      }
    }).map_err(ErrorKind::IOError)?);
    Ok(())
  }
}

#[derive(Clone, Debug)]
pub(crate) struct IoLoopHandle {
  handle: Arc<Mutex<Option<JoinHandle<Result<(), Error>>>>>,
}

impl Default for IoLoopHandle {
  fn default() -> Self {
    Self { handle: Arc::new(Mutex::new(None)) }
  }
}

impl IoLoopHandle {
  pub(crate) fn register(&self, handle: JoinHandle<Result<(), Error>>) {
    *self.handle.lock() = Some(handle);
  }

  pub(crate) fn wait(&self) -> Result<(), Error> {
    if let Some(handle) = self.handle.lock().take() {
      handle.join().expect("io loop")?
    }
    Ok(())
  }
}

#[derive(PartialEq)]
enum Status {
  Initial,
  Setup,
}

struct Inner<T> {
  connection:     Connection,
  socket:         T,
  status:         Status,
  poll:           Poll,
  registration:   Registration,
  set_readiness:  SetReadiness,
  hb_handle:      Option<JoinHandle<()>>,
  frame_size:     usize,
  receive_buffer: Buffer,
  send_buffer:    Buffer,
  can_write:      bool,
  can_read:       bool,
  has_data:       bool,
  send_heartbeat: Arc<AtomicBool>,
}

impl<T> Drop for Inner<T> {
  fn drop(&mut self) {
    self.connection.run().expect("io loop failed");
    if let Some(hb_handle) = self.hb_handle.take() {
      hb_handle.join().expect("heartbeat loop failed");
    }
  }
}

impl<T: Evented + Read + Write + Send + 'static> Inner<T> {
  fn new(connection: Connection, socket: T) -> Result<Self, Error> {
    let frame_size = std::cmp::max(8192, connection.configuration().frame_max() as usize);
    let (registration, set_readiness) = Registration::new2();
    let inner = Self {
      connection,
      socket,
      status:         Status::Initial,
      poll:           Poll::new().map_err(ErrorKind::IOError)?,
      registration,
      set_readiness,
      hb_handle:      None,
      frame_size,
      receive_buffer: Buffer::with_capacity(FRAMES_STORAGE * frame_size),
      send_buffer:    Buffer::with_capacity(FRAMES_STORAGE * frame_size),
      can_write:      false,
      can_read:       false,
      has_data:       false,
      send_heartbeat: Arc::new(AtomicBool::new(false)),
    };
    inner.poll.register(&inner.socket, SOCKET, Ready::readable() | Ready::writable(), PollOpt::edge()).map_err(ErrorKind::IOError)?;
    inner.poll.register(&inner.connection, DATA, Ready::readable(), PollOpt::edge()).map_err(ErrorKind::IOError)?;
    inner.poll.register(&inner.registration, CONTINUE, Ready::readable(), PollOpt::edge()).map_err(ErrorKind::IOError)?;
    Ok(inner)
  }

  fn start_heartbeat(&mut self, interval: Duration) -> Result<(), Error> {
    let send_hartbeat = self.send_heartbeat.clone();
    let hb_handle = ThreadBuilder::new().name("heartbeat".to_owned()).spawn(move || {
      loop {
        let start         = Instant::now();
        let mut remaining = interval;

        loop {
          thread::park_timeout(remaining);
          let elapsed = start.elapsed();
          if elapsed >= remaining {
            break
          }
          remaining -= interval - elapsed;
        }

        send_hartbeat.store(true, Ordering::Relaxed);
      }
    }).map_err(ErrorKind::IOError)?;
    self.hb_handle = Some(hb_handle);
    Ok(())
  }

  fn heartbeat(&mut self) -> Result<(), Error> {
    self.connection.send_heartbeat()?;
    self.send_heartbeat.store(false, Ordering::Relaxed);
    Ok(())
  }

  fn ensure_setup(&mut self) -> Result<(), Error> {
    if self.status != Status::Setup && self.connection.status().connected() {
      let frame_max = self.connection.configuration().frame_max() as usize;
      self.frame_size = std::cmp::max(self.frame_size, frame_max);
      self.receive_buffer.grow(FRAMES_STORAGE * self.frame_size);
      self.send_buffer.grow(FRAMES_STORAGE * self.frame_size);
      let heartbeat = self.connection.configuration().heartbeat();
      if heartbeat != 0 {
        trace!("io_loop: start heartbeat");
        self.start_heartbeat(Duration::from_secs(heartbeat as u64))?;
        trace!("io_loop: heartbeat started");
      }
      self.status = Status::Setup;
    }
    Ok(())
  }

  fn wants_to_write(&self) -> bool {
    self.can_write && self.has_data
  }

  fn wants_to_read(&self) -> bool {
    self.can_read
  }

  fn run(&mut self, events: &mut Events) -> Result<(), Error> {
    // First, update our internal state
    trace!("io_loop run");
    self.ensure_setup()?;
    trace!("io_loop poll");
    self.poll.poll(events, None).map_err(ErrorKind::IOError)?;
    trace!("io_loop poll done");
    for event in events.iter() {
      match event.token() {
        SOCKET    => {
          if event.readiness().is_readable() {
            self.can_read = true;
          }
          if event.readiness().is_writable() {
            self.can_write = true;
          }
        },
        DATA      => self.has_data = true,
        _         => {},
      }
    }

    // Then, actually run an iteration of the event loop
    trace!("io_loop do_run; can_read={}, can_write={}, has_data={}", self.can_read, self.can_write, self.has_data);
    loop {
      if self.send_heartbeat.load(Ordering::Relaxed) {
        self.heartbeat()?;
      }
      if self.wants_to_write() && !self.connection.status().blocked() {
        if let Err(e) = self.write_to_stream() {
          match e.kind() {
            ErrorKind::IOError(e) if e.kind() == io::ErrorKind::WouldBlock => self.can_write = false,
            _ => {
              error!("error writing: {:?}", e);
              self.connection.set_error()?;
              return Err(e);
            }
          }
        }
        self.send_buffer.shift_unless_available(self.frame_size);
      }
      if self.wants_to_read() {
        if let Err(e) = self.read_from_stream() {
          match e.kind() {
            ErrorKind::IOError(e) if e.kind() == io::ErrorKind::WouldBlock => self.can_read = false,
            _ => {
              error!("error reading: {:?}", e);
              self.connection.set_error()?;
              return Err(e);
            }
          }
        }
        self.receive_buffer.shift_unless_available(self.frame_size);
      }
      if self.can_parse() {
        self.parse()?;
      }
      if !self.wants_to_read() || !self.wants_to_write() {
        if self.wants_to_read() || self.can_parse() || self.has_data {
          trace!("io_loop send continue; can_read={}, can_write={}, has_data={}", self.can_read, self.can_write, self.has_data);
          self.set_readiness.set_readiness(Ready::readable()).map_err(ErrorKind::IOError)?;
        }
        break;
      }
    }
    trace!("io_loop do_run done; can_read={}, can_write={}, has_data={}", self.can_read, self.can_write, self.has_data);
    Ok(())
  }

  fn can_parse(&self) -> bool {
    self.receive_buffer.available_data() > 0
  }

  fn write_to_stream(&mut self) -> Result<(), Error> {
    self.serialize()?;

    self.socket.write(&self.send_buffer.data()).map(|sz| {
      trace!("wrote {} bytes", sz);
      self.send_buffer.consume(sz);
    }).map_err(|e| ErrorKind::IOError(e).into())
  }

  fn read_from_stream(&mut self) -> Result<(), Error> {
    let state = self.connection.status().state();
    if state == ConnectionState::Initial || state == ConnectionState::Closed || state == ConnectionState::Error {
      self.connection.set_error()?;
      Err(ErrorKind::InvalidConnectionState(state).into())
    } else {
      self.socket.read(&mut self.receive_buffer.space()).map(|sz| {
        trace!("read {} bytes", sz);
        self.receive_buffer.fill(sz);
      }).map_err(|e| ErrorKind::IOError(e).into())
    }
  }

  fn serialize(&mut self) -> Result<(), Error> {
    if let Some((send_id, next_msg)) = self.connection.next_frame() {
      trace!("will write to buffer: {:?}", next_msg);
      match gen_frame(self.send_buffer.space(), &next_msg).map(|tup| tup.0) {
        Ok(sz) => {
          self.send_buffer.fill(sz);
          self.connection.mark_sent(send_id);
          Ok(())
        },
        Err(e) => {
          match e {
            GenError::BufferTooSmall(_) => {
              // Requeue msg
              self.connection.requeue_frame(send_id, next_msg)?;
              self.send_buffer.shift();
              Ok(())
            },
            GenError::InvalidOffset | GenError::CustomError(_) | GenError::NotYetImplemented => {
              error!("error generating frame: {:?}", e);
              self.connection.set_error()?;
              Err(ErrorKind::SerialisationError(e).into())
            }
          }
        }
      }
    } else {
      self.has_data = false;
      Ok(())
    }
  }

  fn parse(&mut self) -> Result<(), Error> {
    match parse_frame(self.receive_buffer.data()) {
      Ok((i, f)) => {

        let consumed = self.receive_buffer.data().offset(i);
        self.receive_buffer.consume(consumed);

        if let Err(e) = self.connection.handle_frame(f) {
          self.connection.set_error()?;
          Err(e)
        } else {
          Ok(())
        }
      },
      Err(e) => {
        if e.is_incomplete() {
          self.receive_buffer.shift();
          Ok(())
        } else {
          error!("parse error: {:?}", e);
          self.connection.set_error()?;
          Err(ErrorKind::ParsingError(format!("{:?}", e)).into())
        }
      }
    }
  }
}
<|endoftext|>


<commit_before>use crate::FastFieldCodecReader;
use crate::FastFieldCodecSerializer;
use crate::FastFieldDataAccess;
use crate::FastFieldStats;
use common::CountingWriter;
use std::io::{self, Read, Write};
use std::ops::Sub;
use tantivy_bitpacker::compute_num_bits;
use tantivy_bitpacker::BitPacker;

use common::BinarySerializable;
use common::DeserializeFrom;
use tantivy_bitpacker::BitUnpacker;

const CHUNK_SIZE: u64 = 512;

/// Depending on the field type, a different
/// fast field is required.
#[derive(Clone)]
pub struct MultiLinearInterpolFastFieldReader {
    pub footer: MultiLinearInterpolFooter,
}

#[derive(Clone, Debug, Default)]
struct Function {
    // The offset in the data is required, because we have diffrent bit_widths per block
    data_start_offset: u64,
    // start_pos in the block will be CHUNK_SIZE * BLOCK_NUM
    start_pos: u64,
    // only used during serialization, 0 after deserialization
    end_pos: u64,
    // only used during serialization, 0 after deserialization
    value_start_pos: u64,
    // only used during serialization, 0 after deserialization
    value_end_pos: u64,
    slope: f32,
    // The offset so that all values are positive when writing them
    positive_val_offset: u64,
    num_bits: u8,
    bit_unpacker: BitUnpacker,
}

impl Function {
    fn calc_slope(&mut self) {
        let num_vals = self.end_pos - self.start_pos;
        get_slope(self.value_start_pos, self.value_end_pos, num_vals);
    }
    // split the interpolation into two function, change self and return the second split
    fn split(&mut self, split_pos: u64, split_pos_value: u64) -> Function {
        let mut new_function = Function {
            start_pos: split_pos,
            end_pos: self.end_pos,
            value_start_pos: split_pos_value,
            value_end_pos: self.value_end_pos,
            ..Default::default()
        };
        new_function.calc_slope();
        self.end_pos = split_pos;
        self.value_end_pos = split_pos_value;
        self.calc_slope();
        new_function
    }
}

impl BinarySerializable for Function {
    fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
        self.data_start_offset.serialize(write)?;
        self.value_start_pos.serialize(write)?;
        self.positive_val_offset.serialize(write)?;
        self.slope.serialize(write)?;
        self.num_bits.serialize(write)?;
        Ok(())
    }

    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Function> {
        let data_start_offset = u64::deserialize(reader)?;
        let value_start_pos = u64::deserialize(reader)?;
        let offset = u64::deserialize(reader)?;
        let slope = f32::deserialize(reader)?;
        let num_bits = u8::deserialize(reader)?;
        let interpolation = Function {
            data_start_offset,
            value_start_pos,
            positive_val_offset: offset,
            num_bits,
            bit_unpacker: BitUnpacker::new(num_bits),
            slope,
            ..Default::default()
        };

        Ok(interpolation)
    }
}

#[derive(Clone, Debug)]
pub struct MultiLinearInterpolFooter {
    pub num_vals: u64,
    pub min_value: u64,
    pub max_value: u64,
    interpolations: Vec<Function>,
}

impl BinarySerializable for MultiLinearInterpolFooter {
    fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
        let mut out = vec![];
        self.num_vals.serialize(&mut out)?;
        self.min_value.serialize(&mut out)?;
        self.max_value.serialize(&mut out)?;
        self.interpolations.serialize(&mut out)?;
        write.write_all(&out)?;
        (out.len() as u32).serialize(write)?;
        Ok(())
    }

    fn deserialize<R: Read>(reader: &mut R) -> io::Result<MultiLinearInterpolFooter> {
        let mut footer = MultiLinearInterpolFooter {
            num_vals: u64::deserialize(reader)?,
            min_value: u64::deserialize(reader)?,
            max_value: u64::deserialize(reader)?,
            interpolations: Vec::<Function>::deserialize(reader)?,
        };
        for (num, interpol) in footer.interpolations.iter_mut().enumerate() {
            interpol.start_pos = CHUNK_SIZE * num as u64;
        }
        Ok(footer)
    }
}

#[inline]
fn get_interpolation_position(doc: u64) -> usize {
    let index = doc / CHUNK_SIZE;
    index as usize
}

#[inline]
fn get_interpolation_function(doc: u64, interpolations: &[Function]) -> &Function {
    &interpolations[get_interpolation_position(doc)]
}

impl FastFieldCodecReader for MultiLinearInterpolFastFieldReader {
    /// Opens a fast field given a file.
    fn open_from_bytes(bytes: &[u8]) -> io::Result<Self> {
        let footer_len: u32 = (&bytes[bytes.len() - 4..]).deserialize()?;

        let (_data, mut footer) = bytes.split_at(bytes.len() - (4 + footer_len) as usize);
        let footer = MultiLinearInterpolFooter::deserialize(&mut footer)?;

        Ok(MultiLinearInterpolFastFieldReader { footer })
    }

    #[inline]
    fn get_u64(&self, doc: u64, data: &[u8]) -> u64 {
        let interpolation = get_interpolation_function(doc, &self.footer.interpolations);
        let doc = doc - interpolation.start_pos;
        let calculated_value =
            get_calculated_value(interpolation.value_start_pos, doc, interpolation.slope);
        let diff = interpolation
            .bit_unpacker
            .get(doc, &data[interpolation.data_start_offset as usize..]);
        (calculated_value + diff) - interpolation.positive_val_offset
    }

    #[inline]
    fn min_value(&self) -> u64 {
        self.footer.min_value
    }
    #[inline]
    fn max_value(&self) -> u64 {
        self.footer.max_value
    }
}

#[inline]
fn get_slope(first_val: u64, last_val: u64, num_vals: u64) -> f32 {
    ((last_val as f64 - first_val as f64) / (num_vals as u64 - 1) as f64) as f32
}

#[inline]
fn get_calculated_value(first_val: u64, pos: u64, slope: f32) -> u64 {
    (first_val as i64 + (pos as f32 * slope) as i64) as u64
}

/// Same as LinearInterpolFastFieldSerializer, but working on chunks of CHUNK_SIZE elements.
pub struct MultiLinearInterpolFastFieldSerializer {}

impl FastFieldCodecSerializer for MultiLinearInterpolFastFieldSerializer {
    const NAME: &'static str = "MultiLinearInterpol";
    const ID: u8 = 3;
    /// Creates a new fast field serializer.
    fn serialize(
        write: &mut impl Write,
        fastfield_accessor: &impl FastFieldDataAccess,
        stats: FastFieldStats,
        data_iter: impl Iterator<Item = u64>,
        _data_iter1: impl Iterator<Item = u64>,
    ) -> io::Result<()> {
        assert!(stats.min_value <= stats.max_value);

        let first_val = fastfield_accessor.get_val(0);
        let last_val = fastfield_accessor.get_val(stats.num_vals as u64 - 1);

        let mut first_function = Function {
            end_pos: stats.num_vals,
            value_start_pos: first_val,
            value_end_pos: last_val,
            ..Default::default()
        };
        first_function.calc_slope();
        let mut interpolations = vec![first_function];

        // Since we potentially apply multiple passes over the data, the data is cached.
        // Multiple iteration can be expensive (merge with index sorting can add lot of overhead per
        // iteration)
        let data = data_iter.collect::<Vec<_>>();

        //// let's split this into chunks of CHUNK_SIZE
        for data_pos in (0..data.len() as u64).step_by(CHUNK_SIZE as usize).skip(1) {
            let new_fun = {
                let current_interpolation = interpolations.last_mut().unwrap();
                current_interpolation.split(data_pos, data[data_pos as usize])
            };
            interpolations.push(new_fun);
        }
        // calculate offset and max (-> numbits) for each function
        for interpolation in &mut interpolations {
            let mut offset = 0;
            let mut rel_positive_max = 0;
            for (pos, actual_value) in data
                [interpolation.start_pos as usize..interpolation.end_pos as usize]
                .iter()
                .cloned()
                .enumerate()
            {
                let calculated_value = get_calculated_value(
                    interpolation.value_start_pos,
                    pos as u64,
                    interpolation.slope,
                );
                if calculated_value > actual_value {
                    // negative value we need to apply an offset
                    // we ignore negative values in the max value calculation, because negative values
                    // will be offset to 0
                    offset = offset.max(calculated_value - actual_value);
                } else {
                    //positive value no offset reuqired
                    rel_positive_max = rel_positive_max.max(actual_value - calculated_value);
                }
            }

            interpolation.positive_val_offset = offset;
            interpolation.num_bits = compute_num_bits(rel_positive_max + offset);
        }
        let mut bit_packer = BitPacker::new();

        let write = &mut CountingWriter::wrap(write);
        for interpolation in &mut interpolations {
            interpolation.data_start_offset = write.written_bytes();
            let num_bits = interpolation.num_bits;
            for (pos, actual_value) in data
                [interpolation.start_pos as usize..interpolation.end_pos as usize]
                .iter()
                .cloned()
                .enumerate()
            {
                let calculated_value = get_calculated_value(
                    interpolation.value_start_pos,
                    pos as u64,
                    interpolation.slope,
                );
                let diff = (actual_value + interpolation.positive_val_offset) - calculated_value;
                bit_packer.write(diff, num_bits, write)?;
            }
            bit_packer.flush(write)?;
        }
        bit_packer.close(write)?;

        let footer = MultiLinearInterpolFooter {
            num_vals: stats.num_vals,
            min_value: stats.min_value,
            max_value: stats.max_value,
            interpolations,
        };
        footer.serialize(write)?;
        Ok(())
    }

    fn is_applicable(
        _fastfield_accessor: &impl FastFieldDataAccess,
        stats: FastFieldStats,
    ) -> bool {
        if stats.num_vals < 5_000 {
            return false;
        }
        // On serialization the offset is added to the actual value.
        // We need to make sure this won't run into overflow calculation issues.
        // For this we take the maximum theroretical offset and add this to the max value.
        // If this doesn't overflow the algortihm should be fine
        let theorethical_maximum_offset = stats.max_value - stats.min_value;
        if stats
            .max_value
            .checked_add(theorethical_maximum_offset)
            .is_none()
        {
            return false;
        }
        true
    }
    /// estimation for linear interpolation is hard because, you don't know
    /// where the local maxima are for the deviation of the calculated value and
    /// the offset is also unknown.
    fn estimate(fastfield_accessor: &impl FastFieldDataAccess, stats: FastFieldStats) -> f32 {
        let first_val_in_first_block = fastfield_accessor.get_val(0);
        let last_elem_in_first_chunk = CHUNK_SIZE.min(stats.num_vals);
        let last_val_in_first_block =
            fastfield_accessor.get_val(last_elem_in_first_chunk as u64 - 1);
        let slope = get_slope(
            first_val_in_first_block,
            last_val_in_first_block,
            stats.num_vals,
        );

        // let's sample at 0%, 5%, 10% .. 95%, 100%, but for the first block only
        let sample_positions = (0..20)
            .map(|pos| (last_elem_in_first_chunk as f32 / 100.0 * pos as f32 * 5.0) as usize)
            .collect::<Vec<_>>();

        let max_distance = sample_positions
            .iter()
            .map(|pos| {
                let calculated_value =
                    get_calculated_value(first_val_in_first_block, *pos as u64, slope);
                let actual_value = fastfield_accessor.get_val(*pos as u64);
                distance(calculated_value, actual_value)
            })
            .max()
            .unwrap();

        // Estimate one block and extrapolate the cost to all blocks.
        // the theory would be that we don't have the actual max_distance, but we are close within 50%
        // threshold.
        // It is multiplied by 2 because in a log case scenario the line would be as much above as
        // below. So the offset would = max_distance
        //
        let relative_max_value = (max_distance as f32 * 1.5) * 2.0;

        let num_bits = compute_num_bits(relative_max_value as u64) as u64 * stats.num_vals as u64
            // function metadata per block
            + 29 * (stats.num_vals / CHUNK_SIZE);
        let num_bits_uncompressed = 64 * stats.num_vals;
        num_bits as f32 / num_bits_uncompressed as f32
    }
}

fn distance<T: Sub<Output = T> + Ord>(x: T, y: T) -> T {
    if x < y {
        y - x
    } else {
        x - y
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::tests::get_codec_test_data_sets;

    fn create_and_validate(data: &[u64], name: &str) {
        crate::tests::create_and_validate::<
            MultiLinearInterpolFastFieldSerializer,
            MultiLinearInterpolFastFieldReader,
        >(data, name);
    }

    #[test]
    fn test_with_codec_data_sets() {
        let data_sets = get_codec_test_data_sets();
        for (mut data, name) in data_sets {
            create_and_validate(&data, name);
            data.reverse();
            create_and_validate(&data, name);
        }
    }
    #[test]
    fn test_simple() {
        let data = (10..=20_u64).collect::<Vec<_>>();
        create_and_validate(&data, "simple monotonically");
    }

    #[test]
    fn border_cases_1() {
        let data = (0..1024).collect::<Vec<_>>();
        create_and_validate(&data, "border case");
    }
    #[test]
    fn border_case_2() {
        let data = (0..1025).collect::<Vec<_>>();
        create_and_validate(&data, "border case");
    }
    #[test]
    fn rand() {
        for _ in 0..10 {
            let mut data = (5_000..20_000)
                .map(|_| rand::random::<u64>() as u64)
                .collect::<Vec<_>>();
            create_and_validate(&data, "random");

            data.reverse();
            create_and_validate(&data, "random");
        }
    }
}
<commit_msg>add comments on compression (#1119)<commit_after>/*!

MultiLinearInterpol compressor uses linear interpolation to guess a values and stores the offset, but in blocks of 512.

With a CHUNK_SIZE of 512 and 29 byte metadata per block, we get a overhead for metadata of 232 / 512 = 0,45 bits per element.
The additional space required per element in a block is the the maximum deviation of the linear interpolation estimation function.

E.g. if the maximum deviation of an element is 12, all elements cost 4bits.

Size per block:
Num Elements * Maximum Deviation from Interpolation + 29 Byte Metadata

*/

use crate::FastFieldCodecReader;
use crate::FastFieldCodecSerializer;
use crate::FastFieldDataAccess;
use crate::FastFieldStats;
use common::CountingWriter;
use std::io::{self, Read, Write};
use std::ops::Sub;
use tantivy_bitpacker::compute_num_bits;
use tantivy_bitpacker::BitPacker;

use common::BinarySerializable;
use common::DeserializeFrom;
use tantivy_bitpacker::BitUnpacker;

const CHUNK_SIZE: u64 = 512;

/// Depending on the field type, a different
/// fast field is required.
#[derive(Clone)]
pub struct MultiLinearInterpolFastFieldReader {
    pub footer: MultiLinearInterpolFooter,
}

#[derive(Clone, Debug, Default)]
struct Function {
    // The offset in the data is required, because we have diffrent bit_widths per block
    data_start_offset: u64,
    // start_pos in the block will be CHUNK_SIZE * BLOCK_NUM
    start_pos: u64,
    // only used during serialization, 0 after deserialization
    end_pos: u64,
    // only used during serialization, 0 after deserialization
    value_start_pos: u64,
    // only used during serialization, 0 after deserialization
    value_end_pos: u64,
    slope: f32,
    // The offset so that all values are positive when writing them
    positive_val_offset: u64,
    num_bits: u8,
    bit_unpacker: BitUnpacker,
}

impl Function {
    fn calc_slope(&mut self) {
        let num_vals = self.end_pos - self.start_pos;
        get_slope(self.value_start_pos, self.value_end_pos, num_vals);
    }
    // split the interpolation into two function, change self and return the second split
    fn split(&mut self, split_pos: u64, split_pos_value: u64) -> Function {
        let mut new_function = Function {
            start_pos: split_pos,
            end_pos: self.end_pos,
            value_start_pos: split_pos_value,
            value_end_pos: self.value_end_pos,
            ..Default::default()
        };
        new_function.calc_slope();
        self.end_pos = split_pos;
        self.value_end_pos = split_pos_value;
        self.calc_slope();
        new_function
    }
}

impl BinarySerializable for Function {
    fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
        self.data_start_offset.serialize(write)?;
        self.value_start_pos.serialize(write)?;
        self.positive_val_offset.serialize(write)?;
        self.slope.serialize(write)?;
        self.num_bits.serialize(write)?;
        Ok(())
    }

    fn deserialize<R: Read>(reader: &mut R) -> io::Result<Function> {
        let data_start_offset = u64::deserialize(reader)?;
        let value_start_pos = u64::deserialize(reader)?;
        let offset = u64::deserialize(reader)?;
        let slope = f32::deserialize(reader)?;
        let num_bits = u8::deserialize(reader)?;
        let interpolation = Function {
            data_start_offset,
            value_start_pos,
            positive_val_offset: offset,
            num_bits,
            bit_unpacker: BitUnpacker::new(num_bits),
            slope,
            ..Default::default()
        };

        Ok(interpolation)
    }
}

#[derive(Clone, Debug)]
pub struct MultiLinearInterpolFooter {
    pub num_vals: u64,
    pub min_value: u64,
    pub max_value: u64,
    interpolations: Vec<Function>,
}

impl BinarySerializable for MultiLinearInterpolFooter {
    fn serialize<W: Write>(&self, write: &mut W) -> io::Result<()> {
        let mut out = vec![];
        self.num_vals.serialize(&mut out)?;
        self.min_value.serialize(&mut out)?;
        self.max_value.serialize(&mut out)?;
        self.interpolations.serialize(&mut out)?;
        write.write_all(&out)?;
        (out.len() as u32).serialize(write)?;
        Ok(())
    }

    fn deserialize<R: Read>(reader: &mut R) -> io::Result<MultiLinearInterpolFooter> {
        let mut footer = MultiLinearInterpolFooter {
            num_vals: u64::deserialize(reader)?,
            min_value: u64::deserialize(reader)?,
            max_value: u64::deserialize(reader)?,
            interpolations: Vec::<Function>::deserialize(reader)?,
        };
        for (num, interpol) in footer.interpolations.iter_mut().enumerate() {
            interpol.start_pos = CHUNK_SIZE * num as u64;
        }
        Ok(footer)
    }
}

#[inline]
fn get_interpolation_position(doc: u64) -> usize {
    let index = doc / CHUNK_SIZE;
    index as usize
}

#[inline]
fn get_interpolation_function(doc: u64, interpolations: &[Function]) -> &Function {
    &interpolations[get_interpolation_position(doc)]
}

impl FastFieldCodecReader for MultiLinearInterpolFastFieldReader {
    /// Opens a fast field given a file.
    fn open_from_bytes(bytes: &[u8]) -> io::Result<Self> {
        let footer_len: u32 = (&bytes[bytes.len() - 4..]).deserialize()?;

        let (_data, mut footer) = bytes.split_at(bytes.len() - (4 + footer_len) as usize);
        let footer = MultiLinearInterpolFooter::deserialize(&mut footer)?;

        Ok(MultiLinearInterpolFastFieldReader { footer })
    }

    #[inline]
    fn get_u64(&self, doc: u64, data: &[u8]) -> u64 {
        let interpolation = get_interpolation_function(doc, &self.footer.interpolations);
        let doc = doc - interpolation.start_pos;
        let calculated_value =
            get_calculated_value(interpolation.value_start_pos, doc, interpolation.slope);
        let diff = interpolation
            .bit_unpacker
            .get(doc, &data[interpolation.data_start_offset as usize..]);
        (calculated_value + diff) - interpolation.positive_val_offset
    }

    #[inline]
    fn min_value(&self) -> u64 {
        self.footer.min_value
    }
    #[inline]
    fn max_value(&self) -> u64 {
        self.footer.max_value
    }
}

#[inline]
fn get_slope(first_val: u64, last_val: u64, num_vals: u64) -> f32 {
    ((last_val as f64 - first_val as f64) / (num_vals as u64 - 1) as f64) as f32
}

#[inline]
fn get_calculated_value(first_val: u64, pos: u64, slope: f32) -> u64 {
    (first_val as i64 + (pos as f32 * slope) as i64) as u64
}

/// Same as LinearInterpolFastFieldSerializer, but working on chunks of CHUNK_SIZE elements.
pub struct MultiLinearInterpolFastFieldSerializer {}

impl FastFieldCodecSerializer for MultiLinearInterpolFastFieldSerializer {
    const NAME: &'static str = "MultiLinearInterpol";
    const ID: u8 = 3;
    /// Creates a new fast field serializer.
    fn serialize(
        write: &mut impl Write,
        fastfield_accessor: &impl FastFieldDataAccess,
        stats: FastFieldStats,
        data_iter: impl Iterator<Item = u64>,
        _data_iter1: impl Iterator<Item = u64>,
    ) -> io::Result<()> {
        assert!(stats.min_value <= stats.max_value);

        let first_val = fastfield_accessor.get_val(0);
        let last_val = fastfield_accessor.get_val(stats.num_vals as u64 - 1);

        let mut first_function = Function {
            end_pos: stats.num_vals,
            value_start_pos: first_val,
            value_end_pos: last_val,
            ..Default::default()
        };
        first_function.calc_slope();
        let mut interpolations = vec![first_function];

        // Since we potentially apply multiple passes over the data, the data is cached.
        // Multiple iteration can be expensive (merge with index sorting can add lot of overhead per
        // iteration)
        let data = data_iter.collect::<Vec<_>>();

        //// let's split this into chunks of CHUNK_SIZE
        for data_pos in (0..data.len() as u64).step_by(CHUNK_SIZE as usize).skip(1) {
            let new_fun = {
                let current_interpolation = interpolations.last_mut().unwrap();
                current_interpolation.split(data_pos, data[data_pos as usize])
            };
            interpolations.push(new_fun);
        }
        // calculate offset and max (-> numbits) for each function
        for interpolation in &mut interpolations {
            let mut offset = 0;
            let mut rel_positive_max = 0;
            for (pos, actual_value) in data
                [interpolation.start_pos as usize..interpolation.end_pos as usize]
                .iter()
                .cloned()
                .enumerate()
            {
                let calculated_value = get_calculated_value(
                    interpolation.value_start_pos,
                    pos as u64,
                    interpolation.slope,
                );
                if calculated_value > actual_value {
                    // negative value we need to apply an offset
                    // we ignore negative values in the max value calculation, because negative values
                    // will be offset to 0
                    offset = offset.max(calculated_value - actual_value);
                } else {
                    //positive value no offset reuqired
                    rel_positive_max = rel_positive_max.max(actual_value - calculated_value);
                }
            }

            interpolation.positive_val_offset = offset;
            interpolation.num_bits = compute_num_bits(rel_positive_max + offset);
        }
        let mut bit_packer = BitPacker::new();

        let write = &mut CountingWriter::wrap(write);
        for interpolation in &mut interpolations {
            interpolation.data_start_offset = write.written_bytes();
            let num_bits = interpolation.num_bits;
            for (pos, actual_value) in data
                [interpolation.start_pos as usize..interpolation.end_pos as usize]
                .iter()
                .cloned()
                .enumerate()
            {
                let calculated_value = get_calculated_value(
                    interpolation.value_start_pos,
                    pos as u64,
                    interpolation.slope,
                );
                let diff = (actual_value + interpolation.positive_val_offset) - calculated_value;
                bit_packer.write(diff, num_bits, write)?;
            }
            bit_packer.flush(write)?;
        }
        bit_packer.close(write)?;

        let footer = MultiLinearInterpolFooter {
            num_vals: stats.num_vals,
            min_value: stats.min_value,
            max_value: stats.max_value,
            interpolations,
        };
        footer.serialize(write)?;
        Ok(())
    }

    fn is_applicable(
        _fastfield_accessor: &impl FastFieldDataAccess,
        stats: FastFieldStats,
    ) -> bool {
        if stats.num_vals < 5_000 {
            return false;
        }
        // On serialization the offset is added to the actual value.
        // We need to make sure this won't run into overflow calculation issues.
        // For this we take the maximum theroretical offset and add this to the max value.
        // If this doesn't overflow the algortihm should be fine
        let theorethical_maximum_offset = stats.max_value - stats.min_value;
        if stats
            .max_value
            .checked_add(theorethical_maximum_offset)
            .is_none()
        {
            return false;
        }
        true
    }
    /// estimation for linear interpolation is hard because, you don't know
    /// where the local maxima are for the deviation of the calculated value and
    /// the offset is also unknown.
    fn estimate(fastfield_accessor: &impl FastFieldDataAccess, stats: FastFieldStats) -> f32 {
        let first_val_in_first_block = fastfield_accessor.get_val(0);
        let last_elem_in_first_chunk = CHUNK_SIZE.min(stats.num_vals);
        let last_val_in_first_block =
            fastfield_accessor.get_val(last_elem_in_first_chunk as u64 - 1);
        let slope = get_slope(
            first_val_in_first_block,
            last_val_in_first_block,
            stats.num_vals,
        );

        // let's sample at 0%, 5%, 10% .. 95%, 100%, but for the first block only
        let sample_positions = (0..20)
            .map(|pos| (last_elem_in_first_chunk as f32 / 100.0 * pos as f32 * 5.0) as usize)
            .collect::<Vec<_>>();

        let max_distance = sample_positions
            .iter()
            .map(|pos| {
                let calculated_value =
                    get_calculated_value(first_val_in_first_block, *pos as u64, slope);
                let actual_value = fastfield_accessor.get_val(*pos as u64);
                distance(calculated_value, actual_value)
            })
            .max()
            .unwrap();

        // Estimate one block and extrapolate the cost to all blocks.
        // the theory would be that we don't have the actual max_distance, but we are close within 50%
        // threshold.
        // It is multiplied by 2 because in a log case scenario the line would be as much above as
        // below. So the offset would = max_distance
        //
        let relative_max_value = (max_distance as f32 * 1.5) * 2.0;

        let num_bits = compute_num_bits(relative_max_value as u64) as u64 * stats.num_vals as u64
            // function metadata per block
            + 29 * (stats.num_vals / CHUNK_SIZE);
        let num_bits_uncompressed = 64 * stats.num_vals;
        num_bits as f32 / num_bits_uncompressed as f32
    }
}

fn distance<T: Sub<Output = T> + Ord>(x: T, y: T) -> T {
    if x < y {
        y - x
    } else {
        x - y
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::tests::get_codec_test_data_sets;

    fn create_and_validate(data: &[u64], name: &str) {
        crate::tests::create_and_validate::<
            MultiLinearInterpolFastFieldSerializer,
            MultiLinearInterpolFastFieldReader,
        >(data, name);
    }

    #[test]
    fn test_with_codec_data_sets() {
        let data_sets = get_codec_test_data_sets();
        for (mut data, name) in data_sets {
            create_and_validate(&data, name);
            data.reverse();
            create_and_validate(&data, name);
        }
    }
    #[test]
    fn test_simple() {
        let data = (10..=20_u64).collect::<Vec<_>>();
        create_and_validate(&data, "simple monotonically");
    }

    #[test]
    fn border_cases_1() {
        let data = (0..1024).collect::<Vec<_>>();
        create_and_validate(&data, "border case");
    }
    #[test]
    fn border_case_2() {
        let data = (0..1025).collect::<Vec<_>>();
        create_and_validate(&data, "border case");
    }
    #[test]
    fn rand() {
        for _ in 0..10 {
            let mut data = (5_000..20_000)
                .map(|_| rand::random::<u64>() as u64)
                .collect::<Vec<_>>();
            create_and_validate(&data, "random");

            data.reverse();
            create_and_validate(&data, "random");
        }
    }
}
<|endoftext|>