<commit_before>use codespan::{CodeMap, LineIndex, RawIndex};
use std::io;
use termcolor::{Color, ColorSpec, WriteColor};

use crate::{Diagnostic, LabelStyle, Severity};

pub fn emit<W, S>(mut writer: W, codemap: &CodeMap<S>, diagnostic: &Diagnostic) -> io::Result<()>
where
    W: WriteColor,
    S: AsRef<str>,
{
    let severity_color = severity_color(diagnostic.severity);
    let gutter_spec = ColorSpec::new().set_fg(Some(gutter_color())).clone();
    let primary_spec = ColorSpec::new().set_fg(Some(severity_color)).clone();
    let secondary_spec = ColorSpec::new().set_fg(Some(secondary_color())).clone();
    let header_message_spec = ColorSpec::new().set_bold(true).set_intense(true).clone();
    let header_primary_spec = header_message_spec.clone().set_fg(Some(severity_color)).clone();

    // Diagnostic header

    writer.set_color(&header_primary_spec)?;

    // Severity
    //
    // ```
    // error
    // ```
    write!(writer, "{}", severity_name(diagnostic.severity))?;

    // Error code
    //
    // ```
    // [E0001]
    // ```
    if let Some(code) = &diagnostic.code {
        write!(writer, "[{}]", code)?;
    }

    // Diagnostic message
    //
    // ```
    // : Unexpected type in `+` application
    // ```
    writer.set_color(&header_message_spec)?;
    write!(writer, ": {}", diagnostic.message)?;
    write!(writer, "\n")?;
    writer.reset()?;

    // Diagnostic Labels

    for label in &diagnostic.labels {
        match codemap.find_file(label.span.start()) {
            None => {
                if !label.message.is_empty() {
                    write!(writer, "- {}", label.message)?;
                    write!(writer, "\n")?;
                }
            },
            Some(file) => {
                let (start_line, column) =
                    file.location(label.span.start()).expect("location_start");
                let (end_line, _) = file.location(label.span.end()).expect("location_end");
                // Use the length of the last line number as the gutter padding
                let gutter_padding = format!("{}", end_line.number()).len();

                // File name
                //
                // ```
                // - <test>:2:9
                // ```

                write!(
                    writer,
                    "{: >width$} - {file}:{line}:{column}",
                    "",
                    width = gutter_padding,
                    file = file.name(),
                    line = start_line.number(),
                    column = column.number(),
                )?;
                write!(writer, "\n")?;

                // Source code snippet
                //
                // ```
                //   │
                // 2 │ (+ test "")
                //   │         ^^ Expected integer but got string
                //   │
                // ```

                let start_line_span = file.line_span(start_line).expect("line_span");
                let end_line_span = file.line_span(end_line).expect("line_span");

                let label_spec = match label.style {
                    LabelStyle::Primary => &primary_spec,
                    LabelStyle::Secondary => &secondary_spec,
                };

                // Write prefix to marked section

                writer.set_color(&gutter_spec)?;
                write!(writer, "{: >width$} │ ", "", width = gutter_padding)?;
                write!(writer, "\n")?;
                write!(
                    writer,
                    "{: >width$} │ ",
                    start_line.number(),
                    width = gutter_padding,
                )?;
                writer.reset()?;

                let prefix = file
                    .src_slice(start_line_span.with_end(label.span.start()))
                    .expect("prefix");
                write!(writer, "{}", prefix)?;

                // Write marked section
                let mark_len = if start_line == end_line {
                    // Single line

                    let marked = file.src_slice(label.span).expect("marked");
                    writer.set_color(&label_spec)?;
                    write!(writer, "{}", marked)?;
                    writer.reset()?;
                    marked.len()
                } else {
                    // Multiple lines

                    let marked = file
                        .src_slice(start_line_span.with_start(label.span.start()))
                        .expect("start_of_marked");
                    writer.set_color(&label_spec)?;
                    write!(writer, "{}", marked)?;

                    for line_index in ((start_line.to_usize() + 1)..end_line.to_usize())
                        .map(|i| LineIndex::from(i as RawIndex))
                    {
                        writer.set_color(&gutter_spec)?;
                        write!(
                            writer,
                            "{: >width$} │ ",
                            line_index.number(),
                            width = gutter_padding,
                        )?;

                        let line_span = file.line_span(line_index).expect("marked_line_span");
                        let line = file
                            .src_slice(line_span)
                            .expect("line")
                            .trim_end_matches(|ch: char| ch == '\r' || ch == '\n');
                        writer.set_color(&label_spec)?;
                        write!(writer, "{}", line)?;
                        write!(writer, "\n")?;
                    }

                    writer.set_color(&gutter_spec)?;
                    write!(
                        writer,
                        "{: >width$} │ ",
                        end_line.number(),
                        width = gutter_padding,
                    )?;
                    let line = file
                        .src_slice(end_line_span.with_end(label.span.end()))
                        .expect("line");
                    writer.set_color(&label_spec)?;
                    write!(writer, "{}", line)?;
                    writer.reset()?;
                    line.len()
                };

                // Write suffix to marked section

                let suffix = file
                    .src_slice(end_line_span.with_start(label.span.end()))
                    .expect("suffix")
                    .trim_end_matches(|ch: char| ch == '\r' || ch == '\n');
                write!(writer, "{}", suffix)?;
                write!(writer, "\n")?;

                // Write mark and label

                writer.set_color(&gutter_spec)?;
                write!(writer, "{: >width$} │ ", "", width = gutter_padding)?;
                writer.reset()?;

                writer.set_color(&label_spec)?;
                write!(writer, "{: >width$}", "", width = prefix.len())?;
                for _ in 0..mark_len {
                    write!(writer, "{}", underline_mark(label.style))?;
                }
                if !label.message.is_empty() {
                    write!(writer, " {}", label.message)?;
                    write!(writer, "\n")?;
                }
                writer.reset()?;

                writer.set_color(&gutter_spec)?;
                write!(writer, "{: >width$} │", "", width = gutter_padding)?;
                write!(writer, "\n")?;
                writer.reset()?;
            },
        }
    }
    Ok(())
}

// Blue is really difficult to see on the standard windows command line
// FIXME: Make colors configurable
#[cfg(windows)]
const BLUE: Color = Color::Cyan;
#[cfg(not(windows))]
const BLUE: Color = Color::Blue;

/// Return the termcolor to use when rendering messages of this diagnostic severity.
fn severity_color(severity: Severity) -> Color {
    match severity {
        Severity::Bug | Severity::Error => Color::Red,
        Severity::Warning => Color::Yellow,
        Severity::Note => Color::Green,
        Severity::Help => Color::Cyan,
    }
}

/// The color to use for secondary highlights.
fn secondary_color() -> Color {
    BLUE
}

/// The color to use for gutters highlights.
fn gutter_color() -> Color {
    BLUE
}

/// A string that explains this diagnostic severity.
fn severity_name(severity: Severity) -> &'static str {
    match severity {
        Severity::Bug => "bug",
        Severity::Error => "error",
        Severity::Warning => "warning",
        Severity::Note => "note",
        Severity::Help => "help",
    }
}

/// The mark used for the underlined section of code.
fn underline_mark(label_style: LabelStyle) -> char {
    match label_style {
        LabelStyle::Primary => '^',
        LabelStyle::Secondary => '-',
    }
}
<commit_msg>More formatting cleanups<commit_after>use codespan::{CodeMap, LineIndex, RawIndex};
use std::io;
use termcolor::{Color, ColorSpec, WriteColor};

use crate::{Diagnostic, LabelStyle, Severity};

pub fn emit<W, S>(mut writer: W, codemap: &CodeMap<S>, diagnostic: &Diagnostic) -> io::Result<()>
where
    W: WriteColor,
    S: AsRef<str>,
{
    let severity_color = severity_color(diagnostic.severity);
    let gutter_spec = ColorSpec::new().set_fg(Some(gutter_color())).clone();
    let primary_spec = ColorSpec::new().set_fg(Some(severity_color)).clone();
    let secondary_spec = ColorSpec::new().set_fg(Some(secondary_color())).clone();
    let header_message_spec = ColorSpec::new().set_bold(true).set_intense(true).clone();
    let header_primary_spec = ColorSpec::new()
        .set_bold(true)
        .set_intense(true)
        .set_fg(Some(severity_color))
        .clone();

    // Diagnostic header

    writer.set_color(&header_primary_spec)?;

    // Severity
    //
    // ```
    // error
    // ```
    write!(writer, "{}", severity_name(diagnostic.severity))?;

    // Error code
    //
    // ```
    // [E0001]
    // ```
    if let Some(code) = &diagnostic.code {
        write!(writer, "[{}]", code)?;
    }

    // Diagnostic message
    //
    // ```
    // : Unexpected type in `+` application
    // ```
    writer.set_color(&header_message_spec)?;
    write!(writer, ": {}", diagnostic.message)?;
    write!(writer, "\n")?;
    writer.reset()?;

    // Diagnostic Labels

    for label in &diagnostic.labels {
        match codemap.find_file(label.span.start()) {
            None => {
                if !label.message.is_empty() {
                    write!(writer, "- {}", label.message)?;
                    write!(writer, "\n")?;
                }
            },
            Some(file) => {
                let (start_line, column) =
                    file.location(label.span.start()).expect("location_start");
                let (end_line, _) = file.location(label.span.end()).expect("location_end");
                // Use the length of the last line number as the gutter padding
                let gutter_padding = format!("{}", end_line.number()).len();

                // File name
                //
                // ```
                // - <test>:2:9
                // ```

                write!(
                    writer,
                    "{: >width$} - {file}:{line}:{column}",
                    "",
                    width = gutter_padding,
                    file = file.name(),
                    line = start_line.number(),
                    column = column.number(),
                )?;
                write!(writer, "\n")?;

                // Source code snippet
                //
                // ```
                //   │
                // 2 │ (+ test "")
                //   │         ^^ Expected integer but got string
                //   │
                // ```

                let start_line_span = file.line_span(start_line).expect("line_span");
                let end_line_span = file.line_span(end_line).expect("line_span");

                let label_spec = match label.style {
                    LabelStyle::Primary => &primary_spec,
                    LabelStyle::Secondary => &secondary_spec,
                };

                // Write prefix to marked section

                writer.set_color(&gutter_spec)?;
                write!(writer, "{: >width$} │ ", "", width = gutter_padding)?;
                write!(writer, "\n")?;
                write!(
                    writer,
                    "{: >width$} │ ",
                    start_line.number(),
                    width = gutter_padding,
                )?;
                writer.reset()?;

                let prefix = file
                    .src_slice(start_line_span.with_end(label.span.start()))
                    .expect("prefix");
                write!(writer, "{}", prefix)?;

                // Write marked section
                let mark_len = if start_line == end_line {
                    // Single line

                    let marked = file.src_slice(label.span).expect("marked");
                    writer.set_color(&label_spec)?;
                    write!(writer, "{}", marked)?;
                    writer.reset()?;
                    marked.len()
                } else {
                    // Multiple lines

                    let marked = file
                        .src_slice(start_line_span.with_start(label.span.start()))
                        .expect("start_of_marked");
                    writer.set_color(&label_spec)?;
                    write!(writer, "{}", marked)?;

                    for line_index in ((start_line.to_usize() + 1)..end_line.to_usize())
                        .map(|i| LineIndex::from(i as RawIndex))
                    {
                        writer.set_color(&gutter_spec)?;
                        write!(
                            writer,
                            "{: >width$} │ ",
                            line_index.number(),
                            width = gutter_padding,
                        )?;

                        let line_span = file.line_span(line_index).expect("marked_line_span");
                        let line = file
                            .src_slice(line_span)
                            .expect("line")
                            .trim_end_matches(|ch: char| ch == '\r' || ch == '\n');
                        writer.set_color(&label_spec)?;
                        write!(writer, "{}", line)?;
                        write!(writer, "\n")?;
                    }

                    writer.set_color(&gutter_spec)?;
                    write!(
                        writer,
                        "{: >width$} │ ",
                        end_line.number(),
                        width = gutter_padding,
                    )?;
                    let line = file
                        .src_slice(end_line_span.with_end(label.span.end()))
                        .expect("line");
                    writer.set_color(&label_spec)?;
                    write!(writer, "{}", line)?;
                    writer.reset()?;
                    line.len()
                };

                // Write suffix to marked section

                let suffix = file
                    .src_slice(end_line_span.with_start(label.span.end()))
                    .expect("suffix")
                    .trim_end_matches(|ch: char| ch == '\r' || ch == '\n');
                write!(writer, "{}", suffix)?;
                write!(writer, "\n")?;

                // Write mark and label

                writer.set_color(&gutter_spec)?;
                write!(writer, "{: >width$} │ ", "", width = gutter_padding)?;
                writer.reset()?;

                writer.set_color(&label_spec)?;
                write!(writer, "{: >width$}", "", width = prefix.len())?;
                for _ in 0..mark_len {
                    write!(writer, "{}", underline_mark(label.style))?;
                }
                if !label.message.is_empty() {
                    write!(writer, " {}", label.message)?;
                    write!(writer, "\n")?;
                }
                writer.reset()?;

                writer.set_color(&gutter_spec)?;
                write!(writer, "{: >width$} │", "", width = gutter_padding)?;
                write!(writer, "\n")?;
                writer.reset()?;
            },
        }
    }
    Ok(())
}

// Blue is really difficult to see on the standard windows command line
// FIXME: Make colors configurable
#[cfg(windows)]
const BLUE: Color = Color::Cyan;
#[cfg(not(windows))]
const BLUE: Color = Color::Blue;

/// Return the termcolor to use when rendering messages of this diagnostic severity.
fn severity_color(severity: Severity) -> Color {
    match severity {
        Severity::Bug | Severity::Error => Color::Red,
        Severity::Warning => Color::Yellow,
        Severity::Note => Color::Green,
        Severity::Help => Color::Cyan,
    }
}

/// The color to use for secondary highlights.
fn secondary_color() -> Color {
    BLUE
}

/// The color to use for gutters highlights.
fn gutter_color() -> Color {
    BLUE
}

/// A string that explains this diagnostic severity.
fn severity_name(severity: Severity) -> &'static str {
    match severity {
        Severity::Bug => "bug",
        Severity::Error => "error",
        Severity::Warning => "warning",
        Severity::Note => "note",
        Severity::Help => "help",
    }
}

/// The mark used for the underlined section of code.
fn underline_mark(label_style: LabelStyle) -> char {
    match label_style {
        LabelStyle::Primary => '^',
        LabelStyle::Secondary => '-',
    }
}
<|endoftext|>


<commit_before>// Copyright (C) 2016 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Crowbook is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

//! This module contains the `Cleaner` traits and various implementations of it.

use std::borrow::Cow;
use crowbook_text_processing::clean;
use crowbook_text_processing::FrenchFormatter;

/// Contains cleaning parameters
pub struct CleanerParams {
    pub smart_quotes: bool,
    pub ligature_guillemets: bool,
    pub ligature_dashes: bool,
}

/// Trait for cleaning a string.
///
/// This trait must be called for text that is e.g. in a paragraph, a title,
/// NOT for code blocks, hyperlinks and so on!
pub trait Cleaner: Sync {
    /// Cleans a string. The default implementation is to remove multiple consecutive whitespaces
    ///
    /// # Argumets
    ///
    /// * `str`: the string that must be cleaned
    /// * `latex`: a bool specifying whether output is Latex code or not
    fn clean<'a>(&self, str: Cow<'a, str>, _latex: bool) -> Cow<'a, str> {
        str
    }
}

/// Cleaner implementation that does nothing
///
/// # Examples
///
/// ```
/// use crowbook::cleaner::Cleaner;
/// use crowbook::cleaner::Off;
/// use std::borrow::Cow;
/// let off = Off;
/// let s = off.clean(Cow::Borrowed("  A string   that won't be cleaned "), false);
/// assert_eq!(&s, "  A string   that won't be cleaned ");
/// ```
pub struct Off;
impl Cleaner for Off {}

/// Default implementation of cleaner trait.
///
/// Only removes unnecessary whitespaces.
///
/// # Examples
///
/// ```
/// use crowbook::cleaner::Cleaner;
/// use crowbook::cleaner::Default;
/// use std::borrow::Cow;
/// let s = Default.clean(Cow::Borrowed("  A  string   with   more   whitespaces  than  needed   "),
///                                     false);
/// assert_eq!(&s, " A string with more whitespaces than needed ");
/// ```
pub struct Default {
    params: CleanerParams,
}

impl Default {
    /// New Default cleaner
    pub fn new(params: CleanerParams) -> Default {
        Default {
            params: params,
        }
    }
}

impl Cleaner for Default {
    /// Remove unnecessary whitespaces
    fn clean<'a>(&self, input: Cow<'a, str>, _: bool) -> Cow<'a, str> {
        let mut s = clean::whitespaces(input);
        if self.params.smart_quotes {
            s = clean::quotes(s);
        }
        if self.params.ligature_dashes {
            s = clean::dashes(s);
        }
        if self.params.ligature_guillemets {
            s = clean::guillemets(s);
        }
        s
    }
}

/// Implementation for french 'cleaning'
///
/// This implementation replaces spaces before some characters (e.g. `?` or `;`)
/// with non-breaking spaces
///
/// # Examples
///
/// ```
/// use crowbook::cleaner::Cleaner;
/// use crowbook::cleaner::French;
/// use std::borrow::Cow;
/// let s =  French.clean(Cow::Borrowed("  Bonjour ! Comment allez-vous   ?   "), true);
/// assert_eq!(&s, " Bonjour~! Comment allez-vous~? ");
/// ```
pub struct French {
    formatter: FrenchFormatter,
    params: CleanerParams,
}

impl French {
    /// Creates a new french cleaner
    pub fn new(params: CleanerParams) -> French {
        let mut this = French {
            formatter: FrenchFormatter::new(),
            params: params,
        };
        this.formatter.typographic_quotes(this.params.smart_quotes);
        this.formatter.ligature_dashes(this.params.ligature_dashes);
        this.formatter.ligature_guillemets(this.params.ligature_guillemets);
        this
    }
}

impl Cleaner for French {
    /// Puts non breaking spaces before/after `:`, `;`, `?`, `!`, `«`, `»`, `—`
    fn clean<'a>(&self, s: Cow<'a, str>, latex: bool) -> Cow<'a, str> {
        let s = if latex {
            self.formatter.format_tex(s)
        } else {
            self.formatter.format(s)
        };
        s
    }
}
<commit_msg>Remove doctests from cleaner as it is a private module<commit_after>// Copyright (C) 2016 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Crowbook is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

//! This module contains the `Cleaner` traits and various implementations of it.

use std::borrow::Cow;
use crowbook_text_processing::clean;
use crowbook_text_processing::FrenchFormatter;

/// Contains cleaning parameters
pub struct CleanerParams {
    pub smart_quotes: bool,
    pub ligature_guillemets: bool,
    pub ligature_dashes: bool,
}

/// Trait for cleaning a string.
///
/// This trait must be called for text that is e.g. in a paragraph, a title,
/// NOT for code blocks, hyperlinks and so on!
pub trait Cleaner: Sync {
    /// Cleans a string. The default implementation is to remove multiple consecutive whitespaces
    ///
    /// # Argumets
    ///
    /// * `str`: the string that must be cleaned
    /// * `latex`: a bool specifying whether output is Latex code or not
    fn clean<'a>(&self, str: Cow<'a, str>, _latex: bool) -> Cow<'a, str> {
        str
    }
}

/// Cleaner implementation that does nothing
pub struct Off;
impl Cleaner for Off {}

/// Default implementation of cleaner trait.
///
/// Only removes unnecessary whitespaces.
pub struct Default {
    params: CleanerParams,
}

impl Default {
    /// New Default cleaner
    pub fn new(params: CleanerParams) -> Default {
        Default {
            params: params,
        }
    }
}

impl Cleaner for Default {
    /// Remove unnecessary whitespaces
    fn clean<'a>(&self, input: Cow<'a, str>, _: bool) -> Cow<'a, str> {
        let mut s = clean::whitespaces(input);
        if self.params.smart_quotes {
            s = clean::quotes(s);
        }
        if self.params.ligature_dashes {
            s = clean::dashes(s);
        }
        if self.params.ligature_guillemets {
            s = clean::guillemets(s);
        }
        s
    }
}

/// Implementation for french 'cleaning'
///
/// This implementation replaces spaces before some characters (e.g. `?` or `;`)
/// with non-breaking spaces
pub struct French {
    formatter: FrenchFormatter,
    params: CleanerParams,
}

impl French {
    /// Creates a new french cleaner
    pub fn new(params: CleanerParams) -> French {
        let mut this = French {
            formatter: FrenchFormatter::new(),
            params: params,
        };
        this.formatter.typographic_quotes(this.params.smart_quotes);
        this.formatter.ligature_dashes(this.params.ligature_dashes);
        this.formatter.ligature_guillemets(this.params.ligature_guillemets);
        this
    }
}

impl Cleaner for French {
    /// Puts non breaking spaces before/after `:`, `;`, `?`, `!`, `«`, `»`, `—`
    fn clean<'a>(&self, s: Cow<'a, str>, latex: bool) -> Cow<'a, str> {
        let s = if latex {
            self.formatter.format_tex(s)
        } else {
            self.formatter.format(s)
        };
        s
    }
}
<|endoftext|>


<commit_before>use ra_text_edit::{AtomTextEdit};
use ra_syntax::{TextUnit, TextRange};
use crate::{LineIndex, LineCol};
use superslice::Ext;

#[derive(Debug)]
struct OffsetNewlineIter<'a> {
    text: &'a str,
    offset: TextUnit,
}

impl<'a> Iterator for OffsetNewlineIter<'a> {
    type Item = TextUnit;
    fn next(&mut self) -> Option<TextUnit> {
        let next_idx = self
            .text
            .char_indices()
            .filter_map(|(i, c)| if c == '\n' { Some(i + 1) } else { None })
            .next()?;
        let next = self.offset + TextUnit::from_usize(next_idx);
        self.text = &self.text[next_idx..];
        self.offset = next;
        Some(next)
    }
}

#[derive(Debug, Clone, Copy, PartialEq)]
enum TranslatedPos {
    Before,
    After,
}

/// None means it was deleted
type TranslatedOffset = Option<(TranslatedPos, TextUnit)>;

fn translate_offset(offset: TextUnit, edit: &TranslatedAtomEdit) -> TranslatedOffset {
    if offset <= edit.delete.start() {
        Some((TranslatedPos::Before, offset))
    } else if offset <= edit.delete.end() {
        None
    } else {
        let diff = edit.insert.len() as i64 - edit.delete.len().to_usize() as i64;
        let after = TextUnit::from((offset.to_usize() as i64 + diff) as u32);
        Some((TranslatedPos::After, after))
    }
}

trait TranslatedNewlineIterator {
    fn translate(&self, offset: TextUnit) -> TextUnit;
    fn translate_range(&self, range: TextRange) -> TextRange {
        TextRange::from_to(self.translate(range.start()), self.translate(range.end()))
    }
    fn next_translated(&mut self) -> Option<TextUnit>;
    fn boxed<'a>(self) -> Box<TranslatedNewlineIterator + 'a>
    where
        Self: 'a + Sized,
    {
        Box::new(self)
    }
}

#[derive(Debug)]
struct AltEdit<'a> {
    insert_newlines: OffsetNewlineIter<'a>,
    delete: TextRange,
    diff: i64,
}

fn translate_range_by(range: TextRange, diff: i64) -> TextRange {
    if diff == 0 {
        range
    } else {
        let start = translate_by(range.start(), diff);
        let end = translate_by(range.end(), diff);
        TextRange::from_to(start, end)
    }
}

fn translate_by(x: TextUnit, diff: i64) -> TextUnit {
    if diff == 0 {
        x
    } else {
        TextUnit::from((x.to_usize() as i64 + diff) as u32)
    }
}

fn to_alt_edits<'a>(offset: TextUnit, edits: &'a [AtomTextEdit]) -> Vec<AltEdit<'a>> {
    let mut xs: Vec<AltEdit<'a>> = Vec::with_capacity(edits.len());
    // collect and sort edits
    for edit in edits {
        // TODO discard after translating?
        // if edit.delete.start() >= offset {
        //     continue;
        // }
        let insert_index = xs.upper_bound_by_key(&edit.delete.start(), |x| x.delete.start());
        let diff = edit.insert.len() as i64 - edit.delete.len().to_usize() as i64;
        xs.insert(
            insert_index,
            AltEdit {
                insert_newlines: OffsetNewlineIter {
                    offset: edit.delete.start(),
                    text: &edit.insert,
                },
                delete: edit.delete,
                diff: diff,
            },
        );
    }
    // translate edits by previous edits
    for i in 1..xs.len() {
        let (x, prevs) = xs[0..=i].split_last_mut().unwrap();
        for prev in prevs {
            x.delete = translate_range_by(x.delete, prev.diff);
            x.insert_newlines.offset = translate_by(x.insert_newlines.offset, prev.diff);
        }
    }
    xs
}

#[derive(Debug)]
enum NextNewline {
    Use,
    Discard,
    Replace(TextUnit),
    New(TextUnit),
}

fn next_newline(candidate: Option<TextUnit>, edits: &mut [AltEdit]) -> NextNewline {
    let mut candidate = match candidate {
        None => {
            for edit in edits {
                if let Some(inserted) = edit.insert_newlines.next() {
                    return NextNewline::New(inserted);
                }
            }
            return NextNewline::Use; // END
        }
        Some(x) => x,
    };

    for edit in edits {
        if candidate <= edit.delete.start() {
            return NextNewline::Replace(candidate);
        } else if candidate <= edit.delete.end() {
            return match edit.insert_newlines.next() {
                Some(x) => NextNewline::Replace(x),
                None => NextNewline::Discard,
            };
        } else {
            if let Some(inserted) = edit.insert_newlines.next() {
                return NextNewline::New(inserted);
            }
            candidate = translate_by(candidate, edit.diff);
        }
    }
    return NextNewline::Replace(candidate);
}

fn count_newlines(offset: TextUnit, line_index: &LineIndex, edits: &[AtomTextEdit]) -> u32 {
    let mut edits = to_alt_edits(offset, edits);
    let mut orig_newlines = line_index.newlines().iter().map(|x| *x).peekable();

    let mut count = 0;

    loop {
        let res = next_newline(orig_newlines.peek().map(|x| *x), &mut edits);
        let next = match res {
            NextNewline::Use => orig_newlines.next(),
            NextNewline::Discard => {
                orig_newlines.next();
                continue;
            }
            NextNewline::Replace(new) => {
                orig_newlines.next();
                Some(new)
            }
            NextNewline::New(new) => Some(new),
        };
        match next {
            Some(n) if n <= offset => {
                count += 1;
            }
            _ => {
                break;
            }
        }
    }

    count
}

struct TranslatedAtomEdit<'a> {
    delete: TextRange,
    insert: &'a str,
}

struct TranslatedNewlines<'a, T: TranslatedNewlineIterator> {
    inner: T,
    next_inner: Option<TranslatedOffset>,
    edit: TranslatedAtomEdit<'a>,
    insert: OffsetNewlineIter<'a>,
}

impl<'a, T: TranslatedNewlineIterator> TranslatedNewlines<'a, T> {
    fn from(inner: T, edit: &'a AtomTextEdit) -> Self {
        let delete = inner.translate_range(edit.delete);
        let mut res = TranslatedNewlines {
            inner,
            next_inner: None,
            edit: TranslatedAtomEdit {
                delete,
                insert: &edit.insert,
            },
            insert: OffsetNewlineIter {
                offset: delete.start(),
                text: &edit.insert,
            },
        };
        // prepare next_inner
        res.advance_inner();
        res
    }

    fn advance_inner(&mut self) {
        self.next_inner = self
            .inner
            .next_translated()
            .map(|x| translate_offset(x, &self.edit));
    }
}

impl<'a, T: TranslatedNewlineIterator> TranslatedNewlineIterator for TranslatedNewlines<'a, T> {
    fn translate(&self, offset: TextUnit) -> TextUnit {
        let offset = self.inner.translate(offset);
        let (_, offset) =
            translate_offset(offset, &self.edit).expect("translate_unit returned None");
        offset
    }

    fn next_translated(&mut self) -> Option<TextUnit> {
        match self.next_inner {
            None => self.insert.next(),
            Some(next) => match next {
                None => self.insert.next().or_else(|| {
                    self.advance_inner();
                    self.next_translated()
                }),
                Some((TranslatedPos::Before, next)) => {
                    self.advance_inner();
                    Some(next)
                }
                Some((TranslatedPos::After, next)) => self.insert.next().or_else(|| {
                    self.advance_inner();
                    Some(next)
                }),
            },
        }
    }
}

impl<'a> Iterator for Box<dyn TranslatedNewlineIterator + 'a> {
    type Item = TextUnit;
    fn next(&mut self) -> Option<TextUnit> {
        self.next_translated()
    }
}

impl<T: TranslatedNewlineIterator + ?Sized> TranslatedNewlineIterator for Box<T> {
    fn translate(&self, offset: TextUnit) -> TextUnit {
        self.as_ref().translate(offset)
    }
    fn next_translated(&mut self) -> Option<TextUnit> {
        self.as_mut().next_translated()
    }
}

struct IteratorWrapper<T: Iterator<Item = TextUnit>>(T);

impl<T: Iterator<Item = TextUnit>> TranslatedNewlineIterator for IteratorWrapper<T> {
    fn translate(&self, offset: TextUnit) -> TextUnit {
        offset
    }
    fn next_translated(&mut self) -> Option<TextUnit> {
        self.0.next()
    }
}

impl<T: Iterator<Item = TextUnit>> Iterator for IteratorWrapper<T> {
    type Item = TextUnit;
    fn next(&mut self) -> Option<TextUnit> {
        self.0.next()
    }
}

fn translate_newlines<'a>(
    mut newlines: Box<TranslatedNewlineIterator + 'a>,
    edits: &'a [AtomTextEdit],
) -> Box<TranslatedNewlineIterator + 'a> {
    for edit in edits {
        newlines = TranslatedNewlines::from(newlines, edit).boxed();
    }
    newlines
}

pub fn translate_offset_with_edit(
    pre_edit_index: &LineIndex,
    offset: TextUnit,
    edits: &[AtomTextEdit],
) -> LineCol {
    let mut newlines: Box<TranslatedNewlineIterator> = Box::new(IteratorWrapper(
        pre_edit_index.newlines().iter().map(|x| *x),
    ));

    newlines = translate_newlines(newlines, edits);

    let mut line = 0;
    for n in newlines {
        if n > offset {
            break;
        }
        line += 1;
    }

    LineCol {
        line: line,
        col_utf16: 0, // TODO not implemented yet
    }
}

#[cfg(test)]
mod test {
    use proptest::{prelude::*, proptest, proptest_helper};
    use super::*;
    use ra_text_edit::test_utils::{arb_text, arb_offset, arb_edits};

    #[derive(Debug)]
    struct ArbTextWithOffsetAndEdits {
        text: String,
        offset: TextUnit,
        edits: Vec<AtomTextEdit>,
    }

    fn arb_text_with_offset_and_edits() -> BoxedStrategy<ArbTextWithOffsetAndEdits> {
        arb_text()
            .prop_flat_map(|text| {
                (arb_offset(&text), arb_edits(&text), Just(text)).prop_map(
                    |(offset, edits, text)| ArbTextWithOffsetAndEdits {
                        text,
                        offset,
                        edits,
                    },
                )
            })
            .boxed()
    }

    fn edit_text(pre_edit_text: &str, mut edits: Vec<AtomTextEdit>) -> String {
        // apply edits ordered from last to first
        // since they should not overlap we can just use start()
        edits.sort_by_key(|x| -(x.delete.start().to_usize() as isize));

        let mut text = pre_edit_text.to_owned();

        for edit in &edits {
            let range = edit.delete.start().to_usize()..edit.delete.end().to_usize();
            text.replace_range(range, &edit.insert);
        }

        text
    }

    fn translate_after_edit(
        pre_edit_text: &str,
        offset: TextUnit,
        edits: Vec<AtomTextEdit>,
    ) -> LineCol {
        let text = edit_text(pre_edit_text, edits);
        let line_index = LineIndex::new(&text);
        line_index.line_col(offset)
    }

    proptest! {
        #[test]
        fn test_translate_offset_with_edit(x in arb_text_with_offset_and_edits()) {
            let line_index = LineIndex::new(&x.text);
            let expected = translate_after_edit(&x.text, x.offset, x.edits.clone());
            let actual = translate_offset_with_edit(&line_index, x.offset, &x.edits);
            assert_eq!(actual.line, expected.line);
        }

        #[test]
        fn test_translate_offset_with_edit_alt(x in arb_text_with_offset_and_edits()) {
            let line_index = LineIndex::new(&x.text);
            let expected = translate_after_edit(&x.text, x.offset, x.edits.clone());
            let actual_lines = count_newlines(x.offset, &line_index, &x.edits);
            assert_eq!(actual_lines, expected.line);
        }
    }
}
<commit_msg>final iteration, faster a bit simpler the main thing is we iterate over inserted newlines at once for each edit<commit_after>use ra_text_edit::AtomTextEdit;
use ra_syntax::{TextUnit, TextRange};
use crate::{LineIndex, LineCol};
use superslice::Ext;

#[derive(Debug)]
struct OffsetNewlineIter<'a> {
    text: &'a str,
    offset: TextUnit,
}

impl<'a> Iterator for OffsetNewlineIter<'a> {
    type Item = TextUnit;
    fn next(&mut self) -> Option<TextUnit> {
        let next_idx = self
            .text
            .char_indices()
            .filter_map(|(i, c)| if c == '\n' { Some(i + 1) } else { None })
            .next()?;
        let next = self.offset + TextUnit::from_usize(next_idx);
        self.text = &self.text[next_idx..];
        self.offset = next;
        Some(next)
    }
}

#[derive(Debug)]
struct AltEdit<'a> {
    insert_newlines: OffsetNewlineIter<'a>,
    delete: TextRange,
    diff: i64,
}

fn translate_range_by(range: TextRange, diff: i64) -> TextRange {
    if diff == 0 {
        range
    } else {
        let start = translate_by(range.start(), diff);
        let end = translate_by(range.end(), diff);
        TextRange::from_to(start, end)
    }
}

fn translate_by(x: TextUnit, diff: i64) -> TextUnit {
    if diff == 0 {
        x
    } else {
        TextUnit::from((x.to_usize() as i64 + diff) as u32)
    }
}

fn to_alt_edits<'a>(offset: TextUnit, edits: &'a [AtomTextEdit]) -> Vec<AltEdit<'a>> {
    let mut xs: Vec<AltEdit<'a>> = Vec::with_capacity(edits.len());
    // collect and sort edits
    for edit in edits {
        // TODO discard after translating?
        // if edit.delete.start() >= offset {
        //     continue;
        // }
        let insert_index = xs.upper_bound_by_key(&edit.delete.start(), |x| x.delete.start());
        let diff = edit.insert.len() as i64 - edit.delete.len().to_usize() as i64;
        xs.insert(
            insert_index,
            AltEdit {
                insert_newlines: OffsetNewlineIter {
                    offset: edit.delete.start(),
                    text: &edit.insert,
                },
                delete: edit.delete,
                diff: diff,
            },
        );
    }
    // translate edits by previous edits
    for i in 1..xs.len() {
        let (x, prevs) = xs[0..=i].split_last_mut().unwrap();
        for prev in prevs {
            x.delete = translate_range_by(x.delete, prev.diff);
            x.insert_newlines.offset = translate_by(x.insert_newlines.offset, prev.diff);
        }
    }
    xs
}

#[derive(Debug)]
enum NextNewline {
    Use,
    Discard,
    Replace(TextUnit),
    New(TextUnit),
}

fn next_newline(candidate: Option<TextUnit>, edits: &mut [AltEdit]) -> NextNewline {
    let mut candidate = match candidate {
        None => {
            for edit in edits {
                if let Some(inserted) = edit.insert_newlines.next() {
                    return NextNewline::New(inserted);
                }
            }
            return NextNewline::Use; // END
        }
        Some(x) => x,
    };

    for edit in edits {
        if candidate <= edit.delete.start() {
            return NextNewline::Replace(candidate);
        } else if candidate <= edit.delete.end() {
            return match edit.insert_newlines.next() {
                Some(x) => NextNewline::Replace(x),
                None => NextNewline::Discard,
            };
        } else {
            if let Some(inserted) = edit.insert_newlines.next() {
                return NextNewline::New(inserted);
            }
            candidate = translate_by(candidate, edit.diff);
        }
    }
    return NextNewline::Replace(candidate);
}

pub fn count_newlines(offset: TextUnit, line_index: &LineIndex, edits: &[AtomTextEdit]) -> u32 {
    let mut edits = to_alt_edits(offset, edits);
    let mut orig_newlines = line_index.newlines().iter().map(|x| *x).peekable();

    let mut count = 0;

    loop {
        let res = next_newline(orig_newlines.peek().map(|x| *x), &mut edits);
        let next = match res {
            NextNewline::Use => orig_newlines.next(),
            NextNewline::Discard => {
                orig_newlines.next();
                continue;
            }
            NextNewline::Replace(new) => {
                orig_newlines.next();
                Some(new)
            }
            NextNewline::New(new) => Some(new),
        };
        match next {
            Some(n) if n <= offset => {
                count += 1;
            }
            _ => {
                break;
            }
        }
    }

    count
}

#[derive(Debug)]
enum NextNewlines<'a> {
    Use,
    ReplaceMany(OffsetNewlineIter<'a>),
    AddMany(OffsetNewlineIter<'a>),
}

#[derive(Debug)]
struct TranslatedEdit<'a> {
    delete: TextRange,
    insert: &'a str,
    diff: i64,
}

struct Edits<'a, 'b> {
    edits: &'b [&'a AtomTextEdit],
    current: Option<TranslatedEdit<'a>>,
    acc_diff: i64,
}

impl<'a, 'b> Edits<'a, 'b> {
    fn new(sorted_edits: &'b [&'a AtomTextEdit]) -> Edits<'a, 'b> {
        let mut x = Edits {
            edits: sorted_edits,
            current: None,
            acc_diff: 0,
        };
        x.advance_edit();
        x
    }
    fn advance_edit(&mut self) {
        self.acc_diff += self.current.as_ref().map_or(0, |x| x.diff);
        match self.edits.split_first() {
            Some((next, rest)) => {
                let delete = translate_range_by(next.delete, self.acc_diff);
                let diff = next.insert.len() as i64 - next.delete.len().to_usize() as i64;
                self.current = Some(TranslatedEdit {
                    delete,
                    insert: &next.insert,
                    diff,
                });
                self.edits = rest;
            }
            None => {
                self.current = None;
            }
        }
    }

    fn next_inserted_newlines(&mut self) -> Option<OffsetNewlineIter<'a>> {
        let cur = self.current.as_ref()?;
        let res = Some(OffsetNewlineIter {
            offset: cur.delete.start(),
            text: &cur.insert,
        });
        self.advance_edit();
        res
    }

    fn next_newlines(&mut self, candidate: TextUnit) -> NextNewlines {
        let res = match &mut self.current {
            Some(edit) => {
                if candidate <= edit.delete.start() {
                    NextNewlines::Use
                } else if candidate <= edit.delete.end() {
                    let iter = OffsetNewlineIter {
                        offset: edit.delete.start(),
                        text: &edit.insert,
                    };
                    // empty slice
                    edit.insert = &edit.insert[edit.insert.len()..];
                    NextNewlines::ReplaceMany(iter)
                } else {
                    let iter = OffsetNewlineIter {
                        offset: edit.delete.start(),
                        text: &edit.insert,
                    };
                    // empty slice
                    edit.insert = &edit.insert[edit.insert.len()..];
                    self.advance_edit();
                    NextNewlines::AddMany(iter)
                }
            }
            None => NextNewlines::Use,
        };
        res
    }
}

pub fn count_newlines_alt(offset: TextUnit, line_index: &LineIndex, edits: &[AtomTextEdit]) -> u32 {
    let mut sorted_edits: Vec<&AtomTextEdit> = Vec::with_capacity(edits.len());
    for edit in edits {
        let insert_index =
            sorted_edits.upper_bound_by_key(&edit.delete.start(), |x| x.delete.start());
        sorted_edits.insert(insert_index, &edit);
    }

    let mut state = Edits::new(&sorted_edits);

    let mut lines: u32 = 0;

    for &orig_newline in line_index.newlines() {
        loop {
            let translated_newline = translate_by(orig_newline, state.acc_diff);
            match state.next_newlines(translated_newline) {
                NextNewlines::Use => {
                    if offset < translated_newline {
                        return lines;
                    } else {
                        lines += 1;
                    }
                    break;
                }
                NextNewlines::ReplaceMany(ns) => {
                    for n in ns {
                        if offset < n {
                            return lines;
                        } else {
                            lines += 1;
                        }
                    }
                    break;
                }
                NextNewlines::AddMany(ns) => {
                    for n in ns {
                        if offset < n {
                            return lines;
                        } else {
                            lines += 1;
                        }
                    }
                }
            }
        }
    }

    loop {
        match state.next_inserted_newlines() {
            None => break,
            Some(ns) => {
                for n in ns {
                    if offset < n {
                        return lines;
                    } else {
                        lines += 1;
                    }
                }
            }
        }
    }

    lines
}

// for bench
pub fn translate_after_edit(
    pre_edit_text: &str,
    offset: TextUnit,
    edits: Vec<AtomTextEdit>,
) -> LineCol {
    let text = edit_text(pre_edit_text, edits);
    let line_index = LineIndex::new(&text);
    line_index.line_col(offset)
}

fn edit_text(pre_edit_text: &str, mut edits: Vec<AtomTextEdit>) -> String {
    // apply edits ordered from last to first
    // since they should not overlap we can just use start()
    edits.sort_by_key(|x| -(x.delete.start().to_usize() as isize));

    let mut text = pre_edit_text.to_owned();

    for edit in &edits {
        let range = edit.delete.start().to_usize()..edit.delete.end().to_usize();
        text.replace_range(range, &edit.insert);
    }

    text
}

#[cfg(test)]
mod test {
    use proptest::{prelude::*, proptest, proptest_helper};
    use super::*;
    use ra_text_edit::test_utils::{arb_text, arb_offset, arb_edits};

    #[derive(Debug)]
    struct ArbTextWithOffsetAndEdits {
        text: String,
        offset: TextUnit,
        edits: Vec<AtomTextEdit>,
    }

    fn arb_text_with_offset_and_edits() -> BoxedStrategy<ArbTextWithOffsetAndEdits> {
        arb_text()
            .prop_flat_map(|text| {
                (arb_offset(&text), arb_edits(&text), Just(text)).prop_map(
                    |(offset, edits, text)| ArbTextWithOffsetAndEdits {
                        text,
                        offset,
                        edits,
                    },
                )
            })
            .boxed()
    }

    proptest! {
        #[test]
        fn test_translate_offset_with_edit(x in arb_text_with_offset_and_edits()) {
            let line_index = LineIndex::new(&x.text);
            let expected = translate_after_edit(&x.text, x.offset, x.edits.clone());
            let actual_lines = count_newlines(x.offset, &line_index, &x.edits);
            assert_eq!(actual_lines, expected.line);
        }

        #[test]
        fn test_translate_offset_with_edit_alt(x in arb_text_with_offset_and_edits()) {
            let line_index = LineIndex::new(&x.text);
            let expected = translate_after_edit(&x.text, x.offset, x.edits.clone());
            let actual_lines = count_newlines_alt(x.offset, &line_index, &x.edits);
            assert_eq!(actual_lines, expected.line);
        }
    }

    #[test]
    fn test_translate_offset_with_edit_alt_1() {
        let x = ArbTextWithOffsetAndEdits {
            text: String::from("aA\n"),
            offset: 2.into(),
            edits: vec![AtomTextEdit::delete(TextRange::from_to(1.into(), 2.into()))],
        };
        let line_index = LineIndex::new(&x.text);
        let expected = translate_after_edit(&x.text, x.offset, x.edits.clone());
        let actual_lines = count_newlines_alt(x.offset, &line_index, &x.edits);
        assert_eq!(actual_lines, expected.line);
    }

    #[test]
    fn test_translate_offset_with_edit_alt_2() {
        let x = ArbTextWithOffsetAndEdits {
            text: String::from("\nqꀸ#"),
            offset: 5.into(),
            edits: vec![AtomTextEdit::insert(1.into(), "\n".into())],
        };
        let line_index = LineIndex::new(&x.text);
        let expected = translate_after_edit(&x.text, x.offset, x.edits.clone());
        let actual_lines = count_newlines_alt(x.offset, &line_index, &x.edits);
        assert_eq!(actual_lines, expected.line);
    }

    #[test]
    fn test_translate_offset_with_edit_alt_3() {
        let x = ArbTextWithOffsetAndEdits {
            text: String::from("\n\n\n"),
            offset: 0.into(),
            edits: vec![AtomTextEdit::delete(TextRange::from_to(0.into(), 2.into()))],
        };
        let line_index = LineIndex::new(&x.text);
        let expected = translate_after_edit(&x.text, x.offset, x.edits.clone());
        let actual_lines = count_newlines_alt(x.offset, &line_index, &x.edits);
        assert_eq!(actual_lines, expected.line);
    }

    #[test]
    fn test_translate_offset_with_edit_alt_4() {
        let x = ArbTextWithOffsetAndEdits {
            text: String::from("☻54翑\"A"),
            offset: 5.into(),
            edits: vec![
                AtomTextEdit::delete(TextRange::from_to(0.into(), 8.into())),
                AtomTextEdit::insert(9.into(), String::from("\n")),
            ],
        };
        let line_index = LineIndex::new(&x.text);
        let expected = translate_after_edit(&x.text, x.offset, x.edits.clone());
        let actual_lines = count_newlines_alt(x.offset, &line_index, &x.edits);
        assert_eq!(actual_lines, expected.line);
    }

}
<|endoftext|>


<commit_before>#[macro_use] extern crate log;
#[macro_use] extern crate nom;
extern crate env_logger;
extern crate bytes;
extern crate mio;
extern crate rustc_serialize;
extern crate yxorp;
extern crate toml;

mod command;
mod state;

use std::net::{UdpSocket,ToSocketAddrs};
use std::sync::mpsc::{channel};
use std::collections::HashMap;
use yxorp::network;
use yxorp::network::metrics::{METRICS,ProxyMetrics};

fn main() {
  env_logger::init().unwrap();
  info!("starting up");
  let metrics_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
  let metrics_host   = ("192.168.59.103", 8125).to_socket_addrs().unwrap().next().unwrap();
  METRICS.lock().unwrap().set_up_remote(metrics_socket, metrics_host);
  let metrics_guard = ProxyMetrics::run();

  let (sender, _) = channel::<network::ServerMessage>();
  let (tx, jg) = network::http::start_listener("127.0.0.1:8080".parse().unwrap(), 500, sender);

  let (sender2, _) = channel::<network::ServerMessage>();
  let (tx2, jg2) = network::tls::start_listener("127.0.0.1:8443".parse().unwrap(), 500, sender2);

  let mut listeners = HashMap::new();
  listeners.insert(String::from("HTTP"), tx);
  listeners.insert(String::from("TLS"),  tx2);
  command::start(String::from("./command_folder"), listeners);
  let _ = jg.join();
  info!("good bye");
}

<commit_msg>fix compilation<commit_after>#[macro_use] extern crate log;
#[macro_use] extern crate nom;
extern crate env_logger;
extern crate bytes;
extern crate mio;
extern crate rustc_serialize;
extern crate yxorp;
extern crate toml;

mod command;
mod state;

use std::net::{UdpSocket,ToSocketAddrs};
use std::sync::mpsc::{channel};
use std::collections::HashMap;
use yxorp::network;
use yxorp::network::metrics::{METRICS,ProxyMetrics};

fn main() {
  env_logger::init().unwrap();
  info!("starting up");
  let metrics_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
  let metrics_host   = ("192.168.59.103", 8125).to_socket_addrs().unwrap().next().unwrap();
  METRICS.lock().unwrap().set_up_remote(metrics_socket, metrics_host);
  let metrics_guard = ProxyMetrics::run();

  let (sender, _) = channel::<network::ServerMessage>();
  let (tx, jg) = network::http::start_listener("127.0.0.1:8080".parse().unwrap(), 500, sender);

  let (sender2, _) = channel::<network::ServerMessage>();
  let (tx2, jg2) = network::tls::start_listener("127.0.0.1:8443".parse().unwrap(), 500, None, sender2);

  let mut listeners = HashMap::new();
  listeners.insert(String::from("HTTP"), tx);
  listeners.insert(String::from("TLS"),  tx2);
  command::start(String::from("./command_folder"), listeners);
  let _ = jg.join();
  info!("good bye");
}

<|endoftext|>


<commit_before>use address::{LENGTH, Address, Addressable};
use node::Node;
use num::bigint::{BigUint, ToBigUint};
use std::collections::HashMap;
use std::fmt;
use num;

#[derive(Debug, PartialEq)]
pub enum InsertOutcome {
    Inserted,   // Inserted new node
    Updated,    // Updated existing node
    Discarded   // Bucket is full
}

pub type InsertionResult = Result<InsertOutcome, String>;

pub struct NodeBucket {
    k: usize,
    min: BigUint,
    max: BigUint,
    addresses: Vec<Address>,
    nodes: HashMap<Address, Box<Node>>
}

impl fmt::Debug for NodeBucket {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "NodeBucket {{ {:040x} - {:040x}, {} }}", self.min, self.max, self.addresses.len())
    }
}

impl NodeBucket {
    pub fn new(k: usize) -> NodeBucket {
        let min = 0.to_biguint().unwrap();
        let max = num::pow(2.to_biguint().unwrap(), LENGTH);
        NodeBucket {
            k: k,
            min: min,
            max: max,
            addresses: Vec::with_capacity(k),
            nodes: HashMap::with_capacity(k)
        }
    }

    pub fn contains(&self, address: &Address) -> bool {
        self.addresses.contains(address)
    }

    pub fn covers(&self, address: &Address) -> bool {
        let numeric = address.as_numeric();
        self.min <= numeric && numeric < self.max
    }

    pub fn get_nodes(&self) -> Vec<&Box<Node>> {
        self.nodes.iter().map(|(_, node)| node).collect()
    }

    pub fn insert(&mut self, node: Box<Node>) -> InsertionResult {
        let address = node.get_address();
        if self.covers(&address) {
            if let Some(pos) = self.addresses.iter().position(|&a| a == address) {
                self.addresses.remove(pos);
                self.addresses.insert(0, address);
                self.nodes.insert(address, node);
                Ok(InsertOutcome::Updated)
            } else if !self.is_full() {
                self.addresses.insert(0, address);
                self.nodes.insert(address, node);
                Ok(InsertOutcome::Inserted)
            } else {
                Ok(InsertOutcome::Discarded)
            }
        } else {
            Err(format!("Bucket {:?}  does not cover {:?}", self, address))
        }
    }

    pub fn is_full(&self) -> bool {
        self.addresses.len() >= self.k
    }

    pub fn split(&mut self) -> (Self, Self) {
        let difference = self.max.clone() - self.min.clone();
        let partition = difference / 2.to_biguint().unwrap() + self.min.clone();
        let addresses = self.addresses.clone();
        let (a_addresses, b_addresses): (Vec<Address>, Vec<Address>) = addresses
            .into_iter()
            .partition(|&a| a.as_numeric() < partition);
        let mut a_nodes = HashMap::new();
        let mut b_nodes = HashMap::new();
        for (address, node) in self.nodes.drain() {
            if address.as_numeric() < partition {
                a_nodes.insert(address, node);
            } else {
                b_nodes.insert(address, node);
            }
        }

        let a = NodeBucket {
            k: self.k,
            min: self.min.clone(),
            max: partition.clone(),
            addresses: a_addresses,
            nodes: a_nodes
        };
        let b = NodeBucket {
            k: self.k,
            min: partition.clone(),
            max: self.max.clone(),
            addresses: b_addresses,
            nodes: b_nodes
        };
        (a, b)
    }
}

#[cfg(test)]
mod tests {
    use address::{Address, Addressable};
    use node::{Node, Serialize};
    use messages;
    use super::{InsertOutcome, NodeBucket};

    #[derive(Debug)]
    struct TestNode {
        pub address: Address
    }

    impl TestNode {
        fn new(address: Address) -> TestNode {
            TestNode { address: address }
        }
    }

    impl Addressable for TestNode {
        fn get_address(&self) -> Address {
            self.address
        }
    }

    impl Node for TestNode {
        fn update(&mut self) { }
        fn send(&self, _: Vec<u8>) { }
    }

    impl Serialize for TestNode {
        fn serialize(&self) -> messages::protobufs::Node {
            messages::protobufs::Node::new()
        }
    }

    #[test]
    fn test_insert() {
        let mut bucket: NodeBucket = NodeBucket::new(8);
        let node = Box::new(TestNode::new(Address::for_content("some string")));
        assert_eq!(bucket.insert(node).unwrap(), InsertOutcome::Inserted);
        assert_eq!(bucket.addresses.len(), 1);
    }

    #[test]
    fn test_insert_duplicate() {
        let mut bucket: NodeBucket = NodeBucket::new(8);
        let addr_1 = Address::for_content("node 1");
        let addr_2 = Address::for_content("node 2");
        let addr_3 = Address::for_content("node 1");
        let a = Box::new(TestNode::new(addr_1));
        let b = Box::new(TestNode::new(addr_2));
        let c = Box::new(TestNode::new(addr_3));

        bucket.insert(a).unwrap();
        bucket.insert(b).unwrap();
        assert_eq!(bucket.insert(c).unwrap(), InsertOutcome::Updated);

        assert_eq!(bucket.addresses.len(), 2);
        assert_eq!(bucket.nodes[&Address::for_content("node 1")].get_address(), addr_1);
    }

    #[test]
    fn test_insert_full() {
        let mut bucket: NodeBucket = NodeBucket::new(2);
        bucket.insert(Box::new(TestNode::new(Address::for_content("node 1")))).unwrap();
        bucket.insert(Box::new(TestNode::new(Address::for_content("node 2")))).unwrap();
        let result = bucket.insert(Box::new(TestNode::new(Address::for_content("node 3")))).unwrap();
        assert_eq!(result, InsertOutcome::Discarded);
        assert_eq!(bucket.addresses.len(), 2);
        assert_eq!(bucket.nodes.get(&Address::for_content("node 3")), None);
    }

    #[test]
    fn insert_node_outside_of_address_space() {
        use num::bigint::ToBigUint;
        use std::collections::HashMap;

        let mut bucket = NodeBucket {
            k: 8,
            min: 0.to_biguint().unwrap(),
            max: 1.to_biguint().unwrap(),
            addresses: vec![],
            nodes: HashMap::new()
        };
        let result = bucket.insert(Box::new(TestNode::new(Address::for_content("node 3"))));
        assert!(result.is_err());
    }

    #[test]
    fn test_is_full() {
        let mut bucket: NodeBucket = NodeBucket::new(2);
        assert!(!bucket.is_full());

        bucket.insert(Box::new(TestNode::new(Address::for_content("node 1")))).unwrap();
        bucket.insert(Box::new(TestNode::new(Address::for_content("node 2")))).unwrap();
        assert!(bucket.is_full());
    }

    #[test]
    fn test_contains() {
        let mut bucket: NodeBucket = NodeBucket::new(8);
        bucket.insert(Box::new(TestNode::new(Address::for_content("node 1")))).unwrap();
        assert!(bucket.contains(&Address::for_content("node 1")));
    }

    #[test]
    fn test_covers() {
        let bucket: NodeBucket = NodeBucket::new(8);
        let address_1 = Address::from_str("0000000000000000000000000000000000000000");
        let address_2 = Address::from_str("ffffffffffffffffffffffffffffffffffffffff");
        assert!(bucket.covers(&address_1));
        assert!(bucket.covers(&address_2));
    }

    #[test]
    fn test_split() {
        let mut bucket: NodeBucket = NodeBucket::new(4);
        let addr_1 = Address::from_str("0000000000000000000000000000000000000000");
        let addr_2 = Address::from_str("7fffffffffffffffffffffffffffffffffffffff");
        let addr_3 = Address::from_str("8000000000000000000000000000000000000000");
        let addr_4 = Address::from_str("ffffffffffffffffffffffffffffffffffffffff");
        let node_1 = Box::new(TestNode::new(addr_1));
        let node_2 = Box::new(TestNode::new(addr_2));
        let node_3 = Box::new(TestNode::new(addr_3));
        let node_4 = Box::new(TestNode::new(addr_4));

        bucket.insert(node_1).unwrap();
        bucket.insert(node_2).unwrap();
        bucket.insert(node_3).unwrap();
        bucket.insert(node_4).unwrap();

        let (a, b) = bucket.split();

        // Splits up known address
        assert_eq!(a.addresses.len(), 2);
        assert_eq!(b.addresses.len(), 2);

        // Splits up known nodes
        assert_eq!(a.nodes[&Address::from_str("0000000000000000000000000000000000000000")].get_address(), addr_1);
        assert_eq!(a.nodes[&Address::from_str("0000000000000000000000000000000000000000")].get_address(), addr_1);
        assert_eq!(a.nodes[&Address::from_str("7fffffffffffffffffffffffffffffffffffffff")].get_address(), addr_2);
        assert_eq!(b.nodes[&Address::from_str("8000000000000000000000000000000000000000")].get_address(), addr_3);
        assert_eq!(b.nodes[&Address::from_str("ffffffffffffffffffffffffffffffffffffffff")].get_address(), addr_4);

        // Equitably covers address space
        assert!(a.covers(&Address::from_str("0000000000000000000000000000000000000000")));
        assert!(!b.covers(&Address::from_str("0000000000000000000000000000000000000000")));
        assert!(a.covers(&Address::from_str("7fffffffffffffffffffffffffffffffffffffff")));
        assert!(!b.covers(&Address::from_str("7fffffffffffffffffffffffffffffffffffffff")));
        assert!(!a.covers(&Address::from_str("8000000000000000000000000000000000000000")));
        assert!(b.covers(&Address::from_str("8000000000000000000000000000000000000000")));
        assert!(!a.covers(&Address::from_str("ffffffffffffffffffffffffffffffffffffffff")));
        assert!(b.covers(&Address::from_str("ffffffffffffffffffffffffffffffffffffffff")));
    }
}
<commit_msg>Don't re-insert existing nodes<commit_after>use address::{LENGTH, Address, Addressable};
use node::Node;
use num::bigint::{BigUint, ToBigUint};
use std::collections::HashMap;
use std::fmt;
use num;

#[derive(Debug, PartialEq)]
pub enum InsertOutcome {
    Inserted,   // Inserted new node
    Updated,    // Updated existing node
    Discarded   // Bucket is full
}

pub type InsertionResult = Result<InsertOutcome, String>;

pub struct NodeBucket {
    k: usize,
    min: BigUint,
    max: BigUint,
    addresses: Vec<Address>,
    nodes: HashMap<Address, Box<Node>>
}

impl fmt::Debug for NodeBucket {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "NodeBucket {{ {:040x} - {:040x}, {} }}", self.min, self.max, self.addresses.len())
    }
}

impl NodeBucket {
    pub fn new(k: usize) -> NodeBucket {
        let min = 0.to_biguint().unwrap();
        let max = num::pow(2.to_biguint().unwrap(), LENGTH);
        NodeBucket {
            k: k,
            min: min,
            max: max,
            addresses: Vec::with_capacity(k),
            nodes: HashMap::with_capacity(k)
        }
    }

    pub fn contains(&self, address: &Address) -> bool {
        self.addresses.contains(address)
    }

    pub fn covers(&self, address: &Address) -> bool {
        let numeric = address.as_numeric();
        self.min <= numeric && numeric < self.max
    }

    pub fn get_nodes(&self) -> Vec<&Box<Node>> {
        self.nodes.iter().map(|(_, node)| node).collect()
    }

    pub fn insert(&mut self, node: Box<Node>) -> InsertionResult {
        let address = node.get_address();
        if self.covers(&address) {
            if let Some(pos) = self.addresses.iter().position(|&a| a == address) {
                self.addresses.remove(pos);
                self.addresses.insert(0, address);
                Ok(InsertOutcome::Updated)
                // TODO: Should this update the node's socket address incase a devise has changed
                // IPs?
            } else if !self.is_full() {
                self.addresses.insert(0, address);
                self.nodes.insert(address, node);
                Ok(InsertOutcome::Inserted)
            } else {
                Ok(InsertOutcome::Discarded)
            }
        } else {
            Err(format!("Bucket {:?}  does not cover {:?}", self, address))
        }
    }

    pub fn is_full(&self) -> bool {
        self.addresses.len() >= self.k
    }

    pub fn split(&mut self) -> (Self, Self) {
        let difference = self.max.clone() - self.min.clone();
        let partition = difference / 2.to_biguint().unwrap() + self.min.clone();
        let addresses = self.addresses.clone();
        let (a_addresses, b_addresses): (Vec<Address>, Vec<Address>) = addresses
            .into_iter()
            .partition(|&a| a.as_numeric() < partition);
        let mut a_nodes = HashMap::new();
        let mut b_nodes = HashMap::new();
        for (address, node) in self.nodes.drain() {
            if address.as_numeric() < partition {
                a_nodes.insert(address, node);
            } else {
                b_nodes.insert(address, node);
            }
        }

        let a = NodeBucket {
            k: self.k,
            min: self.min.clone(),
            max: partition.clone(),
            addresses: a_addresses,
            nodes: a_nodes
        };
        let b = NodeBucket {
            k: self.k,
            min: partition.clone(),
            max: self.max.clone(),
            addresses: b_addresses,
            nodes: b_nodes
        };
        (a, b)
    }
}

#[cfg(test)]
mod tests {
    use address::{Address, Addressable};
    use node::{Node, Serialize};
    use messages;
    use super::{InsertOutcome, NodeBucket};

    #[derive(Debug)]
    struct TestNode {
        pub address: Address
    }

    impl TestNode {
        fn new(address: Address) -> TestNode {
            TestNode { address: address }
        }
    }

    impl Addressable for TestNode {
        fn get_address(&self) -> Address {
            self.address
        }
    }

    impl Node for TestNode {
        fn update(&mut self) { }
        fn send(&self, _: Vec<u8>) { }
    }

    impl Serialize for TestNode {
        fn serialize(&self) -> messages::protobufs::Node {
            messages::protobufs::Node::new()
        }
    }

    #[test]
    fn test_insert() {
        let mut bucket: NodeBucket = NodeBucket::new(8);
        let node = Box::new(TestNode::new(Address::for_content("some string")));
        assert_eq!(bucket.insert(node).unwrap(), InsertOutcome::Inserted);
        assert_eq!(bucket.addresses.len(), 1);
    }

    #[test]
    fn test_insert_duplicate() {
        let mut bucket: NodeBucket = NodeBucket::new(8);
        let addr_1 = Address::for_content("node 1");
        let addr_2 = Address::for_content("node 2");
        let addr_3 = Address::for_content("node 1");
        let a = Box::new(TestNode::new(addr_1));
        let b = Box::new(TestNode::new(addr_2));
        let c = Box::new(TestNode::new(addr_3));

        bucket.insert(a).unwrap();
        bucket.insert(b).unwrap();
        assert_eq!(bucket.insert(c).unwrap(), InsertOutcome::Updated);

        assert_eq!(bucket.addresses.len(), 2);
        assert_eq!(bucket.nodes[&Address::for_content("node 1")].get_address(), addr_1);
    }

    #[test]
    fn test_insert_full() {
        let mut bucket: NodeBucket = NodeBucket::new(2);
        bucket.insert(Box::new(TestNode::new(Address::for_content("node 1")))).unwrap();
        bucket.insert(Box::new(TestNode::new(Address::for_content("node 2")))).unwrap();
        let result = bucket.insert(Box::new(TestNode::new(Address::for_content("node 3")))).unwrap();
        assert_eq!(result, InsertOutcome::Discarded);
        assert_eq!(bucket.addresses.len(), 2);
        assert_eq!(bucket.nodes.get(&Address::for_content("node 3")), None);
    }

    #[test]
    fn insert_node_outside_of_address_space() {
        use num::bigint::ToBigUint;
        use std::collections::HashMap;

        let mut bucket = NodeBucket {
            k: 8,
            min: 0.to_biguint().unwrap(),
            max: 1.to_biguint().unwrap(),
            addresses: vec![],
            nodes: HashMap::new()
        };
        let result = bucket.insert(Box::new(TestNode::new(Address::for_content("node 3"))));
        assert!(result.is_err());
    }

    #[test]
    fn test_is_full() {
        let mut bucket: NodeBucket = NodeBucket::new(2);
        assert!(!bucket.is_full());

        bucket.insert(Box::new(TestNode::new(Address::for_content("node 1")))).unwrap();
        bucket.insert(Box::new(TestNode::new(Address::for_content("node 2")))).unwrap();
        assert!(bucket.is_full());
    }

    #[test]
    fn test_contains() {
        let mut bucket: NodeBucket = NodeBucket::new(8);
        bucket.insert(Box::new(TestNode::new(Address::for_content("node 1")))).unwrap();
        assert!(bucket.contains(&Address::for_content("node 1")));
    }

    #[test]
    fn test_covers() {
        let bucket: NodeBucket = NodeBucket::new(8);
        let address_1 = Address::from_str("0000000000000000000000000000000000000000");
        let address_2 = Address::from_str("ffffffffffffffffffffffffffffffffffffffff");
        assert!(bucket.covers(&address_1));
        assert!(bucket.covers(&address_2));
    }

    #[test]
    fn test_split() {
        let mut bucket: NodeBucket = NodeBucket::new(4);
        let addr_1 = Address::from_str("0000000000000000000000000000000000000000");
        let addr_2 = Address::from_str("7fffffffffffffffffffffffffffffffffffffff");
        let addr_3 = Address::from_str("8000000000000000000000000000000000000000");
        let addr_4 = Address::from_str("ffffffffffffffffffffffffffffffffffffffff");
        let node_1 = Box::new(TestNode::new(addr_1));
        let node_2 = Box::new(TestNode::new(addr_2));
        let node_3 = Box::new(TestNode::new(addr_3));
        let node_4 = Box::new(TestNode::new(addr_4));

        bucket.insert(node_1).unwrap();
        bucket.insert(node_2).unwrap();
        bucket.insert(node_3).unwrap();
        bucket.insert(node_4).unwrap();

        let (a, b) = bucket.split();

        // Splits up known address
        assert_eq!(a.addresses.len(), 2);
        assert_eq!(b.addresses.len(), 2);

        // Splits up known nodes
        assert_eq!(a.nodes[&Address::from_str("0000000000000000000000000000000000000000")].get_address(), addr_1);
        assert_eq!(a.nodes[&Address::from_str("0000000000000000000000000000000000000000")].get_address(), addr_1);
        assert_eq!(a.nodes[&Address::from_str("7fffffffffffffffffffffffffffffffffffffff")].get_address(), addr_2);
        assert_eq!(b.nodes[&Address::from_str("8000000000000000000000000000000000000000")].get_address(), addr_3);
        assert_eq!(b.nodes[&Address::from_str("ffffffffffffffffffffffffffffffffffffffff")].get_address(), addr_4);

        // Equitably covers address space
        assert!(a.covers(&Address::from_str("0000000000000000000000000000000000000000")));
        assert!(!b.covers(&Address::from_str("0000000000000000000000000000000000000000")));
        assert!(a.covers(&Address::from_str("7fffffffffffffffffffffffffffffffffffffff")));
        assert!(!b.covers(&Address::from_str("7fffffffffffffffffffffffffffffffffffffff")));
        assert!(!a.covers(&Address::from_str("8000000000000000000000000000000000000000")));
        assert!(b.covers(&Address::from_str("8000000000000000000000000000000000000000")));
        assert!(!a.covers(&Address::from_str("ffffffffffffffffffffffffffffffffffffffff")));
        assert!(b.covers(&Address::from_str("ffffffffffffffffffffffffffffffffffffffff")));
    }
}
<|endoftext|>


<commit_before>// Bloom
//
// HTTP REST API caching middleware
// Copyright: 2017, Valerian Saliou <valerian@valeriansaliou.name>
// License: Mozilla Public License v2.0 (MPL v2.0)

use futures::future;
use futures::future::FutureResult;
use hyper;
use hyper::{Method, StatusCode};
use hyper::server::{Request, Response};

use super::header::ProxyHeader;
use super::tunnel::ProxyTunnelBuilder;
use header::request_shard::HeaderRequestBloomRequestShard;
use header::status::HeaderBloomStatusValue;
use cache::read::CacheRead;
use cache::write::CacheWrite;
use cache::route::CacheRoute;

pub struct ProxyServeBuilder;

pub struct ProxyServe;

pub type ProxyServeFuture = FutureResult<Response, hyper::Error>;

impl ProxyServeBuilder {
    pub fn new() -> ProxyServe {
        ProxyServe {}
    }
}

impl ProxyServe {
    pub fn handle(&self, req: Request) -> ProxyServeFuture {
        info!("handled request: {} on {}", req.method(), req.path());

        let mut res = Response::new();

        if req.headers().has::<HeaderRequestBloomRequestShard>() == true {
            match *req.method() {
                Method::Options
                | Method::Head
                | Method::Get
                | Method::Post
                | Method::Patch
                | Method::Put
                | Method::Delete => {
                    self.accept(&req, &mut res)
                }
                _ => {
                    self.reject(&req, &mut res, StatusCode::MethodNotAllowed)
                }
            }
        } else {
            self.reject(&req, &mut res, StatusCode::NotExtended)
        }

        future::ok(res)
    }

    fn accept(&self, req: &Request, res: &mut Response) {
        self.tunnel(req, res);
    }

    fn reject(&self, req: &Request, res: &mut Response, status: StatusCode) {
        res.set_status(status);

        match *req.method() {
            Method::Get | Method::Post | Method::Patch | Method::Put => {
                res.set_body(format!("{}", status));
            }
            _ => {}
        }
    }

    fn tunnel(&self, req: &Request, res: &mut Response) {
        let (auth, shard) = ProxyHeader::parse_from_request(req.headers());

        let ns = CacheRoute::gen_ns(shard, auth, req.version(), req.method(),
            req.path(), req.query());

        // TODO: support for 304 Not Modified here (return empty content \
        //   to ongoing specific client, but still read/populate cache normally)

        // TODO: implement support for Bloom-Response-Bucket
        // CONCERN: how to link this to the gen_ns() utility? We dont \
        //   know about which route is mapped to which bucket in advance. \
        //   so maybe redesign this part.

        info!("tunneling for ns = {}", ns);

        match CacheRead::acquire(ns.as_ref()) {
            Ok(cached_value) => {
                self.dispatch_cached(res, cached_value)
            },
            Err(_) => {
                // TODO -> connect to API using ConfigProxy[:shard].inet
                    // TODO -> enforce timeouts:
                        //   - ConfigProxy.tunnel_connect_timeout
                        //   - ConfigProxy.tunnel_read_timeout
                        //   - ConfigProxy.tunnel_send_timeout
                // TODO -> CacheWrite::save(ns, req, res) (check return value)
                    // TODO -> return == true -> set 'Bloom-Status' as 'MISS'
                    // TODO -> return == false -> set 'Bloom-Status' as 'DIRECT'

                // Acquire response object from client to API

                // TODO: move this to a common factory, ie global.
                // CRITICAL: avoid spawning new threads and destroying them \
                //   for each connection.
                let mut tunnel = ProxyTunnelBuilder::new();

                match tunnel.run(req.method()) {
                    Ok(tunnel_res) => {
                        if CacheWrite::save(ns.as_ref(), req, &tunnel_res) ==
                            true {
                            self.dispatch_direct(res, tunnel_res,
                                HeaderBloomStatusValue::Miss)
                        } else {
                            self.dispatch_direct(res, tunnel_res,
                                HeaderBloomStatusValue::Direct)
                        }
                    }
                    _ => {
                        self.dispatch_failure(res)
                    }
                }
            }
        }

        debug!("done tunneling for ns = {}", ns);
    }

    fn dispatch_cached(&self, res: &mut Response, value: String) {
        // TODO: handle 'tunnel_res: &mut Response' here.

        // TODO: append status
        // TODO: append headers
        res.set_status(StatusCode::Accepted);  // <-- TODO: dynamic status

        // TODO: issue w/ borrow
        // res.with_header(HeaderBloomStatus(HeaderBloomStatusValue::Hit));

        res.set_body(value);
    }

    fn dispatch_direct(&self, res: &mut Response, tunnel_res: Response,
        bloomStatus: HeaderBloomStatusValue) {
        // TODO: handle 'tunnel_res: &mut Response' here.

        // TODO: append status
        // TODO: append headers
        res.set_status(tunnel_res.status());  // <-- TODO: dynamic status

        // res.set_header(HeaderBloomStatus(bloomStatus))

        res.set_body(tunnel_res.body());
    }

    fn dispatch_failure(&self, res: &mut Response) {
        let status = StatusCode::BadGateway;

        res.set_status(status);

        // TODO: issue w/ borrow
        // res.with_header(HeaderBloomStatus(HeaderBloomStatusValue::Offline));

        res.set_body(format!("{}", status));
    }
}
<commit_msg>Notice<commit_after>// Bloom
//
// HTTP REST API caching middleware
// Copyright: 2017, Valerian Saliou <valerian@valeriansaliou.name>
// License: Mozilla Public License v2.0 (MPL v2.0)

use futures::future;
use futures::future::FutureResult;
use hyper;
use hyper::{Method, StatusCode};
use hyper::server::{Request, Response};

use super::header::ProxyHeader;
use super::tunnel::ProxyTunnelBuilder;
use header::request_shard::HeaderRequestBloomRequestShard;
use header::status::HeaderBloomStatusValue;
use cache::read::CacheRead;
use cache::write::CacheWrite;
use cache::route::CacheRoute;

pub struct ProxyServeBuilder;

pub struct ProxyServe;

pub type ProxyServeFuture = FutureResult<Response, hyper::Error>;

impl ProxyServeBuilder {
    pub fn new() -> ProxyServe {
        ProxyServe {}
    }
}

impl ProxyServe {
    pub fn handle(&self, req: Request) -> ProxyServeFuture {
        info!("handled request: {} on {}", req.method(), req.path());

        let mut res = Response::new();

        if req.headers().has::<HeaderRequestBloomRequestShard>() == true {
            match *req.method() {
                Method::Options
                | Method::Head
                | Method::Get
                | Method::Post
                | Method::Patch
                | Method::Put
                | Method::Delete => {
                    self.accept(&req, &mut res)
                }
                _ => {
                    self.reject(&req, &mut res, StatusCode::MethodNotAllowed)
                }
            }
        } else {
            self.reject(&req, &mut res, StatusCode::NotExtended)
        }

        future::ok(res)
    }

    fn accept(&self, req: &Request, res: &mut Response) {
        self.tunnel(req, res);
    }

    fn reject(&self, req: &Request, res: &mut Response, status: StatusCode) {
        res.set_status(status);

        match *req.method() {
            Method::Get | Method::Post | Method::Patch | Method::Put => {
                res.set_body(format!("{}", status));
            }
            _ => {}
        }
    }

    fn tunnel(&self, req: &Request, res: &mut Response) {
        let (auth, shard) = ProxyHeader::parse_from_request(req.headers());

        let ns = CacheRoute::gen_ns(shard, auth, req.version(), req.method(),
            req.path(), req.query());

        // TODO: support for 304 Not Modified here (return empty content \
        //   to ongoing specific client, but still read/populate cache normally)

        // TODO: implement support for Bloom-Response-Bucket
        // CONCERN: how to link this to the gen_ns() utility? We dont \
        //   know about which route is mapped to which bucket in advance. \
        //   so maybe redesign this part.  <--- FOUND OUT
        // WAIT TO GO: any route can be 'tagged' as 'bucket' using a generic \
        //   tagging system. As buckets are only used for cache expiration, \
        //   and not cache storage, they are only useful as 'tags'. This way \
        //   we dont need to know them in advance.

        info!("tunneling for ns = {}", ns);

        match CacheRead::acquire(ns.as_ref()) {
            Ok(cached_value) => {
                self.dispatch_cached(res, cached_value)
            },
            Err(_) => {
                // TODO -> connect to API using ConfigProxy[:shard].inet
                    // TODO -> enforce timeouts:
                        //   - ConfigProxy.tunnel_connect_timeout
                        //   - ConfigProxy.tunnel_read_timeout
                        //   - ConfigProxy.tunnel_send_timeout
                // TODO -> CacheWrite::save(ns, req, res) (check return value)
                    // TODO -> return == true -> set 'Bloom-Status' as 'MISS'
                    // TODO -> return == false -> set 'Bloom-Status' as 'DIRECT'

                // Acquire response object from client to API

                // TODO: move this to a common factory, ie global.
                // CRITICAL: avoid spawning new threads and destroying them \
                //   for each connection.
                let mut tunnel = ProxyTunnelBuilder::new();

                match tunnel.run(req.method()) {
                    Ok(tunnel_res) => {
                        if CacheWrite::save(ns.as_ref(), req, &tunnel_res) ==
                            true {
                            self.dispatch_direct(res, tunnel_res,
                                HeaderBloomStatusValue::Miss)
                        } else {
                            self.dispatch_direct(res, tunnel_res,
                                HeaderBloomStatusValue::Direct)
                        }
                    }
                    _ => {
                        self.dispatch_failure(res)
                    }
                }
            }
        }

        debug!("done tunneling for ns = {}", ns);
    }

    fn dispatch_cached(&self, res: &mut Response, value: String) {
        // TODO: handle 'tunnel_res: &mut Response' here.

        // TODO: append status
        // TODO: append headers
        res.set_status(StatusCode::Accepted);  // <-- TODO: dynamic status

        // TODO: issue w/ borrow
        // res.with_header(HeaderBloomStatus(HeaderBloomStatusValue::Hit));

        res.set_body(value);
    }

    fn dispatch_direct(&self, res: &mut Response, tunnel_res: Response,
        bloomStatus: HeaderBloomStatusValue) {
        // TODO: handle 'tunnel_res: &mut Response' here.

        // TODO: append status
        // TODO: append headers
        res.set_status(tunnel_res.status());  // <-- TODO: dynamic status

        // res.set_header(HeaderBloomStatus(bloomStatus))

        res.set_body(tunnel_res.body());
    }

    fn dispatch_failure(&self, res: &mut Response) {
        let status = StatusCode::BadGateway;

        res.set_status(status);

        // TODO: issue w/ borrow
        // res.with_header(HeaderBloomStatus(HeaderBloomStatusValue::Offline));

        res.set_body(format!("{}", status));
    }
}
<|endoftext|>


<commit_before>use super::Weight;
use crate::core::searcher::Searcher;
use crate::query::Explanation;
use crate::DocAddress;
use crate::Term;
use downcast_rs::impl_downcast;
use std::collections::BTreeSet;
use std::fmt;

/// The `Query` trait defines a set of documents and a scoring method
/// for those documents.
///
/// The `Query` trait is in charge of defining :
///
/// - a set of documents
/// - a way to score these documents
///
/// When performing a [search](#method.search),  these documents will then
/// be pushed to a [Collector](../collector/trait.Collector.html),
/// which will in turn be in charge of deciding what to do with them.
///
/// Concretely, this scored docset is represented by the
/// [`Scorer`](./trait.Scorer.html) trait.
///
/// Because our index is actually split into segments, the
/// query does not actually directly creates `DocSet` object.
/// Instead, the query creates a [`Weight`](./trait.Weight.html)
/// object for a given searcher.
///
/// The weight object, in turn, makes it possible to create
/// a scorer for a specific [`SegmentReader`](../struct.SegmentReader.html).
///
/// So to sum it up :
/// - a `Query` is recipe to define a set of documents as well the way to score them.
/// - a `Weight` is this recipe tied to a specific `Searcher`. It may for instance
/// hold statistics about the different term of the query. It is created by the query.
/// - a `Scorer` is a cursor over the set of matching documents, for a specific
/// [`SegmentReader`](../struct.SegmentReader.html). It is created by the
/// [`Weight`](./trait.Weight.html).
///
/// When implementing a new type of `Query`, it is normal to implement a
/// dedicated `Query`, `Weight` and `Scorer`.
pub trait Query: QueryClone + downcast_rs::Downcast + fmt::Debug {
    /// Create the weight associated to a query.
    ///
    /// If scoring is not required, setting `scoring_enabled` to `false`
    /// can increase performances.
    ///
    /// See [`Weight`](./trait.Weight.html).
    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>>;

    /// Returns an `Explanation` for the score of the document.
    fn explain(&self, searcher: &Searcher, doc_address: DocAddress) -> crate::Result<Explanation> {
        let reader = searcher.segment_reader(doc_address.segment_ord());
        let weight = self.weight(searcher, true)?;
        weight.explain(reader, doc_address.doc())
    }

    /// Returns the number of documents matching the query.
    fn count(&self, searcher: &Searcher) -> crate::Result<usize> {
        let weight = self.weight(searcher, false)?;
        let mut result = 0;
        for reader in searcher.segment_readers() {
            result += weight.count(reader)? as usize;
        }
        Ok(result)
    }

    /// Extract all of the terms associated to the query and insert them in the
    /// term set given in arguments.
    fn query_terms(&self, _term_set: &mut BTreeSet<Term>) {}
}

pub trait QueryClone {
    fn box_clone(&self) -> Box<dyn Query>;
}

impl<T> QueryClone for T
where
    T: 'static + Query + Clone,
{
    fn box_clone(&self) -> Box<dyn Query> {
        Box::new(self.clone())
    }
}

impl Query for Box<dyn Query> {
    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
        self.as_ref().weight(searcher, scoring_enabled)
    }

    fn count(&self, searcher: &Searcher) -> crate::Result<usize> {
        self.as_ref().count(searcher)
    }

    fn query_terms(&self, term_set: &mut BTreeSet<Term<Vec<u8>>>) {
        self.as_ref().query_terms(term_set);
    }
}

impl QueryClone for Box<dyn Query> {
    fn box_clone(&self) -> Box<dyn Query> {
        self.as_ref().box_clone()
    }
}

impl_downcast!(Query);
<commit_msg>Added Send and Sync to the Query trait.<commit_after>use super::Weight;
use crate::core::searcher::Searcher;
use crate::query::Explanation;
use crate::DocAddress;
use crate::Term;
use downcast_rs::impl_downcast;
use std::collections::BTreeSet;
use std::fmt;

/// The `Query` trait defines a set of documents and a scoring method
/// for those documents.
///
/// The `Query` trait is in charge of defining :
///
/// - a set of documents
/// - a way to score these documents
///
/// When performing a [search](#method.search),  these documents will then
/// be pushed to a [Collector](../collector/trait.Collector.html),
/// which will in turn be in charge of deciding what to do with them.
///
/// Concretely, this scored docset is represented by the
/// [`Scorer`](./trait.Scorer.html) trait.
///
/// Because our index is actually split into segments, the
/// query does not actually directly creates `DocSet` object.
/// Instead, the query creates a [`Weight`](./trait.Weight.html)
/// object for a given searcher.
///
/// The weight object, in turn, makes it possible to create
/// a scorer for a specific [`SegmentReader`](../struct.SegmentReader.html).
///
/// So to sum it up :
/// - a `Query` is recipe to define a set of documents as well the way to score them.
/// - a `Weight` is this recipe tied to a specific `Searcher`. It may for instance
/// hold statistics about the different term of the query. It is created by the query.
/// - a `Scorer` is a cursor over the set of matching documents, for a specific
/// [`SegmentReader`](../struct.SegmentReader.html). It is created by the
/// [`Weight`](./trait.Weight.html).
///
/// When implementing a new type of `Query`, it is normal to implement a
/// dedicated `Query`, `Weight` and `Scorer`.
pub trait Query: QueryClone + Send + Sync + downcast_rs::Downcast + fmt::Debug {
    /// Create the weight associated to a query.
    ///
    /// If scoring is not required, setting `scoring_enabled` to `false`
    /// can increase performances.
    ///
    /// See [`Weight`](./trait.Weight.html).
    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>>;

    /// Returns an `Explanation` for the score of the document.
    fn explain(&self, searcher: &Searcher, doc_address: DocAddress) -> crate::Result<Explanation> {
        let reader = searcher.segment_reader(doc_address.segment_ord());
        let weight = self.weight(searcher, true)?;
        weight.explain(reader, doc_address.doc())
    }

    /// Returns the number of documents matching the query.
    fn count(&self, searcher: &Searcher) -> crate::Result<usize> {
        let weight = self.weight(searcher, false)?;
        let mut result = 0;
        for reader in searcher.segment_readers() {
            result += weight.count(reader)? as usize;
        }
        Ok(result)
    }

    /// Extract all of the terms associated to the query and insert them in the
    /// term set given in arguments.
    fn query_terms(&self, _term_set: &mut BTreeSet<Term>) {}
}

pub trait QueryClone {
    fn box_clone(&self) -> Box<dyn Query>;
}

impl<T> QueryClone for T
where
    T: 'static + Query + Clone,
{
    fn box_clone(&self) -> Box<dyn Query> {
        Box::new(self.clone())
    }
}

impl Query for Box<dyn Query> {
    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
        self.as_ref().weight(searcher, scoring_enabled)
    }

    fn count(&self, searcher: &Searcher) -> crate::Result<usize> {
        self.as_ref().count(searcher)
    }

    fn query_terms(&self, term_set: &mut BTreeSet<Term<Vec<u8>>>) {
        self.as_ref().query_terms(term_set);
    }
}

impl QueryClone for Box<dyn Query> {
    fn box_clone(&self) -> Box<dyn Query> {
        self.as_ref().box_clone()
    }
}

impl_downcast!(Query);
<|endoftext|>