<commit_before>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use config::Color;
use diff;
use std::collections::VecDeque;
use std::io;
use term;
use std::io::Write;
use utils::use_colored_tty;

#[derive(Debug, PartialEq)]
pub enum DiffLine {
    Context(String),
    Expected(String),
    Resulting(String),
}

#[derive(Debug, PartialEq)]
pub struct Mismatch {
    /// The line number in the formatted version.
    pub line_number: u32,
    /// The line number in the original version.
    pub line_number_orig: u32,
    /// The set of lines (context and old/new) in the mismatch.
    pub lines: Vec<DiffLine>,
}

impl Mismatch {
    fn new(line_number: u32, line_number_orig: u32) -> Mismatch {
        Mismatch {
            line_number: line_number,
            line_number_orig: line_number_orig,
            lines: Vec::new(),
        }
    }
}

// This struct handles writing output to stdout and abstracts away the logic
// of printing in color, if it's possible in the executing environment.
pub struct OutputWriter {
    terminal: Option<Box<term::Terminal<Output = io::Stdout>>>,
}

impl OutputWriter {
    // Create a new OutputWriter instance based on the caller's preference
    // for colorized output and the capabilities of the terminal.
    pub fn new(color: Color) -> Self {
        if let Some(t) = term::stdout() {
            if use_colored_tty(color) && t.supports_color() {
                return OutputWriter { terminal: Some(t) };
            }
        }
        OutputWriter { terminal: None }
    }

    // Write output in the optionally specified color. The output is written
    // in the specified color if this OutputWriter instance contains a
    // Terminal in its `terminal` field.
    pub fn writeln(&mut self, msg: &str, color: Option<term::color::Color>) {
        match &mut self.terminal {
            Some(ref mut t) => {
                if let Some(color) = color {
                    t.fg(color).unwrap();
                }
                writeln!(t, "{}", msg).unwrap();
                if color.is_some() {
                    t.reset().unwrap();
                }
            }
            None => println!("{}", msg),
        }
    }
}

// Produces a diff between the expected output and actual output of rustfmt.
pub fn make_diff(expected: &str, actual: &str, context_size: usize) -> Vec<Mismatch> {
    let mut line_number = 1;
    let mut line_number_orig = 1;
    let mut context_queue: VecDeque<&str> = VecDeque::with_capacity(context_size);
    let mut lines_since_mismatch = context_size + 1;
    let mut results = Vec::new();
    let mut mismatch = Mismatch::new(0, 0);

    for result in diff::lines(expected, actual) {
        match result {
            diff::Result::Left(str) => {
                if lines_since_mismatch >= context_size && lines_since_mismatch > 0 {
                    results.push(mismatch);
                    mismatch = Mismatch::new(
                        line_number - context_queue.len() as u32,
                        line_number_orig - context_queue.len() as u32,
                    );
                }

                while let Some(line) = context_queue.pop_front() {
                    mismatch.lines.push(DiffLine::Context(line.to_owned()));
                }

                mismatch.lines.push(DiffLine::Resulting(str.to_owned()));
                line_number_orig += 1;
                lines_since_mismatch = 0;
            }
            diff::Result::Right(str) => {
                if lines_since_mismatch >= context_size && lines_since_mismatch > 0 {
                    results.push(mismatch);
                    mismatch = Mismatch::new(
                        line_number - context_queue.len() as u32,
                        line_number_orig - context_queue.len() as u32,
                    );
                }

                while let Some(line) = context_queue.pop_front() {
                    mismatch.lines.push(DiffLine::Context(line.to_owned()));
                }

                mismatch.lines.push(DiffLine::Expected(str.to_owned()));
                line_number += 1;
                lines_since_mismatch = 0;
            }
            diff::Result::Both(str, _) => {
                if context_queue.len() >= context_size {
                    let _ = context_queue.pop_front();
                }

                if lines_since_mismatch < context_size {
                    mismatch.lines.push(DiffLine::Context(str.to_owned()));
                } else if context_size > 0 {
                    context_queue.push_back(str);
                }

                line_number += 1;
                line_number_orig += 1;
                lines_since_mismatch += 1;
            }
        }
    }

    results.push(mismatch);
    results.remove(0);

    results
}

pub fn print_diff<F>(diff: Vec<Mismatch>, get_section_title: F, color: Color)
where
    F: Fn(u32) -> String,
{
    let mut writer = OutputWriter::new(color);

    for mismatch in diff {
        let title = get_section_title(mismatch.line_number);
        writer.writeln(&format!("{}", title), None);

        for line in mismatch.lines {
            match line {
                DiffLine::Context(ref str) => writer.writeln(&format!(" {}⏎", str), None),
                DiffLine::Expected(ref str) => {
                    writer.writeln(&format!("+{}⏎", str), Some(term::color::GREEN))
                }
                DiffLine::Resulting(ref str) => {
                    writer.writeln(&format!("-{}⏎", str), Some(term::color::RED))
                }
            }
        }
    }
}

/// Convert a Mismatch into a serialised form which just includes
/// enough information to modify the original file.
/// Each section starts with a line with three integers, space separated:
///     lineno num_removed num_added
/// followd by (num_added) lines of added text.  The line numbers are
/// relative to the original file.
pub fn output_modified<W>(mut out: W, diff: Vec<Mismatch>)
where
    W: Write,
{
    for mismatch in diff {
        let (num_removed, num_added) = mismatch.lines.iter().fold((0, 0), |(rem, add), line| {
            match *line {
                DiffLine::Context(_) => panic!("No Context expected"),
                DiffLine::Expected(_) => (rem, add + 1),
                DiffLine::Resulting(_) => (rem + 1, add),
            }
        });
        // Write a header with enough information to separate the modified lines.
        writeln!(
            out,
            "{} {} {}",
            mismatch.line_number_orig, num_removed, num_added
        ).unwrap();

        for line in mismatch.lines {
            match line {
                DiffLine::Context(_) | DiffLine::Resulting(_) => (),
                DiffLine::Expected(ref str) => {
                    writeln!(out, "{}", str).unwrap();
                }
            }
        }
    }
}

#[cfg(test)]
mod test {
    use super::{make_diff, Mismatch};
    use super::DiffLine::*;

    #[test]
    fn diff_simple() {
        let src = "one\ntwo\nthree\nfour\nfive\n";
        let dest = "one\ntwo\ntrois\nfour\nfive\n";
        let diff = make_diff(src, dest, 1);
        assert_eq!(
            diff,
            vec![
                Mismatch {
                    line_number: 2,
                    line_number_orig: 2,
                    lines: vec![
                        Context("two".to_owned()),
                        Resulting("three".to_owned()),
                        Expected("trois".to_owned()),
                        Context("four".to_owned()),
                    ],
                },
            ]
        );
    }

    #[test]
    fn diff_simple2() {
        let src = "one\ntwo\nthree\nfour\nfive\nsix\nseven\n";
        let dest = "one\ntwo\ntrois\nfour\ncinq\nsix\nseven\n";
        let diff = make_diff(src, dest, 1);
        assert_eq!(
            diff,
            vec![
                Mismatch {
                    line_number: 2,
                    line_number_orig: 2,
                    lines: vec![
                        Context("two".to_owned()),
                        Resulting("three".to_owned()),
                        Expected("trois".to_owned()),
                        Context("four".to_owned()),
                    ],
                },
                Mismatch {
                    line_number: 5,
                    line_number_orig: 5,
                    lines: vec![
                        Resulting("five".to_owned()),
                        Expected("cinq".to_owned()),
                        Context("six".to_owned()),
                    ],
                },
            ]
        );
    }

    #[test]
    fn diff_zerocontext() {
        let src = "one\ntwo\nthree\nfour\nfive\n";
        let dest = "one\ntwo\ntrois\nfour\nfive\n";
        let diff = make_diff(src, dest, 0);
        assert_eq!(
            diff,
            vec![
                Mismatch {
                    line_number: 3,
                    line_number_orig: 3,
                    lines: vec![Resulting("three".to_owned()), Expected("trois".to_owned())],
                },
            ]
        );
    }

    #[test]
    fn diff_trailing_newline() {
        let src = "one\ntwo\nthree\nfour\nfive";
        let dest = "one\ntwo\nthree\nfour\nfive\n";
        let diff = make_diff(src, dest, 1);
        assert_eq!(
            diff,
            vec![
                Mismatch {
                    line_number: 5,
                    line_number_orig: 5,
                    lines: vec![Context("five".to_owned()), Expected("".to_owned())],
                },
            ]
        );
    }
}
<commit_msg>Fix typo in comment.<commit_after>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use config::Color;
use diff;
use std::collections::VecDeque;
use std::io;
use term;
use std::io::Write;
use utils::use_colored_tty;

#[derive(Debug, PartialEq)]
pub enum DiffLine {
    Context(String),
    Expected(String),
    Resulting(String),
}

#[derive(Debug, PartialEq)]
pub struct Mismatch {
    /// The line number in the formatted version.
    pub line_number: u32,
    /// The line number in the original version.
    pub line_number_orig: u32,
    /// The set of lines (context and old/new) in the mismatch.
    pub lines: Vec<DiffLine>,
}

impl Mismatch {
    fn new(line_number: u32, line_number_orig: u32) -> Mismatch {
        Mismatch {
            line_number: line_number,
            line_number_orig: line_number_orig,
            lines: Vec::new(),
        }
    }
}

// This struct handles writing output to stdout and abstracts away the logic
// of printing in color, if it's possible in the executing environment.
pub struct OutputWriter {
    terminal: Option<Box<term::Terminal<Output = io::Stdout>>>,
}

impl OutputWriter {
    // Create a new OutputWriter instance based on the caller's preference
    // for colorized output and the capabilities of the terminal.
    pub fn new(color: Color) -> Self {
        if let Some(t) = term::stdout() {
            if use_colored_tty(color) && t.supports_color() {
                return OutputWriter { terminal: Some(t) };
            }
        }
        OutputWriter { terminal: None }
    }

    // Write output in the optionally specified color. The output is written
    // in the specified color if this OutputWriter instance contains a
    // Terminal in its `terminal` field.
    pub fn writeln(&mut self, msg: &str, color: Option<term::color::Color>) {
        match &mut self.terminal {
            Some(ref mut t) => {
                if let Some(color) = color {
                    t.fg(color).unwrap();
                }
                writeln!(t, "{}", msg).unwrap();
                if color.is_some() {
                    t.reset().unwrap();
                }
            }
            None => println!("{}", msg),
        }
    }
}

// Produces a diff between the expected output and actual output of rustfmt.
pub fn make_diff(expected: &str, actual: &str, context_size: usize) -> Vec<Mismatch> {
    let mut line_number = 1;
    let mut line_number_orig = 1;
    let mut context_queue: VecDeque<&str> = VecDeque::with_capacity(context_size);
    let mut lines_since_mismatch = context_size + 1;
    let mut results = Vec::new();
    let mut mismatch = Mismatch::new(0, 0);

    for result in diff::lines(expected, actual) {
        match result {
            diff::Result::Left(str) => {
                if lines_since_mismatch >= context_size && lines_since_mismatch > 0 {
                    results.push(mismatch);
                    mismatch = Mismatch::new(
                        line_number - context_queue.len() as u32,
                        line_number_orig - context_queue.len() as u32,
                    );
                }

                while let Some(line) = context_queue.pop_front() {
                    mismatch.lines.push(DiffLine::Context(line.to_owned()));
                }

                mismatch.lines.push(DiffLine::Resulting(str.to_owned()));
                line_number_orig += 1;
                lines_since_mismatch = 0;
            }
            diff::Result::Right(str) => {
                if lines_since_mismatch >= context_size && lines_since_mismatch > 0 {
                    results.push(mismatch);
                    mismatch = Mismatch::new(
                        line_number - context_queue.len() as u32,
                        line_number_orig - context_queue.len() as u32,
                    );
                }

                while let Some(line) = context_queue.pop_front() {
                    mismatch.lines.push(DiffLine::Context(line.to_owned()));
                }

                mismatch.lines.push(DiffLine::Expected(str.to_owned()));
                line_number += 1;
                lines_since_mismatch = 0;
            }
            diff::Result::Both(str, _) => {
                if context_queue.len() >= context_size {
                    let _ = context_queue.pop_front();
                }

                if lines_since_mismatch < context_size {
                    mismatch.lines.push(DiffLine::Context(str.to_owned()));
                } else if context_size > 0 {
                    context_queue.push_back(str);
                }

                line_number += 1;
                line_number_orig += 1;
                lines_since_mismatch += 1;
            }
        }
    }

    results.push(mismatch);
    results.remove(0);

    results
}

pub fn print_diff<F>(diff: Vec<Mismatch>, get_section_title: F, color: Color)
where
    F: Fn(u32) -> String,
{
    let mut writer = OutputWriter::new(color);

    for mismatch in diff {
        let title = get_section_title(mismatch.line_number);
        writer.writeln(&format!("{}", title), None);

        for line in mismatch.lines {
            match line {
                DiffLine::Context(ref str) => writer.writeln(&format!(" {}⏎", str), None),
                DiffLine::Expected(ref str) => {
                    writer.writeln(&format!("+{}⏎", str), Some(term::color::GREEN))
                }
                DiffLine::Resulting(ref str) => {
                    writer.writeln(&format!("-{}⏎", str), Some(term::color::RED))
                }
            }
        }
    }
}

/// Convert a Mismatch into a serialised form which just includes
/// enough information to modify the original file.
/// Each section starts with a line with three integers, space separated:
///     lineno num_removed num_added
/// followed by (num_added) lines of added text.  The line numbers are
/// relative to the original file.
pub fn output_modified<W>(mut out: W, diff: Vec<Mismatch>)
where
    W: Write,
{
    for mismatch in diff {
        let (num_removed, num_added) = mismatch.lines.iter().fold((0, 0), |(rem, add), line| {
            match *line {
                DiffLine::Context(_) => panic!("No Context expected"),
                DiffLine::Expected(_) => (rem, add + 1),
                DiffLine::Resulting(_) => (rem + 1, add),
            }
        });
        // Write a header with enough information to separate the modified lines.
        writeln!(
            out,
            "{} {} {}",
            mismatch.line_number_orig, num_removed, num_added
        ).unwrap();

        for line in mismatch.lines {
            match line {
                DiffLine::Context(_) | DiffLine::Resulting(_) => (),
                DiffLine::Expected(ref str) => {
                    writeln!(out, "{}", str).unwrap();
                }
            }
        }
    }
}

#[cfg(test)]
mod test {
    use super::{make_diff, Mismatch};
    use super::DiffLine::*;

    #[test]
    fn diff_simple() {
        let src = "one\ntwo\nthree\nfour\nfive\n";
        let dest = "one\ntwo\ntrois\nfour\nfive\n";
        let diff = make_diff(src, dest, 1);
        assert_eq!(
            diff,
            vec![
                Mismatch {
                    line_number: 2,
                    line_number_orig: 2,
                    lines: vec![
                        Context("two".to_owned()),
                        Resulting("three".to_owned()),
                        Expected("trois".to_owned()),
                        Context("four".to_owned()),
                    ],
                },
            ]
        );
    }

    #[test]
    fn diff_simple2() {
        let src = "one\ntwo\nthree\nfour\nfive\nsix\nseven\n";
        let dest = "one\ntwo\ntrois\nfour\ncinq\nsix\nseven\n";
        let diff = make_diff(src, dest, 1);
        assert_eq!(
            diff,
            vec![
                Mismatch {
                    line_number: 2,
                    line_number_orig: 2,
                    lines: vec![
                        Context("two".to_owned()),
                        Resulting("three".to_owned()),
                        Expected("trois".to_owned()),
                        Context("four".to_owned()),
                    ],
                },
                Mismatch {
                    line_number: 5,
                    line_number_orig: 5,
                    lines: vec![
                        Resulting("five".to_owned()),
                        Expected("cinq".to_owned()),
                        Context("six".to_owned()),
                    ],
                },
            ]
        );
    }

    #[test]
    fn diff_zerocontext() {
        let src = "one\ntwo\nthree\nfour\nfive\n";
        let dest = "one\ntwo\ntrois\nfour\nfive\n";
        let diff = make_diff(src, dest, 0);
        assert_eq!(
            diff,
            vec![
                Mismatch {
                    line_number: 3,
                    line_number_orig: 3,
                    lines: vec![Resulting("three".to_owned()), Expected("trois".to_owned())],
                },
            ]
        );
    }

    #[test]
    fn diff_trailing_newline() {
        let src = "one\ntwo\nthree\nfour\nfive";
        let dest = "one\ntwo\nthree\nfour\nfive\n";
        let diff = make_diff(src, dest, 1);
        assert_eq!(
            diff,
            vec![
                Mismatch {
                    line_number: 5,
                    line_number_orig: 5,
                    lines: vec![Context("five".to_owned()), Expected("".to_owned())],
                },
            ]
        );
    }
}
<|endoftext|>


<commit_before>// Copyright (C) 2019 Inderjit Gill

// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::error::Error;
use crate::result::Result;

#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum Token<'a> {
    BackQuote,
    Colon,
    Comment(&'a str),
    CurlyBracketEnd,
    CurlyBracketStart,
    DoubleQuote,
    Name(&'a str),
    Number(&'a str),
    ParenEnd,
    ParenStart,
    Quote,
    SquareBracketEnd,
    SquareBracketStart,
    Whitespace(&'a str),
    End,
}

pub fn tokenize(s: &str) -> Result<Vec<Token>> {
    let mut lex = Lexer::new(s);
    let mut res = Vec::new();

    loop {
        match lex.eat_token()? {
            Token::End => break,
            tok => res.push(tok),
        }
    }

    Ok(res)
}

struct Lexer<'a> {
    input: &'a str,
}

impl<'a> Lexer<'a> {
    pub fn new(input: &str) -> Lexer {
        Lexer { input }
    }

    pub fn eat_token(&mut self) -> Result<Token<'a>> {
        if self.input.is_empty() {
            self.input = &self.input[..0];
            return Ok(Token::End);
        }

        if let Some(ch) = self.input.chars().nth(0) {
            let res = match ch {
                '(' => Ok((Token::ParenStart, 1)),
                ')' => Ok((Token::ParenEnd, 1)),
                '[' => Ok((Token::SquareBracketStart, 1)),
                ']' => Ok((Token::SquareBracketEnd, 1)),
                '{' => Ok((Token::CurlyBracketStart, 1)),
                '}' => Ok((Token::CurlyBracketEnd, 1)),
                ':' => Ok((Token::Colon, 1)),
                '\'' => Ok((Token::Quote, 1)),
                '"' => Ok((Token::DoubleQuote, 1)),
                '`' => Ok((Token::BackQuote, 1)),
                ';' => eat_comment(&self.input),
                '-' | '0'...'9' => eat_number(&self.input),
                ch if ch.is_whitespace() => eat_whitespace(&self.input),
                _ if is_name(ch) => eat_name(&self.input),
                ch => Err(Error::ParserInvalidChar(ch)),
            };

            let (tok, size) = match res {
                Ok(v) => v,
                Err(kind) => return Err(kind),
            };

            self.input = &self.input[size..];

            Ok(tok)
        } else {
            Err(Error::GeneralError)
        }
    }
}

fn is_name(ch: char) -> bool {
    ch.is_alphanumeric() || is_symbol(ch)
}

fn is_symbol(ch: char) -> bool {
    match ch {
        '+' | '-' | '*' | '/' | '=' | '!' | '@' | '#' | '$' | '%' | '^' | '&' | '<' | '>' | '?' => {
            true
        }
        _ => false,
    }
}

fn eat_name(input: &str) -> Result<(Token, usize)> {
    for (ind, ch) in input.char_indices() {
        if !is_name(ch) {
            return Ok((Token::Name(&input[..ind]), ind));
        }
    }

    Ok((Token::Name(input), input.len()))
}

fn eat_whitespace(input: &str) -> Result<(Token, usize)> {
    for (ind, ch) in input.char_indices() {
        if !ch.is_whitespace() {
            return Ok((Token::Whitespace(&input[..ind]), ind));
        }
    }

    Ok((Token::Whitespace(input), input.len()))
}

fn eat_number(input: &str) -> Result<(Token, usize)> {
    let mut digits = false;
    let mut dot = false;
    let mut size = input.len();

    let (prefix_offset, rest) = if input.starts_with('-') {
        match input[1..].chars().next() {
            Some(ch) if ch.is_digit(10) => (1, &input[1..]),
            // Actually a name beginning with '-' rather a number
            _ => return eat_name(input),
        }
    } else {
        (0, input)
    };

    for (ind, ch) in rest.char_indices() {
        match ch {
            '.' => {
                if dot {
                    return Err(Error::ParserInvalidLiteral);
                }
                dot = true;
            }
            _ if ch.is_digit(10) => {
                digits = true;
            }
            _ => {
                size = prefix_offset + ind;
                break;
            }
        }
    }

    if !digits {
        Err(Error::ParserInvalidLiteral)
    } else {
        Ok((Token::Number(&input[..size]), size))
    }
}

fn eat_comment(input: &str) -> Result<(Token, usize)> {
    let rest = &input[1..]; // remove the first character (;)
    let mut size = rest.len();

    for (ind, ch) in rest.char_indices() {
        match ch {
            '\n' => {
                size = ind;
                break;
            }
            _ => {
                continue;
            }
        }
    }
    Ok((Token::Comment(&rest[..size]), size + 1))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_lexer() {
        assert_eq!(
            tokenize("()").unwrap(),
            [Token::ParenStart, Token::ParenEnd]
        );

        assert_eq!(
            tokenize("( )").unwrap(),
            [Token::ParenStart, Token::Whitespace(" "), Token::ParenEnd]
        );

        assert_eq!(
            tokenize("[]").unwrap(),
            [Token::SquareBracketStart, Token::SquareBracketEnd]
        );

        assert_eq!(
            tokenize("{}").unwrap(),
            [Token::CurlyBracketStart, Token::CurlyBracketEnd]
        );

        assert_eq!(
            tokenize("'(1)").unwrap(),
            [
                Token::Quote,
                Token::ParenStart,
                Token::Number("1"),
                Token::ParenEnd
            ]
        );

        assert_eq!(tokenize("5").unwrap(), [Token::Number("5")]);
        assert_eq!(tokenize("-3").unwrap(), [Token::Number("-3")]);
        assert_eq!(tokenize("3.14").unwrap(), [Token::Number("3.14")]);
        assert_eq!(tokenize("-0.34").unwrap(), [Token::Number("-0.34")]);

        assert_eq!(
            tokenize("1 foo 3").unwrap(),
            [
                Token::Number("1"),
                Token::Whitespace(" "),
                Token::Name("foo"),
                Token::Whitespace(" "),
                Token::Number("3")
            ]
        );

        assert_eq!(
            tokenize("hello\nworld").unwrap(),
            [
                Token::Name("hello"),
                Token::Whitespace("\n"),
                Token::Name("world")
            ]
        );

        assert_eq!(
            tokenize("hello ; some comment").unwrap(),
            [
                Token::Name("hello"),
                Token::Whitespace(" "),
                Token::Comment(" some comment")
            ]
        );

        assert_eq!(
            tokenize("hello ; some comment\n(valid)").unwrap(),
            [
                Token::Name("hello"),
                Token::Whitespace(" "),
                Token::Comment(" some comment"),
                Token::Whitespace("\n"),
                Token::ParenStart,
                Token::Name("valid"),
                Token::ParenEnd
            ]
        );

        assert_eq!(
            tokenize("{ 23 (gen/scalar min: 3 max: 100)}").unwrap(),
            [
                Token::CurlyBracketStart,
                Token::Whitespace(" "),
                Token::Number("23"),
                Token::Whitespace(" "),
                Token::ParenStart,
                Token::Name("gen/scalar"),
                Token::Whitespace(" "),
                Token::Name("min"),
                Token::Colon,
                Token::Whitespace(" "),
                Token::Number("3"),
                Token::Whitespace(" "),
                Token::Name("max"),
                Token::Colon,
                Token::Whitespace(" "),
                Token::Number("100"),
                Token::ParenEnd,
                Token::CurlyBracketEnd
            ]
        );
    }
}
<commit_msg>lexer outputs String<commit_after>// Copyright (C) 2019 Inderjit Gill

// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::error::Error;
use crate::result::Result;

#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum Token<'a> {
    BackQuote,
    Colon,
    Comment(&'a str),
    CurlyBracketEnd,
    CurlyBracketStart,
    String(&'a str),
    Name(&'a str),
    Number(&'a str),
    ParenEnd,
    ParenStart,
    Quote,
    SquareBracketEnd,
    SquareBracketStart,
    Whitespace(&'a str),
    End,
}

pub fn tokenize(s: &str) -> Result<Vec<Token>> {
    let mut lex = Lexer::new(s);
    let mut res = Vec::new();

    loop {
        match lex.eat_token()? {
            Token::End => break,
            tok => res.push(tok),
        }
    }

    Ok(res)
}

struct Lexer<'a> {
    input: &'a str,
}

impl<'a> Lexer<'a> {
    pub fn new(input: &str) -> Lexer {
        Lexer { input }
    }

    pub fn eat_token(&mut self) -> Result<Token<'a>> {
        if self.input.is_empty() {
            self.input = &self.input[..0];
            return Ok(Token::End);
        }

        if let Some(ch) = self.input.chars().nth(0) {
            let res = match ch {
                '(' => Ok((Token::ParenStart, 1)),
                ')' => Ok((Token::ParenEnd, 1)),
                '[' => Ok((Token::SquareBracketStart, 1)),
                ']' => Ok((Token::SquareBracketEnd, 1)),
                '{' => Ok((Token::CurlyBracketStart, 1)),
                '}' => Ok((Token::CurlyBracketEnd, 1)),
                ':' => Ok((Token::Colon, 1)),
                '\'' => Ok((Token::Quote, 1)),
                '`' => Ok((Token::BackQuote, 1)),
                '"' => eat_string(&self.input),
                ';' => eat_comment(&self.input),
                '-' | '0'...'9' => eat_number(&self.input),
                ch if ch.is_whitespace() => eat_whitespace(&self.input),
                _ if is_name(ch) => eat_name(&self.input),
                ch => Err(Error::ParserInvalidChar(ch)),
            };

            let (tok, size) = match res {
                Ok(v) => v,
                Err(kind) => return Err(kind),
            };

            self.input = &self.input[size..];

            Ok(tok)
        } else {
            Err(Error::GeneralError)
        }
    }
}

fn is_name(ch: char) -> bool {
    ch.is_alphanumeric() || is_symbol(ch)
}

fn is_symbol(ch: char) -> bool {
    match ch {
        '+' | '-' | '*' | '/' | '=' | '!' | '@' | '#' | '$' | '%' | '^' | '&' | '<' | '>' | '?' => {
            true
        }
        _ => false,
    }
}

fn eat_name(input: &str) -> Result<(Token, usize)> {
    for (ind, ch) in input.char_indices() {
        if !is_name(ch) {
            return Ok((Token::Name(&input[..ind]), ind));
        }
    }

    Ok((Token::Name(input), input.len()))
}

fn eat_whitespace(input: &str) -> Result<(Token, usize)> {
    for (ind, ch) in input.char_indices() {
        if !ch.is_whitespace() {
            return Ok((Token::Whitespace(&input[..ind]), ind));
        }
    }

    Ok((Token::Whitespace(input), input.len()))
}

fn eat_number(input: &str) -> Result<(Token, usize)> {
    let mut digits = false;
    let mut dot = false;
    let mut size = input.len();

    let (prefix_offset, rest) = if input.starts_with('-') {
        match input[1..].chars().next() {
            Some(ch) if ch.is_digit(10) => (1, &input[1..]),
            // Actually a name beginning with '-' rather a number
            _ => return eat_name(input),
        }
    } else {
        (0, input)
    };

    for (ind, ch) in rest.char_indices() {
        match ch {
            '.' => {
                if dot {
                    return Err(Error::ParserInvalidLiteral);
                }
                dot = true;
            }
            _ if ch.is_digit(10) => {
                digits = true;
            }
            _ => {
                size = prefix_offset + ind;
                break;
            }
        }
    }

    if !digits {
        Err(Error::ParserInvalidLiteral)
    } else {
        Ok((Token::Number(&input[..size]), size))
    }
}

fn eat_string(input: &str) -> Result<(Token, usize)> {
    let rest = &input[1..]; // remove the first doublequote
    let mut size = rest.len();

    for (ind, ch) in rest.char_indices() {
        match ch {
            '"' => {
                size = ind;
                break;
            }
            _ => {
                continue;
            }
        }
    }
    Ok((Token::String(&rest[..size]), size + 2))
}

fn eat_comment(input: &str) -> Result<(Token, usize)> {
    let rest = &input[1..]; // remove the first character (;)
    let mut size = rest.len();

    for (ind, ch) in rest.char_indices() {
        match ch {
            '\n' => {
                size = ind;
                break;
            }
            _ => {
                continue;
            }
        }
    }
    Ok((Token::Comment(&rest[..size]), size + 1))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_lexer() {
        assert_eq!(
            tokenize("()").unwrap(),
            [Token::ParenStart, Token::ParenEnd]
        );

        assert_eq!(
            tokenize("( )").unwrap(),
            [Token::ParenStart, Token::Whitespace(" "), Token::ParenEnd]
        );

        assert_eq!(
            tokenize("[]").unwrap(),
            [Token::SquareBracketStart, Token::SquareBracketEnd]
        );

        assert_eq!(
            tokenize("{}").unwrap(),
            [Token::CurlyBracketStart, Token::CurlyBracketEnd]
        );

        assert_eq!(
            tokenize("'(1)").unwrap(),
            [
                Token::Quote,
                Token::ParenStart,
                Token::Number("1"),
                Token::ParenEnd
            ]
        );

        assert_eq!(tokenize("5").unwrap(), [Token::Number("5")]);
        assert_eq!(tokenize("-3").unwrap(), [Token::Number("-3")]);
        assert_eq!(tokenize("3.14").unwrap(), [Token::Number("3.14")]);
        assert_eq!(tokenize("-0.34").unwrap(), [Token::Number("-0.34")]);

        assert_eq!(
            tokenize("1 foo 3").unwrap(),
            [
                Token::Number("1"),
                Token::Whitespace(" "),
                Token::Name("foo"),
                Token::Whitespace(" "),
                Token::Number("3")
            ]
        );

        assert_eq!(
            tokenize("hello\nworld").unwrap(),
            [
                Token::Name("hello"),
                Token::Whitespace("\n"),
                Token::Name("world")
            ]
        );

        assert_eq!(
            tokenize("hello ; some comment").unwrap(),
            [
                Token::Name("hello"),
                Token::Whitespace(" "),
                Token::Comment(" some comment")
            ]
        );

        assert_eq!(
            tokenize("\"hello.png\" ; some comment").unwrap(),
            [
                Token::String("hello.png"),
                Token::Whitespace(" "),
                Token::Comment(" some comment")
            ]
        );

        assert_eq!(
            tokenize("hello ; some comment\n(valid)").unwrap(),
            [
                Token::Name("hello"),
                Token::Whitespace(" "),
                Token::Comment(" some comment"),
                Token::Whitespace("\n"),
                Token::ParenStart,
                Token::Name("valid"),
                Token::ParenEnd
            ]
        );

        assert_eq!(
            tokenize("{ 23 (gen/scalar min: 3 max: 100)}").unwrap(),
            [
                Token::CurlyBracketStart,
                Token::Whitespace(" "),
                Token::Number("23"),
                Token::Whitespace(" "),
                Token::ParenStart,
                Token::Name("gen/scalar"),
                Token::Whitespace(" "),
                Token::Name("min"),
                Token::Colon,
                Token::Whitespace(" "),
                Token::Number("3"),
                Token::Whitespace(" "),
                Token::Name("max"),
                Token::Colon,
                Token::Whitespace(" "),
                Token::Number("100"),
                Token::ParenEnd,
                Token::CurlyBracketEnd
            ]
        );
    }
}
<|endoftext|>


<commit_before>use std::path;
use std::env;
use ::powerline::*;
use ::color::Color;

fn pb_to_str(path: path::PathBuf) -> String {
    path.to_str().unwrap().to_string()
}

pub fn add_segment(prompt : &mut Powerline, special: &str) {
    let cwd = pb_to_str(env::current_dir().unwrap());
    let mut cwd_slice = cwd.get(0..).unwrap();
    match env::home_dir() {
        Some(home_path) => {
            let home = pb_to_str(home_path);
            match cwd.find(&home) {
                    Some(pos) => {
                    cwd_slice = cwd.get((pos + home.len() )..).unwrap();
                    prompt.add_segment(Segment::simple(&format!(" {} ", special), Color::HOME_FG, Color::HOME_BG) )
                    },
                None => {},
            }
        },
        None => {},
    }
    let path: Vec<&str> = cwd_slice.split("/").skip(1).collect();
    let size = path.len();
    for idx in 0..size {
        let el = &path[idx];
        if idx != size - 1 {
            prompt.add_segment(Segment::special(&format!(" {} ", el), Color::PATH_FG, Color::PATH_BG, '\u{E0B1}', Color::SEPARATOR_FG ) );
        }
        else {
            prompt.add_segment(Segment::simple(&format!(" {} ", el), Color::PATH_FG, Color::PATH_BG) );
        }
    }
}
<commit_msg>Fixed root directory issue<commit_after>use std::path;
use std::env;
use ::powerline::*;
use ::color::Color;

fn pb_to_str(path: path::PathBuf) -> String {
    path.to_str().unwrap().to_string()
}

pub fn add_segment(prompt : &mut Powerline, special: &str) {
    let cwd = pb_to_str(env::current_dir().unwrap());
    let mut cwd_slice = cwd.get(0..).unwrap();
    match env::home_dir() {
        Some(home_path) => {
            let home = pb_to_str(home_path);
            match cwd.find(&home) {
                    Some(pos) => {
                    cwd_slice = cwd.get((pos + home.len() )..).unwrap();
                    prompt.add_segment(Segment::simple(&format!(" {} ", special), Color::HOME_FG, Color::HOME_BG) )
                    },
                None => {},
            }
        },
        None => {},
    }
    let path: Vec<&str> = cwd_slice.split("/").skip(1).collect();
    let size = path.len();
    for idx in 0..size {
        let mut el = &path[idx];
        if el == &"" { el = &"/" }
        if idx != size - 1 {
            prompt.add_segment(Segment::special(&format!(" {} ", el), Color::PATH_FG, Color::PATH_BG, '\u{E0B1}', Color::SEPARATOR_FG ) );
        }
        else {
            prompt.add_segment(Segment::simple(&format!(" {} ", el), Color::PATH_FG, Color::PATH_BG) );
        }
    }
}
<|endoftext|>


<commit_before>use clap::{App, Arg};

fn main() {
    // Of the three argument types, flags are the most simple. Flags are simple switches which can
    // be either "on" or "off"
    //
    // clap also supports multiple occurrences of flags, the common example is "verbosity" where a
    // user could want a little information with "-v" or tons of information with "-v -v" or "-vv"
    let matches = App::new("MyApp")
        // Regular App configuration goes here...
        // We'll add a flag that represents an awesome meter...
        //
        // I'll explain each possible setting that "flags" accept. Keep in mind
        // that you DO NOT need to set each of these for every flag, only the ones
        // you want for your individual case.
        .arg(
            Arg::new("awesome")
                .about("turns up the awesome") // Displayed when showing help info
                .short('a') // Trigger this arg with "-a"
                .long("awesome") // Trigger this arg with "--awesome"
                .multiple_occurrences(true) // This flag should allow multiple
                // occurrences such as "-aaa" or "-a -a"
                .requires("config") // Says, "If the user uses -a, they MUST
                // also use this other 'config' arg too"
                // Can also specify a list using
                // requires_all(Vec<&str>)
                .conflicts_with("output"), // Opposite of requires(), says "if the
                                           // user uses -a, they CANNOT use 'output'"
                                           // also has a conflicts_with_all(Vec<&str>)
                                           // and an exclusive(true)
        )
        .arg(
            // Sometimes we might want to accept one of the following: `--pseudo-flag`, `--pseudo-flag=true`, `--pseudo-flag=false.`
            // The following is the `pseudo-flag` pattern stated in https://github.com/clap-rs/clap/issues/1649#issuecomment-661274943
            Arg::new("pseudo-flag") // Create a "pesudo-flag" with optional value
                .possible_values(&["true", "false"]) // Limit that value to `true` of `false`
                .long("pseudo-flag")
                .min_values(0)
                .max_values(1),
        )
        .arg("-c, --config=[FILE] 'sets a custom config file'")
        .arg("<output> 'sets an output file'")
        .get_matches();

    // We can find out whether or not awesome was used
    if matches.is_present("awesome") {
        println!("Awesomeness is turned on");
    }

    // Same thing with `pseudo-flag`
    if app.value_of("pseudo-flag") != Some("false") {
        println!("Pseudo-flag is turned on");
    }

    // If we set the multiple option of a flag we can check how many times the user specified
    //
    // Note: if we did not specify the multiple option, and the user used "awesome" we would get
    // a 1 (no matter how many times they actually used it), or a 0 if they didn't use it at all
    match matches.occurrences_of("awesome") {
        0 => println!("Nothing is awesome"),
        1 => println!("Some things are awesome"),
        2 => println!("Lots of things are awesome"),
        _ => println!("EVERYTHING is awesome!"),
    }

    // Continued program logic goes here...
}
<commit_msg>docs: fix typo in example<commit_after>use clap::{App, Arg};

fn main() {
    // Of the three argument types, flags are the most simple. Flags are simple switches which can
    // be either "on" or "off"
    //
    // clap also supports multiple occurrences of flags, the common example is "verbosity" where a
    // user could want a little information with "-v" or tons of information with "-v -v" or "-vv"
    let matches = App::new("MyApp")
        // Regular App configuration goes here...
        // We'll add a flag that represents an awesome meter...
        //
        // I'll explain each possible setting that "flags" accept. Keep in mind
        // that you DO NOT need to set each of these for every flag, only the ones
        // you want for your individual case.
        .arg(
            Arg::new("awesome")
                .about("turns up the awesome") // Displayed when showing help info
                .short('a') // Trigger this arg with "-a"
                .long("awesome") // Trigger this arg with "--awesome"
                .multiple_occurrences(true) // This flag should allow multiple
                // occurrences such as "-aaa" or "-a -a"
                .requires("config") // Says, "If the user uses -a, they MUST
                // also use this other 'config' arg too"
                // Can also specify a list using
                // requires_all(Vec<&str>)
                .conflicts_with("output"), // Opposite of requires(), says "if the
                                           // user uses -a, they CANNOT use 'output'"
                                           // also has a conflicts_with_all(Vec<&str>)
                                           // and an exclusive(true)
        )
        .arg(
            // Sometimes we might want to accept one of the following: `--pseudo-flag`, `--pseudo-flag=true`, `--pseudo-flag=false.`
            // The following is the `pseudo-flag` pattern stated in https://github.com/clap-rs/clap/issues/1649#issuecomment-661274943
            Arg::new("pseudo-flag") // Create a "pesudo-flag" with optional value
                .possible_values(&["true", "false"]) // Limit that value to `true` of `false`
                .long("pseudo-flag")
                .min_values(0)
                .max_values(1),
        )
        .arg("-c, --config=[FILE] 'sets a custom config file'")
        .arg("<output> 'sets an output file'")
        .get_matches();

    // We can find out whether or not awesome was used
    if matches.is_present("awesome") {
        println!("Awesomeness is turned on");
    }

    // Same thing with `pseudo-flag`
    if matches.value_of("pseudo-flag") != Some("false") {
        println!("Pseudo-flag is turned on");
    }

    // If we set the multiple option of a flag we can check how many times the user specified
    //
    // Note: if we did not specify the multiple option, and the user used "awesome" we would get
    // a 1 (no matter how many times they actually used it), or a 0 if they didn't use it at all
    match matches.occurrences_of("awesome") {
        0 => println!("Nothing is awesome"),
        1 => println!("Some things are awesome"),
        2 => println!("Lots of things are awesome"),
        _ => println!("EVERYTHING is awesome!"),
    }

    // Continued program logic goes here...
}
<|endoftext|>


<commit_before>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! A split symbol reader is used when two targets want to read from the same stream.
//!

use std::cell::*;
use std::rc::*;
use std::collections::*;

use super::symbol_reader::*;

///
/// Shares a symbol reader between two targets
///
pub struct SplitSymbolReader<'a, Symbol: Clone+'a> {
    // The buffer for this reader
    buffer: Rc<RefCell<SplitSymbolReaderBuffer<'a, Symbol>>>,

    // The ID of this reader within the buffer
    reader_id: usize
}

///
/// Stores the shared data between two symbol readers
///
/// Rust really does a poor job of making it so you can share things. The issue is that you can only borrow something
/// mutably once, so we need to faff around with RefCells and so on, which add overhead solely for dealing with language
/// semantics rather than actual issues with sharing data.
///
struct SplitSymbolReaderBuffer<'a, Symbol: 'a> {
    /// The underlying symbol reader
    symbol_reader: &'a mut SymbolReader<Symbol>,

    /// Positions for the various split readers that are using this object, relative to the buffer
    positions: Vec<usize>,

    /// Buffer of symbols that are waiting to be consumed by other readers
    buffer: VecDeque<Option<Symbol>>
}

impl<'a, Symbol: Clone> SplitSymbolReaderBuffer<'a, Symbol> {
    fn new(reader: &'a mut SymbolReader<Symbol>) -> SplitSymbolReaderBuffer<'a, Symbol> {
        SplitSymbolReaderBuffer { symbol_reader: reader, positions: vec![], buffer: VecDeque::new() }
    }

    ///
    /// Clears out any buffered items that have been read by all readers
    ///
    fn clear_buffer(&mut self) {
        let lowest_pos = self.positions.iter().min().map(|x| *x);

        if let Some(lowest_pos) = lowest_pos {
            for _ in 0..lowest_pos {
                self.buffer.pop_front();
            }

            for pos in self.positions.iter_mut() {
                *pos -= lowest_pos;
            }
        }
    }

    ///
    /// Reads a symbol for the reader with a particular ID
    ///
    fn read(&mut self, reader_id: usize) -> Option<Symbol> {
        let buf_pos = self.positions[reader_id];

        if buf_pos >= self.buffer.len() {
            // This reader is at the end of the buffer, so just read the next symbol
            let next_symbol = self.symbol_reader.next_symbol();
            self.buffer.push_back(next_symbol.clone());

            self.positions[reader_id] = buf_pos+1;

            // If this reader was at the start of the buffer, then it might need clearing (in this case, only if this is the only active buffer)
            if buf_pos == 0 {
                self.clear_buffer();
            }

            next_symbol
        } else {
            // This reader is in the middle of the buffer, so just return a buffered value
            let next_symbol = self.buffer[buf_pos].clone();

            self.positions[reader_id] = buf_pos + 1;

            // If this reader was at the end of the buffer, then it might need clearing
            if buf_pos == 0 {
                self.clear_buffer();
            }

            next_symbol
        }
    }
}

///
/// Trait that provides 'splittability' for symbol streams
///
pub trait SplittableSymbolReader<Symbol: Clone> : SymbolReader<Symbol> {
    ///
    /// Returns two symbol readers that will independently read the stream of symbols from this reader
    ///
    fn split<'a>(&'a mut self) -> (SplitSymbolReader<'a, Symbol>, SplitSymbolReader<'a, Symbol>);
}

impl<Symbol: Clone, Reader: SymbolReader<Symbol>> SplittableSymbolReader<Symbol> for Reader {
    fn split<'a>(&'a mut self) -> (SplitSymbolReader<'a, Symbol>, SplitSymbolReader<'a, Symbol>) {
        // Generate the buffer that gets shared between the readers
        let mut buffer = SplitSymbolReaderBuffer::new(self);
        buffer.positions = vec![0,0];

        let buffer_ref = Rc::new(RefCell::new(buffer));

        // The readers share the buffer but have different IDs so they can read the same stream twice
        (SplitSymbolReader { buffer: buffer_ref.clone(), reader_id: 0 }, SplitSymbolReader { buffer: buffer_ref.clone(), reader_id: 1 })
    }
}

impl<'a, Symbol: Clone+'a> SymbolReader<Symbol> for SplitSymbolReader<'a, Symbol> {
    fn next_symbol(&mut self) -> Option<Symbol> {
        (*self.buffer).borrow_mut().read(self.reader_id)
    }
}

impl<'a, Symbol: Clone+'a> Drop for SplitSymbolReader<'a, Symbol> {
    fn drop(&mut self) {
        let mut buffer_ref = (*self.buffer).borrow_mut();

        buffer_ref.positions[self.reader_id] = usize::max_value();
        buffer_ref.clear_buffer();
    }
}

#[cfg(test)]
mod test {
    use super::super::*;

    #[test]
    fn can_split_stream_and_read_both_interleaved() {
        let source = vec![1,2,3];
        let mut stream = source.read_symbols();

        /* -- TODO: stream.split crashes (WHHHHHY?)
        let (mut first, mut second) = stream.split();

        assert!(first.next_symbol() == Some(1));
        assert!(second.next_symbol() == Some(1));
        assert!(first.next_symbol() == Some(2));
        assert!(second.next_symbol() == Some(2));
        assert!(first.next_symbol() == Some(3));
        assert!(second.next_symbol() == Some(3));
        assert!(first.next_symbol() == None);
        assert!(second.next_symbol() == None);
        */
    }

    /*
    #[test]
    fn can_split_stream_and_read_both_sequentially() {
        let source = vec![1,2,3];
        let mut stream = source.read_symbols();

        let (mut first, mut second) = stream.split();

        assert!(first.to_vec() == vec![1,2,3]);
        assert!(second.to_vec() == vec![1,2,3]);
    }
    */
}
<commit_msg>Fix issue caused when trying to flush the buffered items of the last split stream in existence<commit_after>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! A split symbol reader is used when two targets want to read from the same stream.
//!

use std::cell::*;
use std::rc::*;
use std::collections::*;

use super::symbol_reader::*;

///
/// Shares a symbol reader between two targets
///
pub struct SplitSymbolReader<'a, Symbol: Clone+'a> {
    // The buffer for this reader
    buffer: Rc<RefCell<SplitSymbolReaderBuffer<'a, Symbol>>>,

    // The ID of this reader within the buffer
    reader_id: usize
}

///
/// Stores the shared data between two symbol readers
///
/// Rust really does a poor job of making it so you can share things. The issue is that you can only borrow something
/// mutably once, so we need to faff around with RefCells and so on, which add overhead solely for dealing with language
/// semantics rather than actual issues with sharing data.
///
struct SplitSymbolReaderBuffer<'a, Symbol: 'a> {
    /// The underlying symbol reader
    symbol_reader: &'a mut SymbolReader<Symbol>,

    /// Positions for the various split readers that are using this object, relative to the buffer
    positions: Vec<usize>,

    /// Buffer of symbols that are waiting to be consumed by other readers
    buffer: VecDeque<Option<Symbol>>
}

impl<'a, Symbol: Clone> SplitSymbolReaderBuffer<'a, Symbol> {
    fn new(reader: &'a mut SymbolReader<Symbol>) -> SplitSymbolReaderBuffer<'a, Symbol> {
        SplitSymbolReaderBuffer { symbol_reader: reader, positions: vec![], buffer: VecDeque::new() }
    }

    ///
    /// Clears out any buffered items that have been read by all readers
    ///
    fn clear_buffer(&mut self) {
        let lowest_pos = self.positions.iter()
            .min()
            .map(|x| *x);

        if let Some(lowest_pos) = lowest_pos {
            if lowest_pos < usize::max_value() {
                for _ in 0..lowest_pos {
                    self.buffer.pop_front();
                }

                for pos in self.positions.iter_mut() {
                    *pos -= lowest_pos;
                }
            }
        }
    }

    ///
    /// Reads a symbol for the reader with a particular ID
    ///
    fn read(&mut self, reader_id: usize) -> Option<Symbol> {
        let buf_pos = self.positions[reader_id];

        if buf_pos >= self.buffer.len() {
            // This reader is at the end of the buffer, so just read the next symbol
            let next_symbol = self.symbol_reader.next_symbol();
            self.buffer.push_back(next_symbol.clone());

            self.positions[reader_id] = buf_pos+1;

            // If this reader was at the start of the buffer, then it might need clearing (in this case, only if this is the only active buffer)
            if buf_pos == 0 {
                self.clear_buffer();
            }

            next_symbol
        } else {
            // This reader is in the middle of the buffer, so just return a buffered value
            let next_symbol = self.buffer[buf_pos].clone();

            self.positions[reader_id] = buf_pos + 1;

            // If this reader was at the end of the buffer, then it might need clearing
            if buf_pos == 0 {
                self.clear_buffer();
            }

            next_symbol
        }
    }
}

///
/// Trait that provides 'splittability' for symbol streams
///
pub trait SplittableSymbolReader<Symbol: Clone> : SymbolReader<Symbol> {
    ///
    /// Returns two symbol readers that will independently read the stream of symbols from this reader
    ///
    fn split<'a>(&'a mut self) -> (SplitSymbolReader<'a, Symbol>, SplitSymbolReader<'a, Symbol>);
}

impl<Symbol: Clone, Reader: SymbolReader<Symbol>> SplittableSymbolReader<Symbol> for Reader {
    fn split<'a>(&'a mut self) -> (SplitSymbolReader<'a, Symbol>, SplitSymbolReader<'a, Symbol>) {
        // Generate the buffer that gets shared between the readers
        let mut buffer = SplitSymbolReaderBuffer::new(self);

        buffer.positions = vec![0,0];

        let buffer_ref = Rc::new(RefCell::new(buffer));

        // The readers share the buffer but have different IDs so they can read the same stream twice
        (SplitSymbolReader { buffer: buffer_ref.clone(), reader_id: 0 }, SplitSymbolReader { buffer: buffer_ref.clone(), reader_id: 1 })
    }
}

impl<'a, Symbol: Clone+'a> SymbolReader<Symbol> for SplitSymbolReader<'a, Symbol> {
    fn next_symbol(&mut self) -> Option<Symbol> {
        (*self.buffer).borrow_mut().read(self.reader_id)
    }
}

impl<'a, Symbol: Clone+'a> Drop for SplitSymbolReader<'a, Symbol> {
    fn drop(&mut self) {
        let mut buffer_ref = (*self.buffer).borrow_mut();

        buffer_ref.positions[self.reader_id] = usize::max_value();
        buffer_ref.clear_buffer();
    }
}

#[cfg(test)]
mod test {
    use super::super::*;

    #[test]
    fn can_split_stream_and_read_both_interleaved() {
        let source = vec![1,2,3];
        let mut stream = source.read_symbols();

        let (mut first, mut second) = stream.split();

        assert!(first.next_symbol() == Some(1));
        assert!(second.next_symbol() == Some(1));
        assert!(first.next_symbol() == Some(2));
        assert!(second.next_symbol() == Some(2));
        assert!(first.next_symbol() == Some(3));
        assert!(second.next_symbol() == Some(3));
        assert!(first.next_symbol() == None);
        assert!(second.next_symbol() == None);
    }

    #[test]
    fn can_split_stream_and_read_both_sequentially() {
        let source = vec![1,2,3];
        let mut stream = source.read_symbols();

        let (mut first, mut second) = stream.split();

        assert!(first.to_vec() == vec![1,2,3]);
        assert!(second.to_vec() == vec![1,2,3]);
    }
}
<|endoftext|>


<commit_before>extern crate itertools;

use std::collections::{HashMap, HashSet};
use std::fmt;
use itertools::Itertools;
use std::cmp;

pub fn solve(puzzle: &str) -> Option<HashMap<char, u8>> {
    let puzzle_split = puzzle.split("==").map(|s| s.trim()).collect::<Vec<_>>();
    if puzzle_split.len() != 2 {
        return None;
    }
    let result = puzzle_split[1];
    let mut data = puzzle_split[0]
        .split('+')
        .map(|s| s.trim())
        .collect::<Vec<_>>();

    data.push(result);

    let max_lgth = data.iter().map(|s| s.len()).max().unwrap();

    let data = normalize_tokens(data.as_mut_slice(), max_lgth);
    let data = data.iter().map(|s| s.as_str()).collect::<Vec<_>>();
    println!("data: {:?}", data);
    let mut leading_zero_letters = data.iter()
        .map(|s| s.chars().take(1).next().unwrap())
        .unique()
        .collect::<Vec<_>>();

    leading_zero_letters.push(result.chars().take(1).next().unwrap());

    let leading_zero_letters = leading_zero_letters
        .into_iter()
        .collect::<HashSet<_>>()
        .iter()
        .map(|&x| x)
        .collect::<String>();

    let mut m = HashMap::new();
    m.entry('_').or_insert(0);
    if find_solution(
        data.as_slice(),
        &mut m,
        0,
        0,
        0,
        data.len(),
        (data[0].len() - 1) as i8,
        leading_zero_letters.as_str(),
    )
    {
        m.remove(&'_');
        return Some(m);
    }

    None
}

fn assign_letter(m: &mut HashMap<char, u8>, leading_zeroes: &str, letter: char, digit: u8) -> bool {
    if leading_zeroes
        .chars()
        .find(|&c| c == letter && digit == 0)
        .is_some() || m.iter().find(|&(_, &d)| digit == d).is_some()
    {
        return false;
    }

    m.insert(letter, digit);

    return true;
}

pub fn unassign_letter(m: &mut HashMap<char, u8>, letter: char, digit: u8) {
    if let Some(idx) = m.iter().position(|(&x, &y)| x == letter && digit == y) {
        m.remove(&letter);
    }
}

fn find_solution(
    data: &[&str],
    m: &mut HashMap<char, u8>,
    carry: u8,
    sum: u8,
    curr_row: usize,
    max_rows: usize,
    curr_col: i8,
    leading_zero_letters: &str,
) -> bool {
    // // this has the leftmost digit of the terms
    // // If we try to assign a char in one of the addends
    let mut mm = HashMap::new();
    mm.entry('_').or_insert(0);
    mm.entry('D').or_insert(7);
    mm.entry('E').or_insert(5);
    // mm.entry('N').or_insert(6);
    // mm.entry('R').or_insert(8);
    if *m == mm {
        println!("hello");
    }
    if curr_col == -1 {
        if carry == 0 {
            return true;
        } else {
            return false;
        }
    }

    if curr_row != max_rows - 1 {
        // println!("M: {:?}", m);
        // println!("CARRY: {:?}", carry);
        // println!("SUM: {:?}", sum);
        // println!("CURR_ROW: {:?}", curr_row);
        // println!("CURR_COL: {:?}", curr_col);
        // // If we are on the last iterm of data, we are in the SUM, this means that
        let mut letter = data[curr_row]
            .chars()
            .skip(curr_col as usize)
            .next()
            .unwrap();

        if m.contains_key(&letter) {
            let mut v = 0;
            {
                v = m[&letter];
            }
            if find_solution(
                data,
                m,
                carry,
                sum + v,
                curr_row + 1,
                max_rows,
                curr_col,
                leading_zero_letters,
            )
            {
                return true;
            }
        } else {
            // letter is not already assigned, iterate over available digits
            let mut avail_digits = (0..10u8)
                .into_iter()
                .collect::<HashSet<_>>()
                .difference(&m.iter()
                    .filter(|&(&x, &y)| x != '_')
                    .map(|(&x, &y)| y)
                    .collect::<HashSet<_>>())
                .cloned()
                .collect::<Vec<_>>();
            avail_digits.sort();

            for d in avail_digits {
                if d == 0 {
                    if let Some(_) = leading_zero_letters.chars().find(|&c| c == letter) {
                        continue;
                    }
                }
                m.insert(letter, d);
                if find_solution(
                    data,
                    m,
                    carry,
                    sum + d,
                    curr_row + 1,
                    max_rows,
                    curr_col,
                    leading_zero_letters,
                )
                {
                    return true;
                } else {
                    m.remove(&letter);
                }
            }
            return false;

        }

    } else if curr_row == max_rows - 1 {
        let mut local_carry = 0;
        // we are evaluating the SUM now
        let mut letter = data[curr_row]
            .chars()
            .skip(curr_col as usize)
            .next()
            .unwrap();
        if m.contains_key(&letter) {
            let mut v = 0;
            {
                v = m[&letter];
            }
            // if char is assigned and matches the sum value
            let mut calc_val = sum + carry;
            local_carry = calc_val / 10;
            calc_val %= 10;
            if v == calc_val {
                if find_solution(
                    data,
                    m,
                    local_carry,
                    0,
                    0,
                    max_rows,
                    curr_col - 1,
                    leading_zero_letters,
                )
                {
                    return true;
                }
            } else {
                // char is assigned and doesn't match the sum value
                return false;
            }

        } else {
            // char is not assigned and correct digit is already in use
            let mut calc_val = sum + carry;
            local_carry = calc_val / 10;
            calc_val %= 10;
            let letter = data[curr_row]
                .chars()
                .skip(curr_col as usize)
                .next()
                .unwrap();

            if calc_val == 0 {
                if let Some(_) = leading_zero_letters.chars().find(|&c| c == letter) {
                    return false;
                }
            }
            if let Some(_) = m.iter().find(|&(&c, &d)| d == calc_val && c != '_') {
                return false;
            } else {
                // if calc_val >= 10 {
                //     println!("M: {:?}", m);
                //     println!("CARRY: {:?}", *carry);
                //     println!("SUM: {:?}", sum);
                //     println!("CURR_ROW: {:?}", curr_row);
                //     println!("CURR_COL: {:?}", curr_col);
                //     panic!("boom");
                // }
                m.insert(letter, calc_val);

                if find_solution(
                    data,
                    m,
                    local_carry,
                    0,
                    0,
                    max_rows,
                    curr_col - 1,
                    leading_zero_letters,
                )
                {
                    return true;
                } else {
                    m.remove(&letter);
                }
            }
        }
    }
    false
}

fn normalize_tokens(tkns: &mut [&str], max_len: usize) -> Vec<String> {
    let mut output = Vec::new();

    for tkn in tkns.iter_mut() {
        if tkn.len() < max_len {
            let mut padded_tkn = "_".repeat(max_len - tkn.len());
            padded_tkn.extend(tkn.chars());
            output.push(padded_tkn);
        } else {
            output.push(tkn.to_string());
        }
    }
    output
}
<commit_msg>Rust: Fix clippy warnings<commit_after>extern crate itertools;

use std::collections::{HashMap, HashSet};
use itertools::Itertools;

pub fn solve(puzzle: &str) -> Option<HashMap<char, u8>> {
    let puzzle_split = puzzle.split("==").map(|s| s.trim()).collect::<Vec<_>>();
    if puzzle_split.len() != 2 {
        return None;
    }
    let result = puzzle_split[1];
    let mut data = puzzle_split[0]
        .split('+')
        .map(|s| s.trim())
        .collect::<Vec<_>>();

    data.push(result);

    let max_lgth = data.iter().map(|s| s.len()).max().unwrap();

    let data = normalize_tokens(data.as_mut_slice(), max_lgth);
    let data = data.iter().map(|s| s.as_str()).collect::<Vec<_>>();
    println!("data: {:?}", data);
    let mut leading_zero_letters = data.iter()
        .map(|s| s.chars().take(1).next().unwrap())
        .unique()
        .collect::<Vec<_>>();

    leading_zero_letters.push(result.chars().take(1).next().unwrap());

    let leading_zero_letters = leading_zero_letters
        .into_iter()
        .collect::<HashSet<_>>()
        .iter()
        .cloned()
        .collect::<String>();

    let mut m = HashMap::new();

    m.entry('_').or_insert(0);
    if find_solution(
        data.as_slice(),
        &mut m,
        0,
        0,
        0,
        data.len(),
        (data[0].len() - 1) as i8,
        leading_zero_letters.as_str(),
    )
    {
        m.remove(&'_');
        return Some(m);
    }

    None
}

fn find_solution(
    data: &[&str],
    m: &mut HashMap<char, u8>,
    carry: u8,
    sum: u8,
    curr_row: usize,
    max_rows: usize,
    curr_col: i8,
    leading_zero_letters: &str,
) -> bool {
    if curr_col == -1 {
        return carry == 0;
    }

    if curr_row != max_rows - 1 {
        let letter = data[curr_row].chars().nth(curr_col as usize).unwrap();

        if m.get(&letter).is_some() {
            let v: u8;
            {
                v = m[&letter];
            }
            if find_solution(
                data,
                m,
                carry,
                sum + v,
                curr_row + 1,
                max_rows,
                curr_col,
                leading_zero_letters,
            )
            {
                return true;
            }
        } else {
            // letter is not already assigned, iterate over available digits
            let mut avail_digits = (0..10u8)
                .into_iter()
                .collect::<HashSet<_>>()
                .difference(&m.iter()
                    .filter(|&(&x, _)| x != '_')
                    .map(|(_, &y)| y)
                    .collect::<HashSet<_>>())
                .cloned()
                .collect::<Vec<_>>();
            avail_digits.sort();

            for d in avail_digits {
                if d == 0 {
                    if leading_zero_letters.chars().any(|c| c == letter) {
                        continue;
                    }
                }
                m.insert(letter, d);
                if find_solution(
                    data,
                    m,
                    carry,
                    sum + d,
                    curr_row + 1,
                    max_rows,
                    curr_col,
                    leading_zero_letters,
                )
                {
                    return true;
                } else {
                    m.remove(&letter);
                }
            }
            return false;

        }

    } else if curr_row == max_rows - 1 {
        let local_carry: u8;
        // we are evaluating the SUM now
        let letter = data[curr_row].chars().nth(curr_col as usize).unwrap();
        if m.get(&letter).is_some() {
            let v = m[&letter];
            // if char is assigned and matches the sum value
            let mut calc_val = sum + carry;
            local_carry = calc_val / 10;
            calc_val %= 10;
            if v == calc_val {
                if find_solution(
                    data,
                    m,
                    local_carry,
                    0,
                    0,
                    max_rows,
                    curr_col - 1,
                    leading_zero_letters,
                )
                {
                    return true;
                }
            } else {
                // char is assigned and doesn't match the sum value
                return false;
            }

        } else {
            // char is not assigned and correct digit is already in use
            let mut calc_val = sum + carry;
            local_carry = calc_val / 10;
            calc_val %= 10;

            if calc_val == 0 {
                if leading_zero_letters.chars().any(|c| c == letter) {
                    return false;
                }
            }
            if m.iter().any(|(&c, &d)| d == calc_val && c != '_') {
                return false;
            } else {
                // if calc_val >= 10 {
                //     println!("M: {:?}", m);
                //     println!("CARRY: {:?}", *carry);
                //     println!("SUM: {:?}", sum);
                //     println!("CURR_ROW: {:?}", curr_row);
                //     println!("CURR_COL: {:?}", curr_col);
                //     panic!("boom");
                // }
                m.insert(letter, calc_val);

                if find_solution(
                    data,
                    m,
                    local_carry,
                    0,
                    0,
                    max_rows,
                    curr_col - 1,
                    leading_zero_letters,
                )
                {
                    return true;
                } else {
                    m.remove(&letter);
                }
            }
        }
    }
    false
}

fn normalize_tokens(tkns: &mut [&str], max_len: usize) -> Vec<String> {
    let mut output = Vec::new();

    for tkn in tkns.iter_mut() {
        if tkn.len() < max_len {
            let mut padded_tkn = "_".repeat(max_len - tkn.len());
            padded_tkn.push_str(tkn);
            output.push(padded_tkn);
        } else {
            output.push(tkn.to_string());
        }
    }
    output
}
<|endoftext|>


<commit_before>use std::cell::Cell;
use std::rc::Rc;
use std::sync::Arc;

fn rc_cell() -> Rc<Cell<i32>> {
    let r = Rc::new(Cell::new(42));
    let x = r.get();
    r.set(x + x);
    r
}

// TODO(solson): also requires destructors to run for the second borrow to work
// TODO(solson): needs StructWrappedNullablePointer support
// fn rc_refcell() -> i32 {
//     let r = Rc::new(RefCell::new(42));
//     *r.borrow_mut() += 10;
//     let x = *r.borrow();
//     x
// }

fn arc() -> Arc<i32> {
    let a = Arc::new(42);
    a
}

fn true_assert() {
    assert_eq!(1, 1);
}

fn main() {
    assert_eq!(*arc(), 42);
    assert_eq!(rc_cell().get(), 84);
    true_assert();
}
<commit_msg>Enable an old test that works now!<commit_after>use std::cell::{Cell, RefCell};
use std::rc::Rc;
use std::sync::Arc;

fn rc_cell() -> Rc<Cell<i32>> {
    let r = Rc::new(Cell::new(42));
    let x = r.get();
    r.set(x + x);
    r
}

fn rc_refcell() -> i32 {
    let r = Rc::new(RefCell::new(42));
    *r.borrow_mut() += 10;
    let x = *r.borrow();
    x
}

fn arc() -> Arc<i32> {
    let a = Arc::new(42);
    a
}

fn true_assert() {
    assert_eq!(1, 1);
}

fn main() {
    assert_eq!(*arc(), 42);
    assert_eq!(rc_cell().get(), 84);
    assert_eq!(rc_refcell(), 52);
    true_assert();
}
<|endoftext|>


<commit_before>use core::SegmentId;
use error::{ErrorKind, ResultExt};
use schema::Schema;
use serde_json;
use std::borrow::BorrowMut;
use std::fmt;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use Result;

use super::pool::LeasedItem;
use super::pool::Pool;
use super::segment::create_segment;
use super::segment::Segment;
use core::searcher::Searcher;
use core::IndexMeta;
use core::SegmentMeta;
use core::SegmentReader;
use core::META_FILEPATH;
use directory::{ManagedDirectory, DirectoryClone};
#[cfg(feature = "mmap")]
use directory::MmapDirectory;
use directory::{Directory, RAMDirectory};
use indexer::index_writer::open_index_writer;
use indexer::index_writer::HEAP_SIZE_MIN;
use indexer::segment_updater::save_new_metas;
use indexer::DirectoryLock;
use num_cpus;
use std::path::Path;
use tokenizer::TokenizerManager;
use IndexWriter;

fn load_metas(directory: &Directory) -> Result<IndexMeta> {
    let meta_data = directory.atomic_read(&META_FILEPATH)?;
    let meta_string = String::from_utf8_lossy(&meta_data);
    serde_json::from_str(&meta_string).chain_err(|| ErrorKind::CorruptedFile(META_FILEPATH.clone()))
}

/// Search Index
pub struct Index {
    directory: ManagedDirectory,
    schema: Schema,
    num_searchers: Arc<AtomicUsize>,
    searcher_pool: Arc<Pool<Searcher>>,
    tokenizers: TokenizerManager,
}

impl Index {
    /// Creates a new index using the `RAMDirectory`.
    ///
    /// The index will be allocated in anonymous memory.
    /// This should only be used for unit tests.
    pub fn create_in_ram(schema: Schema) -> Index {
        let ram_directory = RAMDirectory::create();
        Index::create(ram_directory, schema).expect("Creating a RAMDirectory should never fail")
    }

    /// Creates a new index in a given filepath.
    /// The index will use the `MMapDirectory`.
    ///
    /// If a previous index was in this directory, then its meta file will be destroyed.
    #[cfg(feature = "mmap")]
    pub fn create_in_dir<P: AsRef<Path>>(directory_path: P, schema: Schema) -> Result<Index> {
        let mmap_directory = MmapDirectory::open(directory_path)?;
        Index::create(mmap_directory, schema)
    }

    /// Creates a new index in a temp directory.
    ///
    /// The index will use the `MMapDirectory` in a newly created directory.
    /// The temp directory will be destroyed automatically when the `Index` object
    /// is destroyed.
    ///
    /// The temp directory is only used for testing the `MmapDirectory`.
    /// For other unit tests, prefer the `RAMDirectory`, see: `create_in_ram`.
    #[cfg(feature = "mmap")]
    pub fn create_from_tempdir(schema: Schema) -> Result<Index> {
        let mmap_directory = MmapDirectory::create_from_tempdir()?;
        Index::create(mmap_directory, schema)
    }

    /// Creates a new index given an implementation of the trait `Directory`
    pub fn create<Dir: Directory>(dir: Dir, schema: Schema) -> Result<Index> {
        let directory = ManagedDirectory::new(dir)?;
        Index::from_directory(directory, schema)
    }

    /// Create a new index from a directory.
    fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> Result<Index> {
        save_new_metas(schema.clone(), 0, directory.borrow_mut())?;
        let metas = IndexMeta::with_schema(schema);
        Index::create_from_metas(directory, &metas)
    }

    /// Creates a new index given a directory and an `IndexMeta`.
    fn create_from_metas(directory: ManagedDirectory, metas: &IndexMeta) -> Result<Index> {
        let schema = metas.schema.clone();
        let n_cpus = num_cpus::get();
        let index = Index {
            directory,
            schema,
            num_searchers: Arc::new(AtomicUsize::new(n_cpus)),
            searcher_pool: Arc::new(Pool::new()),
            tokenizers: TokenizerManager::default(),
        };
        index.load_searchers()?;
        Ok(index)
    }

    /// Accessor for the tokenizer manager.
    pub fn tokenizers(&self) -> &TokenizerManager {
        &self.tokenizers
    }

    /// Opens a new directory from an index path.
    #[cfg(feature = "mmap")]
    pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> Result<Index> {
        let mmap_directory = MmapDirectory::open(directory_path)?;
        Index::open(mmap_directory)
    }

    /// Open the index using the provided directory
    pub fn open<D: Directory>(directory: D) -> Result<Index> {
        let directory = ManagedDirectory::new(directory)?;
        let metas = load_metas(&directory)?;
        Index::create_from_metas(directory, &metas)
    }

    /// Reads the index meta file from the directory.
    pub fn load_metas(&self) -> Result<IndexMeta> {
        load_metas(self.directory())
    }

    /// Open a new index writer. Attempts to acquire a lockfile.
    ///
    /// The lockfile should be deleted on drop, but it is possible
    /// that due to a panic or other error, a stale lockfile will be
    /// left in the index directory. If you are sure that no other
    /// `IndexWriter` on the system is accessing the index directory,
    /// it is safe to manually delete the lockfile.
    ///
    /// - `num_threads` defines the number of indexing workers that
    /// should work at the same time.
    ///
    /// - `overall_heap_size_in_bytes` sets the amount of memory
    /// allocated for all indexing thread.
    /// Each thread will receive a budget of  `overall_heap_size_in_bytes / num_threads`.
    ///
    /// # Errors
    /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
    /// # Panics
    /// If the heap size per thread is too small, panics.
    pub fn writer_with_num_threads(
        &self,
        num_threads: usize,
        overall_heap_size_in_bytes: usize,
    ) -> Result<IndexWriter> {
        let directory_lock = DirectoryLock::lock(self.directory().box_clone())?;
        let heap_size_in_bytes_per_thread = overall_heap_size_in_bytes / num_threads;
        open_index_writer(
            self,
            num_threads,
            heap_size_in_bytes_per_thread,
            directory_lock,
        )
    }

    /// Creates a multithreaded writer
    ///
    /// Tantivy will automatically define the number of threads to use.
    /// `overall_heap_size_in_bytes` is the total target memory usage that will be split
    /// between a given number of threads.
    ///
    /// # Errors
    /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
    /// # Panics
    /// If the heap size per thread is too small, panics.
    pub fn writer(&self, overall_heap_size_in_bytes: usize) -> Result<IndexWriter> {
        let mut num_threads = num_cpus::get();
        let heap_size_in_bytes_per_thread = overall_heap_size_in_bytes / num_threads;
        if heap_size_in_bytes_per_thread < HEAP_SIZE_MIN {
            num_threads = (overall_heap_size_in_bytes / HEAP_SIZE_MIN).max(1);
        }
        self.writer_with_num_threads(num_threads, overall_heap_size_in_bytes)
    }

    /// Accessor to the index schema
    ///
    /// The schema is actually cloned.
    pub fn schema(&self) -> Schema {
        self.schema.clone()
    }

    /// Returns the list of segments that are searchable
    pub fn searchable_segments(&self) -> Result<Vec<Segment>> {
        Ok(self.searchable_segment_metas()?
            .into_iter()
            .map(|segment_meta| self.segment(segment_meta))
            .collect())
    }

    #[doc(hidden)]
    pub fn segment(&self, segment_meta: SegmentMeta) -> Segment {
        create_segment(self.clone(), segment_meta)
    }

    /// Creates a new segment.
    pub fn new_segment(&self) -> Segment {
        let segment_meta = SegmentMeta::new(SegmentId::generate_random(), 0);
        self.segment(segment_meta)
    }

    /// Return a reference to the index directory.
    pub fn directory(&self) -> &ManagedDirectory {
        &self.directory
    }

    /// Return a mutable reference to the index directory.
    pub fn directory_mut(&mut self) -> &mut ManagedDirectory {
        &mut self.directory
    }

    /// Reads the meta.json and returns the list of
    /// `SegmentMeta` from the last commit.
    pub fn searchable_segment_metas(&self) -> Result<Vec<SegmentMeta>> {
        Ok(self.load_metas()?.segments)
    }

    /// Returns the list of segment ids that are searchable.
    pub fn searchable_segment_ids(&self) -> Result<Vec<SegmentId>> {
        Ok(self.searchable_segment_metas()?
            .iter()
            .map(|segment_meta| segment_meta.id())
            .collect())
    }

    /// Sets the number of searchers to use

    /// Only works after the next call to `load_searchers`
    pub fn set_num_searchers(&mut self, num_searchers: usize) {
        self.num_searchers = Arc::new(AtomicUsize::new(num_searchers));
    }

    /// Creates a new generation of searchers after

    /// a change of the set of searchable indexes.
    ///
    /// This needs to be called when a new segment has been
    /// published or after a merge.
    pub fn load_searchers(&self) -> Result<()> {
        let searchable_segments = self.searchable_segments()?;
        let segment_readers: Vec<SegmentReader> = searchable_segments
            .iter()
            .map(SegmentReader::open)
            .collect::<Result<_>>()?;
        let schema = self.schema();
        let num_searchers: usize = self.num_searchers.load(Ordering::Relaxed);
        let searchers = (0..num_searchers)
            .map(|_| Searcher::new(schema.clone(), segment_readers.clone()))
            .collect();
        self.searcher_pool.publish_new_generation(searchers);
        Ok(())
    }

    /// Returns a searcher
    ///
    /// This method should be called every single time a search
    /// query is performed.
    /// The searchers are taken from a pool of `num_searchers` searchers.
    /// If no searcher is available
    /// this may block.
    ///
    /// The same searcher must be used for a given query, as it ensures
    /// the use of a consistent segment set.
    pub fn searcher(&self) -> LeasedItem<Searcher> {
        self.searcher_pool.acquire()
    }
}

impl fmt::Debug for Index {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Index({:?})", self.directory)
    }
}

impl Clone for Index {
    fn clone(&self) -> Index {
        Index {
            directory: self.directory.clone(),
            schema: self.schema.clone(),
            num_searchers: Arc::clone(&self.num_searchers),
            searcher_pool: Arc::clone(&self.searcher_pool),
            tokenizers: self.tokenizers.clone(),
        }
    }
}
<commit_msg>N370 pr fix num searchers (#371)<commit_after>use core::SegmentId;
use error::{ErrorKind, ResultExt};
use schema::Schema;
use serde_json;
use std::borrow::BorrowMut;
use std::fmt;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use Result;

use super::pool::LeasedItem;
use super::pool::Pool;
use super::segment::create_segment;
use super::segment::Segment;
use core::searcher::Searcher;
use core::IndexMeta;
use core::SegmentMeta;
use core::SegmentReader;
use core::META_FILEPATH;
use directory::{ManagedDirectory, DirectoryClone};
#[cfg(feature = "mmap")]
use directory::MmapDirectory;
use directory::{Directory, RAMDirectory};
use indexer::index_writer::open_index_writer;
use indexer::index_writer::HEAP_SIZE_MIN;
use indexer::segment_updater::save_new_metas;
use indexer::DirectoryLock;
use num_cpus;
use std::path::Path;
use tokenizer::TokenizerManager;
use IndexWriter;

fn load_metas(directory: &Directory) -> Result<IndexMeta> {
    let meta_data = directory.atomic_read(&META_FILEPATH)?;
    let meta_string = String::from_utf8_lossy(&meta_data);
    serde_json::from_str(&meta_string).chain_err(|| ErrorKind::CorruptedFile(META_FILEPATH.clone()))
}

/// Search Index
pub struct Index {
    directory: ManagedDirectory,
    schema: Schema,
    num_searchers: Arc<AtomicUsize>,
    searcher_pool: Arc<Pool<Searcher>>,
    tokenizers: TokenizerManager,
}

impl Index {
    /// Creates a new index using the `RAMDirectory`.
    ///
    /// The index will be allocated in anonymous memory.
    /// This should only be used for unit tests.
    pub fn create_in_ram(schema: Schema) -> Index {
        let ram_directory = RAMDirectory::create();
        Index::create(ram_directory, schema).expect("Creating a RAMDirectory should never fail")
    }

    /// Creates a new index in a given filepath.
    /// The index will use the `MMapDirectory`.
    ///
    /// If a previous index was in this directory, then its meta file will be destroyed.
    #[cfg(feature = "mmap")]
    pub fn create_in_dir<P: AsRef<Path>>(directory_path: P, schema: Schema) -> Result<Index> {
        let mmap_directory = MmapDirectory::open(directory_path)?;
        Index::create(mmap_directory, schema)
    }

    /// Creates a new index in a temp directory.
    ///
    /// The index will use the `MMapDirectory` in a newly created directory.
    /// The temp directory will be destroyed automatically when the `Index` object
    /// is destroyed.
    ///
    /// The temp directory is only used for testing the `MmapDirectory`.
    /// For other unit tests, prefer the `RAMDirectory`, see: `create_in_ram`.
    #[cfg(feature = "mmap")]
    pub fn create_from_tempdir(schema: Schema) -> Result<Index> {
        let mmap_directory = MmapDirectory::create_from_tempdir()?;
        Index::create(mmap_directory, schema)
    }

    /// Creates a new index given an implementation of the trait `Directory`
    pub fn create<Dir: Directory>(dir: Dir, schema: Schema) -> Result<Index> {
        let directory = ManagedDirectory::new(dir)?;
        Index::from_directory(directory, schema)
    }

    /// Create a new index from a directory.
    fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> Result<Index> {
        save_new_metas(schema.clone(), 0, directory.borrow_mut())?;
        let metas = IndexMeta::with_schema(schema);
        Index::create_from_metas(directory, &metas)
    }

    /// Creates a new index given a directory and an `IndexMeta`.
    fn create_from_metas(directory: ManagedDirectory, metas: &IndexMeta) -> Result<Index> {
        let schema = metas.schema.clone();
        let n_cpus = num_cpus::get();
        let index = Index {
            directory,
            schema,
            num_searchers: Arc::new(AtomicUsize::new(n_cpus)),
            searcher_pool: Arc::new(Pool::new()),
            tokenizers: TokenizerManager::default(),
        };
        index.load_searchers()?;
        Ok(index)
    }

    /// Accessor for the tokenizer manager.
    pub fn tokenizers(&self) -> &TokenizerManager {
        &self.tokenizers
    }

    /// Opens a new directory from an index path.
    #[cfg(feature = "mmap")]
    pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> Result<Index> {
        let mmap_directory = MmapDirectory::open(directory_path)?;
        Index::open(mmap_directory)
    }

    /// Open the index using the provided directory
    pub fn open<D: Directory>(directory: D) -> Result<Index> {
        let directory = ManagedDirectory::new(directory)?;
        let metas = load_metas(&directory)?;
        Index::create_from_metas(directory, &metas)
    }

    /// Reads the index meta file from the directory.
    pub fn load_metas(&self) -> Result<IndexMeta> {
        load_metas(self.directory())
    }

    /// Open a new index writer. Attempts to acquire a lockfile.
    ///
    /// The lockfile should be deleted on drop, but it is possible
    /// that due to a panic or other error, a stale lockfile will be
    /// left in the index directory. If you are sure that no other
    /// `IndexWriter` on the system is accessing the index directory,
    /// it is safe to manually delete the lockfile.
    ///
    /// - `num_threads` defines the number of indexing workers that
    /// should work at the same time.
    ///
    /// - `overall_heap_size_in_bytes` sets the amount of memory
    /// allocated for all indexing thread.
    /// Each thread will receive a budget of  `overall_heap_size_in_bytes / num_threads`.
    ///
    /// # Errors
    /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
    /// # Panics
    /// If the heap size per thread is too small, panics.
    pub fn writer_with_num_threads(
        &self,
        num_threads: usize,
        overall_heap_size_in_bytes: usize,
    ) -> Result<IndexWriter> {
        let directory_lock = DirectoryLock::lock(self.directory().box_clone())?;
        let heap_size_in_bytes_per_thread = overall_heap_size_in_bytes / num_threads;
        open_index_writer(
            self,
            num_threads,
            heap_size_in_bytes_per_thread,
            directory_lock,
        )
    }

    /// Creates a multithreaded writer
    ///
    /// Tantivy will automatically define the number of threads to use.
    /// `overall_heap_size_in_bytes` is the total target memory usage that will be split
    /// between a given number of threads.
    ///
    /// # Errors
    /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
    /// # Panics
    /// If the heap size per thread is too small, panics.
    pub fn writer(&self, overall_heap_size_in_bytes: usize) -> Result<IndexWriter> {
        let mut num_threads = num_cpus::get();
        let heap_size_in_bytes_per_thread = overall_heap_size_in_bytes / num_threads;
        if heap_size_in_bytes_per_thread < HEAP_SIZE_MIN {
            num_threads = (overall_heap_size_in_bytes / HEAP_SIZE_MIN).max(1);
        }
        self.writer_with_num_threads(num_threads, overall_heap_size_in_bytes)
    }

    /// Accessor to the index schema
    ///
    /// The schema is actually cloned.
    pub fn schema(&self) -> Schema {
        self.schema.clone()
    }

    /// Returns the list of segments that are searchable
    pub fn searchable_segments(&self) -> Result<Vec<Segment>> {
        Ok(self.searchable_segment_metas()?
            .into_iter()
            .map(|segment_meta| self.segment(segment_meta))
            .collect())
    }

    #[doc(hidden)]
    pub fn segment(&self, segment_meta: SegmentMeta) -> Segment {
        create_segment(self.clone(), segment_meta)
    }

    /// Creates a new segment.
    pub fn new_segment(&self) -> Segment {
        let segment_meta = SegmentMeta::new(SegmentId::generate_random(), 0);
        self.segment(segment_meta)
    }

    /// Return a reference to the index directory.
    pub fn directory(&self) -> &ManagedDirectory {
        &self.directory
    }

    /// Return a mutable reference to the index directory.
    pub fn directory_mut(&mut self) -> &mut ManagedDirectory {
        &mut self.directory
    }

    /// Reads the meta.json and returns the list of
    /// `SegmentMeta` from the last commit.
    pub fn searchable_segment_metas(&self) -> Result<Vec<SegmentMeta>> {
        Ok(self.load_metas()?.segments)
    }

    /// Returns the list of segment ids that are searchable.
    pub fn searchable_segment_ids(&self) -> Result<Vec<SegmentId>> {
        Ok(self.searchable_segment_metas()?
            .iter()
            .map(|segment_meta| segment_meta.id())
            .collect())
    }

    /// Sets the number of searchers to use

    /// Only works after the next call to `load_searchers`
    pub fn set_num_searchers(&mut self, num_searchers: usize) {
        self.num_searchers.store(num_searchers, Ordering::Release);
    }

    /// Creates a new generation of searchers after

    /// a change of the set of searchable indexes.
    ///
    /// This needs to be called when a new segment has been
    /// published or after a merge.
    pub fn load_searchers(&self) -> Result<()> {
        let searchable_segments = self.searchable_segments()?;
        let segment_readers: Vec<SegmentReader> = searchable_segments
            .iter()
            .map(SegmentReader::open)
            .collect::<Result<_>>()?;
        let schema = self.schema();
        let num_searchers: usize = self.num_searchers.load(Ordering::Acquire);
        let searchers = (0..num_searchers)
            .map(|_| Searcher::new(schema.clone(), segment_readers.clone()))
            .collect();
        self.searcher_pool.publish_new_generation(searchers);
        Ok(())
    }

    /// Returns a searcher
    ///
    /// This method should be called every single time a search
    /// query is performed.
    /// The searchers are taken from a pool of `num_searchers` searchers.
    /// If no searcher is available
    /// this may block.
    ///
    /// The same searcher must be used for a given query, as it ensures
    /// the use of a consistent segment set.
    pub fn searcher(&self) -> LeasedItem<Searcher> {
        self.searcher_pool.acquire()
    }
}

impl fmt::Debug for Index {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Index({:?})", self.directory)
    }
}

impl Clone for Index {
    fn clone(&self) -> Index {
        Index {
            directory: self.directory.clone(),
            schema: self.schema.clone(),
            num_searchers: Arc::clone(&self.num_searchers),
            searcher_pool: Arc::clone(&self.searcher_pool),
            tokenizers: self.tokenizers.clone(),
        }
    }
}
<|endoftext|>


<commit_before>extern crate ispc;

fn main() {
    ispc::compile_library("simple", &["src/simple.ispc"]);
}

<commit_msg>test isa selection on simple example<commit_after>extern crate ispc;

fn main() {
    // ispc::compile_library("simple", &["src/simple.ispc"]);
    // We need to use a custom config to explicitly not generate debug info
    // for ISPC code on windows otherwise we get repeated symbol declarations
    let mut cfg = ispc::Config::new();
    if cfg!(windows) {
        cfg.debug(false);
    }
    let ispc_files = vec!["src/simple.ispc"];
    for s in &ispc_files[..] {
        cfg.file(*s);
    }
    cfg.target_isas(vec![ispc::opt::TargetISA::SSE4i32x4,
                    ispc::opt::TargetISA::AVX2i32x8,
                    ispc::opt::TargetISA::AVX11i32x8,
                    ispc::opt::TargetISA::AVX512KNLi32x16]);
    cfg.compile("simple");
}

<|endoftext|>