<commit_before>// Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Semver parsing and logic

#[allow(missing_doc)];

use std::char;
use std::cmp;
use std::io::{ReaderUtil};
use std::io;
use std::option::{Option, Some, None};
use std::to_str::ToStr;

#[deriving(Clone, Eq)]
pub enum Identifier {
    Numeric(uint),
    AlphaNumeric(~str)
}

impl cmp::Ord for Identifier {
    #[inline]
    fn lt(&self, other: &Identifier) -> bool {
        match (self, other) {
            (&Numeric(a), &Numeric(b)) => a < b,
            (&Numeric(_), _) => true,
            (&AlphaNumeric(ref a), &AlphaNumeric(ref b)) => *a < *b,
            (&AlphaNumeric(_), _) => false
        }
    }
}

impl ToStr for Identifier {
    #[inline]
    fn to_str(&self) -> ~str {
        match self {
            &Numeric(n) => n.to_str(),
            &AlphaNumeric(ref s) => s.to_str()
        }
    }
}

#[deriving(Clone, Eq)]
pub struct Version {
    major: uint,
    minor: uint,
    patch: uint,
    pre: ~[Identifier],
    build: ~[Identifier],
}

impl ToStr for Version {
    #[inline]
    fn to_str(&self) -> ~str {
        let s = fmt!("%u.%u.%u", self.major, self.minor, self.patch);
        let s = if self.pre.is_empty() {
            s
        } else {
            fmt!("%s-%s", s, self.pre.map(|i| i.to_str()).connect("."))
        };
        if self.build.is_empty() {
            s
        } else {
            fmt!("%s+%s", s, self.build.map(|i| i.to_str()).connect("."))
        }
    }
}

impl cmp::Ord for Version {
    #[inline]
    fn lt(&self, other: &Version) -> bool {

        self.major < other.major ||

            (self.major == other.major &&
             self.minor < other.minor) ||

            (self.major == other.major &&
             self.minor == other.minor &&
             self.patch < other.patch) ||

            (self.major == other.major &&
             self.minor == other.minor &&
             self.patch == other.patch &&
             // NB: semver spec says 0.0.0-pre < 0.0.0
             // but the version of ord defined for vec
             // says that [] < [pre], so we alter it
             // here.
             (match (self.pre.len(), other.pre.len()) {
                 (0, 0) => false,
                 (0, _) => false,
                 (_, 0) => true,
                 (_, _) => self.pre < other.pre
             }))
    }

    #[inline]
    fn le(&self, other: &Version) -> bool {
        ! (other < self)
    }
    #[inline]
    fn gt(&self, other: &Version) -> bool {
        other < self
    }
    #[inline]
    fn ge(&self, other: &Version) -> bool {
        ! (self < other)
    }
}

condition! {
    bad_parse: () -> ();
}

fn take_nonempty_prefix(rdr: @io::Reader,
                        ch: char,
                        pred: &fn(char) -> bool) -> (~str, char) {
    let mut buf = ~"";
    let mut ch = ch;
    while pred(ch) {
        buf.push_char(ch);
        ch = rdr.read_char();
    }
    if buf.is_empty() {
        bad_parse::cond.raise(())
    }
    debug!("extracted nonempty prefix: %s", buf);
    (buf, ch)
}

fn take_num(rdr: @io::Reader, ch: char) -> (uint, char) {
    let (s, ch) = take_nonempty_prefix(rdr, ch, char::is_digit);
    match from_str::<uint>(s) {
        None => { bad_parse::cond.raise(()); (0, ch) },
        Some(i) => (i, ch)
    }
}

fn take_ident(rdr: @io::Reader, ch: char) -> (Identifier, char) {
    let (s,ch) = take_nonempty_prefix(rdr, ch, char::is_alphanumeric);
    if s.iter().all(char::is_digit) {
        match from_str::<uint>(s) {
            None => { bad_parse::cond.raise(()); (Numeric(0), ch) },
            Some(i) => (Numeric(i), ch)
        }
    } else {
        (AlphaNumeric(s), ch)
    }
}

fn expect(ch: char, c: char) {
    if ch != c {
        bad_parse::cond.raise(())
    }
}

fn parse_reader(rdr: @io::Reader) -> Version {
    let (major, ch) = take_num(rdr, rdr.read_char());
    expect(ch, '.');
    let (minor, ch) = take_num(rdr, rdr.read_char());
    expect(ch, '.');
    let (patch, ch) = take_num(rdr, rdr.read_char());

    let mut pre = ~[];
    let mut build = ~[];

    let mut ch = ch;
    if ch == '-' {
        loop {
            let (id, c) = take_ident(rdr, rdr.read_char());
            pre.push(id);
            ch = c;
            if ch != '.' { break; }
        }
    }

    if ch == '+' {
        loop {
            let (id, c) = take_ident(rdr, rdr.read_char());
            build.push(id);
            ch = c;
            if ch != '.' { break; }
        }
    }

    Version {
        major: major,
        minor: minor,
        patch: patch,
        pre: pre,
        build: build,
    }
}

pub fn parse(s: &str) -> Option<Version> {
    if !s.is_ascii() {
        return None;
    }
    let s = s.trim();
    let mut bad = false;
    do bad_parse::cond.trap(|_| { debug!("bad"); bad = true }).inside {
        do io::with_str_reader(s) |rdr| {
            let v = parse_reader(rdr);
            if bad || v.to_str() != s.to_owned() {
                None
            } else {
                Some(v)
            }
        }
    }
}

#[test]
fn test_parse() {
    assert_eq!(parse(""), None);
    assert_eq!(parse("  "), None);
    assert_eq!(parse("1"), None);
    assert_eq!(parse("1.2"), None);
    assert_eq!(parse("1.2"), None);
    assert_eq!(parse("1"), None);
    assert_eq!(parse("1.2"), None);
    assert_eq!(parse("1.2.3-"), None);
    assert_eq!(parse("a.b.c"), None);
    assert_eq!(parse("1.2.3 abc"), None);

    assert!(parse("1.2.3") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[],
        build: ~[],
    }));
    assert!(parse("  1.2.3  ") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[],
        build: ~[],
    }));
    assert!(parse("1.2.3-alpha1") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[AlphaNumeric(~"alpha1")],
        build: ~[]
    }));
    assert!(parse("  1.2.3-alpha1  ") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[AlphaNumeric(~"alpha1")],
        build: ~[]
    }));
    assert!(parse("1.2.3+build5") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[],
        build: ~[AlphaNumeric(~"build5")]
    }));
    assert!(parse("  1.2.3+build5  ") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[],
        build: ~[AlphaNumeric(~"build5")]
    }));
    assert!(parse("1.2.3-alpha1+build5") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[AlphaNumeric(~"alpha1")],
        build: ~[AlphaNumeric(~"build5")]
    }));
    assert!(parse("  1.2.3-alpha1+build5  ") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[AlphaNumeric(~"alpha1")],
        build: ~[AlphaNumeric(~"build5")]
    }));
    assert!(parse("1.2.3-1.alpha1.9+build5.7.3aedf  ") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[Numeric(1),AlphaNumeric(~"alpha1"),Numeric(9)],
        build: ~[AlphaNumeric(~"build5"),
                 Numeric(7),
                 AlphaNumeric(~"3aedf")]
    }));

}

#[test]
fn test_eq() {
    assert_eq!(parse("1.2.3"), parse("1.2.3"));
    assert_eq!(parse("1.2.3-alpha1"), parse("1.2.3-alpha1"));
    assert_eq!(parse("1.2.3+build.42"), parse("1.2.3+build.42"));
    assert_eq!(parse("1.2.3-alpha1+42"), parse("1.2.3-alpha1+42"));
}

#[test]
fn test_ne() {
    assert!(parse("0.0.0")       != parse("0.0.1"));
    assert!(parse("0.0.0")       != parse("0.1.0"));
    assert!(parse("0.0.0")       != parse("1.0.0"));
    assert!(parse("1.2.3-alpha") != parse("1.2.3-beta"));
    assert!(parse("1.2.3+23")    != parse("1.2.3+42"));
}

#[test]
fn test_lt() {
    assert!(parse("0.0.0")        < parse("1.2.3-alpha2"));
    assert!(parse("1.0.0")        < parse("1.2.3-alpha2"));
    assert!(parse("1.2.0")        < parse("1.2.3-alpha2"));
    assert!(parse("1.2.3-alpha1") < parse("1.2.3"));
    assert!(parse("1.2.3-alpha1") < parse("1.2.3-alpha2"));
    assert!(!(parse("1.2.3-alpha2") < parse("1.2.3-alpha2")));
    assert!(!(parse("1.2.3+23")     < parse("1.2.3+42")));
}

#[test]
fn test_le() {
    assert!(parse("0.0.0")        <= parse("1.2.3-alpha2"));
    assert!(parse("1.0.0")        <= parse("1.2.3-alpha2"));
    assert!(parse("1.2.0")        <= parse("1.2.3-alpha2"));
    assert!(parse("1.2.3-alpha1") <= parse("1.2.3-alpha2"));
    assert!(parse("1.2.3-alpha2") <= parse("1.2.3-alpha2"));
    assert!(parse("1.2.3+23")     <= parse("1.2.3+42"));
}

#[test]
fn test_gt() {
    assert!(parse("1.2.3-alpha2") > parse("0.0.0"));
    assert!(parse("1.2.3-alpha2") > parse("1.0.0"));
    assert!(parse("1.2.3-alpha2") > parse("1.2.0"));
    assert!(parse("1.2.3-alpha2") > parse("1.2.3-alpha1"));
    assert!(parse("1.2.3")        > parse("1.2.3-alpha2"));
    assert!(!(parse("1.2.3-alpha2") > parse("1.2.3-alpha2")));
    assert!(!(parse("1.2.3+23")     > parse("1.2.3+42")));
}

#[test]
fn test_ge() {
    assert!(parse("1.2.3-alpha2") >= parse("0.0.0"));
    assert!(parse("1.2.3-alpha2") >= parse("1.0.0"));
    assert!(parse("1.2.3-alpha2") >= parse("1.2.0"));
    assert!(parse("1.2.3-alpha2") >= parse("1.2.3-alpha1"));
    assert!(parse("1.2.3-alpha2") >= parse("1.2.3-alpha2"));
    assert!(parse("1.2.3+23")     >= parse("1.2.3+42"));
}

#[test]
fn test_spec_order() {

    let vs = ["1.0.0-alpha",
              "1.0.0-alpha.1",
              "1.0.0-alpha.beta",
              "1.0.0-beta",
              "1.0.0-beta.2",
              "1.0.0-beta.11",
              "1.0.0-rc.1",
              "1.0.0"];
    let mut i = 1;
    while i < vs.len() {
        let a = parse(vs[i-1]).unwrap();
        let b = parse(vs[i]).unwrap();
        assert!(a < b);
        i += 1;
    }
}
<commit_msg>Add docs for extra::semver<commit_after>// Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Semantic version parsing and comparison.
//!
//! Semantic versioning (see http://semver.org/) is a set of rules for
//! assigning version numbers intended to convey meaning about what has
//! changed, and how much. A version number has five parts:
//!
//!  * Major number, updated for incompatible API changes
//!  * Minor number, updated for backwards-compatible API additions
//!  * Patch number, updated for backwards-compatible bugfixes
//!  * Pre-release information (optional), preceded by a hyphen (`-`)
//!  * Build metadata (optional), preceded by a plus sign (`+`)
//!
//! The three mandatory components are required to be decimal numbers. The
//! pre-release information and build metadata are required to be a
//! period-separated list of identifiers containing only alphanumeric
//! characters and hyphens.
//!
//! An example version number with all five components is
//! `0.8.1-rc.3.0+20130922.linux`.

use std::char;
use std::cmp;
use std::io::{ReaderUtil};
use std::io;
use std::option::{Option, Some, None};
use std::to_str::ToStr;

/// An identifier in the pre-release or build metadata. If the identifier can
/// be parsed as a decimal value, it will be represented with `Numeric`.
#[deriving(Clone, Eq)]
pub enum Identifier {
    Numeric(uint),
    AlphaNumeric(~str)
}

impl cmp::Ord for Identifier {
    #[inline]
    fn lt(&self, other: &Identifier) -> bool {
        match (self, other) {
            (&Numeric(a), &Numeric(b)) => a < b,
            (&Numeric(_), _) => true,
            (&AlphaNumeric(ref a), &AlphaNumeric(ref b)) => *a < *b,
            (&AlphaNumeric(_), _) => false
        }
    }
}

impl ToStr for Identifier {
    #[inline]
    fn to_str(&self) -> ~str {
        match self {
            &Numeric(n) => n.to_str(),
            &AlphaNumeric(ref s) => s.to_str()
        }
    }
}

/// Represents a version number conforming to the semantic versioning scheme.
#[deriving(Clone, Eq)]
pub struct Version {
    /// The major version, to be incremented on incompatible changes.
    major: uint,
    /// The minor version, to be incremented when functionality is added in a
    /// backwards-compatible manner.
    minor: uint,
    /// The patch version, to be incremented when backwards-compatible bug
    /// fixes are made.
    patch: uint,
    /// The pre-release version identifier, if one exists.
    pre: ~[Identifier],
    /// The build metadata, ignored when determining version precedence.
    build: ~[Identifier],
}

impl ToStr for Version {
    #[inline]
    fn to_str(&self) -> ~str {
        let s = fmt!("%u.%u.%u", self.major, self.minor, self.patch);
        let s = if self.pre.is_empty() {
            s
        } else {
            fmt!("%s-%s", s, self.pre.map(|i| i.to_str()).connect("."))
        };
        if self.build.is_empty() {
            s
        } else {
            fmt!("%s+%s", s, self.build.map(|i| i.to_str()).connect("."))
        }
    }
}

impl cmp::Ord for Version {
    #[inline]
    fn lt(&self, other: &Version) -> bool {

        self.major < other.major ||

            (self.major == other.major &&
             self.minor < other.minor) ||

            (self.major == other.major &&
             self.minor == other.minor &&
             self.patch < other.patch) ||

            (self.major == other.major &&
             self.minor == other.minor &&
             self.patch == other.patch &&
             // NB: semver spec says 0.0.0-pre < 0.0.0
             // but the version of ord defined for vec
             // says that [] < [pre], so we alter it
             // here.
             (match (self.pre.len(), other.pre.len()) {
                 (0, 0) => false,
                 (0, _) => false,
                 (_, 0) => true,
                 (_, _) => self.pre < other.pre
             }))
    }

    #[inline]
    fn le(&self, other: &Version) -> bool {
        ! (other < self)
    }
    #[inline]
    fn gt(&self, other: &Version) -> bool {
        other < self
    }
    #[inline]
    fn ge(&self, other: &Version) -> bool {
        ! (self < other)
    }
}

condition! {
    bad_parse: () -> ();
}

fn take_nonempty_prefix(rdr: @io::Reader,
                        ch: char,
                        pred: &fn(char) -> bool) -> (~str, char) {
    let mut buf = ~"";
    let mut ch = ch;
    while pred(ch) {
        buf.push_char(ch);
        ch = rdr.read_char();
    }
    if buf.is_empty() {
        bad_parse::cond.raise(())
    }
    debug!("extracted nonempty prefix: %s", buf);
    (buf, ch)
}

fn take_num(rdr: @io::Reader, ch: char) -> (uint, char) {
    let (s, ch) = take_nonempty_prefix(rdr, ch, char::is_digit);
    match from_str::<uint>(s) {
        None => { bad_parse::cond.raise(()); (0, ch) },
        Some(i) => (i, ch)
    }
}

fn take_ident(rdr: @io::Reader, ch: char) -> (Identifier, char) {
    let (s,ch) = take_nonempty_prefix(rdr, ch, char::is_alphanumeric);
    if s.iter().all(char::is_digit) {
        match from_str::<uint>(s) {
            None => { bad_parse::cond.raise(()); (Numeric(0), ch) },
            Some(i) => (Numeric(i), ch)
        }
    } else {
        (AlphaNumeric(s), ch)
    }
}

fn expect(ch: char, c: char) {
    if ch != c {
        bad_parse::cond.raise(())
    }
}

fn parse_reader(rdr: @io::Reader) -> Version {
    let (major, ch) = take_num(rdr, rdr.read_char());
    expect(ch, '.');
    let (minor, ch) = take_num(rdr, rdr.read_char());
    expect(ch, '.');
    let (patch, ch) = take_num(rdr, rdr.read_char());

    let mut pre = ~[];
    let mut build = ~[];

    let mut ch = ch;
    if ch == '-' {
        loop {
            let (id, c) = take_ident(rdr, rdr.read_char());
            pre.push(id);
            ch = c;
            if ch != '.' { break; }
        }
    }

    if ch == '+' {
        loop {
            let (id, c) = take_ident(rdr, rdr.read_char());
            build.push(id);
            ch = c;
            if ch != '.' { break; }
        }
    }

    Version {
        major: major,
        minor: minor,
        patch: patch,
        pre: pre,
        build: build,
    }
}

/// Parse a string into a semver object.
pub fn parse(s: &str) -> Option<Version> {
    if !s.is_ascii() {
        return None;
    }
    let s = s.trim();
    let mut bad = false;
    do bad_parse::cond.trap(|_| { debug!("bad"); bad = true }).inside {
        do io::with_str_reader(s) |rdr| {
            let v = parse_reader(rdr);
            if bad || v.to_str() != s.to_owned() {
                None
            } else {
                Some(v)
            }
        }
    }
}

#[test]
fn test_parse() {
    assert_eq!(parse(""), None);
    assert_eq!(parse("  "), None);
    assert_eq!(parse("1"), None);
    assert_eq!(parse("1.2"), None);
    assert_eq!(parse("1.2"), None);
    assert_eq!(parse("1"), None);
    assert_eq!(parse("1.2"), None);
    assert_eq!(parse("1.2.3-"), None);
    assert_eq!(parse("a.b.c"), None);
    assert_eq!(parse("1.2.3 abc"), None);

    assert!(parse("1.2.3") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[],
        build: ~[],
    }));
    assert!(parse("  1.2.3  ") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[],
        build: ~[],
    }));
    assert!(parse("1.2.3-alpha1") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[AlphaNumeric(~"alpha1")],
        build: ~[]
    }));
    assert!(parse("  1.2.3-alpha1  ") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[AlphaNumeric(~"alpha1")],
        build: ~[]
    }));
    assert!(parse("1.2.3+build5") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[],
        build: ~[AlphaNumeric(~"build5")]
    }));
    assert!(parse("  1.2.3+build5  ") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[],
        build: ~[AlphaNumeric(~"build5")]
    }));
    assert!(parse("1.2.3-alpha1+build5") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[AlphaNumeric(~"alpha1")],
        build: ~[AlphaNumeric(~"build5")]
    }));
    assert!(parse("  1.2.3-alpha1+build5  ") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[AlphaNumeric(~"alpha1")],
        build: ~[AlphaNumeric(~"build5")]
    }));
    assert!(parse("1.2.3-1.alpha1.9+build5.7.3aedf  ") == Some(Version {
        major: 1u,
        minor: 2u,
        patch: 3u,
        pre: ~[Numeric(1),AlphaNumeric(~"alpha1"),Numeric(9)],
        build: ~[AlphaNumeric(~"build5"),
                 Numeric(7),
                 AlphaNumeric(~"3aedf")]
    }));

}

#[test]
fn test_eq() {
    assert_eq!(parse("1.2.3"), parse("1.2.3"));
    assert_eq!(parse("1.2.3-alpha1"), parse("1.2.3-alpha1"));
    assert_eq!(parse("1.2.3+build.42"), parse("1.2.3+build.42"));
    assert_eq!(parse("1.2.3-alpha1+42"), parse("1.2.3-alpha1+42"));
}

#[test]
fn test_ne() {
    assert!(parse("0.0.0")       != parse("0.0.1"));
    assert!(parse("0.0.0")       != parse("0.1.0"));
    assert!(parse("0.0.0")       != parse("1.0.0"));
    assert!(parse("1.2.3-alpha") != parse("1.2.3-beta"));
    assert!(parse("1.2.3+23")    != parse("1.2.3+42"));
}

#[test]
fn test_lt() {
    assert!(parse("0.0.0")        < parse("1.2.3-alpha2"));
    assert!(parse("1.0.0")        < parse("1.2.3-alpha2"));
    assert!(parse("1.2.0")        < parse("1.2.3-alpha2"));
    assert!(parse("1.2.3-alpha1") < parse("1.2.3"));
    assert!(parse("1.2.3-alpha1") < parse("1.2.3-alpha2"));
    assert!(!(parse("1.2.3-alpha2") < parse("1.2.3-alpha2")));
    assert!(!(parse("1.2.3+23")     < parse("1.2.3+42")));
}

#[test]
fn test_le() {
    assert!(parse("0.0.0")        <= parse("1.2.3-alpha2"));
    assert!(parse("1.0.0")        <= parse("1.2.3-alpha2"));
    assert!(parse("1.2.0")        <= parse("1.2.3-alpha2"));
    assert!(parse("1.2.3-alpha1") <= parse("1.2.3-alpha2"));
    assert!(parse("1.2.3-alpha2") <= parse("1.2.3-alpha2"));
    assert!(parse("1.2.3+23")     <= parse("1.2.3+42"));
}

#[test]
fn test_gt() {
    assert!(parse("1.2.3-alpha2") > parse("0.0.0"));
    assert!(parse("1.2.3-alpha2") > parse("1.0.0"));
    assert!(parse("1.2.3-alpha2") > parse("1.2.0"));
    assert!(parse("1.2.3-alpha2") > parse("1.2.3-alpha1"));
    assert!(parse("1.2.3")        > parse("1.2.3-alpha2"));
    assert!(!(parse("1.2.3-alpha2") > parse("1.2.3-alpha2")));
    assert!(!(parse("1.2.3+23")     > parse("1.2.3+42")));
}

#[test]
fn test_ge() {
    assert!(parse("1.2.3-alpha2") >= parse("0.0.0"));
    assert!(parse("1.2.3-alpha2") >= parse("1.0.0"));
    assert!(parse("1.2.3-alpha2") >= parse("1.2.0"));
    assert!(parse("1.2.3-alpha2") >= parse("1.2.3-alpha1"));
    assert!(parse("1.2.3-alpha2") >= parse("1.2.3-alpha2"));
    assert!(parse("1.2.3+23")     >= parse("1.2.3+42"));
}

#[test]
fn test_spec_order() {

    let vs = ["1.0.0-alpha",
              "1.0.0-alpha.1",
              "1.0.0-alpha.beta",
              "1.0.0-beta",
              "1.0.0-beta.2",
              "1.0.0-beta.11",
              "1.0.0-rc.1",
              "1.0.0"];
    let mut i = 1;
    while i < vs.len() {
        let a = parse(vs[i-1]).unwrap();
        let b = parse(vs[i]).unwrap();
        assert!(a < b);
        i += 1;
    }
}
<|endoftext|>


<commit_before>#[cfg(all(feature="syntax-highlight", not(windows)))]
mod syntect;
#[cfg(all(feature="syntax-highlight", not(windows)))]
pub use self::syntect::*;

#[cfg(any(not(feature="syntax-highlight"), windows))]
mod null;
#[cfg(any(not(feature="syntax-highlight"), windows))]
pub use self::null::*;
<commit_msg>chore: Remove platform check for syntax-highlight<commit_after>#[cfg(feature="syntax-highlight")]
mod syntect;
#[cfg(feature="syntax-highlight")]
pub use self::syntect::*;

#[cfg(not(feature="syntax-highlight"))]
mod null;
#[cfg(not(feature="syntax-highlight"))]
pub use self::null::*;
<|endoftext|>


<commit_before>//! A filter for cross-site request forgery (CSRF).
//!
//! This middleware is stateless and [based on request
//! headers](https://www.owasp.org/index.php/Cross-Site_Request_Forgery_(CSRF)_Prevention_Cheat_Sheet#Verifying_Same_Origin_with_Standard_Headers).
//!
//! By default requests are allowed only if one of these is true:
//!
//! * The request method is safe (`GET`, `HEAD`, `OPTIONS`). It is the
//!   applications responsibility to ensure these methods cannot be used to
//!   execute unwanted actions. Note that upgrade requests for websockets are
//!   also considered safe.
//! * The `Origin` header (added automatically by the browser) matches one
//!   of the allowed origins.
//! * There is no `Origin` header but the `Referer` header matches one of
//!   the allowed origins.
//!
//! Use [`CsrfFilterBuilder::allow_xhr()`](struct.CsrfFilterBuilder.html#method.allow_xhr)
//! if you want to allow requests with unsafe methods via
//! [CORS](../cors/struct.Cors.html).
//!
//! # Example
//!
//! ```
//! # extern crate actix_web;
//! # use actix_web::*;
//!
//! use actix_web::middleware::csrf;
//!
//! fn handle_post(_req: HttpRequest) -> &'static str {
//!     "This action should only be triggered with requests from the same site"
//! }
//!
//! fn main() {
//!     let app = Application::new()
//!         .middleware(
//!             csrf::CsrfFilter::build()
//!                 .allowed_origin("https://www.example.com")
//!                 .finish())
//!         .resource("/", |r| {
//!             r.method(Method::GET).f(|_| httpcodes::HttpOk);
//!             r.method(Method::POST).f(handle_post);
//!         })
//!         .finish();
//! }
//! ```
//!
//! In this example the entire application is protected from CSRF.

use std::borrow::Cow;
use std::collections::HashSet;

use bytes::Bytes;
use error::{Result, ResponseError};
use http::{HeaderMap, HttpTryFrom, Uri, header};
use httprequest::HttpRequest;
use httpresponse::HttpResponse;
use httpmessage::HttpMessage;
use httpcodes::HttpForbidden;
use middleware::{Middleware, Started};

/// Potential cross-site request forgery detected.
#[derive(Debug, Fail)]
pub enum CsrfError {
    /// The HTTP request header `Origin` was required but not provided.
    #[fail(display="Origin header required")]
    MissingOrigin,
    /// The HTTP request header `Origin` could not be parsed correctly.
    #[fail(display="Could not parse Origin header")]
    BadOrigin,
    /// The cross-site request was denied.
    #[fail(display="Cross-site request denied")]
    CsrDenied,
}

impl ResponseError for CsrfError {
    fn error_response(&self) -> HttpResponse {
        HttpForbidden.build().body(self.to_string()).unwrap()
    }
}

fn uri_origin(uri: &Uri) -> Option<String> {
    match (uri.scheme_part(), uri.host(), uri.port()) {
        (Some(scheme), Some(host), Some(port)) => {
            Some(format!("{}://{}:{}", scheme, host, port))
        }
        (Some(scheme), Some(host), None) => {
            Some(format!("{}://{}", scheme, host))
        }
        _ => None
    }
}

fn origin(headers: &HeaderMap) -> Option<Result<Cow<str>, CsrfError>> {
    headers.get(header::ORIGIN)
        .map(|origin| {
            origin
                .to_str()
                .map_err(|_| CsrfError::BadOrigin)
                .map(|o| o.into())
        })
        .or_else(|| {
            headers.get(header::REFERER)
                .map(|referer| {
                    Uri::try_from(Bytes::from(referer.as_bytes()))
                        .ok()
                        .as_ref()
                        .and_then(uri_origin)
                        .ok_or(CsrfError::BadOrigin)
                        .map(|o| o.into())
                })
        })
}

/// A middleware that filters cross-site requests.
pub struct CsrfFilter {
    origins: HashSet<String>,
    allow_xhr: bool,
    allow_missing_origin: bool,
}

impl CsrfFilter {
    /// Start building a `CsrfFilter`.
    pub fn build() -> CsrfFilterBuilder {
        CsrfFilterBuilder {
            csrf: CsrfFilter {
                origins: HashSet::new(),
                allow_xhr: false,
                allow_missing_origin: false,
            }
        }
    }

    fn validate<S>(&self, req: &mut HttpRequest<S>) -> Result<(), CsrfError> {
        if req.method().is_safe() || (self.allow_xhr && req.headers().contains_key("x-requested-with")) {
            Ok(())
        } else if let Some(header) = origin(req.headers()) {
            match header {
                Ok(ref origin) if self.origins.contains(origin.as_ref()) => Ok(()),
                Ok(_) => Err(CsrfError::CsrDenied),
                Err(err) => Err(err),
            }
        } else if self.allow_missing_origin {
            Ok(())
        } else {
            Err(CsrfError::MissingOrigin)
        }
    }
}

impl<S> Middleware<S> for CsrfFilter {
    fn start(&self, req: &mut HttpRequest<S>) -> Result<Started> {
        self.validate(req)?;
        Ok(Started::Done)
    }
}

/// Used to build a `CsrfFilter`.
///
/// To construct a CSRF filter:
///
/// 1. Call [`CsrfFilter::build`](struct.CsrfFilter.html#method.build) to
///    start building.
/// 2. [Add](struct.CsrfFilterBuilder.html#method.allowed_origin) allowed
///    origins.
/// 3. Call [finish](struct.CsrfFilterBuilder.html#method.finish) to retrieve
///    the constructed filter.
///
/// # Example
///
/// ```
/// use actix_web::middleware::csrf;
///
/// let csrf = csrf::CsrfFilter::build()
///     .allowed_origin("https://www.example.com")
///     .finish();
/// ```
pub struct CsrfFilterBuilder {
    csrf: CsrfFilter,
}

impl CsrfFilterBuilder {
    /// Add an origin that is allowed to make requests. Will be verified
    /// against the `Origin` request header.
    pub fn allowed_origin(mut self, origin: &str) -> CsrfFilterBuilder {
        self.csrf.origins.insert(origin.to_owned());
        self
    }

    /// Allow all requests with an `X-Requested-With` header.
    ///
    /// A cross-site attacker should not be able to send requests with custom
    /// headers unless a CORS policy whitelists them. Therefore it should be
    /// safe to allow requests with an `X-Requested-With` header (added
    /// automatically by many JavaScript libraries).
    ///
    /// This is disabled by default, because in Safari it is possible to
    /// circumvent this using redirects and Flash.
    ///
    /// Use this method to enable more lax filtering.
    pub fn allow_xhr(mut self) -> CsrfFilterBuilder {
        self.csrf.allow_xhr = true;
        self
    }

    /// Allow requests if the expected `Origin` header is missing (and
    /// there is no `Referer` to fall back on).
    ///
    /// The filter is conservative by default, but it should be safe to allow
    /// missing `Origin` headers because a cross-site attacker cannot prevent
    /// the browser from sending `Origin` on unsafe requests.
    pub fn allow_missing_origin(mut self) -> CsrfFilterBuilder {
        self.csrf.allow_missing_origin = true;
        self
    }

    /// Finishes building the `CsrfFilter` instance.
    pub fn finish(self) -> CsrfFilter {
        self.csrf
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use http::Method;
    use test::TestRequest;

    #[test]
    fn test_safe() {
        let csrf = CsrfFilter::build()
            .allowed_origin("https://www.example.com")
            .finish();

        let mut req = TestRequest::with_header("Origin", "https://www.w3.org")
            .method(Method::HEAD)
            .finish();

        assert!(csrf.start(&mut req).is_ok());
    }

    #[test]
    fn test_csrf() {
        let csrf = CsrfFilter::build()
            .allowed_origin("https://www.example.com")
            .finish();

        let mut req = TestRequest::with_header("Origin", "https://www.w3.org")
            .method(Method::POST)
            .finish();

        assert!(csrf.start(&mut req).is_err());
    }

    #[test]
    fn test_referer() {
        let csrf = CsrfFilter::build()
            .allowed_origin("https://www.example.com")
            .finish();

        let mut req = TestRequest::with_header("Referer", "https://www.example.com/some/path?query=param")
            .method(Method::POST)
            .finish();

        assert!(csrf.start(&mut req).is_ok());
    }
}
<commit_msg>filter cross-site upgrades in csrf middleware<commit_after>//! A filter for cross-site request forgery (CSRF).
//!
//! This middleware is stateless and [based on request
//! headers](https://www.owasp.org/index.php/Cross-Site_Request_Forgery_(CSRF)_Prevention_Cheat_Sheet#Verifying_Same_Origin_with_Standard_Headers).
//!
//! By default requests are allowed only if one of these is true:
//!
//! * The request method is safe (`GET`, `HEAD`, `OPTIONS`). It is the
//!   applications responsibility to ensure these methods cannot be used to
//!   execute unwanted actions. Note that upgrade requests for websockets are
//!   also considered safe.
//! * The `Origin` header (added automatically by the browser) matches one
//!   of the allowed origins.
//! * There is no `Origin` header but the `Referer` header matches one of
//!   the allowed origins.
//!
//! Use [`CsrfFilterBuilder::allow_xhr()`](struct.CsrfFilterBuilder.html#method.allow_xhr)
//! if you want to allow requests with unsafe methods via
//! [CORS](../cors/struct.Cors.html).
//!
//! # Example
//!
//! ```
//! # extern crate actix_web;
//! # use actix_web::*;
//!
//! use actix_web::middleware::csrf;
//!
//! fn handle_post(_req: HttpRequest) -> &'static str {
//!     "This action should only be triggered with requests from the same site"
//! }
//!
//! fn main() {
//!     let app = Application::new()
//!         .middleware(
//!             csrf::CsrfFilter::build()
//!                 .allowed_origin("https://www.example.com")
//!                 .finish())
//!         .resource("/", |r| {
//!             r.method(Method::GET).f(|_| httpcodes::HttpOk);
//!             r.method(Method::POST).f(handle_post);
//!         })
//!         .finish();
//! }
//! ```
//!
//! In this example the entire application is protected from CSRF.

use std::borrow::Cow;
use std::collections::HashSet;

use bytes::Bytes;
use error::{Result, ResponseError};
use http::{HeaderMap, HttpTryFrom, Uri, header};
use httprequest::HttpRequest;
use httpresponse::HttpResponse;
use httpmessage::HttpMessage;
use httpcodes::HttpForbidden;
use middleware::{Middleware, Started};

/// Potential cross-site request forgery detected.
#[derive(Debug, Fail)]
pub enum CsrfError {
    /// The HTTP request header `Origin` was required but not provided.
    #[fail(display="Origin header required")]
    MissingOrigin,
    /// The HTTP request header `Origin` could not be parsed correctly.
    #[fail(display="Could not parse Origin header")]
    BadOrigin,
    /// The cross-site request was denied.
    #[fail(display="Cross-site request denied")]
    CsrDenied,
}

impl ResponseError for CsrfError {
    fn error_response(&self) -> HttpResponse {
        HttpForbidden.build().body(self.to_string()).unwrap()
    }
}

fn uri_origin(uri: &Uri) -> Option<String> {
    match (uri.scheme_part(), uri.host(), uri.port()) {
        (Some(scheme), Some(host), Some(port)) => {
            Some(format!("{}://{}:{}", scheme, host, port))
        }
        (Some(scheme), Some(host), None) => {
            Some(format!("{}://{}", scheme, host))
        }
        _ => None
    }
}

fn origin(headers: &HeaderMap) -> Option<Result<Cow<str>, CsrfError>> {
    headers.get(header::ORIGIN)
        .map(|origin| {
            origin
                .to_str()
                .map_err(|_| CsrfError::BadOrigin)
                .map(|o| o.into())
        })
        .or_else(|| {
            headers.get(header::REFERER)
                .map(|referer| {
                    Uri::try_from(Bytes::from(referer.as_bytes()))
                        .ok()
                        .as_ref()
                        .and_then(uri_origin)
                        .ok_or(CsrfError::BadOrigin)
                        .map(|o| o.into())
                })
        })
}

/// A middleware that filters cross-site requests.
pub struct CsrfFilter {
    origins: HashSet<String>,
    allow_xhr: bool,
    allow_missing_origin: bool,
    allow_upgrade: bool,
}

impl CsrfFilter {
    /// Start building a `CsrfFilter`.
    pub fn build() -> CsrfFilterBuilder {
        CsrfFilterBuilder {
            csrf: CsrfFilter {
                origins: HashSet::new(),
                allow_xhr: false,
                allow_missing_origin: false,
                allow_upgrade: false,
            }
        }
    }

    fn validate<S>(&self, req: &mut HttpRequest<S>) -> Result<(), CsrfError> {
        let is_upgrade = req.headers().contains_key(header::UPGRADE);
        let is_safe = req.method().is_safe() && (self.allow_upgrade || !is_upgrade);

        if is_safe || (self.allow_xhr && req.headers().contains_key("x-requested-with")) {
            Ok(())
        } else if let Some(header) = origin(req.headers()) {
            match header {
                Ok(ref origin) if self.origins.contains(origin.as_ref()) => Ok(()),
                Ok(_) => Err(CsrfError::CsrDenied),
                Err(err) => Err(err),
            }
        } else if self.allow_missing_origin {
            Ok(())
        } else {
            Err(CsrfError::MissingOrigin)
        }
    }
}

impl<S> Middleware<S> for CsrfFilter {
    fn start(&self, req: &mut HttpRequest<S>) -> Result<Started> {
        self.validate(req)?;
        Ok(Started::Done)
    }
}

/// Used to build a `CsrfFilter`.
///
/// To construct a CSRF filter:
///
/// 1. Call [`CsrfFilter::build`](struct.CsrfFilter.html#method.build) to
///    start building.
/// 2. [Add](struct.CsrfFilterBuilder.html#method.allowed_origin) allowed
///    origins.
/// 3. Call [finish](struct.CsrfFilterBuilder.html#method.finish) to retrieve
///    the constructed filter.
///
/// # Example
///
/// ```
/// use actix_web::middleware::csrf;
///
/// let csrf = csrf::CsrfFilter::build()
///     .allowed_origin("https://www.example.com")
///     .finish();
/// ```
pub struct CsrfFilterBuilder {
    csrf: CsrfFilter,
}

impl CsrfFilterBuilder {
    /// Add an origin that is allowed to make requests. Will be verified
    /// against the `Origin` request header.
    pub fn allowed_origin(mut self, origin: &str) -> CsrfFilterBuilder {
        self.csrf.origins.insert(origin.to_owned());
        self
    }

    /// Allow all requests with an `X-Requested-With` header.
    ///
    /// A cross-site attacker should not be able to send requests with custom
    /// headers unless a CORS policy whitelists them. Therefore it should be
    /// safe to allow requests with an `X-Requested-With` header (added
    /// automatically by many JavaScript libraries).
    ///
    /// This is disabled by default, because in Safari it is possible to
    /// circumvent this using redirects and Flash.
    ///
    /// Use this method to enable more lax filtering.
    pub fn allow_xhr(mut self) -> CsrfFilterBuilder {
        self.csrf.allow_xhr = true;
        self
    }

    /// Allow requests if the expected `Origin` header is missing (and
    /// there is no `Referer` to fall back on).
    ///
    /// The filter is conservative by default, but it should be safe to allow
    /// missing `Origin` headers because a cross-site attacker cannot prevent
    /// the browser from sending `Origin` on unsafe requests.
    pub fn allow_missing_origin(mut self) -> CsrfFilterBuilder {
        self.csrf.allow_missing_origin = true;
        self
    }

    /// Allow cross-site upgrade requests (for example to open a WebSocket).
    pub fn allow_upgrade(mut self) -> CsrfFilterBuilder {
        self.csrf.allow_upgrade = true;
        self
    }

    /// Finishes building the `CsrfFilter` instance.
    pub fn finish(self) -> CsrfFilter {
        self.csrf
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use http::Method;
    use test::TestRequest;

    #[test]
    fn test_safe() {
        let csrf = CsrfFilter::build()
            .allowed_origin("https://www.example.com")
            .finish();

        let mut req = TestRequest::with_header("Origin", "https://www.w3.org")
            .method(Method::HEAD)
            .finish();

        assert!(csrf.start(&mut req).is_ok());
    }

    #[test]
    fn test_csrf() {
        let csrf = CsrfFilter::build()
            .allowed_origin("https://www.example.com")
            .finish();

        let mut req = TestRequest::with_header("Origin", "https://www.w3.org")
            .method(Method::POST)
            .finish();

        assert!(csrf.start(&mut req).is_err());
    }

    #[test]
    fn test_referer() {
        let csrf = CsrfFilter::build()
            .allowed_origin("https://www.example.com")
            .finish();

        let mut req = TestRequest::with_header("Referer", "https://www.example.com/some/path?query=param")
            .method(Method::POST)
            .finish();

        assert!(csrf.start(&mut req).is_ok());
    }
}
<|endoftext|>


<commit_before>use std::hash::Hasher;
use byteorder::{LittleEndian, ByteOrder};
use std::io::Read;
use std::error::Error;

pub struct MurmurHasher{
    h1: u64,
    h2: u64,
    buf: [u8; 16],
    index: usize,
    processed: usize
}

impl Default for MurmurHasher {

    fn default() -> Self{
        MurmurHasher{
            h1: 0,
            h2: 0,
            buf: [0; 16],
            index: 0,
            processed: 0
        }
    }
}

pub fn murmur3_x64_128<T :Read>(source: &mut T, seed: u32) -> Result<u128, String> {
    let mut buffer:[u8;16] = [0; 16];
    let mut hasher = MurmurHasher::new(seed);
    loop {
        match source.read(&mut buffer) {
            Ok(16) => {
                hasher.write(&buffer);
            }
            Ok(0) => {
                return Ok(hasher.build_murmur_hash());
            }
            Err(e) => {
                return Err(String::from(e.description()))
            }
            Ok(i) => {
                hasher.write(&buffer[..i]);
                return Ok(hasher.build_murmur_hash());
            }
        }
    }
}

impl Hasher for MurmurHasher{
    fn finish(&self) -> u64 {
        self.build_murmur_hash() as u64
    }

    fn write(&mut self, bytes: &[u8]){
        self.processed += bytes.len();
        let to_split = if self.index == 0 {
            bytes
        }else{
            if bytes.len() + self.index >= 16 {
                let t = bytes.split_at(16 - self.index);
                for i in 0 .. (16 - self.index) {
                    self.buf[self.index + i] = t.0[i];
                }
                let r = process_16_bytes(self.h1, self.h2, &self.buf);
                self.h1 = r.0;
                self.h2 = r.1;
                self.index = 0;
                t.1
            }else{
                bytes
            }
        };
        for chunk in to_split.chunks(16) {
            if chunk.len() == 16{
                let t = process_16_bytes(self.h1, self.h2, chunk);
                self.h1 = t.0;
                self.h2 = t.1;
            }else{
                self.push_odd_bytes(chunk);
            }
        }
    }
}

impl MurmurHasher {
    pub fn new(seed:u32) -> Self{
        MurmurHasher{
            h1: seed as u64,
            h2: seed as u64,
            ..MurmurHasher::default()
        }
    }

    pub fn build_murmur_hash(&self) -> u128{
        let state = if self.index != 0 {
            process_odd_bytes(self.h1, self.h2, self.index, &self.buf)
        }else{
            (self.h1, self.h2)
        };
        finish(state.0, state.1, self.processed)
    }

    fn push_odd_bytes(&mut self, to_push: &[u8]){
        for x in to_push {
            self.buf[self.index] = *x;
            self.index += 1;
        }
    }
}

fn process_16_bytes(h1: u64, h2: u64, chunk:&[u8]) -> (u64,u64){
    const C1: u64 = 0x52dc_e729;
    const C2: u64 = 0x3849_5ab5;
    const R1: u32 = 27;
    const R2: u32 = 31;
    const M: u64 = 5;
    let mut h1 = h1;
    let mut h2 = h2;
    let k1 = LittleEndian::read_u64(&chunk[0..8]);
    let k2 = LittleEndian::read_u64(&chunk[8..]);
    h1 ^= process_h1_k_x64(k1);
    h1 = h1.rotate_left(R1).wrapping_add(h2).wrapping_mul(M).wrapping_add(C1);
    h2 ^= process_h2_k_x64(k2);
    h2 = h2.rotate_left(R2).wrapping_add(h1).wrapping_mul(M).wrapping_add(C2);
    (h1,h2)
}

fn process_odd_bytes(h1: u64, h2:u64, index: usize, buf:&[u8]) -> (u64,u64){
    match index {
        9...15 => {
            (h1 ^ process_h1_k_x64(LittleEndian::read_u64(&buf[0..8])),
            h2 ^ process_h2_k_x64(LittleEndian::read_uint(&buf[8..], index  - 8) as u64))
        }
        8 => {
            (h1 ^ process_h1_k_x64(LittleEndian::read_u64(&buf)), h2)
        }
        1...7 =>{
            (h1 ^ process_h1_k_x64(LittleEndian::read_uint(&buf, index ) as u64), h2)
        }
        _ => {
            panic!("Invalid index on process_odd_bytes");
        }
    }
}

fn finish(h1: u64, h2:u64, processed: usize) -> u128 {
    let mut h1 = h1 ^ (processed as u64);
    let mut h2 = h2 ^ (processed as u64);
    h1 = h1.wrapping_add(h2);
    h2 = h2.wrapping_add(h1);
    h1 = fmix64(h1);
    h2 = fmix64(h2);
    h1 = h1.wrapping_add(h2);
    h2 = h2.wrapping_add(h1);
    ((h2 as u128) << 64) + (h1 as u128)
}

fn process_h1_k_x64(k: u64) -> u64 {
    const C1: u64 = 0x87c37b91114253d5;
    const C2: u64 = 0x4cf5ad432745937f;
    const R: u32 = 31;
    k.wrapping_mul(C1).rotate_left(R).wrapping_mul(C2)
}

fn process_h2_k_x64(k: u64) -> u64 {
    const C1: u64 = 0x87c37b91114253d5;
    const C2: u64 = 0x4cf5ad432745937f;
    const R: u32 = 33;
    k.wrapping_mul(C2).rotate_left(R).wrapping_mul(C1)
}

fn fmix64(k: u64) -> u64 {
    const C1: u64 = 0xff51_afd7_ed55_8ccd;
    const C2: u64 = 0xc4ce_b9fe_1a85_ec53;
    const R: u32 = 33;
    let mut tmp = k;
    tmp ^= tmp >> R;
    tmp = tmp.wrapping_mul(C1);
    tmp ^= tmp >> R;
    tmp = tmp.wrapping_mul(C2);
    tmp ^= tmp >> R;
    tmp
}<commit_msg>Small perf tweak<commit_after>use std::hash::Hasher;
use byteorder::{LittleEndian, ByteOrder};
use std::io::Read;
use std::error::Error;

pub struct MurmurHasher{
    h1: u64,
    h2: u64,
    buf: [u8; 16],
    index: usize,
    processed: usize
}

impl Default for MurmurHasher {

    fn default() -> Self{
        MurmurHasher{
            h1: 0,
            h2: 0,
            buf: [0; 16],
            index: 0,
            processed: 0
        }
    }
}

pub fn murmur3_x64_128<T :Read>(source: &mut T, seed: u32) -> Result<u128, String> {
    let mut buffer:[u8;16] = [0; 16];
    let mut hasher = MurmurHasher::new(seed);
    loop {
        match source.read(&mut buffer) {
            Ok(16) => {
                hasher.write(&buffer);
            }
            Ok(0) => {
                return Ok(hasher.build_murmur_hash());
            }
            Err(e) => {
                return Err(String::from(e.description()))
            }
            Ok(i) => {
                hasher.write(&buffer[..i]);
                return Ok(hasher.build_murmur_hash());
            }
        }
    }
}

impl Hasher for MurmurHasher{
    fn finish(&self) -> u64 {
        self.build_murmur_hash() as u64
    }

    fn write(&mut self, bytes: &[u8]){
        self.processed += bytes.len();
        let to_split = if self.index == 0 {
            bytes
        }else{
            if bytes.len() + self.index >= 16 {
                let t = bytes.split_at(16 - self.index);
                for i in 0 .. (16 - self.index) {
                    self.buf[self.index + i] = t.0[i];
                }
                let r = process_16_bytes(self.h1, self.h2, &self.buf);
                self.h1 = r.0;
                self.h2 = r.1;
                self.index = 0;
                t.1
            }else{
                bytes
            }
        };
        for chunk in to_split.chunks(16) {
            if chunk.len() == 16{
                let t = process_16_bytes(self.h1, self.h2, chunk);
                self.h1 = t.0;
                self.h2 = t.1;
            }else{
                self.push_odd_bytes(chunk);
            }
        }
    }
}

impl MurmurHasher {
    pub fn new(seed:u32) -> Self{
        MurmurHasher{
            h1: seed as u64,
            h2: seed as u64,
            ..MurmurHasher::default()
        }
    }

    pub fn build_murmur_hash(&self) -> u128{
        let state = if self.index != 0 {
            process_odd_bytes(self.h1, self.h2, self.index, &self.buf)
        }else{
            (self.h1, self.h2)
        };
        finish(state.0, state.1, self.processed)
    }

    fn push_odd_bytes(&mut self, to_push: &[u8]){
        let l = to_push.len();
        self.buf[self.index..(self.index + l)].clone_from_slice(&to_push);
        self.index += l;
    }
}

fn process_16_bytes(h1: u64, h2: u64, chunk:&[u8]) -> (u64,u64){
    const C1: u64 = 0x52dc_e729;
    const C2: u64 = 0x3849_5ab5;
    const R1: u32 = 27;
    const R2: u32 = 31;
    const M: u64 = 5;
    let mut h1 = h1;
    let mut h2 = h2;
    let k1 = LittleEndian::read_u64(&chunk[0..8]);
    let k2 = LittleEndian::read_u64(&chunk[8..]);
    h1 ^= process_h1_k_x64(k1);
    h1 = h1.rotate_left(R1).wrapping_add(h2).wrapping_mul(M).wrapping_add(C1);
    h2 ^= process_h2_k_x64(k2);
    h2 = h2.rotate_left(R2).wrapping_add(h1).wrapping_mul(M).wrapping_add(C2);
    (h1,h2)
}

fn process_odd_bytes(h1: u64, h2:u64, index: usize, buf:&[u8]) -> (u64,u64){
    match index {
        9...15 => {
            (h1 ^ process_h1_k_x64(LittleEndian::read_u64(&buf[0..8])),
            h2 ^ process_h2_k_x64(LittleEndian::read_uint(&buf[8..], index  - 8) as u64))
        }
        8 => {
            (h1 ^ process_h1_k_x64(LittleEndian::read_u64(&buf)), h2)
        }
        1...7 =>{
            (h1 ^ process_h1_k_x64(LittleEndian::read_uint(&buf, index ) as u64), h2)
        }
        _ => {
            panic!("Invalid index on process_odd_bytes");
        }
    }
}

fn finish(h1: u64, h2:u64, processed: usize) -> u128 {
    let mut h1 = h1 ^ (processed as u64);
    let mut h2 = h2 ^ (processed as u64);
    h1 = h1.wrapping_add(h2);
    h2 = h2.wrapping_add(h1);
    h1 = fmix64(h1);
    h2 = fmix64(h2);
    h1 = h1.wrapping_add(h2);
    h2 = h2.wrapping_add(h1);
    ((h2 as u128) << 64) | (h1 as u128)
}

fn process_h1_k_x64(k: u64) -> u64 {
    const C1: u64 = 0x87c37b91114253d5;
    const C2: u64 = 0x4cf5ad432745937f;
    const R: u32 = 31;
    k.wrapping_mul(C1).rotate_left(R).wrapping_mul(C2)
}

fn process_h2_k_x64(k: u64) -> u64 {
    const C1: u64 = 0x87c37b91114253d5;
    const C2: u64 = 0x4cf5ad432745937f;
    const R: u32 = 33;
    k.wrapping_mul(C2).rotate_left(R).wrapping_mul(C1)
}

fn fmix64(k: u64) -> u64 {
    const C1: u64 = 0xff51_afd7_ed55_8ccd;
    const C2: u64 = 0xc4ce_b9fe_1a85_ec53;
    const R: u32 = 33;
    let mut tmp = k;
    tmp ^= tmp >> R;
    tmp = tmp.wrapping_mul(C1);
    tmp ^= tmp >> R;
    tmp = tmp.wrapping_mul(C2);
    tmp ^= tmp >> R;
    tmp
}<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

#![forbid(unsafe_code)]

//! This library implements a schematized DB on top of [RocksDB](https://rocksdb.org/). It makes
//! sure all data passed in and out are structured according to predefined schemas and prevents
//! access to raw keys and values. This library also enforces a set of Libra specific DB options,
//! like custom comparators and schema-to-column-family mapping.
//!
//! It requires that different kinds of key-value pairs be stored in separate column
//! families.  To use this library to store a kind of key-value pairs, the user needs to use the
//! [`define_schema!`] macro to define the schema name, the types of key and value, and name of the
//! column family.

#[macro_use]
pub mod schema;

use crate::schema::{KeyCodec, Schema, SeekKeyCodec, ValueCodec};
use anyhow::{format_err, Result};
use lazy_static::lazy_static;
use libra_metrics::OpMetrics;
use rocksdb::{
    rocksdb_options::ColumnFamilyDescriptor, CFHandle, DBOptions, Writable, WriteOptions,
};
use std::{
    collections::{BTreeMap, HashMap},
    iter::Iterator,
    marker::PhantomData,
    path::Path,
};

lazy_static! {
    static ref OP_COUNTER: OpMetrics = OpMetrics::new_and_registered("schemadb");
}

/// Type alias to `rocksdb::ColumnFamilyOptions`. See [`rocksdb doc`](https://github.com/pingcap/rust-rocksdb/blob/master/src/rocksdb_options.rs)
pub type ColumnFamilyOptions = rocksdb::ColumnFamilyOptions;
/// Type alias to `rocksdb::ReadOptions`. See [`rocksdb doc`](https://github.com/pingcap/rust-rocksdb/blob/master/src/rocksdb_options.rs)
pub type ReadOptions = rocksdb::ReadOptions;

/// Type alias to improve readability.
pub type ColumnFamilyName = &'static str;
/// Type alias to improve readability.
pub type ColumnFamilyOptionsMap = HashMap<ColumnFamilyName, ColumnFamilyOptions>;

/// Name for the `default` column family that's always open by RocksDB. We use it to store
/// [`LedgerInfo`](../types/ledger_info/struct.LedgerInfo.html).
pub const DEFAULT_CF_NAME: ColumnFamilyName = "default";

#[derive(Debug)]
enum WriteOp {
    Value(Vec<u8>),
    Deletion,
}

/// `SchemaBatch` holds a collection of updates that can be applied to a DB atomically. The updates
/// will be applied in the order in which they are added to the `SchemaBatch`.
#[derive(Debug, Default)]
pub struct SchemaBatch {
    rows: HashMap<ColumnFamilyName, BTreeMap<Vec<u8>, WriteOp>>,
}

impl SchemaBatch {
    /// Creates an empty batch.
    pub fn new() -> Self {
        Self::default()
    }

    /// Adds an insert/update operation to the batch.
    pub fn put<S: Schema>(&mut self, key: &S::Key, value: &S::Value) -> Result<()> {
        let key = <S::Key as KeyCodec<S>>::encode_key(key)?;
        let value = <S::Value as ValueCodec<S>>::encode_value(value)?;
        self.rows
            .entry(S::COLUMN_FAMILY_NAME)
            .or_insert_with(BTreeMap::new)
            .insert(key, WriteOp::Value(value));

        Ok(())
    }

    /// Adds a delete operation to the batch.
    pub fn delete<S: Schema>(&mut self, key: &S::Key) -> Result<()> {
        let key = <S::Key as KeyCodec<S>>::encode_key(key)?;
        self.rows
            .entry(S::COLUMN_FAMILY_NAME)
            .or_insert_with(BTreeMap::new)
            .insert(key, WriteOp::Deletion);

        Ok(())
    }
}

/// DB Iterator parameterized on [`Schema`] that seeks with [`Schema::Key`] and yields
/// [`Schema::Key`] and [`Schema::Value`]
pub struct SchemaIterator<'a, S> {
    db_iter: rocksdb::DBIterator<&'a rocksdb::DB>,
    phantom: PhantomData<S>,
}

impl<'a, S> SchemaIterator<'a, S>
where
    S: Schema,
{
    fn new(db_iter: rocksdb::DBIterator<&'a rocksdb::DB>) -> Self {
        SchemaIterator {
            db_iter,
            phantom: PhantomData,
        }
    }

    /// Seeks to the first key.
    pub fn seek_to_first(&mut self) -> bool {
        self.db_iter.seek(rocksdb::SeekKey::Start)
    }

    /// Seeks to the last key.
    pub fn seek_to_last(&mut self) -> bool {
        self.db_iter.seek(rocksdb::SeekKey::End)
    }

    /// Seeks to the first key whose binary representation is equal to or greater than that of the
    /// `seek_key`.
    pub fn seek<SK>(&mut self, seek_key: &SK) -> Result<bool>
    where
        SK: SeekKeyCodec<S>,
    {
        let key = <SK as SeekKeyCodec<S>>::encode_seek_key(seek_key)?;
        Ok(self.db_iter.seek(rocksdb::SeekKey::Key(&key)))
    }

    /// Seeks to the last key whose binary representation is less than or equal to that of the
    /// `seek_key`.
    ///
    /// See example in [`RocksDB doc`](https://github.com/facebook/rocksdb/wiki/SeekForPrev).
    pub fn seek_for_prev<SK>(&mut self, seek_key: &SK) -> Result<bool>
    where
        SK: SeekKeyCodec<S>,
    {
        let key = <SK as SeekKeyCodec<S>>::encode_seek_key(seek_key)?;
        Ok(self.db_iter.seek_for_prev(rocksdb::SeekKey::Key(&key)))
    }
}

impl<'a, S> Iterator for SchemaIterator<'a, S>
where
    S: Schema,
{
    type Item = Result<(S::Key, S::Value)>;

    fn next(&mut self) -> Option<Self::Item> {
        self.db_iter.kv().map(|(raw_key, raw_value)| {
            self.db_iter.next();
            Ok((
                <S::Key as KeyCodec<S>>::decode_key(&raw_key)?,
                <S::Value as ValueCodec<S>>::decode_value(&raw_value)?,
            ))
        })
    }
}

/// Checks underlying Rocksdb instance existence by checking `CURRENT` file existence, the same way
/// Rocksdb adopts to detect db existence.
fn db_exists(path: &Path) -> bool {
    let rocksdb_current_file = path.join("CURRENT");
    rocksdb_current_file.is_file()
}

/// All the RocksDB methods return `std::result::Result<T, String>`. Since our methods return
/// `anyhow::Result<T>`, manual conversion is needed.
fn convert_rocksdb_err(msg: String) -> anyhow::Error {
    format_err!("RocksDB internal error: {}.", msg)
}

/// This DB is a schematized RocksDB wrapper where all data passed in and out are typed according to
/// [`Schema`]s.
#[derive(Debug)]
pub struct DB {
    inner: rocksdb::DB,
}

impl DB {
    /// Create db with all the column families provided if it doesn't exist at `path`; Otherwise,
    /// try to open it with all the column families.
    pub fn open<P: AsRef<Path>>(path: P, mut cf_opts_map: ColumnFamilyOptionsMap) -> Result<Self> {
        let mut db_opts = DBOptions::new();

        // For now we set the max total WAL size to be 1G. This config can be useful when column
        // families are updated at non-uniform frequencies.
        db_opts.set_max_total_wal_size(1 << 30);

        // If db exists, just open it with all cfs.
        if db_exists(path.as_ref()) {
            return DB::open_cf(db_opts, &path, cf_opts_map.into_iter().collect());
        }

        // If db doesn't exist, create a db first with all column families.
        db_opts.create_if_missing(true);

        let mut db = DB::open_cf(
            db_opts,
            path,
            vec![cf_opts_map
                .remove_entry(&DEFAULT_CF_NAME)
                .ok_or_else(|| format_err!("No \"default\" column family name found"))?],
        )?;
        cf_opts_map
            .into_iter()
            .map(|(cf_name, cf_opts)| db.create_cf((cf_name, cf_opts)))
            .collect::<Result<Vec<_>>>()?;
        Ok(db)
    }

    fn open_cf<'a, P, T>(opts: DBOptions, path: P, cfds: Vec<T>) -> Result<DB>
    where
        P: AsRef<Path>,
        T: Into<ColumnFamilyDescriptor<'a>>,
    {
        let inner = rocksdb::DB::open_cf(
            opts,
            path.as_ref().to_str().ok_or_else(|| {
                format_err!("Path {:?} can not be converted to string.", path.as_ref())
            })?,
            cfds,
        )
        .map_err(convert_rocksdb_err)?;

        Ok(DB { inner })
    }

    fn create_cf<'a, T>(&mut self, cfd: T) -> Result<()>
    where
        T: Into<ColumnFamilyDescriptor<'a>>,
    {
        let _cf_handle = self.inner.create_cf(cfd).map_err(convert_rocksdb_err)?;
        Ok(())
    }

    /// Reads single record by key.
    pub fn get<S: Schema>(&self, schema_key: &S::Key) -> Result<Option<S::Value>> {
        let k = <S::Key as KeyCodec<S>>::encode_key(&schema_key)?;
        let cf_handle = self.get_cf_handle(S::COLUMN_FAMILY_NAME)?;
        let time = std::time::Instant::now();

        let result = self
            .inner
            .get_cf(cf_handle, &k)
            .map_err(convert_rocksdb_err)?;
        OP_COUNTER.observe_duration(&format!("db_get_{}", S::COLUMN_FAMILY_NAME), time.elapsed());
        result
            .map(|raw_value| <S::Value as ValueCodec<S>>::decode_value(&raw_value))
            .transpose()
    }

    /// Writes single record.
    pub fn put<S: Schema>(&self, key: &S::Key, value: &S::Value) -> Result<()> {
        let k = <S::Key as KeyCodec<S>>::encode_key(&key)?;
        let v = <S::Value as ValueCodec<S>>::encode_value(&value)?;
        let cf_handle = self.get_cf_handle(S::COLUMN_FAMILY_NAME)?;

        self.inner
            .put_cf_opt(cf_handle, &k, &v, &default_write_options())
            .map_err(convert_rocksdb_err)
    }

    /// Delete all keys in range [begin, end).
    ///
    /// `SK` has to be an explict type parameter since
    /// https://github.com/rust-lang/rust/issues/44721
    pub fn range_delete<S, SK>(&self, begin: &SK, end: &SK) -> Result<()>
    where
        S: Schema,
        SK: SeekKeyCodec<S>,
    {
        let raw_begin = begin.encode_seek_key()?;
        let raw_end = end.encode_seek_key()?;
        let cf_handle = self.get_cf_handle(S::COLUMN_FAMILY_NAME)?;

        self.inner
            .delete_range_cf(&cf_handle, &raw_begin, &raw_end)
            .map_err(convert_rocksdb_err)
    }

    /// Returns a [`SchemaIterator`] on a certain schema.
    pub fn iter<S: Schema>(&self, opts: ReadOptions) -> Result<SchemaIterator<S>> {
        let cf_handle = self.get_cf_handle(S::COLUMN_FAMILY_NAME)?;
        Ok(SchemaIterator::new(self.inner.iter_cf_opt(cf_handle, opts)))
    }

    /// Writes a group of records wrapped in a [`SchemaBatch`].
    pub fn write_schemas(&self, batch: SchemaBatch) -> Result<()> {
        let db_batch = rocksdb::WriteBatch::new();
        for (cf_name, rows) in &batch.rows {
            let cf_handle = self.get_cf_handle(cf_name)?;
            for (key, write_op) in rows {
                match write_op {
                    WriteOp::Value(value) => db_batch.put_cf(cf_handle, key, value),
                    WriteOp::Deletion => db_batch.delete_cf(cf_handle, key),
                }
                .map_err(convert_rocksdb_err)?;
            }
        }

        self.inner
            .write_opt(&db_batch, &default_write_options())
            .map_err(convert_rocksdb_err)?;

        // Bump counters only after DB write succeeds.
        for (cf_name, rows) in &batch.rows {
            for (key, write_op) in rows {
                match write_op {
                    WriteOp::Value(value) => OP_COUNTER.observe(
                        &format!("db_put_bytes_{}", cf_name),
                        (key.len() + value.len()) as f64,
                    ),
                    WriteOp::Deletion => OP_COUNTER.inc(&format!("db_delete_{}", cf_name)),
                }
            }
        }

        Ok(())
    }

    fn get_cf_handle(&self, cf_name: &str) -> Result<&CFHandle> {
        self.inner.cf_handle(cf_name).ok_or_else(|| {
            format_err!(
                "DB::cf_handle not found for column family name: {}",
                cf_name
            )
        })
    }

    /// Returns the approximate size of each non-empty column family in bytes.
    pub fn get_approximate_sizes_cf(&self) -> Result<BTreeMap<String, u64>> {
        let mut cf_sizes = BTreeMap::new();

        for cf_name in self.inner.cf_names().into_iter().map(ToString::to_string) {
            let cf_handle = self.get_cf_handle(&cf_name)?;
            let size = self
                .inner
                .get_property_int_cf(cf_handle, "rocksdb.estimate-live-data-size")
                .ok_or_else(|| {
                    format_err!(
                        "Unable to get approximate size of {} column family.",
                        cf_name,
                    )
                })?;
            cf_sizes.insert(cf_name, size);
        }

        Ok(cf_sizes)
    }

    /// Flushes all memtable data. If `sync` is true, the flush will wait until it's done. This is
    /// only used for testing `get_approximate_sizes_cf` in unit tests.
    pub fn flush_all(&self, sync: bool) -> Result<()> {
        for cf_name in self.inner.cf_names() {
            let cf_handle = self.get_cf_handle(cf_name)?;
            self.inner
                .flush_cf(cf_handle, sync)
                .map_err(convert_rocksdb_err)?;
        }
        Ok(())
    }
}

/// For now we always use synchronous writes. This makes sure that once the operation returns
/// `Ok(())` the data is persisted even if the machine crashes. In the future we might consider
/// selectively turning this off for some non-critical writes to improve performance.
fn default_write_options() -> WriteOptions {
    let mut opts = WriteOptions::new();
    opts.set_sync(true);
    opts
}
<commit_msg>typo fix<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

#![forbid(unsafe_code)]

//! This library implements a schematized DB on top of [RocksDB](https://rocksdb.org/). It makes
//! sure all data passed in and out are structured according to predefined schemas and prevents
//! access to raw keys and values. This library also enforces a set of Libra specific DB options,
//! like custom comparators and schema-to-column-family mapping.
//!
//! It requires that different kinds of key-value pairs be stored in separate column
//! families.  To use this library to store a kind of key-value pairs, the user needs to use the
//! [`define_schema!`] macro to define the schema name, the types of key and value, and name of the
//! column family.

#[macro_use]
pub mod schema;

use crate::schema::{KeyCodec, Schema, SeekKeyCodec, ValueCodec};
use anyhow::{format_err, Result};
use lazy_static::lazy_static;
use libra_metrics::OpMetrics;
use rocksdb::{
    rocksdb_options::ColumnFamilyDescriptor, CFHandle, DBOptions, Writable, WriteOptions,
};
use std::{
    collections::{BTreeMap, HashMap},
    iter::Iterator,
    marker::PhantomData,
    path::Path,
};

lazy_static! {
    static ref OP_COUNTER: OpMetrics = OpMetrics::new_and_registered("schemadb");
}

/// Type alias to `rocksdb::ColumnFamilyOptions`. See [`rocksdb doc`](https://github.com/pingcap/rust-rocksdb/blob/master/src/rocksdb_options.rs)
pub type ColumnFamilyOptions = rocksdb::ColumnFamilyOptions;
/// Type alias to `rocksdb::ReadOptions`. See [`rocksdb doc`](https://github.com/pingcap/rust-rocksdb/blob/master/src/rocksdb_options.rs)
pub type ReadOptions = rocksdb::ReadOptions;

/// Type alias to improve readability.
pub type ColumnFamilyName = &'static str;
/// Type alias to improve readability.
pub type ColumnFamilyOptionsMap = HashMap<ColumnFamilyName, ColumnFamilyOptions>;

/// Name for the `default` column family that's always open by RocksDB. We use it to store
/// [`LedgerInfo`](../types/ledger_info/struct.LedgerInfo.html).
pub const DEFAULT_CF_NAME: ColumnFamilyName = "default";

#[derive(Debug)]
enum WriteOp {
    Value(Vec<u8>),
    Deletion,
}

/// `SchemaBatch` holds a collection of updates that can be applied to a DB atomically. The updates
/// will be applied in the order in which they are added to the `SchemaBatch`.
#[derive(Debug, Default)]
pub struct SchemaBatch {
    rows: HashMap<ColumnFamilyName, BTreeMap<Vec<u8>, WriteOp>>,
}

impl SchemaBatch {
    /// Creates an empty batch.
    pub fn new() -> Self {
        Self::default()
    }

    /// Adds an insert/update operation to the batch.
    pub fn put<S: Schema>(&mut self, key: &S::Key, value: &S::Value) -> Result<()> {
        let key = <S::Key as KeyCodec<S>>::encode_key(key)?;
        let value = <S::Value as ValueCodec<S>>::encode_value(value)?;
        self.rows
            .entry(S::COLUMN_FAMILY_NAME)
            .or_insert_with(BTreeMap::new)
            .insert(key, WriteOp::Value(value));

        Ok(())
    }

    /// Adds a delete operation to the batch.
    pub fn delete<S: Schema>(&mut self, key: &S::Key) -> Result<()> {
        let key = <S::Key as KeyCodec<S>>::encode_key(key)?;
        self.rows
            .entry(S::COLUMN_FAMILY_NAME)
            .or_insert_with(BTreeMap::new)
            .insert(key, WriteOp::Deletion);

        Ok(())
    }
}

/// DB Iterator parameterized on [`Schema`] that seeks with [`Schema::Key`] and yields
/// [`Schema::Key`] and [`Schema::Value`]
pub struct SchemaIterator<'a, S> {
    db_iter: rocksdb::DBIterator<&'a rocksdb::DB>,
    phantom: PhantomData<S>,
}

impl<'a, S> SchemaIterator<'a, S>
where
    S: Schema,
{
    fn new(db_iter: rocksdb::DBIterator<&'a rocksdb::DB>) -> Self {
        SchemaIterator {
            db_iter,
            phantom: PhantomData,
        }
    }

    /// Seeks to the first key.
    pub fn seek_to_first(&mut self) -> bool {
        self.db_iter.seek(rocksdb::SeekKey::Start)
    }

    /// Seeks to the last key.
    pub fn seek_to_last(&mut self) -> bool {
        self.db_iter.seek(rocksdb::SeekKey::End)
    }

    /// Seeks to the first key whose binary representation is equal to or greater than that of the
    /// `seek_key`.
    pub fn seek<SK>(&mut self, seek_key: &SK) -> Result<bool>
    where
        SK: SeekKeyCodec<S>,
    {
        let key = <SK as SeekKeyCodec<S>>::encode_seek_key(seek_key)?;
        Ok(self.db_iter.seek(rocksdb::SeekKey::Key(&key)))
    }

    /// Seeks to the last key whose binary representation is less than or equal to that of the
    /// `seek_key`.
    ///
    /// See example in [`RocksDB doc`](https://github.com/facebook/rocksdb/wiki/SeekForPrev).
    pub fn seek_for_prev<SK>(&mut self, seek_key: &SK) -> Result<bool>
    where
        SK: SeekKeyCodec<S>,
    {
        let key = <SK as SeekKeyCodec<S>>::encode_seek_key(seek_key)?;
        Ok(self.db_iter.seek_for_prev(rocksdb::SeekKey::Key(&key)))
    }
}

impl<'a, S> Iterator for SchemaIterator<'a, S>
where
    S: Schema,
{
    type Item = Result<(S::Key, S::Value)>;

    fn next(&mut self) -> Option<Self::Item> {
        self.db_iter.kv().map(|(raw_key, raw_value)| {
            self.db_iter.next();
            Ok((
                <S::Key as KeyCodec<S>>::decode_key(&raw_key)?,
                <S::Value as ValueCodec<S>>::decode_value(&raw_value)?,
            ))
        })
    }
}

/// Checks underlying Rocksdb instance existence by checking `CURRENT` file existence, the same way
/// Rocksdb adopts to detect db existence.
fn db_exists(path: &Path) -> bool {
    let rocksdb_current_file = path.join("CURRENT");
    rocksdb_current_file.is_file()
}

/// All the RocksDB methods return `std::result::Result<T, String>`. Since our methods return
/// `anyhow::Result<T>`, manual conversion is needed.
fn convert_rocksdb_err(msg: String) -> anyhow::Error {
    format_err!("RocksDB internal error: {}.", msg)
}

/// This DB is a schematized RocksDB wrapper where all data passed in and out are typed according to
/// [`Schema`]s.
#[derive(Debug)]
pub struct DB {
    inner: rocksdb::DB,
}

impl DB {
    /// Create db with all the column families provided if it doesn't exist at `path`; Otherwise,
    /// try to open it with all the column families.
    pub fn open<P: AsRef<Path>>(path: P, mut cf_opts_map: ColumnFamilyOptionsMap) -> Result<Self> {
        let mut db_opts = DBOptions::new();

        // For now we set the max total WAL size to be 1G. This config can be useful when column
        // families are updated at non-uniform frequencies.
        db_opts.set_max_total_wal_size(1 << 30);

        // If db exists, just open it with all cfs.
        if db_exists(path.as_ref()) {
            return DB::open_cf(db_opts, &path, cf_opts_map.into_iter().collect());
        }

        // If db doesn't exist, create a db first with all column families.
        db_opts.create_if_missing(true);

        let mut db = DB::open_cf(
            db_opts,
            path,
            vec![cf_opts_map
                .remove_entry(&DEFAULT_CF_NAME)
                .ok_or_else(|| format_err!("No \"default\" column family name found"))?],
        )?;
        cf_opts_map
            .into_iter()
            .map(|(cf_name, cf_opts)| db.create_cf((cf_name, cf_opts)))
            .collect::<Result<Vec<_>>>()?;
        Ok(db)
    }

    fn open_cf<'a, P, T>(opts: DBOptions, path: P, cfds: Vec<T>) -> Result<DB>
    where
        P: AsRef<Path>,
        T: Into<ColumnFamilyDescriptor<'a>>,
    {
        let inner = rocksdb::DB::open_cf(
            opts,
            path.as_ref().to_str().ok_or_else(|| {
                format_err!("Path {:?} can not be converted to string.", path.as_ref())
            })?,
            cfds,
        )
        .map_err(convert_rocksdb_err)?;

        Ok(DB { inner })
    }

    fn create_cf<'a, T>(&mut self, cfd: T) -> Result<()>
    where
        T: Into<ColumnFamilyDescriptor<'a>>,
    {
        let _cf_handle = self.inner.create_cf(cfd).map_err(convert_rocksdb_err)?;
        Ok(())
    }

    /// Reads single record by key.
    pub fn get<S: Schema>(&self, schema_key: &S::Key) -> Result<Option<S::Value>> {
        let k = <S::Key as KeyCodec<S>>::encode_key(&schema_key)?;
        let cf_handle = self.get_cf_handle(S::COLUMN_FAMILY_NAME)?;
        let time = std::time::Instant::now();

        let result = self
            .inner
            .get_cf(cf_handle, &k)
            .map_err(convert_rocksdb_err)?;
        OP_COUNTER.observe_duration(&format!("db_get_{}", S::COLUMN_FAMILY_NAME), time.elapsed());
        result
            .map(|raw_value| <S::Value as ValueCodec<S>>::decode_value(&raw_value))
            .transpose()
    }

    /// Writes single record.
    pub fn put<S: Schema>(&self, key: &S::Key, value: &S::Value) -> Result<()> {
        let k = <S::Key as KeyCodec<S>>::encode_key(&key)?;
        let v = <S::Value as ValueCodec<S>>::encode_value(&value)?;
        let cf_handle = self.get_cf_handle(S::COLUMN_FAMILY_NAME)?;

        self.inner
            .put_cf_opt(cf_handle, &k, &v, &default_write_options())
            .map_err(convert_rocksdb_err)
    }

    /// Delete all keys in range [begin, end).
    ///
    /// `SK` has to be an explicit type parameter since
    /// https://github.com/rust-lang/rust/issues/44721
    pub fn range_delete<S, SK>(&self, begin: &SK, end: &SK) -> Result<()>
    where
        S: Schema,
        SK: SeekKeyCodec<S>,
    {
        let raw_begin = begin.encode_seek_key()?;
        let raw_end = end.encode_seek_key()?;
        let cf_handle = self.get_cf_handle(S::COLUMN_FAMILY_NAME)?;

        self.inner
            .delete_range_cf(&cf_handle, &raw_begin, &raw_end)
            .map_err(convert_rocksdb_err)
    }

    /// Returns a [`SchemaIterator`] on a certain schema.
    pub fn iter<S: Schema>(&self, opts: ReadOptions) -> Result<SchemaIterator<S>> {
        let cf_handle = self.get_cf_handle(S::COLUMN_FAMILY_NAME)?;
        Ok(SchemaIterator::new(self.inner.iter_cf_opt(cf_handle, opts)))
    }

    /// Writes a group of records wrapped in a [`SchemaBatch`].
    pub fn write_schemas(&self, batch: SchemaBatch) -> Result<()> {
        let db_batch = rocksdb::WriteBatch::new();
        for (cf_name, rows) in &batch.rows {
            let cf_handle = self.get_cf_handle(cf_name)?;
            for (key, write_op) in rows {
                match write_op {
                    WriteOp::Value(value) => db_batch.put_cf(cf_handle, key, value),
                    WriteOp::Deletion => db_batch.delete_cf(cf_handle, key),
                }
                .map_err(convert_rocksdb_err)?;
            }
        }

        self.inner
            .write_opt(&db_batch, &default_write_options())
            .map_err(convert_rocksdb_err)?;

        // Bump counters only after DB write succeeds.
        for (cf_name, rows) in &batch.rows {
            for (key, write_op) in rows {
                match write_op {
                    WriteOp::Value(value) => OP_COUNTER.observe(
                        &format!("db_put_bytes_{}", cf_name),
                        (key.len() + value.len()) as f64,
                    ),
                    WriteOp::Deletion => OP_COUNTER.inc(&format!("db_delete_{}", cf_name)),
                }
            }
        }

        Ok(())
    }

    fn get_cf_handle(&self, cf_name: &str) -> Result<&CFHandle> {
        self.inner.cf_handle(cf_name).ok_or_else(|| {
            format_err!(
                "DB::cf_handle not found for column family name: {}",
                cf_name
            )
        })
    }

    /// Returns the approximate size of each non-empty column family in bytes.
    pub fn get_approximate_sizes_cf(&self) -> Result<BTreeMap<String, u64>> {
        let mut cf_sizes = BTreeMap::new();

        for cf_name in self.inner.cf_names().into_iter().map(ToString::to_string) {
            let cf_handle = self.get_cf_handle(&cf_name)?;
            let size = self
                .inner
                .get_property_int_cf(cf_handle, "rocksdb.estimate-live-data-size")
                .ok_or_else(|| {
                    format_err!(
                        "Unable to get approximate size of {} column family.",
                        cf_name,
                    )
                })?;
            cf_sizes.insert(cf_name, size);
        }

        Ok(cf_sizes)
    }

    /// Flushes all memtable data. If `sync` is true, the flush will wait until it's done. This is
    /// only used for testing `get_approximate_sizes_cf` in unit tests.
    pub fn flush_all(&self, sync: bool) -> Result<()> {
        for cf_name in self.inner.cf_names() {
            let cf_handle = self.get_cf_handle(cf_name)?;
            self.inner
                .flush_cf(cf_handle, sync)
                .map_err(convert_rocksdb_err)?;
        }
        Ok(())
    }
}

/// For now we always use synchronous writes. This makes sure that once the operation returns
/// `Ok(())` the data is persisted even if the machine crashes. In the future we might consider
/// selectively turning this off for some non-critical writes to improve performance.
fn default_write_options() -> WriteOptions {
    let mut opts = WriteOptions::new();
    opts.set_sync(true);
    opts
}
<|endoftext|>


<commit_before>// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#![unstable(feature = "futures_api",
            reason = "futures in libcore are unstable",
            issue = "50547")]

use fmt;
use future::Future;
use marker::PhantomData;
use mem::PinMut;
use task::{Context, Poll};

/// A custom trait object for polling futures, roughly akin to
/// `Box<dyn Future<Output = T> + 'a>`.
///
/// This custom trait object was introduced for two reasons:
/// - Currently it is not possible to take `dyn Trait` by value and
///   `Box<dyn Trait>` is not available in no_std contexts.
/// - The `Future` trait is currently not object safe: The `Future::poll`
///   method makes uses the arbitrary self types feature and traits in which
///   this feature is used are currently not object safe due to current compiler
///   limitations. (See tracking issue for arbitray self types for more
///   information #44874)
pub struct LocalFutureObj<'a, T> {
    ptr: *mut (),
    poll_fn: unsafe fn(*mut (), &mut Context) -> Poll<T>,
    drop_fn: unsafe fn(*mut ()),
    _marker1: PhantomData<T>,
    _marker2: PhantomData<&'a ()>,
}

impl<'a, T> LocalFutureObj<'a, T> {
    /// Create a `LocalFutureObj` from a custom trait object representation.
    #[inline]
    pub fn new<F: UnsafeFutureObj<'a, T> + 'a>(f: F) -> LocalFutureObj<'a, T> {
        LocalFutureObj {
            ptr: f.into_raw(),
            poll_fn: F::poll,
            drop_fn: F::drop,
            _marker1: PhantomData,
            _marker2: PhantomData,
        }
    }

    /// Converts the `LocalFutureObj` into a `FutureObj`
    /// To make this operation safe one has to ensure that the `UnsafeFutureObj`
    /// instance from which this `LocalFutureObj` was created actually
    /// implements `Send`.
    #[inline]
    pub unsafe fn into_future_obj(self) -> FutureObj<'a, T> {
        FutureObj(self)
    }
}

impl<'a, T> fmt::Debug for LocalFutureObj<'a, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.debug_struct("LocalFutureObj")
            .finish()
    }
}

impl<'a, T> From<FutureObj<'a, T>> for LocalFutureObj<'a, T> {
    #[inline]
    fn from(f: FutureObj<'a, T>) -> LocalFutureObj<'a, T> {
        f.0
    }
}

impl<'a, T> Future for LocalFutureObj<'a, T> {
    type Output = T;

    #[inline]
    fn poll(self: PinMut<Self>, cx: &mut Context) -> Poll<T> {
        unsafe {
            (self.poll_fn)(self.ptr, cx)
        }
    }
}

impl<'a, T> Drop for LocalFutureObj<'a, T> {
    fn drop(&mut self) {
        unsafe {
            (self.drop_fn)(self.ptr)
        }
    }
}

/// A custom trait object for polling futures, roughly akin to
/// `Box<dyn Future<Output = T> + Send + 'a>`.
///
/// This custom trait object was introduced for two reasons:
/// - Currently it is not possible to take `dyn Trait` by value and
///   `Box<dyn Trait>` is not available in no_std contexts.
/// - The `Future` trait is currently not object safe: The `Future::poll`
///   method makes uses the arbitrary self types feature and traits in which
///   this feature is used are currently not object safe due to current compiler
///   limitations. (See tracking issue for arbitray self types for more
///   information #44874)
pub struct FutureObj<'a, T>(LocalFutureObj<'a, T>);

unsafe impl<'a, T> Send for FutureObj<'a, T> {}

impl<'a, T> FutureObj<'a, T> {
    /// Create a `FutureObj` from a custom trait object representation.
    #[inline]
    pub fn new<F: UnsafeFutureObj<'a, T> + Send>(f: F) -> FutureObj<'a, T> {
        FutureObj(LocalFutureObj::new(f))
    }
}

impl<'a, T> fmt::Debug for FutureObj<'a, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.debug_struct("FutureObj")
            .finish()
    }
}

impl<'a, T> Future for FutureObj<'a, T> {
    type Output = T;

    #[inline]
    fn poll(self: PinMut<Self>, cx: &mut Context) -> Poll<T> {
        let pinned_field = unsafe { PinMut::map_unchecked(self, |x| &mut x.0) };
        pinned_field.poll(cx)
    }
}

/// A custom implementation of a future trait object for `FutureObj`, providing
/// a hand-rolled vtable.
///
/// This custom representation is typically used only in `no_std` contexts,
/// where the default `Box`-based implementation is not available.
///
/// The implementor must guarantee that it is safe to call `poll` repeatedly (in
/// a non-concurrent fashion) with the result of `into_raw` until `drop` is
/// called.
pub unsafe trait UnsafeFutureObj<'a, T>: 'a {
    /// Convert an owned instance into a (conceptually owned) void pointer.
    fn into_raw(self) -> *mut ();

    /// Poll the future represented by the given void pointer.
    ///
    /// # Safety
    ///
    /// The trait implementor must guarantee that it is safe to repeatedly call
    /// `poll` with the result of `into_raw` until `drop` is called; such calls
    /// are not, however, allowed to race with each other or with calls to
    /// `drop`.
    unsafe fn poll(ptr: *mut (), cx: &mut Context) -> Poll<T>;

    /// Drops the future represented by the given void pointer.
    ///
    /// # Safety
    ///
    /// The trait implementor must guarantee that it is safe to call this
    /// function once per `into_raw` invocation; that call cannot race with
    /// other calls to `drop` or `poll`.
    unsafe fn drop(ptr: *mut ());
}
<commit_msg>Remove unnecessary `PhantomData` field<commit_after>// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#![unstable(feature = "futures_api",
            reason = "futures in libcore are unstable",
            issue = "50547")]

use fmt;
use future::Future;
use marker::PhantomData;
use mem::PinMut;
use task::{Context, Poll};

/// A custom trait object for polling futures, roughly akin to
/// `Box<dyn Future<Output = T> + 'a>`.
///
/// This custom trait object was introduced for two reasons:
/// - Currently it is not possible to take `dyn Trait` by value and
///   `Box<dyn Trait>` is not available in no_std contexts.
/// - The `Future` trait is currently not object safe: The `Future::poll`
///   method makes uses the arbitrary self types feature and traits in which
///   this feature is used are currently not object safe due to current compiler
///   limitations. (See tracking issue for arbitray self types for more
///   information #44874)
pub struct LocalFutureObj<'a, T> {
    ptr: *mut (),
    poll_fn: unsafe fn(*mut (), &mut Context) -> Poll<T>,
    drop_fn: unsafe fn(*mut ()),
    _marker: PhantomData<&'a ()>,
}

impl<'a, T> LocalFutureObj<'a, T> {
    /// Create a `LocalFutureObj` from a custom trait object representation.
    #[inline]
    pub fn new<F: UnsafeFutureObj<'a, T> + 'a>(f: F) -> LocalFutureObj<'a, T> {
        LocalFutureObj {
            ptr: f.into_raw(),
            poll_fn: F::poll,
            drop_fn: F::drop,
            _marker: PhantomData,
        }
    }

    /// Converts the `LocalFutureObj` into a `FutureObj`
    /// To make this operation safe one has to ensure that the `UnsafeFutureObj`
    /// instance from which this `LocalFutureObj` was created actually
    /// implements `Send`.
    #[inline]
    pub unsafe fn into_future_obj(self) -> FutureObj<'a, T> {
        FutureObj(self)
    }
}

impl<'a, T> fmt::Debug for LocalFutureObj<'a, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.debug_struct("LocalFutureObj")
            .finish()
    }
}

impl<'a, T> From<FutureObj<'a, T>> for LocalFutureObj<'a, T> {
    #[inline]
    fn from(f: FutureObj<'a, T>) -> LocalFutureObj<'a, T> {
        f.0
    }
}

impl<'a, T> Future for LocalFutureObj<'a, T> {
    type Output = T;

    #[inline]
    fn poll(self: PinMut<Self>, cx: &mut Context) -> Poll<T> {
        unsafe {
            (self.poll_fn)(self.ptr, cx)
        }
    }
}

impl<'a, T> Drop for LocalFutureObj<'a, T> {
    fn drop(&mut self) {
        unsafe {
            (self.drop_fn)(self.ptr)
        }
    }
}

/// A custom trait object for polling futures, roughly akin to
/// `Box<dyn Future<Output = T> + Send + 'a>`.
///
/// This custom trait object was introduced for two reasons:
/// - Currently it is not possible to take `dyn Trait` by value and
///   `Box<dyn Trait>` is not available in no_std contexts.
/// - The `Future` trait is currently not object safe: The `Future::poll`
///   method makes uses the arbitrary self types feature and traits in which
///   this feature is used are currently not object safe due to current compiler
///   limitations. (See tracking issue for arbitray self types for more
///   information #44874)
pub struct FutureObj<'a, T>(LocalFutureObj<'a, T>);

unsafe impl<'a, T> Send for FutureObj<'a, T> {}

impl<'a, T> FutureObj<'a, T> {
    /// Create a `FutureObj` from a custom trait object representation.
    #[inline]
    pub fn new<F: UnsafeFutureObj<'a, T> + Send>(f: F) -> FutureObj<'a, T> {
        FutureObj(LocalFutureObj::new(f))
    }
}

impl<'a, T> fmt::Debug for FutureObj<'a, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.debug_struct("FutureObj")
            .finish()
    }
}

impl<'a, T> Future for FutureObj<'a, T> {
    type Output = T;

    #[inline]
    fn poll(self: PinMut<Self>, cx: &mut Context) -> Poll<T> {
        let pinned_field = unsafe { PinMut::map_unchecked(self, |x| &mut x.0) };
        pinned_field.poll(cx)
    }
}

/// A custom implementation of a future trait object for `FutureObj`, providing
/// a hand-rolled vtable.
///
/// This custom representation is typically used only in `no_std` contexts,
/// where the default `Box`-based implementation is not available.
///
/// The implementor must guarantee that it is safe to call `poll` repeatedly (in
/// a non-concurrent fashion) with the result of `into_raw` until `drop` is
/// called.
pub unsafe trait UnsafeFutureObj<'a, T>: 'a {
    /// Convert an owned instance into a (conceptually owned) void pointer.
    fn into_raw(self) -> *mut ();

    /// Poll the future represented by the given void pointer.
    ///
    /// # Safety
    ///
    /// The trait implementor must guarantee that it is safe to repeatedly call
    /// `poll` with the result of `into_raw` until `drop` is called; such calls
    /// are not, however, allowed to race with each other or with calls to
    /// `drop`.
    unsafe fn poll(ptr: *mut (), cx: &mut Context) -> Poll<T>;

    /// Drops the future represented by the given void pointer.
    ///
    /// # Safety
    ///
    /// The trait implementor must guarantee that it is safe to call this
    /// function once per `into_raw` invocation; that call cannot race with
    /// other calls to `drop` or `poll`.
    unsafe fn drop(ptr: *mut ());
}
<|endoftext|>


<commit_before>use std::str;
use std::ops::Deref;
use std::fmt;

use bytes::Bytes;

use clear::Clear;

/// Thin wrapper around `Bytes` which guarantees that bytes are valid UTF-8 string.
#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct Chars(Bytes);

impl Chars {
    /// New empty object.
    pub fn new() -> Chars {
        Chars(Bytes::new())
    }

    /// Try convert from `Bytes`
    pub fn from_bytes(bytes: Bytes) -> Result<Chars, str::Utf8Error> {
        str::from_utf8(&bytes)?;

        Ok(Chars(bytes))
    }

    /// Len in bytes.
    pub fn len(&self) -> usize {
        self.0.len()
    }

    pub fn is_empty(&self) -> bool {
        self.0.is_empty()
    }
}

impl<'a> From<&'a str> for Chars {
    fn from(src: &'a str) -> Chars {
        Chars(Bytes::from(src))
    }
}

impl Default for Chars {
    fn default() -> Self {
        Chars::new()
    }
}

impl Deref for Chars {
    type Target = str;

    fn deref(&self) -> &str {
        unsafe { str::from_utf8_unchecked(&self.0) }
    }
}

impl Clear for Chars {
    fn clear(&mut self) {
        self.0.clear();
    }
}

impl fmt::Display for Chars {
    #[inline]
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        fmt::Display::fmt(&**self, f)
    }
}

impl fmt::Debug for Chars {
    #[inline]
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        fmt::Debug::fmt(&**self, f)
    }
}

#[cfg(test)]
mod test {
    use super::Chars;

    #[test]
    fn test_display_and_debug() {
        let s = "test";
        let string: String = s.into();
        let chars: Chars = s.into();

        assert_eq!(format!("{}", string), format!("{}", chars));
        assert_eq!(format!("{:?}", string), format!("{:?}", chars));
    }
}
<commit_msg>convert String to Chars<commit_after>use std::str;
use std::ops::Deref;
use std::fmt;

use bytes::Bytes;

use clear::Clear;

/// Thin wrapper around `Bytes` which guarantees that bytes are valid UTF-8 string.
#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct Chars(Bytes);

impl Chars {
    /// New empty object.
    pub fn new() -> Chars {
        Chars(Bytes::new())
    }

    /// Try convert from `Bytes`
    pub fn from_bytes(bytes: Bytes) -> Result<Chars, str::Utf8Error> {
        str::from_utf8(&bytes)?;

        Ok(Chars(bytes))
    }

    /// Len in bytes.
    pub fn len(&self) -> usize {
        self.0.len()
    }

    pub fn is_empty(&self) -> bool {
        self.0.is_empty()
    }
}

impl<'a> From<&'a str> for Chars {
    fn from(src: &'a str) -> Chars {
        Chars(Bytes::from(src))
    }
}

impl From<String> for Chars {
    fn from(src: String) -> Chars {
        Chars(Bytes::from(src))
    }
}

impl Default for Chars {
    fn default() -> Self {
        Chars::new()
    }
}

impl Deref for Chars {
    type Target = str;

    fn deref(&self) -> &str {
        unsafe { str::from_utf8_unchecked(&self.0) }
    }
}

impl Clear for Chars {
    fn clear(&mut self) {
        self.0.clear();
    }
}

impl fmt::Display for Chars {
    #[inline]
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        fmt::Display::fmt(&**self, f)
    }
}

impl fmt::Debug for Chars {
    #[inline]
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        fmt::Debug::fmt(&**self, f)
    }
}

#[cfg(test)]
mod test {
    use super::Chars;

    #[test]
    fn test_display_and_debug() {
        let s = "test";
        let string: String = s.into();
        let chars: Chars = s.into();

        assert_eq!(format!("{}", string), format!("{}", chars));
        assert_eq!(format!("{:?}", string), format!("{:?}", chars));
    }
}
<|endoftext|>