<commit_before>use crate::io::{BufReader, BufWriter};
use crate::{AsyncBufRead, AsyncRead, AsyncWrite};
use pin_project::pin_project;
use std::io::{self};
use std::{
    pin::Pin,
    task::{Context, Poll},
};

/// Wraps a type that is [`AsyncWrite`] and [`AsyncRead`], and buffers its input and output.
///
/// It can be excessively inefficient to work directly with something that implements [`AsyncWrite`]
/// and [`AsyncRead`]. For example, every `write`, however small, has to traverse the syscall
/// interface, and similarly, every read has to do the same. The [`BufWriter`] and [`BufReader`]
/// types aid with these problems respectively, but do so in only one direction. `BufStream` wraps
/// one in the other so that both directions are buffered. See their documentation for details.
#[pin_project]
#[derive(Debug)]
pub struct BufStream<RW>(#[pin] BufReader<BufWriter<RW>>);

impl<RW: AsyncRead + AsyncWrite> BufStream<RW> {
    /// Wrap a type in both [`BufWriter`] and [`BufReader`].
    ///
    /// See the documentation for those types and [`BufStream`] for details.
    pub fn new(stream: RW) -> BufStream<RW> {
        BufStream(BufReader::new(BufWriter::new(stream)))
    }
}

impl<RW: AsyncRead + AsyncWrite> AsyncWrite for BufStream<RW> {
    fn poll_write(
        self: Pin<&mut Self>,
        cx: &mut Context<'_>,
        buf: &[u8],
    ) -> Poll<io::Result<usize>> {
        self.project().0.poll_write(cx, buf)
    }

    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
        self.project().0.poll_flush(cx)
    }

    fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
        self.project().0.poll_shutdown(cx)
    }
}

impl<RW: AsyncRead + AsyncWrite> AsyncRead for BufStream<RW> {
    fn poll_read(
        self: Pin<&mut Self>,
        cx: &mut Context<'_>,
        buf: &mut [u8],
    ) -> Poll<io::Result<usize>> {
        self.project().0.poll_read(cx, buf)
    }

    // we can't skip unconditionally because of the large buffer case in read.
    unsafe fn prepare_uninitialized_buffer(&self, buf: &mut [u8]) -> bool {
        self.0.prepare_uninitialized_buffer(buf)
    }
}

impl<RW: AsyncBufRead + AsyncRead + AsyncWrite> AsyncBufRead for BufStream<RW> {
    fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<&[u8]>> {
        self.project().0.poll_fill_buf(cx)
    }

    fn consume(self: Pin<&mut Self>, amt: usize) {
        self.project().0.consume(amt)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn assert_unpin() {
        crate::is_unpin::<BufStream<()>>();
    }
}
<commit_msg>io: add get_*/into_inner methods to BufStream (#1598)<commit_after>use crate::io::{BufReader, BufWriter};
use crate::{AsyncBufRead, AsyncRead, AsyncWrite};
use pin_project::pin_project;
use std::io::{self};
use std::{
    pin::Pin,
    task::{Context, Poll},
};

/// Wraps a type that is [`AsyncWrite`] and [`AsyncRead`], and buffers its input and output.
///
/// It can be excessively inefficient to work directly with something that implements [`AsyncWrite`]
/// and [`AsyncRead`]. For example, every `write`, however small, has to traverse the syscall
/// interface, and similarly, every read has to do the same. The [`BufWriter`] and [`BufReader`]
/// types aid with these problems respectively, but do so in only one direction. `BufStream` wraps
/// one in the other so that both directions are buffered. See their documentation for details.
#[pin_project]
#[derive(Debug)]
pub struct BufStream<RW>(#[pin] BufReader<BufWriter<RW>>);

impl<RW: AsyncRead + AsyncWrite> BufStream<RW> {
    /// Wrap a type in both [`BufWriter`] and [`BufReader`].
    ///
    /// See the documentation for those types and [`BufStream`] for details.
    pub fn new(stream: RW) -> BufStream<RW> {
        BufStream(BufReader::new(BufWriter::new(stream)))
    }

    /// Gets a reference to the underlying I/O object.
    ///
    /// It is inadvisable to directly read from the underlying I/O object.
    pub fn get_ref(&self) -> &RW {
        self.0.get_ref().get_ref()
    }

    /// Gets a mutable reference to the underlying I/O object.
    ///
    /// It is inadvisable to directly read from the underlying I/O object.
    pub fn get_mut(&mut self) -> &mut RW {
        self.0.get_mut().get_mut()
    }

    /// Gets a pinned mutable reference to the underlying I/O object.
    ///
    /// It is inadvisable to directly read from the underlying I/O object.
    pub fn get_pin_mut(self: Pin<&mut Self>) -> Pin<&mut RW> {
        self.project().0.get_pin_mut().get_pin_mut()
    }

    /// Consumes this `BufStream`, returning the underlying I/O object.
    ///
    /// Note that any leftover data in the internal buffer is lost.
    pub fn into_inner(self) -> RW {
        self.0.into_inner().into_inner()
    }
}

impl<RW: AsyncRead + AsyncWrite> AsyncWrite for BufStream<RW> {
    fn poll_write(
        self: Pin<&mut Self>,
        cx: &mut Context<'_>,
        buf: &[u8],
    ) -> Poll<io::Result<usize>> {
        self.project().0.poll_write(cx, buf)
    }

    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
        self.project().0.poll_flush(cx)
    }

    fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
        self.project().0.poll_shutdown(cx)
    }
}

impl<RW: AsyncRead + AsyncWrite> AsyncRead for BufStream<RW> {
    fn poll_read(
        self: Pin<&mut Self>,
        cx: &mut Context<'_>,
        buf: &mut [u8],
    ) -> Poll<io::Result<usize>> {
        self.project().0.poll_read(cx, buf)
    }

    // we can't skip unconditionally because of the large buffer case in read.
    unsafe fn prepare_uninitialized_buffer(&self, buf: &mut [u8]) -> bool {
        self.0.prepare_uninitialized_buffer(buf)
    }
}

impl<RW: AsyncBufRead + AsyncRead + AsyncWrite> AsyncBufRead for BufStream<RW> {
    fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<&[u8]>> {
        self.project().0.poll_fill_buf(cx)
    }

    fn consume(self: Pin<&mut Self>, amt: usize) {
        self.project().0.consume(amt)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn assert_unpin() {
        crate::is_unpin::<BufStream<()>>();
    }
}
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use std::collections::VecDeque;
use std::error;
use std::fmt::{self, Display, Formatter};
use std::sync::Arc;
use std::time::Instant;

use crate::storage::CF_WRITE;
use crate::util::escape;
use crate::util::rocksdb_util::{self, compact_range, stats::get_range_entries_and_versions};
use crate::util::worker::Runnable;
use rocksdb::DB;

use super::metrics::COMPACT_RANGE_CF;

type Key = Vec<u8>;

pub enum Task {
    Compact {
        cf_name: String,
        start_key: Option<Key>, // None means smallest key
        end_key: Option<Key>,   // None means largest key
    },

    CheckAndCompact {
        cf_names: Vec<String>,         // Column families need to compact
        ranges: Vec<Key>,              // Ranges need to check
        tombstones_num_threshold: u64, // The minimum RocksDB tombstones a range that need compacting has
        tombstones_percent_threshold: u64,
    },
}

impl Display for Task {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        match *self {
            Task::Compact {
                ref cf_name,
                ref start_key,
                ref end_key,
            } => f
                .debug_struct("Compact")
                .field("cf_name", cf_name)
                .field("start_key", &start_key.as_ref().map(|k| escape(k)))
                .field("end_key", &end_key.as_ref().map(|k| escape(k)))
                .finish(),
            Task::CheckAndCompact {
                ref cf_names,
                ref ranges,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            } => f
                .debug_struct("CheckAndCompact")
                .field("cf_names", cf_names)
                .field(
                    "ranges",
                    &(
                        ranges.first().as_ref().map(|k| escape(k)),
                        ranges.last().as_ref().map(|k| escape(k)),
                    ),
                )
                .field("tombstones_num_threshold", &tombstones_num_threshold)
                .field(
                    "tombstones_percent_threshold",
                    &tombstones_percent_threshold,
                )
                .finish(),
        }
    }
}

quick_error! {
    #[derive(Debug)]
    pub enum Error {
        Other(err: Box<dyn error::Error + Sync + Send>) {
            from()
            cause(err.as_ref())
            description(err.description())
            display("compact failed {:?}", err)
        }
    }
}

pub struct Runner {
    engine: Arc<DB>,
}

impl Runner {
    pub fn new(engine: Arc<DB>) -> Runner {
        Runner { engine }
    }

    /// Sends a compact range command to RocksDB to compact the range of the cf.
    pub fn compact_range_cf(
        &mut self,
        cf_name: String,
        start_key: Option<Vec<u8>>,
        end_key: Option<Vec<u8>>,
    ) -> Result<(), Error> {
        let handle = box_try!(rocksdb_util::get_cf_handle(&self.engine, &cf_name));
        let timer = Instant::now();
        let compact_range_timer = COMPACT_RANGE_CF
            .with_label_values(&[&cf_name])
            .start_coarse_timer();
        let start = start_key.as_ref().map(Vec::as_slice);
        let end = end_key.as_ref().map(Vec::as_slice);
        compact_range(
            &self.engine,
            handle,
            start,
            end,
            false,
            1, /* threads */
        );
        compact_range_timer.observe_duration();
        info!(
            "compact range finished";
            "range_start" => start.map(::log_wrappers::Key),
            "range_end" => end.map(::log_wrappers::Key),
            "cf" => cf_name,
            "time_takes" => ?timer.elapsed(),
        );
        Ok(())
    }
}

impl Runnable<Task> for Runner {
    fn run(&mut self, task: Task) {
        match task {
            Task::Compact {
                cf_name,
                start_key,
                end_key,
            } => {
                let cf = cf_name.clone();
                if let Err(e) = self.compact_range_cf(cf_name, start_key, end_key) {
                    error!("execute compact range failed"; "cf" => cf, "err" => %e);
                }
            }
            Task::CheckAndCompact {
                cf_names,
                ranges,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            } => match collect_ranges_need_compact(
                &self.engine,
                ranges,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            ) {
                Ok(mut ranges) => {
                    for (start, end) in ranges.drain(..) {
                        for cf in &cf_names {
                            if let Err(e) = self.compact_range_cf(
                                cf.clone(),
                                Some(start.clone()),
                                Some(end.clone()),
                            ) {
                                error!(
                                    "compact range failed";
                                    "range_start" => log_wrappers::Key(&start),
                                    "range_end" => log_wrappers::Key(&end),
                                    "cf" => cf,
                                    "err" => %e,
                                );
                            }
                        }
                    }
                }
                Err(e) => warn!("check ranges need reclaim failed, err: {:?}", e),
            },
        }
    }
}

fn need_compact(
    num_entires: u64,
    num_versions: u64,
    tombstones_num_threshold: u64,
    tombstones_percent_threshold: u64,
) -> bool {
    if num_entires <= num_versions {
        return false;
    }

    // When the number of tombstones exceed threshold and ratio, this range need compacting.
    let estimate_num_del = num_entires - num_versions;
    estimate_num_del >= tombstones_num_threshold
        && estimate_num_del * 100 >= tombstones_percent_threshold * num_entires
}

fn collect_ranges_need_compact(
    engine: &DB,
    ranges: Vec<Key>,
    tombstones_num_threshold: u64,
    tombstones_percent_threshold: u64,
) -> Result<VecDeque<(Key, Key)>, Error> {
    // Check the SST properties for each range, and TiKV will compact a range if the range
    // contains too many RocksDB tombstones. TiKV will merge multiple neighboring ranges
    // that need compacting into a single range.
    let mut ranges_need_compact = VecDeque::new();
    let cf = box_try!(rocksdb_util::get_cf_handle(engine, CF_WRITE));
    let mut compact_start = None;
    let mut compact_end = None;
    for range in ranges.windows(2) {
        // Get total entries and total versions in this range and checks if it needs to be compacted.
        if let Some((num_ent, num_ver)) =
            get_range_entries_and_versions(engine, cf, &range[0], &range[1])
        {
            if need_compact(
                num_ent,
                num_ver,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            ) {
                if compact_start.is_none() {
                    // The previous range doesn't need compacting.
                    compact_start = Some(range[0].clone());
                }
                compact_end = Some(range[1].clone());
                // Move to next range.
                continue;
            }
        }

        // Current range doesn't need compacting, save previous range that need compacting.
        if compact_start.is_some() {
            ranges_need_compact.push_back((compact_start.unwrap(), compact_end.unwrap()));
            compact_start = None;
            compact_end = None;
        }
    }

    // Save the last range that needs to be compacted.
    if compact_start.is_some() {
        ranges_need_compact.push_back((compact_start.unwrap(), compact_end.unwrap()));
    }

    Ok(ranges_need_compact)
}

#[cfg(test)]
mod tests {
    use std::thread::sleep;
    use std::time::Duration;

    use tempdir::TempDir;

    use crate::raftstore::store::keys::data_key;
    use crate::storage::mvcc::{Write, WriteType};
    use crate::storage::types::Key as MvccKey;
    use crate::storage::{CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE};
    use crate::util::rocksdb_util::{
        get_cf_handle, new_engine, new_engine_opt, properties::MvccPropertiesCollectorFactory,
        stats::get_range_entries_and_versions, CFOptions,
    };
    use rocksdb::{self, Writable, WriteBatch, DB};

    use super::*;

    const ROCKSDB_TOTAL_SST_FILES_SIZE: &str = "rocksdb.total-sst-files-size";

    #[test]
    fn test_compact_range() {
        let path = TempDir::new("compact-range-test").unwrap();
        let db = new_engine(path.path().to_str().unwrap(), None, &[CF_DEFAULT], None).unwrap();
        let db = Arc::new(db);

        let mut runner = Runner::new(Arc::clone(&db));

        let handle = get_cf_handle(&db, CF_DEFAULT).unwrap();

        // Generate the first SST file.
        let wb = WriteBatch::new();
        for i in 0..1000 {
            let k = format!("key_{}", i);
            wb.put_cf(handle, k.as_bytes(), b"whatever content")
                .unwrap();
        }
        db.write(wb).unwrap();
        db.flush_cf(handle, true).unwrap();

        // Generate another SST file has the same content with first SST file.
        let wb = WriteBatch::new();
        for i in 0..1000 {
            let k = format!("key_{}", i);
            wb.put_cf(handle, k.as_bytes(), b"whatever content")
                .unwrap();
        }
        db.write(wb).unwrap();
        db.flush_cf(handle, true).unwrap();

        // Get the total SST files size.
        let old_sst_files_size = db
            .get_property_int_cf(handle, ROCKSDB_TOTAL_SST_FILES_SIZE)
            .unwrap();

        // Schedule compact range task.
        runner.run(Task::Compact {
            cf_name: String::from(CF_DEFAULT),
            start_key: None,
            end_key: None,
        });
        sleep(Duration::from_secs(5));

        // Get the total SST files size after compact range.
        let new_sst_files_size = db
            .get_property_int_cf(handle, ROCKSDB_TOTAL_SST_FILES_SIZE)
            .unwrap();
        assert!(old_sst_files_size > new_sst_files_size);
    }

    fn mvcc_put(db: &DB, k: &[u8], v: &[u8], start_ts: u64, commit_ts: u64) {
        let cf = get_cf_handle(db, CF_WRITE).unwrap();
        let k = MvccKey::from_encoded(data_key(k)).append_ts(commit_ts);
        let w = Write::new(WriteType::Put, start_ts, Some(v.to_vec()));
        db.put_cf(cf, k.as_encoded(), &w.to_bytes()).unwrap();
    }

    fn delete(db: &DB, k: &[u8], commit_ts: u64) {
        let cf = get_cf_handle(db, CF_WRITE).unwrap();
        let k = MvccKey::from_encoded(data_key(k)).append_ts(commit_ts);
        db.delete_cf(cf, k.as_encoded()).unwrap();
    }

    fn open_db(path: &str) -> DB {
        let db_opts = rocksdb::DBOptions::new();
        let mut cf_opts = rocksdb::ColumnFamilyOptions::new();
        cf_opts.set_level_zero_file_num_compaction_trigger(8);
        let f = Box::new(MvccPropertiesCollectorFactory::default());
        cf_opts.add_table_properties_collector_factory("tikv.test-collector", f);
        let cfs_opts = vec![
            CFOptions::new(CF_DEFAULT, rocksdb::ColumnFamilyOptions::new()),
            CFOptions::new(CF_RAFT, rocksdb::ColumnFamilyOptions::new()),
            CFOptions::new(CF_LOCK, rocksdb::ColumnFamilyOptions::new()),
            CFOptions::new(CF_WRITE, cf_opts),
        ];
        new_engine_opt(path, db_opts, cfs_opts).unwrap()
    }

    #[test]
    fn test_check_space_redundancy() {
        let p = TempDir::new("test").unwrap();
        let engine = open_db(p.path().to_str().unwrap());
        let cf = get_cf_handle(&engine, CF_WRITE).unwrap();

        // mvcc_put 0..5
        for i in 0..5 {
            let (k, v) = (format!("k{}", i), format!("value{}", i));
            mvcc_put(&engine, k.as_bytes(), v.as_bytes(), 1, 2);
        }
        engine.flush_cf(cf, true).unwrap();

        // gc 0..5
        for i in 0..5 {
            let k = format!("k{}", i);
            delete(&engine, k.as_bytes(), 2);
        }
        engine.flush_cf(cf, true).unwrap();

        let (s, e) = (data_key(b"k0"), data_key(b"k5"));
        let (entries, version) = get_range_entries_and_versions(&engine, cf, &s, &e).unwrap();
        assert_eq!(entries, 10);
        assert_eq!(version, 5);

        // mvcc_put 5..10
        for i in 5..10 {
            let (k, v) = (format!("k{}", i), format!("value{}", i));
            mvcc_put(&engine, k.as_bytes(), v.as_bytes(), 1, 2);
        }
        engine.flush_cf(cf, true).unwrap();

        let (s, e) = (data_key(b"k5"), data_key(b"k9"));
        let (entries, version) = get_range_entries_and_versions(&engine, cf, &s, &e).unwrap();
        assert_eq!(entries, 5);
        assert_eq!(version, 5);

        let ranges_need_to_compact = collect_ranges_need_compact(
            &engine,
            vec![data_key(b"k0"), data_key(b"k5"), data_key(b"k9")],
            1,
            50,
        )
        .unwrap();
        let (s, e) = (data_key(b"k0"), data_key(b"k5"));
        let mut expected_ranges = VecDeque::new();
        expected_ranges.push_back((s, e));
        assert_eq!(ranges_need_to_compact, expected_ranges);

        // gc 5..10
        for i in 5..10 {
            let k = format!("k{}", i);
            delete(&engine, k.as_bytes(), 2);
        }
        engine.flush_cf(cf, true).unwrap();

        let (s, e) = (data_key(b"k5"), data_key(b"k9"));
        let (entries, version) = get_range_entries_and_versions(&engine, cf, &s, &e).unwrap();
        assert_eq!(entries, 10);
        assert_eq!(version, 5);

        let ranges_need_to_compact = collect_ranges_need_compact(
            &engine,
            vec![data_key(b"k0"), data_key(b"k5"), data_key(b"k9")],
            1,
            50,
        )
        .unwrap();
        let (s, e) = (data_key(b"k0"), data_key(b"k9"));
        let mut expected_ranges = VecDeque::new();
        expected_ranges.push_back((s, e));
        assert_eq!(ranges_need_to_compact, expected_ranges);
    }
}
<commit_msg>raftstore: reduce clones when compact regions (#4115)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use std::collections::VecDeque;
use std::error;
use std::fmt::{self, Display, Formatter};
use std::sync::Arc;
use std::time::Instant;

use crate::storage::CF_WRITE;
use crate::util::escape;
use crate::util::rocksdb_util::{self, compact_range, stats::get_range_entries_and_versions};
use crate::util::worker::Runnable;
use rocksdb::DB;

use super::metrics::COMPACT_RANGE_CF;

type Key = Vec<u8>;

pub enum Task {
    Compact {
        cf_name: String,
        start_key: Option<Key>, // None means smallest key
        end_key: Option<Key>,   // None means largest key
    },

    CheckAndCompact {
        cf_names: Vec<String>,         // Column families need to compact
        ranges: Vec<Key>,              // Ranges need to check
        tombstones_num_threshold: u64, // The minimum RocksDB tombstones a range that need compacting has
        tombstones_percent_threshold: u64,
    },
}

impl Display for Task {
    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
        match *self {
            Task::Compact {
                ref cf_name,
                ref start_key,
                ref end_key,
            } => f
                .debug_struct("Compact")
                .field("cf_name", cf_name)
                .field("start_key", &start_key.as_ref().map(|k| escape(k)))
                .field("end_key", &end_key.as_ref().map(|k| escape(k)))
                .finish(),
            Task::CheckAndCompact {
                ref cf_names,
                ref ranges,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            } => f
                .debug_struct("CheckAndCompact")
                .field("cf_names", cf_names)
                .field(
                    "ranges",
                    &(
                        ranges.first().as_ref().map(|k| escape(k)),
                        ranges.last().as_ref().map(|k| escape(k)),
                    ),
                )
                .field("tombstones_num_threshold", &tombstones_num_threshold)
                .field(
                    "tombstones_percent_threshold",
                    &tombstones_percent_threshold,
                )
                .finish(),
        }
    }
}

quick_error! {
    #[derive(Debug)]
    pub enum Error {
        Other(err: Box<dyn error::Error + Sync + Send>) {
            from()
            cause(err.as_ref())
            description(err.description())
            display("compact failed {:?}", err)
        }
    }
}

pub struct Runner {
    engine: Arc<DB>,
}

impl Runner {
    pub fn new(engine: Arc<DB>) -> Runner {
        Runner { engine }
    }

    /// Sends a compact range command to RocksDB to compact the range of the cf.
    pub fn compact_range_cf(
        &mut self,
        cf_name: &str,
        start_key: Option<&[u8]>,
        end_key: Option<&[u8]>,
    ) -> Result<(), Error> {
        let handle = box_try!(rocksdb_util::get_cf_handle(&self.engine, cf_name));
        let timer = Instant::now();
        let compact_range_timer = COMPACT_RANGE_CF
            .with_label_values(&[cf_name])
            .start_coarse_timer();
        compact_range(
            &self.engine,
            handle,
            start_key,
            end_key,
            false,
            1, /* threads */
        );
        compact_range_timer.observe_duration();
        info!(
            "compact range finished";
            "range_start" => start_key.map(::log_wrappers::Key),
            "range_end" => end_key.map(::log_wrappers::Key),
            "cf" => cf_name,
            "time_takes" => ?timer.elapsed(),
        );
        Ok(())
    }
}

impl Runnable<Task> for Runner {
    fn run(&mut self, task: Task) {
        match task {
            Task::Compact {
                cf_name,
                start_key,
                end_key,
            } => {
                let cf = &cf_name;
                if let Err(e) = self.compact_range_cf(
                    cf,
                    start_key.as_ref().map(Vec::as_slice),
                    end_key.as_ref().map(Vec::as_slice),
                ) {
                    error!("execute compact range failed"; "cf" => cf, "err" => %e);
                }
            }
            Task::CheckAndCompact {
                cf_names,
                ranges,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            } => match collect_ranges_need_compact(
                &self.engine,
                ranges,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            ) {
                Ok(mut ranges) => {
                    for (start, end) in ranges.drain(..) {
                        for cf in &cf_names {
                            if let Err(e) = self.compact_range_cf(cf, Some(&start), Some(&end)) {
                                error!(
                                    "compact range failed";
                                    "range_start" => log_wrappers::Key(&start),
                                    "range_end" => log_wrappers::Key(&end),
                                    "cf" => cf,
                                    "err" => %e,
                                );
                            }
                        }
                    }
                }
                Err(e) => warn!("check ranges need reclaim failed"; "err" => %e),
            },
        }
    }
}

fn need_compact(
    num_entires: u64,
    num_versions: u64,
    tombstones_num_threshold: u64,
    tombstones_percent_threshold: u64,
) -> bool {
    if num_entires <= num_versions {
        return false;
    }

    // When the number of tombstones exceed threshold and ratio, this range need compacting.
    let estimate_num_del = num_entires - num_versions;
    estimate_num_del >= tombstones_num_threshold
        && estimate_num_del * 100 >= tombstones_percent_threshold * num_entires
}

fn collect_ranges_need_compact(
    engine: &DB,
    ranges: Vec<Key>,
    tombstones_num_threshold: u64,
    tombstones_percent_threshold: u64,
) -> Result<VecDeque<(Key, Key)>, Error> {
    // Check the SST properties for each range, and TiKV will compact a range if the range
    // contains too many RocksDB tombstones. TiKV will merge multiple neighboring ranges
    // that need compacting into a single range.
    let mut ranges_need_compact = VecDeque::new();
    let cf = box_try!(rocksdb_util::get_cf_handle(engine, CF_WRITE));
    let mut compact_start = None;
    let mut compact_end = None;
    for range in ranges.windows(2) {
        // Get total entries and total versions in this range and checks if it needs to be compacted.
        if let Some((num_ent, num_ver)) =
            get_range_entries_and_versions(engine, cf, &range[0], &range[1])
        {
            if need_compact(
                num_ent,
                num_ver,
                tombstones_num_threshold,
                tombstones_percent_threshold,
            ) {
                if compact_start.is_none() {
                    // The previous range doesn't need compacting.
                    compact_start = Some(range[0].clone());
                }
                compact_end = Some(range[1].clone());
                // Move to next range.
                continue;
            }
        }

        // Current range doesn't need compacting, save previous range that need compacting.
        if compact_start.is_some() {
            ranges_need_compact.push_back((compact_start.unwrap(), compact_end.unwrap()));
            compact_start = None;
            compact_end = None;
        }
    }

    // Save the last range that needs to be compacted.
    if compact_start.is_some() {
        ranges_need_compact.push_back((compact_start.unwrap(), compact_end.unwrap()));
    }

    Ok(ranges_need_compact)
}

#[cfg(test)]
mod tests {
    use std::thread::sleep;
    use std::time::Duration;

    use tempdir::TempDir;

    use crate::raftstore::store::keys::data_key;
    use crate::storage::mvcc::{Write, WriteType};
    use crate::storage::types::Key as MvccKey;
    use crate::storage::{CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE};
    use crate::util::rocksdb_util::{
        get_cf_handle, new_engine, new_engine_opt, properties::MvccPropertiesCollectorFactory,
        stats::get_range_entries_and_versions, CFOptions,
    };
    use rocksdb::{self, Writable, WriteBatch, DB};

    use super::*;

    const ROCKSDB_TOTAL_SST_FILES_SIZE: &str = "rocksdb.total-sst-files-size";

    #[test]
    fn test_compact_range() {
        let path = TempDir::new("compact-range-test").unwrap();
        let db = new_engine(path.path().to_str().unwrap(), None, &[CF_DEFAULT], None).unwrap();
        let db = Arc::new(db);

        let mut runner = Runner::new(Arc::clone(&db));

        let handle = get_cf_handle(&db, CF_DEFAULT).unwrap();

        // Generate the first SST file.
        let wb = WriteBatch::new();
        for i in 0..1000 {
            let k = format!("key_{}", i);
            wb.put_cf(handle, k.as_bytes(), b"whatever content")
                .unwrap();
        }
        db.write(wb).unwrap();
        db.flush_cf(handle, true).unwrap();

        // Generate another SST file has the same content with first SST file.
        let wb = WriteBatch::new();
        for i in 0..1000 {
            let k = format!("key_{}", i);
            wb.put_cf(handle, k.as_bytes(), b"whatever content")
                .unwrap();
        }
        db.write(wb).unwrap();
        db.flush_cf(handle, true).unwrap();

        // Get the total SST files size.
        let old_sst_files_size = db
            .get_property_int_cf(handle, ROCKSDB_TOTAL_SST_FILES_SIZE)
            .unwrap();

        // Schedule compact range task.
        runner.run(Task::Compact {
            cf_name: String::from(CF_DEFAULT),
            start_key: None,
            end_key: None,
        });
        sleep(Duration::from_secs(5));

        // Get the total SST files size after compact range.
        let new_sst_files_size = db
            .get_property_int_cf(handle, ROCKSDB_TOTAL_SST_FILES_SIZE)
            .unwrap();
        assert!(old_sst_files_size > new_sst_files_size);
    }

    fn mvcc_put(db: &DB, k: &[u8], v: &[u8], start_ts: u64, commit_ts: u64) {
        let cf = get_cf_handle(db, CF_WRITE).unwrap();
        let k = MvccKey::from_encoded(data_key(k)).append_ts(commit_ts);
        let w = Write::new(WriteType::Put, start_ts, Some(v.to_vec()));
        db.put_cf(cf, k.as_encoded(), &w.to_bytes()).unwrap();
    }

    fn delete(db: &DB, k: &[u8], commit_ts: u64) {
        let cf = get_cf_handle(db, CF_WRITE).unwrap();
        let k = MvccKey::from_encoded(data_key(k)).append_ts(commit_ts);
        db.delete_cf(cf, k.as_encoded()).unwrap();
    }

    fn open_db(path: &str) -> DB {
        let db_opts = rocksdb::DBOptions::new();
        let mut cf_opts = rocksdb::ColumnFamilyOptions::new();
        cf_opts.set_level_zero_file_num_compaction_trigger(8);
        let f = Box::new(MvccPropertiesCollectorFactory::default());
        cf_opts.add_table_properties_collector_factory("tikv.test-collector", f);
        let cfs_opts = vec![
            CFOptions::new(CF_DEFAULT, rocksdb::ColumnFamilyOptions::new()),
            CFOptions::new(CF_RAFT, rocksdb::ColumnFamilyOptions::new()),
            CFOptions::new(CF_LOCK, rocksdb::ColumnFamilyOptions::new()),
            CFOptions::new(CF_WRITE, cf_opts),
        ];
        new_engine_opt(path, db_opts, cfs_opts).unwrap()
    }

    #[test]
    fn test_check_space_redundancy() {
        let p = TempDir::new("test").unwrap();
        let engine = open_db(p.path().to_str().unwrap());
        let cf = get_cf_handle(&engine, CF_WRITE).unwrap();

        // mvcc_put 0..5
        for i in 0..5 {
            let (k, v) = (format!("k{}", i), format!("value{}", i));
            mvcc_put(&engine, k.as_bytes(), v.as_bytes(), 1, 2);
        }
        engine.flush_cf(cf, true).unwrap();

        // gc 0..5
        for i in 0..5 {
            let k = format!("k{}", i);
            delete(&engine, k.as_bytes(), 2);
        }
        engine.flush_cf(cf, true).unwrap();

        let (s, e) = (data_key(b"k0"), data_key(b"k5"));
        let (entries, version) = get_range_entries_and_versions(&engine, cf, &s, &e).unwrap();
        assert_eq!(entries, 10);
        assert_eq!(version, 5);

        // mvcc_put 5..10
        for i in 5..10 {
            let (k, v) = (format!("k{}", i), format!("value{}", i));
            mvcc_put(&engine, k.as_bytes(), v.as_bytes(), 1, 2);
        }
        engine.flush_cf(cf, true).unwrap();

        let (s, e) = (data_key(b"k5"), data_key(b"k9"));
        let (entries, version) = get_range_entries_and_versions(&engine, cf, &s, &e).unwrap();
        assert_eq!(entries, 5);
        assert_eq!(version, 5);

        let ranges_need_to_compact = collect_ranges_need_compact(
            &engine,
            vec![data_key(b"k0"), data_key(b"k5"), data_key(b"k9")],
            1,
            50,
        )
        .unwrap();
        let (s, e) = (data_key(b"k0"), data_key(b"k5"));
        let mut expected_ranges = VecDeque::new();
        expected_ranges.push_back((s, e));
        assert_eq!(ranges_need_to_compact, expected_ranges);

        // gc 5..10
        for i in 5..10 {
            let k = format!("k{}", i);
            delete(&engine, k.as_bytes(), 2);
        }
        engine.flush_cf(cf, true).unwrap();

        let (s, e) = (data_key(b"k5"), data_key(b"k9"));
        let (entries, version) = get_range_entries_and_versions(&engine, cf, &s, &e).unwrap();
        assert_eq!(entries, 10);
        assert_eq!(version, 5);

        let ranges_need_to_compact = collect_ranges_need_compact(
            &engine,
            vec![data_key(b"k0"), data_key(b"k5"), data_key(b"k9")],
            1,
            50,
        )
        .unwrap();
        let (s, e) = (data_key(b"k0"), data_key(b"k9"));
        let mut expected_ranges = VecDeque::new();
        expected_ranges.push_back((s, e));
        assert_eq!(ranges_need_to_compact, expected_ranges);
    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate redis_async;
#[macro_use]
extern crate serde_derive;

use std::io;

use actix::prelude::Addr;
use actix_files::{Files, NamedFile};
use actix_redis::{Command, RedisActor, RespValue};
use actix_web::{middleware, web, App, HttpRequest, HttpResponse, HttpServer, Result};
use dotenv::dotenv;
use futures::future;
use futures::future::Future;
use serde::Serialize;
use uuid::Uuid;

#[derive(Serialize, Deserialize)]
pub struct SecretBody {
    content: String,
}

#[derive(Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct CreateSecretPostBody {
    content: String,
    expiration_seconds: usize,
}

#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub struct CreateSecretResponseBody {
    id: Uuid,
}

#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub struct ApiError<T: Serialize> {
    error: T,
}

impl<T: Serialize> ApiError<T> {
    pub fn from(err: T) -> ApiError<T> {
        ApiError { error: err }
    }
}

#[derive(Debug, Clone, Copy)]
struct AppConfig {
    max_secret_length: usize,
    max_secret_expiration: usize,
}

struct AppState {
    redis: Addr<RedisActor>,
    config: AppConfig,
}

impl AppState {
    fn create_secret(
        self: &AppState,
        id: Uuid,
        body: String,
        expiration_seconds: usize,
    ) -> impl Future<Item = bool, Error = actix_web::Error> {
        self.redis
            .send(Command(resp_array![
                "SETEX",
                id.to_string(),
                expiration_seconds.to_string(),
                body
            ]))
            .map_err(actix_web::Error::from)
            .and_then(|res: Result<RespValue, actix_redis::Error>| match res {
                Ok(RespValue::SimpleString(_)) => Ok(true),

                err => {
                    println!("unexpected response to creating secret: {:?}", err);
                    Err(actix_web::error::ErrorInternalServerError(
                        "failed to create secret",
                    ))
                }
            })
    }

    fn fetch_secret(
        self: &AppState,
        id: String,
    ) -> impl Future<Item = Option<String>, Error = actix_web::Error> {
        self.redis
            .send(Command(resp_array!["GET", id]))
            .map_err(actix_web::Error::from)
            .map(|res| match res {
                Ok(RespValue::BulkString(bytes)) => {
                    let str = String::from_utf8(bytes).expect("non-utf8 string in redis");
                    Some(str)
                }

                _ => None,
            })
    }

    fn consume_secret(
        self: &AppState,
        id: String,
    ) -> impl Future<Item = Option<String>, Error = actix_web::Error> {
        let remove_secret = self
            .redis
            .send(Command(resp_array!["DEL", id.clone()]))
            .map_err(actix_web::Error::from);

        self.fetch_secret(id)
            .and_then(|result| remove_secret.map(|_| result))
    }
}

fn create_secret(
    body: web::Json<CreateSecretPostBody>,
    state: web::Data<AppState>,
) -> impl Future<Item = HttpResponse, Error = actix_web::Error> {
    let body = body.into_inner();

    let id = Uuid::new_v4();
    let expiration = body.expiration_seconds.to_string();

    let validate = if body.content.len() > state.config.max_secret_length {
        Err("content too large")
    } else if body.expiration_seconds > state.config.max_secret_expiration {
        Err("expiration too long")
    } else {
        Ok(())
    };

    future::result(validate)
        .map_err(|err| HttpResponse::BadRequest().json(ApiError::from(err)))
        .map_err(actix_web::Error::from)
        .and_then(move |_| {
            println!("Writing: {:?}: {:?}", id.to_string(), expiration);

            // TODO: Possibly re-encrypt secret here? libsodium or something.

            state
                .create_secret(id, body.content, body.expiration_seconds)
                .map(move |_| HttpResponse::Ok().json(CreateSecretResponseBody { id }))
        })
}

fn get_secret(
    path: web::Path<String>,
    state: web::Data<AppState>,
) -> impl Future<Item = HttpResponse, Error = actix_web::Error> {
    let secret_id = path.to_string();
    println!("Fetching: {:?}", secret_id);

    state
        .consume_secret(secret_id)
        .map(|res: Option<String>| match res {
            Some(secret) => HttpResponse::Ok().json(SecretBody { content: secret }),
            None => HttpResponse::NotFound().json(ApiError::from("not found")),
        })
}

// fn view_secret_page(
//     path: web::Path<String>,
//     state: web::Data<AppState>,
// ) -> impl Future<Item = HttpResponse, Error = actix_web::Error> {
// }

fn main() -> io::Result<()> {
    env_logger::init();
    dotenv().ok();

    let config = AppConfig {
        max_secret_length: std::env::var("MAX_SECRET_LENGTH")
            .unwrap_or("4096".to_string())
            .parse()
            .expect("unvalid integer value"),
        max_secret_expiration: std::env::var("MAX_SECRET_EXPIRATION")
            .unwrap_or("4096".to_string())
            .parse()
            .expect("unvalid integer value"),
    };

    let bind_addr = std::env::var("BIND_ADDR").unwrap_or("127.0.0.1:8080".to_string());

    println!("Booting with config: {:?}", config);

    HttpServer::new(move || {
        let state = AppState {
            redis: RedisActor::start("localhost:6379"),
            config: config,
        };

        App::new()
            .data(state)
            .wrap(middleware::Logger::default())
            .service(
                web::resource("/api/secret").route(web::post().to_async(create_secret)),
            )
            .service(
                web::resource("/api/secret/{secret_id}")
                    .route(web::get().to_async(get_secret)),
            )
            .service(web::resource("/view/{secret_id}").route(web::get().to(
                |_: HttpRequest| -> Result<NamedFile> {
                    Ok(NamedFile::open("./static/view.html")?)
                },
            )))
            .service(Files::new("/", "./static/").index_file("index.html"))
    })
    .bind(bind_addr)?
    .run()
}
<commit_msg>Minimal wiring for view_secret page<commit_after>#[macro_use]
extern crate redis_async;
#[macro_use]
extern crate serde_derive;

use std::io;

use actix::prelude::Addr;
use actix_files::{Files, NamedFile};
use actix_redis::{Command, RedisActor, RespValue};
use actix_web::{middleware, web, App, HttpResponse, HttpServer, Result};
use dotenv::dotenv;
use futures::future;
use futures::future::Future;
use serde::Serialize;
use uuid::Uuid;

#[derive(Serialize, Deserialize)]
pub struct SecretBody {
    content: String,
}

#[derive(Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct CreateSecretPostBody {
    content: String,
    expiration_seconds: usize,
}

#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub struct CreateSecretResponseBody {
    id: Uuid,
}

#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub struct ApiError<T: Serialize> {
    error: T,
}

impl<T: Serialize> ApiError<T> {
    pub fn from(err: T) -> ApiError<T> {
        ApiError { error: err }
    }
}

#[derive(Debug, Clone, Copy)]
struct AppConfig {
    max_secret_length: usize,
    max_secret_expiration: usize,
}

struct AppState {
    redis: Addr<RedisActor>,
    config: AppConfig,
}

impl AppState {
    fn create_secret(
        self: &AppState,
        id: Uuid,
        body: String,
        expiration_seconds: usize,
    ) -> impl Future<Item = bool, Error = actix_web::Error> {
        self.redis
            .send(Command(resp_array![
                "SETEX",
                id.to_string(),
                expiration_seconds.to_string(),
                body
            ]))
            .map_err(actix_web::Error::from)
            .and_then(|res: Result<RespValue, actix_redis::Error>| match res {
                Ok(RespValue::SimpleString(_)) => Ok(true),

                err => {
                    println!("unexpected response to creating secret: {:?}", err);
                    Err(actix_web::error::ErrorInternalServerError(
                        "failed to create secret",
                    ))
                }
            })
    }

    fn fetch_secret(
        self: &AppState,
        id: String,
    ) -> impl Future<Item = Option<String>, Error = actix_web::Error> {
        self.redis
            .send(Command(resp_array!["GET", id]))
            .map_err(actix_web::Error::from)
            .map(|res| match res {
                Ok(RespValue::BulkString(bytes)) => {
                    let str = String::from_utf8(bytes).expect("non-utf8 string in redis");
                    Some(str)
                }

                _ => None,
            })
    }

    fn consume_secret(
        self: &AppState,
        id: String,
    ) -> impl Future<Item = Option<String>, Error = actix_web::Error> {
        let remove_secret = self
            .redis
            .send(Command(resp_array!["DEL", id.clone()]))
            .map_err(actix_web::Error::from);

        self.fetch_secret(id)
            .and_then(|result| remove_secret.map(|_| result))
    }
}

fn create_secret(
    body: web::Json<CreateSecretPostBody>,
    state: web::Data<AppState>,
) -> impl Future<Item = HttpResponse, Error = actix_web::Error> {
    let body = body.into_inner();

    let id = Uuid::new_v4();
    let expiration = body.expiration_seconds.to_string();

    let validate = if body.content.len() > state.config.max_secret_length {
        Err("content too large")
    } else if body.expiration_seconds > state.config.max_secret_expiration {
        Err("expiration too long")
    } else {
        Ok(())
    };

    future::result(validate)
        .map_err(|err| HttpResponse::BadRequest().json(ApiError::from(err)))
        .map_err(actix_web::Error::from)
        .and_then(move |_| {
            println!("Writing: {:?}: {:?}", id.to_string(), expiration);

            // TODO: Possibly re-encrypt secret here? libsodium or something.

            state
                .create_secret(id, body.content, body.expiration_seconds)
                .map(move |_| HttpResponse::Ok().json(CreateSecretResponseBody { id }))
        })
}

fn get_secret(
    path: web::Path<String>,
    state: web::Data<AppState>,
) -> impl Future<Item = HttpResponse, Error = actix_web::Error> {
    let secret_id = path.to_string();
    println!("Fetching: {:?}", secret_id);

    state
        .consume_secret(secret_id)
        .map(|res: Option<String>| match res {
            Some(secret) => HttpResponse::Ok().json(SecretBody { content: secret }),
            None => HttpResponse::NotFound().json(ApiError::from("not found")),
        })
}

fn view_secret_page(
    path: web::Path<String>,
    state: web::Data<AppState>,
) -> impl Future<Item = NamedFile, Error = actix_web::Error> {
    let secret_id = path.to_string();

    state.fetch_secret(secret_id).and_then(|res| match res {
        Some(_) => Ok(NamedFile::open("./static/view.html")?),

        None => {
            let file = NamedFile::open("./static/404.html")?
                // TODO: (requires HTTP crate).set_status_code()
                ;

            Ok(file)
        }
    })
}

fn main() -> io::Result<()> {
    env_logger::init();
    dotenv().ok();

    let config = AppConfig {
        max_secret_length: std::env::var("MAX_SECRET_LENGTH")
            .unwrap_or("4096".to_string())
            .parse()
            .expect("unvalid integer value"),
        max_secret_expiration: std::env::var("MAX_SECRET_EXPIRATION")
            .unwrap_or("4096".to_string())
            .parse()
            .expect("unvalid integer value"),
    };

    let bind_addr = std::env::var("BIND_ADDR").unwrap_or("127.0.0.1:8080".to_string());

    println!("Booting with config: {:?}", config);

    HttpServer::new(move || {
        let state = AppState {
            redis: RedisActor::start("localhost:6379"),
            config: config,
        };

        App::new()
            .data(state)
            .wrap(middleware::Logger::default())
            .service(
                web::resource("/api/secret").route(web::post().to_async(create_secret)),
            )
            .service(
                web::resource("/api/secret/{secret_id}")
                    .route(web::get().to_async(get_secret)),
            )
            .service(
                web::resource("/view/{secret_id}")
                    .route(web::get().to_async(view_secret_page)),
            )
            .service(Files::new("/", "./static/").index_file("index.html"))
    })
    .bind(bind_addr)?
    .run()
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//! The `ByteString` struct.

use std::ascii::AsciiExt;
use std::borrow::{ToOwned, Cow};
use std::fmt;
use std::hash::{Hash, Hasher};
use std::ops;
use std::ops::{Deref, DerefMut};
use std::str;
use std::str::{Bytes, FromStr};
use string_cache::Atom;

/// Encapsulates the IDL `ByteString` type.
#[derive(JSTraceable, Clone, Eq, PartialEq, HeapSizeOf)]
pub struct ByteString(Vec<u8>);

impl ByteString {
    /// Creates a new `ByteString`.
    pub fn new(value: Vec<u8>) -> ByteString {
        ByteString(value)
    }

    /// Returns `self` as a string, if it encodes valid UTF-8, and `None`
    /// otherwise.
    pub fn as_str(&self) -> Option<&str> {
        str::from_utf8(&self.0).ok()
    }

    /// Returns the length.
    pub fn len(&self) -> usize {
        self.0.len()
    }

    /// Compare `self` to `other`, matching A–Z and a–z as equal.
    pub fn eq_ignore_case(&self, other: &ByteString) -> bool {
        self.0.eq_ignore_ascii_case(&other.0)
    }

    /// Returns `self` with A–Z replaced by a–z.
    pub fn to_lower(&self) -> ByteString {
        ByteString::new(self.0.to_ascii_lowercase())
    }
}

impl Into<Vec<u8>> for ByteString {
    fn into(self) -> Vec<u8> {
        self.0
    }
}

impl Hash for ByteString {
    fn hash<H: Hasher>(&self, state: &mut H) {
        self.0.hash(state);
    }
}

impl FromStr for ByteString {
    type Err = ();
    fn from_str(s: &str) -> Result<ByteString, ()> {
        Ok(ByteString::new(s.to_owned().into_bytes()))
    }
}

impl ops::Deref for ByteString {
    type Target = [u8];
    fn deref(&self) -> &[u8] {
        &self.0
    }
}

/// A string that is constructed from a UCS-2 buffer by replacing invalid code
/// points with the replacement character.
#[derive(Clone, HeapSizeOf)]
pub struct USVString(pub String);

/// Returns whether `s` is a `token`, as defined by
/// [RFC 2616](http://tools.ietf.org/html/rfc2616#page-17).
pub fn is_token(s: &[u8]) -> bool {
    if s.is_empty() {
        return false; // A token must be at least a single character
    }
    s.iter().all(|&x| {
        // http://tools.ietf.org/html/rfc2616#section-2.2
        match x {
            0...31 | 127 => false, // CTLs
            40 |
            41 |
            60 |
            62 |
            64 |
            44 |
            59 |
            58 |
            92 |
            34 |
            47 |
            91 |
            93 |
            63 |
            61 |
            123 |
            125 |
            32 => false, // separators
            x if x > 127 => false, // non-CHARs
            _ => true,
        }
    })
}

/// A DOMString.
#[derive(Clone, Debug, Eq, Hash, HeapSizeOf, Ord, PartialEq, PartialOrd)]
pub struct DOMString(String);

impl !Send for DOMString {}

impl DOMString {
    /// Creates a new `DOMString`.
    pub fn new() -> DOMString {
        DOMString(String::new())
    }

    /// Creates a new `DOMString` from a `String`.
    pub fn from_string(s: String) -> DOMString {
        DOMString(s)
    }

    /// Appends a given string slice onto the end of this String.
    pub fn push_str(&mut self, string: &str) {
        self.0.push_str(string)
    }

    /// Truncates this `DOMString`, removing all contents.
    pub fn clear(&mut self) {
        self.0.clear()
    }

    /// An iterator over the bytes of this `DOMString`.
    pub fn bytes(&self) -> Bytes {
        self.0.bytes()
    }
}

impl Default for DOMString {
    fn default() -> Self {
        DOMString(String::new())
    }
}

impl Deref for DOMString {
    type Target = str;

    #[inline]
    fn deref(&self) -> &str {
        &self.0
    }
}

impl DerefMut for DOMString {
    #[inline]
    fn deref_mut(&mut self) -> &mut str {
        &mut self.0
    }
}

impl AsRef<str> for DOMString {
    fn as_ref(&self) -> &str {
        &self.0
    }
}

impl fmt::Display for DOMString {
    #[inline]
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        fmt::Display::fmt(&**self, f)
    }
}

impl PartialEq<str> for DOMString {
    fn eq(&self, other: &str) -> bool {
        &**self == other
    }
}

impl<'a> PartialEq<&'a str> for DOMString {
    fn eq(&self, other: &&'a str) -> bool {
        &**self == *other
    }
}

impl From<String> for DOMString {
    fn from(contents: String) -> DOMString {
        DOMString(contents)
    }
}

impl<'a> From<&'a str> for DOMString {
    fn from(contents: &str) -> DOMString {
        DOMString::from(String::from(contents))
    }
}

impl<'a> From<Cow<'a, str>> for DOMString {
    fn from(contents: Cow<'a, str>) -> DOMString {
        match contents {
            Cow::Owned(s) => DOMString::from(s),
            Cow::Borrowed(s) => DOMString::from(s),
        }
    }
}

impl From<DOMString> for Atom {
    fn from(contents: DOMString) -> Atom {
        Atom::from(contents.0)
    }
}

impl From<DOMString> for String {
    fn from(contents: DOMString) -> String {
        contents.0
    }
}

impl Into<Vec<u8>> for DOMString {
    fn into(self) -> Vec<u8> {
        self.0.into()
    }
}

impl Extend<char> for DOMString {
    fn extend<I>(&mut self, iterable: I) where I: IntoIterator<Item=char> {
        self.0.extend(iterable)
    }
}
<commit_msg>Auto merge of #12500 - Ms2ger:DOMString, r=nox<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//! The `ByteString` struct.

use std::ascii::AsciiExt;
use std::borrow::{ToOwned, Cow};
use std::fmt;
use std::hash::{Hash, Hasher};
use std::ops;
use std::ops::{Deref, DerefMut};
use std::str;
use std::str::{Bytes, FromStr};
use string_cache::Atom;

/// Encapsulates the IDL `ByteString` type.
#[derive(JSTraceable, Clone, Eq, PartialEq, HeapSizeOf)]
pub struct ByteString(Vec<u8>);

impl ByteString {
    /// Creates a new `ByteString`.
    pub fn new(value: Vec<u8>) -> ByteString {
        ByteString(value)
    }

    /// Returns `self` as a string, if it encodes valid UTF-8, and `None`
    /// otherwise.
    pub fn as_str(&self) -> Option<&str> {
        str::from_utf8(&self.0).ok()
    }

    /// Returns the length.
    pub fn len(&self) -> usize {
        self.0.len()
    }

    /// Compare `self` to `other`, matching A–Z and a–z as equal.
    pub fn eq_ignore_case(&self, other: &ByteString) -> bool {
        self.0.eq_ignore_ascii_case(&other.0)
    }

    /// Returns `self` with A–Z replaced by a–z.
    pub fn to_lower(&self) -> ByteString {
        ByteString::new(self.0.to_ascii_lowercase())
    }
}

impl Into<Vec<u8>> for ByteString {
    fn into(self) -> Vec<u8> {
        self.0
    }
}

impl Hash for ByteString {
    fn hash<H: Hasher>(&self, state: &mut H) {
        self.0.hash(state);
    }
}

impl FromStr for ByteString {
    type Err = ();
    fn from_str(s: &str) -> Result<ByteString, ()> {
        Ok(ByteString::new(s.to_owned().into_bytes()))
    }
}

impl ops::Deref for ByteString {
    type Target = [u8];
    fn deref(&self) -> &[u8] {
        &self.0
    }
}

/// A string that is constructed from a UCS-2 buffer by replacing invalid code
/// points with the replacement character.
#[derive(Clone, HeapSizeOf)]
pub struct USVString(pub String);

/// Returns whether `s` is a `token`, as defined by
/// [RFC 2616](http://tools.ietf.org/html/rfc2616#page-17).
pub fn is_token(s: &[u8]) -> bool {
    if s.is_empty() {
        return false; // A token must be at least a single character
    }
    s.iter().all(|&x| {
        // http://tools.ietf.org/html/rfc2616#section-2.2
        match x {
            0...31 | 127 => false, // CTLs
            40 |
            41 |
            60 |
            62 |
            64 |
            44 |
            59 |
            58 |
            92 |
            34 |
            47 |
            91 |
            93 |
            63 |
            61 |
            123 |
            125 |
            32 => false, // separators
            x if x > 127 => false, // non-CHARs
            _ => true,
        }
    })
}

/// A DOMString.
///
/// This type corresponds to the [`DOMString`](idl) type in WebIDL.
///
/// [idl]: https://heycam.github.io/webidl/#idl-DOMString
///
/// Cenceptually, a DOMString has the same value space as a JavaScript String,
/// i.e., an array of 16-bit *code units* representing UTF-16, potentially with
/// unpaired surrogates present (also sometimes called WTF-16).
///
/// Currently, this type stores a Rust `String`, in order to avoid issues when
/// integrating with the rest of the Rust ecosystem and even the rest of the
/// browser itself.
///
/// However, Rust `String`s are guaranteed to be valid UTF-8, and as such have
/// a *smaller value space* than WTF-16 (i.e., some JavaScript String values
/// can not be represented as a Rust `String`). This introduces the question of
/// what to do with values being passed from JavaScript to Rust that contain
/// unpaired surrogates.
///
/// The hypothesis is that it does not matter much how exactly those values are
/// transformed, because passing unpaired surrogates into the DOM is very rare.
/// In order to test this hypothesis, Servo will panic when encountering any
/// unpaired surrogates on conversion to `DOMString` by default. (The command
/// line option `-Z replace-surrogates` instead causes Servo to replace the
/// unpaired surrogate by a U+FFFD replacement character.)
///
/// Currently, the lack of crash reports about this issue provides some
/// evidence to support the hypothesis. This evidence will hopefully be used to
/// convince other browser vendors that it would be safe to replace unpaired
/// surrogates at the boundary between JavaScript and native code. (This would
/// unify the `DOMString` and `USVString` types, both in the WebIDL standard
/// and in Servo.)
///
/// This type is currently `!Send`, in order to help with an independent
/// experiment to store `JSString`s rather than Rust `String`s.
#[derive(Clone, Debug, Eq, Hash, HeapSizeOf, Ord, PartialEq, PartialOrd)]
pub struct DOMString(String);

impl !Send for DOMString {}

impl DOMString {
    /// Creates a new `DOMString`.
    pub fn new() -> DOMString {
        DOMString(String::new())
    }

    /// Creates a new `DOMString` from a `String`.
    pub fn from_string(s: String) -> DOMString {
        DOMString(s)
    }

    /// Appends a given string slice onto the end of this String.
    pub fn push_str(&mut self, string: &str) {
        self.0.push_str(string)
    }

    /// Truncates this `DOMString`, removing all contents.
    pub fn clear(&mut self) {
        self.0.clear()
    }

    /// An iterator over the bytes of this `DOMString`.
    pub fn bytes(&self) -> Bytes {
        self.0.bytes()
    }
}

impl Default for DOMString {
    fn default() -> Self {
        DOMString(String::new())
    }
}

impl Deref for DOMString {
    type Target = str;

    #[inline]
    fn deref(&self) -> &str {
        &self.0
    }
}

impl DerefMut for DOMString {
    #[inline]
    fn deref_mut(&mut self) -> &mut str {
        &mut self.0
    }
}

impl AsRef<str> for DOMString {
    fn as_ref(&self) -> &str {
        &self.0
    }
}

impl fmt::Display for DOMString {
    #[inline]
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        fmt::Display::fmt(&**self, f)
    }
}

impl PartialEq<str> for DOMString {
    fn eq(&self, other: &str) -> bool {
        &**self == other
    }
}

impl<'a> PartialEq<&'a str> for DOMString {
    fn eq(&self, other: &&'a str) -> bool {
        &**self == *other
    }
}

impl From<String> for DOMString {
    fn from(contents: String) -> DOMString {
        DOMString(contents)
    }
}

impl<'a> From<&'a str> for DOMString {
    fn from(contents: &str) -> DOMString {
        DOMString::from(String::from(contents))
    }
}

impl<'a> From<Cow<'a, str>> for DOMString {
    fn from(contents: Cow<'a, str>) -> DOMString {
        match contents {
            Cow::Owned(s) => DOMString::from(s),
            Cow::Borrowed(s) => DOMString::from(s),
        }
    }
}

impl From<DOMString> for Atom {
    fn from(contents: DOMString) -> Atom {
        Atom::from(contents.0)
    }
}

impl From<DOMString> for String {
    fn from(contents: DOMString) -> String {
        contents.0
    }
}

impl Into<Vec<u8>> for DOMString {
    fn into(self) -> Vec<u8> {
        self.0.into()
    }
}

impl Extend<char> for DOMString {
    fn extend<I>(&mut self, iterable: I) where I: IntoIterator<Item=char> {
        self.0.extend(iterable)
    }
}
<|endoftext|>


<commit_before>use assert_cmd::Command;
use tempfile::tempdir;

#[test]
fn build_x11() {
    logger();
    let tmp = tempdir().unwrap();

    Command::cargo_bin("kbdgen").unwrap()
        .arg("build")
        .arg("x11")
        .arg("examples/sme.kbdgen")
        .arg("--output")
        .arg(tmp.path())
        .assert()
        .success();

    assert!(tmp.path().join("linux").join("se-SE.xkb").exists());
}

pub fn logger() {
    let _ = env_logger::Builder::from_default_env()
        .filter(None, log::LevelFilter::Debug)
        .target(env_logger::Target::Stderr)
        .is_test(true)
        .try_init();
}
<commit_msg>Add M17N smoketest<commit_after>use assert_cmd::Command;
use tempfile::tempdir;

#[test]
fn build_x11() {
    logger();
    let tmp = tempdir().unwrap();

    Command::cargo_bin("kbdgen").unwrap()
        .arg("build")
        .arg("x11")
        .arg("examples/sme.kbdgen")
        .arg("--output")
        .arg(tmp.path())
        .assert()
        .success();

    assert!(tmp.path().join("linux").join("se-SE.xkb").exists());
}

#[test]
fn build_m17n() {
    logger();
    let tmp = tempdir().unwrap();

    Command::cargo_bin("kbdgen").unwrap()
        .arg("build")
        .arg("m17n")
        .arg("examples/sme.kbdgen")
        .arg("--output")
        .arg(tmp.path())
        .assert()
        .success();

    assert!(tmp.path().join("se-SE").join("win.mim").exists());
}

pub fn logger() {
    let _ = env_logger::Builder::from_default_env()
        .filter(None, log::LevelFilter::Debug)
        .target(env_logger::Target::Stderr)
        .is_test(true)
        .try_init();
}
<|endoftext|>


<commit_before>#![cfg_attr(feature = "i128", feature(i128_type))]

#[cfg(feature = "num-complex")]
extern crate num_complex;

#[cfg(feature = "ndarray")]
extern crate ndarray;

#[cfg(feature = "num-rational")]
extern crate num_rational;

#[macro_use]
extern crate nearly_eq;

#[cfg(feature = "num-complex")]
use num_complex::Complex;

#[cfg(feature = "num-rational")]
use num_rational::Rational;

#[cfg(feature = "ndarray")]
use ndarray::arr1;

#[cfg(feature = "ndarray")]
use ndarray::arr2;

#[cfg(feature = "ndarray")]
use ndarray::arr3;

#[test]
fn it_should_not_panic_if_values_are_nearly_equal() {
    assert_nearly_eq!(8f32, 8f32 + 1e-7);
}

#[test]
fn it_should_not_panic_if_values_are_nearly_equal_f64() {
    assert_nearly_eq!(0f64, 1e-12 as f64);
}

#[test]
#[should_panic]
fn it_should_panic_if_values_are_not_nearly_equal() {
    assert_nearly_eq!(8f32, 8f32 - 1e-5);
}

#[test]
fn compare_with_explicit_eps() {
    assert_nearly_eq!(3f64, 4f64, 2f64);
}

#[test]
#[should_panic]
fn bad_compare_with_explicit_eps() {
    assert_nearly_eq!(3f64, 4f64, 1e-3f64);
}

#[test]
fn compare_with_vector() {
    let left = vec![1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    let right = vec![1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
fn bad_compare_with_vector() {
    let left = vec![1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.01];
    let right = vec![1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
fn bad_len_compare_with_vector() {
    let left = vec![1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0];
    let right = vec![1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(left, right);
}

#[test]
fn compare_with_slice() {
    let left = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    let right = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(&left as &[f32], &right as &[f32]);
}

#[test]
#[should_panic]
fn bad_compare_with_slice() {
    let left = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.01];
    let right = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(&left as &[f32], &right as &[f32]);
}

#[test]
#[should_panic]
fn bad_len_compare_with_slice() {
    let left = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0];
    let right = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(&left as &[f32], &right as &[f32]);
}

#[test]
fn compare_with_array() {
    let left = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    let right = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
fn bad_compare_with_array() {
    let left = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.01];
    let right = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(left, right);
}

#[test]
#[cfg(feature = "num-complex")]
fn compare_with_complex() {
    let left = Complex::new(1.0f64, 0.0);
    let right = Complex::new(1.0f64, 1e-12);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
#[cfg(feature = "num-complex")]
fn bad_compare_with_complex() {
    let left = Complex::new(1.0f64, 0.0);
    let right = Complex::new(1.0f64, 1e-8);
    assert_nearly_eq!(left, right);
}

#[test]
#[cfg(feature = "ndarray")]
fn compare_with_ndarray1d() {
    let left = arr1(&[1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]);
    let right = arr1(&[1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
#[cfg(feature = "ndarray")]
fn bad_compare_with_ndarray1d() {
    let left = arr1(&[1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0001]);
    let right = arr1(&[1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]);
    assert_nearly_eq!(left, right);
}

#[test]
#[cfg(feature = "ndarray")]
fn compare_with_ndarray2d() {
    let left = arr2(&[[1f64, 2.0, 3.0, 4.0, 5.0], [6.0, 7.0, 8.0, 9.0, 10.0]]);
    let right = arr2(&[[1f64, 2.0, 3.0, 4.0, 5.0], [6.0, 7.0, 8.0, 9.0, 10.0]]);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
#[cfg(feature = "ndarray")]
fn bad_compare_with_ndarray2d_val() {
    let left = arr2(&[[1f64, 2.0, 3.0, 4.0, 5.0], [6.0, 7.0, 8.0, 9.0, 10.0001]]);
    let right = arr2(&[[1f64, 2.0, 3.0, 4.0, 5.0], [6.0, 7.0, 8.0, 9.0, 10.0]]);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
#[cfg(feature = "ndarray")]
fn bad_compare_with_ndarray2d_len() {
    let left = arr2(&[[1f64, 2.0, 3.0, 4.0, 5.0], [6.0, 7.0, 8.0, 9.0, 10.0]]);
    let right = arr2(&[
        [1f64, 2.0],
        [3.0, 4.0],
        [5.0, 6.0],
        [7.0, 8.0],
        [9.0, 10.0],
    ]);
    assert_nearly_eq!(left, right);
}

#[test]
#[cfg(feature = "ndarray")]
fn compare_with_ndarray3d() {
    let left = arr3(&[[[1f64, 2.0], [4.0, 5.0]], [[6.0, 7.0], [9.0, 10.0]]]);
    let right = arr3(&[[[1f64, 2.0], [4.0, 5.0]], [[6.0, 7.0], [9.0, 10.0]]]);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
#[cfg(feature = "ndarray")]
fn bad_compare_with_ndarray3d() {
    let left = arr3(&[[[1f64, 2.0], [4.0, 5.0]], [[6.0, 7.0], [9.0, 10.0001]]]);
    let right = arr3(&[[[1f64, 2.0], [4.0, 5.0]], [[6.0, 7.0], [9.0, 10.0]]]);
    assert_nearly_eq!(left, right);
}

macro_rules! type_impls {
    ($($T:ident)+) => {
        $(
            mod $T {
                #[test]
                fn it_should_not_panic_if_values_are_nearly_equal() {
                    assert_nearly_eq!(0 as $T, 0 as $T);
                }

                #[test]
                #[should_panic]
                fn it_should_panic_if_values_are_not_nearly_equal() {
                    assert_nearly_eq!(0 as $T, 1 as $T);
                }
            }
        )+
    }
}

type_impls! { i8 i16 i32 i64 u8 u16 u32 u64 }

#[cfg(feature = "i128")]
type_impls! { i128 u128 }

#[test]
fn compare_with_option_both_some() {
    let left = Option::Some(0f64);
    let right = Option::Some(1e-12);
    assert_nearly_eq!(left, right);
}

#[test]
fn compare_with_option_both_none() {
    let left: Option<f64> = Option::None;
    let right = Option::None;
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
fn bad_compare_with_option_both_some() {
    let left = Option::Some(0f64);
    let right = Option::Some(1f64);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
fn bad_compare_with_option_left_some() {
    let left = Option::Some(0f64);
    let right = Option::None;
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
fn bad_compare_with_option_right_some() {
    let left: Option<f64> = Option::None;
    let right = Option::Some(0f64);
    assert_nearly_eq!(left, right);
}

#[test]
#[cfg(feature = "num-rational")]
fn compare_with_ratio_mindiff() {
    let left = Rational::new(1, 1000);
    let right = Rational::new(1, 1001);
    let eps = Rational::new(1, 10000);
    assert_nearly_eq!(left, right, eps);
}

#[test]
#[cfg(feature = "num-rational")]
fn compare_with_ratio_equal() {
    let left = Rational::new(1, 1000);
    let right = Rational::new(1, 1000);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
#[cfg(feature = "num-rational")]
fn bad_compare_with_ratio() {
    let left = Rational::new(1, 1000);
    let right = Rational::new(1, 1001);
    let eps = Rational::new(1, 1000000000);
    assert_nearly_eq!(left, right, eps);
}
<commit_msg>Add ndarray test<commit_after>#![cfg_attr(feature = "i128", feature(i128_type))]

#[cfg(feature = "num-complex")]
extern crate num_complex;

#[cfg(feature = "ndarray")]
extern crate ndarray;

#[cfg(feature = "num-rational")]
extern crate num_rational;

#[macro_use]
extern crate nearly_eq;

#[cfg(feature = "num-complex")]
use num_complex::Complex;

#[cfg(feature = "num-rational")]
use num_rational::Rational;

#[cfg(feature = "ndarray")]
use ndarray::{arr1, arr2, arr3, ArrayD, IxDyn};

#[test]
fn it_should_not_panic_if_values_are_nearly_equal() {
    assert_nearly_eq!(8f32, 8f32 + 1e-7);
}

#[test]
fn it_should_not_panic_if_values_are_nearly_equal_f64() {
    assert_nearly_eq!(0f64, 1e-12 as f64);
}

#[test]
#[should_panic]
fn it_should_panic_if_values_are_not_nearly_equal() {
    assert_nearly_eq!(8f32, 8f32 - 1e-5);
}

#[test]
fn compare_with_explicit_eps() {
    assert_nearly_eq!(3f64, 4f64, 2f64);
}

#[test]
#[should_panic]
fn bad_compare_with_explicit_eps() {
    assert_nearly_eq!(3f64, 4f64, 1e-3f64);
}

#[test]
fn compare_with_vector() {
    let left = vec![1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    let right = vec![1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
fn bad_compare_with_vector() {
    let left = vec![1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.01];
    let right = vec![1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
fn bad_len_compare_with_vector() {
    let left = vec![1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0];
    let right = vec![1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(left, right);
}

#[test]
fn compare_with_slice() {
    let left = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    let right = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(&left as &[f32], &right as &[f32]);
}

#[test]
#[should_panic]
fn bad_compare_with_slice() {
    let left = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.01];
    let right = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(&left as &[f32], &right as &[f32]);
}

#[test]
#[should_panic]
fn bad_len_compare_with_slice() {
    let left = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0];
    let right = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(&left as &[f32], &right as &[f32]);
}

#[test]
fn compare_with_array() {
    let left = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    let right = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
fn bad_compare_with_array() {
    let left = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.01];
    let right = [1f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    assert_nearly_eq!(left, right);
}

#[test]
#[cfg(feature = "num-complex")]
fn compare_with_complex() {
    let left = Complex::new(1.0f64, 0.0);
    let right = Complex::new(1.0f64, 1e-12);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
#[cfg(feature = "num-complex")]
fn bad_compare_with_complex() {
    let left = Complex::new(1.0f64, 0.0);
    let right = Complex::new(1.0f64, 1e-8);
    assert_nearly_eq!(left, right);
}

#[test]
#[cfg(feature = "ndarray")]
fn compare_with_ndarray1d() {
    let left = arr1(&[1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]);
    let right = arr1(&[1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
#[cfg(feature = "ndarray")]
fn bad_compare_with_ndarray1d() {
    let left = arr1(&[1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0001]);
    let right = arr1(&[1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]);
    assert_nearly_eq!(left, right);
}

#[test]
#[cfg(feature = "ndarray")]
fn compare_with_ndarray2d() {
    let left = arr2(&[[1f64, 2.0, 3.0, 4.0, 5.0], [6.0, 7.0, 8.0, 9.0, 10.0]]);
    let right = arr2(&[[1f64, 2.0, 3.0, 4.0, 5.0], [6.0, 7.0, 8.0, 9.0, 10.0]]);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
#[cfg(feature = "ndarray")]
fn bad_compare_with_ndarray2d_val() {
    let left = arr2(&[[1f64, 2.0, 3.0, 4.0, 5.0], [6.0, 7.0, 8.0, 9.0, 10.0001]]);
    let right = arr2(&[[1f64, 2.0, 3.0, 4.0, 5.0], [6.0, 7.0, 8.0, 9.0, 10.0]]);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
#[cfg(feature = "ndarray")]
fn bad_compare_with_ndarray2d_len() {
    let left = arr2(&[[1f64, 2.0, 3.0, 4.0, 5.0], [6.0, 7.0, 8.0, 9.0, 10.0]]);
    let right = arr2(&[
        [1f64, 2.0],
        [3.0, 4.0],
        [5.0, 6.0],
        [7.0, 8.0],
        [9.0, 10.0],
    ]);
    assert_nearly_eq!(left, right);
}

#[test]
#[cfg(feature = "ndarray")]
fn compare_with_ndarray3d() {
    let left = arr3(&[[[1f64, 2.0], [4.0, 5.0]], [[6.0, 7.0], [9.0, 10.0]]]);
    let right = arr3(&[[[1f64, 2.0], [4.0, 5.0]], [[6.0, 7.0], [9.0, 10.0]]]);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
#[cfg(feature = "ndarray")]
fn bad_compare_with_ndarray3d() {
    let left = arr3(&[[[1f64, 2.0], [4.0, 5.0]], [[6.0, 7.0], [9.0, 10.0001]]]);
    let right = arr3(&[[[1f64, 2.0], [4.0, 5.0]], [[6.0, 7.0], [9.0, 10.0]]]);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
#[cfg(feature = "ndarray")]
fn bad_compare_with_ndarraynd() {
    let left = ArrayD::<f64>::zeros(IxDyn(&[2, 3, 4, 5, 6, 7]));
    let right = ArrayD::<f64>::zeros(IxDyn(&[2, 3, 4, 5, 6]));
    assert_nearly_eq!(left, right);
}

macro_rules! type_impls {
    ($($T:ident)+) => {
        $(
            mod $T {
                #[test]
                fn it_should_not_panic_if_values_are_nearly_equal() {
                    assert_nearly_eq!(0 as $T, 0 as $T);
                }

                #[test]
                #[should_panic]
                fn it_should_panic_if_values_are_not_nearly_equal() {
                    assert_nearly_eq!(0 as $T, 1 as $T);
                }
            }
        )+
    }
}

type_impls! { i8 i16 i32 i64 u8 u16 u32 u64 }

#[cfg(feature = "i128")]
type_impls! { i128 u128 }

#[test]
fn compare_with_option_both_some() {
    let left = Option::Some(0f64);
    let right = Option::Some(1e-12);
    assert_nearly_eq!(left, right);
}

#[test]
fn compare_with_option_both_none() {
    let left: Option<f64> = Option::None;
    let right = Option::None;
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
fn bad_compare_with_option_both_some() {
    let left = Option::Some(0f64);
    let right = Option::Some(1f64);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
fn bad_compare_with_option_left_some() {
    let left = Option::Some(0f64);
    let right = Option::None;
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
fn bad_compare_with_option_right_some() {
    let left: Option<f64> = Option::None;
    let right = Option::Some(0f64);
    assert_nearly_eq!(left, right);
}

#[test]
#[cfg(feature = "num-rational")]
fn compare_with_ratio_mindiff() {
    let left = Rational::new(1, 1000);
    let right = Rational::new(1, 1001);
    let eps = Rational::new(1, 10000);
    assert_nearly_eq!(left, right, eps);
}

#[test]
#[cfg(feature = "num-rational")]
fn compare_with_ratio_equal() {
    let left = Rational::new(1, 1000);
    let right = Rational::new(1, 1000);
    assert_nearly_eq!(left, right);
}

#[test]
#[should_panic]
#[cfg(feature = "num-rational")]
fn bad_compare_with_ratio() {
    let left = Rational::new(1, 1000);
    let right = Rational::new(1, 1001);
    let eps = Rational::new(1, 1000000000);
    assert_nearly_eq!(left, right, eps);
}
<|endoftext|>


<commit_before>#![feature(collections)]

extern crate macaroons;
pub use macaroons::{Token, Tag};
pub use macaroons::caveat::{Caveat, Predicate};

fn example_key() -> Vec<u8> {
  String::from_str("this is our super secret key; only we should know it").into_bytes()
}

fn example_id() -> Vec<u8> {
  String::from_str("we used our secret key").into_bytes()
}

fn example_uri() -> Vec<u8> {
  String::from_str("http://mybank/").into_bytes()
}

fn example_predicate() -> Predicate {
  Predicate(String::from_str("test = caveat").into_bytes())
}

#[test]
fn empty_macaroon_signature() {
  let token = Token::new(example_key(), example_id(), example_uri());

  let expected_tag = [0xe3,0xd9,0xe0,0x29,0x08,0x52,0x6c,0x4c
                     ,0x00,0x39,0xae,0x15,0x11,0x41,0x15,0xd9
                     ,0x7f,0xdd,0x68,0xbf,0x2b,0xa3,0x79,0xb3
                     ,0x42,0xaa,0xf0,0xf6,0x17,0xd0,0x55,0x2f];

  let Tag(actual_tag) = token.tag;
  assert_eq!(expected_tag, actual_tag)
}

#[test]
fn signature_with_first_party_caveat() {
  let token = Token::new(example_key(), example_id(), example_uri());
  let new_token = token.add_caveat(Caveat::new(example_predicate()));

  let expected_tag = [0x19,0x7b,0xac,0x7a,0x04,0x4a,0xf3,0x33
                     ,0x32,0x86,0x5b,0x92,0x66,0xe2,0x6d,0x49
                     ,0x3b,0xdd,0x66,0x8a,0x66,0x0e,0x44,0xd8
                     ,0x8c,0xe1,0xa9,0x98,0xc2,0x3d,0xbd,0x67];

  let Tag(actual_tag) = new_token.tag;
  assert_eq!(expected_tag, actual_tag)
}

#[test]
fn binary_serialization() {
  let token = Token::new(example_key(), example_id(), example_uri());
  let new_token = token.add_caveat(Caveat::new(example_predicate()));

  let expected_macaroon = String::from_str("MDAxY2xvY2F0aW9uIGh0dHA6Ly9teWJhbmsvCjAwMjZpZGVudGlmaWVyIHdlIHVzZWQgb3VyIHNlY3JldCBrZXkKMDAxNmNpZCB0ZXN0ID0gY2F2ZWF0CjAwMmZzaWduYXR1cmUgGXusegRK8zMyhluSZuJtSTvdZopmDkTYjOGpmMI9vWcK").into_bytes();
  assert_eq!(expected_macaroon, new_token.serialize());
}
<commit_msg>Factor out example_macaroon()<commit_after>#![feature(collections)]

extern crate macaroons;
pub use macaroons::{Token, Tag};
pub use macaroons::caveat::{Caveat, Predicate};

fn example_key() -> Vec<u8> {
  String::from_str("this is our super secret key; only we should know it").into_bytes()
}

fn example_id() -> Vec<u8> {
  String::from_str("we used our secret key").into_bytes()
}

fn example_uri() -> Vec<u8> {
  String::from_str("http://mybank/").into_bytes()
}

fn example_predicate() -> Predicate {
  Predicate(String::from_str("test = caveat").into_bytes())
}

fn example_macaroon() -> Vec<u8> {
  String::from_str("MDAxY2xvY2F0aW9uIGh0dHA6Ly9teWJhbmsvCjAwMjZpZGVudGlmaWVyIHdlIHVzZWQgb3VyIHNlY3JldCBrZXkKMDAxNmNpZCB0ZXN0ID0gY2F2ZWF0CjAwMmZzaWduYXR1cmUgGXusegRK8zMyhluSZuJtSTvdZopmDkTYjOGpmMI9vWcK").into_bytes()
}

#[test]
fn empty_macaroon_signature() {
  let token = Token::new(example_key(), example_id(), example_uri());

  let expected_tag = [0xe3,0xd9,0xe0,0x29,0x08,0x52,0x6c,0x4c
                     ,0x00,0x39,0xae,0x15,0x11,0x41,0x15,0xd9
                     ,0x7f,0xdd,0x68,0xbf,0x2b,0xa3,0x79,0xb3
                     ,0x42,0xaa,0xf0,0xf6,0x17,0xd0,0x55,0x2f];

  let Tag(actual_tag) = token.tag;
  assert_eq!(expected_tag, actual_tag)
}

#[test]
fn signature_with_first_party_caveat() {
  let token = Token::new(example_key(), example_id(), example_uri());
  let new_token = token.add_caveat(Caveat::new(example_predicate()));

  let expected_tag = [0x19,0x7b,0xac,0x7a,0x04,0x4a,0xf3,0x33
                     ,0x32,0x86,0x5b,0x92,0x66,0xe2,0x6d,0x49
                     ,0x3b,0xdd,0x66,0x8a,0x66,0x0e,0x44,0xd8
                     ,0x8c,0xe1,0xa9,0x98,0xc2,0x3d,0xbd,0x67];

  let Tag(actual_tag) = new_token.tag;
  assert_eq!(expected_tag, actual_tag)
}

#[test]
fn binary_serialization() {
  let token = Token::new(example_key(), example_id(), example_uri());
  let new_token = token.add_caveat(Caveat::new(example_predicate()));

  assert_eq!(example_macaroon(), new_token.serialize());
}
<|endoftext|>


<commit_before>#[macro_use(owned_tree, shared_tree)]
extern crate entmut;

// This will define macros for generalized tests of Nav and Editor impls.
#[cfg(test)]
#[macro_use]
mod view_tests;

#[cfg(test)]
#[macro_use]
mod owned {
    view_tests!(owned_tree);
}

#[cfg(test)]
#[macro_use]
mod shared {
    view_tests!(shared_tree);
}
<commit_msg>Tweak test definitions to clarify what's going on.<commit_after>#[macro_use(owned_tree, shared_tree)]
extern crate entmut;

/// Defines macros for generalized tests of Nav impls.
#[macro_use]
mod view_tests;

#[macro_use]
mod owned {
    view_tests!(owned_tree);
}

#[macro_use]
mod shared {
    view_tests!(shared_tree);
}
<|endoftext|>


<commit_before>extern crate time;

extern crate stopwatch;
use stopwatch::{Stopwatch};

use std::io::timer;
use std::time::Duration;

#[test]
fn it_works() {
	let mut sw = Stopwatch::start_new();

	timer::sleep(Duration::milliseconds(200));

	sw.stop();

	println!("{}", sw.elapsed_ms());
}

#[test]
fn is_running() {
	let mut sw = Stopwatch::new();
	assert!(!sw.is_running());
	sw.start();
	assert!(sw.is_running());
	sw.stop();
	assert!(!sw.is_running());
}

#[test]
fn restart() {
	let mut sw = Stopwatch::start_new();
	timer::sleep(Duration::milliseconds(50));
	sw.restart();
	timer::sleep(Duration::milliseconds(50));
	println!("{}", sw.elapsed_ms());
}
<commit_msg>improving tests<commit_after>extern crate time;
extern crate stopwatch;

use stopwatch::{Stopwatch};
use std::num::SignedInt;
use std::io::timer;
use std::time::Duration;

static SLEEP_MS: i64 = 50;
static TOLERANCE_PERCENTAGE: f64 = 0.2;

#[test]
fn elapsed_ms() {
	let sw = Stopwatch::start_new();
	timer::sleep(Duration::milliseconds(SLEEP_MS));
	assert_sw_near(sw, SLEEP_MS);
}

#[test]
fn stop() {
	let mut sw = Stopwatch::start_new();
	timer::sleep(Duration::milliseconds(SLEEP_MS));
	sw.stop();
	assert_sw_near(sw, SLEEP_MS);

	timer::sleep(Duration::milliseconds(SLEEP_MS));
	assert_sw_near(sw, SLEEP_MS);
}

#[test]
fn resume_once() {
	let mut sw = Stopwatch::start_new();
	timer::sleep(Duration::milliseconds(SLEEP_MS));
	sw.stop();
	assert_sw_near(sw, SLEEP_MS);
	sw.start();
	timer::sleep(Duration::milliseconds(SLEEP_MS));
	assert_sw_near(sw, 2 * SLEEP_MS);
}

#[test]
fn resume_twice() {
	let mut sw = Stopwatch::start_new();
	timer::sleep(Duration::milliseconds(SLEEP_MS));
	sw.stop();
	assert_sw_near(sw, SLEEP_MS);
	sw.start();
	timer::sleep(Duration::milliseconds(SLEEP_MS));
	sw.stop();
	assert_sw_near(sw, 2 * SLEEP_MS);
	sw.start();
	timer::sleep(Duration::milliseconds(SLEEP_MS));
	assert_sw_near(sw, 3 * SLEEP_MS);
}

#[test]
fn is_running() {
	let mut sw = Stopwatch::new();
	assert!(!sw.is_running());
	sw.start();
	assert!(sw.is_running());
	sw.stop();
	assert!(!sw.is_running());
}

#[test]
fn restart() {
	let mut sw = Stopwatch::start_new();
	timer::sleep(Duration::milliseconds(SLEEP_MS));
	sw.restart();
	timer::sleep(Duration::milliseconds(SLEEP_MS));
	assert_sw_near(sw, SLEEP_MS);
}

#[test]
fn reset() {
	let mut sw = Stopwatch::start_new();
	timer::sleep(Duration::milliseconds(SLEEP_MS));
	sw.reset();
	assert!(!sw.is_running());
	assert_eq!(sw.elapsed_ms(), 0);
}

/////////////// helpers

fn assert_near(x: i64, y: i64, tolerance: i64) {
	let diff = (x - y).abs();
	if diff > tolerance {
		panic!("Expected {}, got {}", x, y);
	}
}

fn assert_sw_near(mut sw: Stopwatch, elapsed: i64) {
	let tolerance_value = (TOLERANCE_PERCENTAGE * elapsed as f64) as i64;
	assert_near(elapsed, sw.elapsed_ms().to_i64().unwrap(), tolerance_value);
}<|endoftext|>


<commit_before>#[macro_use]
extern crate nom;
//extern crate serialize;

use nom::{IResult,FlatMapper,Mapper,Producer,ProducerState,FileProducer};
use nom::IResult::*;

use std::str;
use std::collections::HashMap;
//use serialize::hex::ToHex;

pub trait HexDisplay {
      /// Converts the value of `self` to a hex value, returning the owned
      ///     /// string.
  fn to_hex(&self, chunk_size: usize) -> String;
}

static CHARS: &'static[u8] = b"0123456789abcdef";

impl HexDisplay for [u8] {
    fn to_hex(&self, chunk_size: usize) -> String {
      let mut v = Vec::with_capacity(self.len() * 3);
      let mut i = 0;
      for chunk in self.chunks(chunk_size) {
        let s = format!("{:08x}", i);
        let s2: &str = &s;
        for &ch in s.as_bytes().iter() {
          v.push(ch);
        }
        v.push('\t' as u8);

        i = i + chunk_size;

        for &byte in chunk {
          v.push(CHARS[(byte >> 4) as usize]);
          v.push(CHARS[(byte & 0xf) as usize]);
          v.push(' ' as u8);
        }
        if chunk_size > chunk.len() {
          for i in 0..(chunk_size - chunk.len()) {
            v.push(' ' as u8);
            v.push(' ' as u8);
            v.push(' ' as u8);
          }
        }
        v.push('\t' as u8);

        for &byte in chunk {
          if (byte >=32 && byte <= 126) || byte >= 128 {
            v.push(byte);
          } else {
            v.push('.' as u8);
          }
        }
        v.push('\n' as u8);
      }

      unsafe {
        String::from_utf8_unchecked(v)
      }
    }
}

take!(offset_parser 4);
fn mp4_box(input:&[u8]) -> IResult<&[u8], &[u8]> {

  match offset_parser(input) {
    Done(i, offset_bytes) => {
      let offset:u32 = (offset_bytes[3] as u32) + (offset_bytes[2] as u32) * 0x100 + (offset_bytes[1] as u32) * 0x10000 + (offset_bytes[0] as u32) * 0x1000000;
      let sz: usize = offset as usize;
      if i.len() >= sz {
        return Done(&i[(sz-4)..], &i[0..(sz-4)])
      } else {
        return Error(42)//Incomplete(0)
      }
    },
    e => e
  }
}

#[derive(PartialEq,Eq,Debug)]
struct FileType<'a> {
  major_brand:         &'a str,
  major_brand_version: &'a [u8],
  compatible_brands:   Vec<&'a str>
}

#[derive(PartialEq,Eq,Debug)]
enum MP4Box<'a> {
  Ftyp(FileType<'a>),
  Moov,
  Free
}

take!(offset 4);
tag!(ftyp    "ftyp".as_bytes());

fn brand_name(input:&[u8]) -> IResult<&[u8],&str> {
  take!(major_brand_bytes 4);
  major_brand_bytes(input).map_res(str::from_utf8)
}
take!(major_brand_version 4);
many0!(compatible_brands<&[u8],&str> brand_name);

fn filetype_parser<'a>(input: &'a[u8]) -> IResult<&'a [u8], FileType<'a> > {
  chaining_parser!(input, ||{FileType{major_brand: m, major_brand_version:v, compatible_brands: c}},
    m: brand_name,
    v: major_brand_version,
    c: compatible_brands,)
}

o!(filetype <&[u8], FileType>  ftyp ~ [ filetype_parser ]);

fn filetype_box(input:&[u8]) -> IResult<&[u8], MP4Box> {
  //FIXME: flat_map should work here
  //mp4_box(input).flat_map(filetype)
  match mp4_box(input) {
    Done(i, o) => {
      match filetype(o) {
        Done(i2, o2) => {
          Done(i, MP4Box::Ftyp(o2))
        },
        a => Error(0)
      }
    },
    a => Error(0)
  }
}

tag!(free    "ftyp".as_bytes());
fn free_box(input:&[u8]) -> IResult<&[u8], MP4Box> {
  match mp4_box(input) {
    Done(i, o) => {
      match free(o) {
        Done(i2, o2) => {
          Done(i, MP4Box::Free)
        },
        a => Error(0)
      }
    },
    a => Error(0)
  }
}

alt!(box_parser<&[u8], MP4Box>, filetype_box | free_box);

fn parse_mp4_file(filename: &str) {
  FileProducer::new(filename, 100).map(|producer: FileProducer| {
    let mut p = producer;
    match p.produce() {
      ProducerState::Data(bytes) => {
        println!("bytes:\n{}", bytes.to_hex(8));
        match filetype_box(bytes) {
          Done(i, o) => {
            match o {
              MP4Box::Ftyp(f) => println!("parsed ftyp: {:?}", f),
              MP4Box::Moov    => println!("found moov box"),
              MP4Box::Free    => println!("found free box")
            }
            println!("remaining:\n{}", i.to_hex(8))
          },
          a => println!("error: {:?}", a)
        }
      },
      _ => println!("got error")
    }
    /*//p.push(|par| {println!("parsed file: {}", par); par});
    //p.push(|par| par.flat_map(print));
    fn pr<'a,'b,'c>(data: &[u8]) -> IResult<'b,&[u8], &[u8]> {
      println!("bytes: {:?}", &data[0..20]);
      //Done("".as_bytes(), data).map_res(str::from_utf8);
      Done("".as_bytes(),"".as_bytes())
    }
    pusher!(ps, pr);
    ps(&mut p);
    */
    assert!(false);
  });

}
#[test]
fn file_test() {
  parse_mp4_file("small.mp4");
}
#[test]
fn bunny_test() {
  parse_mp4_file("bigbuckbunny.mp4");
}
<commit_msg>augment the chunk size<commit_after>#[macro_use]
extern crate nom;
//extern crate serialize;

use nom::{IResult,FlatMapper,Mapper,Producer,ProducerState,FileProducer};
use nom::IResult::*;

use std::str;
use std::collections::HashMap;
//use serialize::hex::ToHex;

pub trait HexDisplay {
      /// Converts the value of `self` to a hex value, returning the owned
      ///     /// string.
  fn to_hex(&self, chunk_size: usize) -> String;
}

static CHARS: &'static[u8] = b"0123456789abcdef";

impl HexDisplay for [u8] {
    fn to_hex(&self, chunk_size: usize) -> String {
      let mut v = Vec::with_capacity(self.len() * 3);
      let mut i = 0;
      for chunk in self.chunks(chunk_size) {
        let s = format!("{:08x}", i);
        let s2: &str = &s;
        for &ch in s.as_bytes().iter() {
          v.push(ch);
        }
        v.push('\t' as u8);

        i = i + chunk_size;

        for &byte in chunk {
          v.push(CHARS[(byte >> 4) as usize]);
          v.push(CHARS[(byte & 0xf) as usize]);
          v.push(' ' as u8);
        }
        if chunk_size > chunk.len() {
          for i in 0..(chunk_size - chunk.len()) {
            v.push(' ' as u8);
            v.push(' ' as u8);
            v.push(' ' as u8);
          }
        }
        v.push('\t' as u8);

        for &byte in chunk {
          if (byte >=32 && byte <= 126) || byte >= 128 {
            v.push(byte);
          } else {
            v.push('.' as u8);
          }
        }
        v.push('\n' as u8);
      }

      unsafe {
        String::from_utf8_unchecked(v)
      }
    }
}

take!(offset_parser 4);
fn mp4_box(input:&[u8]) -> IResult<&[u8], &[u8]> {

  match offset_parser(input) {
    Done(i, offset_bytes) => {
      let offset:u32 = (offset_bytes[3] as u32) + (offset_bytes[2] as u32) * 0x100 + (offset_bytes[1] as u32) * 0x10000 + (offset_bytes[0] as u32) * 0x1000000;
      let sz: usize = offset as usize;
      if i.len() >= sz {
        return Done(&i[(sz-4)..], &i[0..(sz-4)])
      } else {
        return Error(42)//Incomplete(0)
      }
    },
    e => e
  }
}

#[derive(PartialEq,Eq,Debug)]
struct FileType<'a> {
  major_brand:         &'a str,
  major_brand_version: &'a [u8],
  compatible_brands:   Vec<&'a str>
}

#[derive(PartialEq,Eq,Debug)]
enum MP4Box<'a> {
  Ftyp(FileType<'a>),
  Moov,
  Free
}

take!(offset 4);
tag!(ftyp    "ftyp".as_bytes());

fn brand_name(input:&[u8]) -> IResult<&[u8],&str> {
  take!(major_brand_bytes 4);
  major_brand_bytes(input).map_res(str::from_utf8)
}
take!(major_brand_version 4);
many0!(compatible_brands<&[u8],&str> brand_name);

fn filetype_parser<'a>(input: &'a[u8]) -> IResult<&'a [u8], FileType<'a> > {
  chaining_parser!(input, ||{FileType{major_brand: m, major_brand_version:v, compatible_brands: c}},
    m: brand_name,
    v: major_brand_version,
    c: compatible_brands,)
}

o!(filetype <&[u8], FileType>  ftyp ~ [ filetype_parser ]);

fn filetype_box(input:&[u8]) -> IResult<&[u8], MP4Box> {
  //FIXME: flat_map should work here
  //mp4_box(input).flat_map(filetype)
  match mp4_box(input) {
    Done(i, o) => {
      match filetype(o) {
        Done(i2, o2) => {
          Done(i, MP4Box::Ftyp(o2))
        },
        a => Error(0)
      }
    },
    a => Error(0)
  }
}

tag!(free    "ftyp".as_bytes());
fn free_box(input:&[u8]) -> IResult<&[u8], MP4Box> {
  match mp4_box(input) {
    Done(i, o) => {
      match free(o) {
        Done(i2, o2) => {
          Done(i, MP4Box::Free)
        },
        a => Error(0)
      }
    },
    a => Error(0)
  }
}

alt!(box_parser<&[u8], MP4Box>, filetype_box | free_box);

fn parse_mp4_file(filename: &str) {
  FileProducer::new(filename, 150).map(|producer: FileProducer| {
    let mut p = producer;
    match p.produce() {
      ProducerState::Data(bytes) => {
        println!("bytes:\n{}", bytes.to_hex(8));
        match filetype_box(bytes) {
          Done(i, o) => {
            match o {
              MP4Box::Ftyp(f) => println!("parsed ftyp: {:?}", f),
              MP4Box::Moov    => println!("found moov box"),
              MP4Box::Free    => println!("found free box")
            }
            println!("remaining:\n{}", i.to_hex(8))
          },
          a => println!("error: {:?}", a)
        }
      },
      _ => println!("got error")
    }
    /*//p.push(|par| {println!("parsed file: {}", par); par});
    //p.push(|par| par.flat_map(print));
    fn pr<'a,'b,'c>(data: &[u8]) -> IResult<'b,&[u8], &[u8]> {
      println!("bytes: {:?}", &data[0..20]);
      //Done("".as_bytes(), data).map_res(str::from_utf8);
      Done("".as_bytes(),"".as_bytes())
    }
    pusher!(ps, pr);
    ps(&mut p);
    */
    assert!(false);
  });

}
#[test]
fn file_test() {
  parse_mp4_file("small.mp4");
}
#[test]
fn bunny_test() {
  parse_mp4_file("bigbuckbunny.mp4");
}
<|endoftext|>