<commit_before>//! The fields that can be used in queries.
//!
//! Some field types can be used for multiple entities, to make it more user
//! friendly the types are reexported to the submodules corresponding to the
//! names of the entities which they can be used to query.
//!
//! Link to [MusicBrainz
//! documentation](https://musicbrainz.org/doc/Indexed_Search_Syntax).

// TODO: To verify we actually implemented all fields it might make more sense to swap our type
// and the string value in the following declarations and then sort the entries alphabetically
// again.

use super::{full_entities};
// use super::query::QueryExpression;
use super::full_entities::{Mbid, PartialDate};
// use super::entities;

pub trait SearchField {
    type Value: ToString;

    fn to_string(&self) -> String;
}

macro_rules! define_fields {
    ( $( $type:ident, $value:ty );* ) => {
        $(
            pub struct $type ( pub $value );

            impl SearchField for $type {
                type Value = $value;

                fn to_string(&self) -> String {
                    self.0.to_string()
                }
            }
        )*
    }
}

// TODO consider whether we should rename `Comment` to `Disambiguation` or something like that to
// be more consistent with the rest of the crate.
//
// TODO: enums for quality, lang, script, etc
define_fields!(
    Alias, String;
    AreaId, Mbid;
    AreaIso, String;
    AreaIso1, String;
    AreaIso2, String;
    AreaIso3, String;
    AreaName, String;
    AreaType, full_entities::AreaType;
    ArtistCredit, String;
    ArtistId, Mbid;
    ArtistName, String;
    ArtistNameAccent, String;
    ArtistType, full_entities::ArtistType;
    Asin, String;
    Barcode, String;
    BeginArea, String;
    BeginDate, PartialDate;
    CatalogNumber, String;
    Comment, String;
    Country, String;
    CreditName, String;
    DataQuality, String;
    EndArea, String;
    EndDate, PartialDate;
    Ended, bool;
    Gender, String;
    IpiCode, String;
    LabelId, String;
    Language, String;
    MediumCount, u32;
    MediumFormat, String;
    NumDiscIds, u32;
    NumDiscIdsMedium, u32;
    NumTracks, u32;
    NumTracksMedium, u32;
    PrimaryType, full_entities::ReleaseGroupPrimaryType;
    ReleaseDate, full_entities::PartialDate;
    ReleaseGroupId, Mbid;
    ReleaseGroupName, String;
    ReleaseGroupNameAccent, String;
    ReleaseId, Mbid;
    ReleaseName, String;
    ReleaseNameAccent, String;
    ReleaseNumber, u16;
    ReleaseStatus, full_entities::ReleaseStatus;
    Script, String;
    SecondaryType, String;
    SortName, String;
    Tag, String
);

macro_rules! define_entity_fields {
    (
        $field_trait:ident, $modname:ident;
        $(
            $field_type:ident, $strname:expr
        );*
    )
        =>
    {
        /// Acceptable fields searching for instances of the entity.
        pub trait $field_trait : SearchField {
            fn name() -> &'static str;
        }

        pub mod $modname {
            pub use super::$field_trait;

            $(
                pub use super::$field_type;

                impl $field_trait for $field_type {
                    fn name() -> &'static str { $strname }
                }
            )*
        }

    }
}

define_entity_fields!(
    AreaSearchField, area;

    Alias, "alias";
    AreaId, "aid";
    AreaIso, "iso";
    AreaIso1, "iso1";
    AreaIso2, "iso2";
    AreaIso3, "iso3";
    AreaName, "area";
    AreaType, "type";
    BeginDate, "begin";
    Comment, "comment";
    EndDate, "end";
    Ended, "ended";
    SortName, "sortname"
);

define_entity_fields!(
    ArtistSearchField, artist;

    Alias, "alias";
    AreaName, "area";
    ArtistId, "arid";
    ArtistName, "artist";
    ArtistNameAccent, "artistaccent";
    ArtistType, "type";
    BeginArea, "beginarea";
    BeginDate, "begin";
    Comment, "comment";
    Country, "country";
    EndArea, "endarea";
    EndDate, "end";
    Ended, "ended";
    Gender, "gender";
    IpiCode, "ipi";
    SortName, "sortname";
    Tag, "tag"
);

// TODO what are puids?
define_entity_fields!(
    ReleaseSearchField, release;

    ArtistId, "arid";
    ArtistName, "artist";
    Asin, "asin";
    Barcode, "barcode";
    CatalogNumber, "catno";
    Comment, "comment";
    Country, "country";
    CreditName, "creditname";
    ReleaseDate, "date";
    NumDiscIds, "discids";
    NumDiscIdsMedium, "discidsmedium";
    MediumFormat, "format";
    LabelId, "laid";
    Language, "lang";
    MediumCount, "mediums";
    PrimaryType, "primarytype";
    DataQuality, "quality";
    ReleaseId, "reid";
    ReleaseName, "release";
    ReleaseNameAccent, "releaseaccent";
    ReleaseGroupId, "rgid";
    Script, "script";
    SecondaryType, "secondarytype";
    ReleaseStatus, "status";
    Tag, "tag";
    NumTracks, "tracks";
    NumTracksMedium, "tracksmedium"
);

define_entity_fields!(
    ReleaseGroupSearchField, release_group;

    ArtistCredit, "artist";
    ArtistId, "arid";
    ArtistName, "artistname";
    Comment, "comment";
    CreditName, "creditname";
    PrimaryType, "primarytype";
    ReleaseGroupId, "rgid";
    ReleaseGroupName, "releasegroup";
    ReleaseGroupNameAccent, "releasegroupaccent";
    ReleaseId, "reid";
    ReleaseName, "release";
    ReleaseNumber, "releases";
    ReleaseStatus, "status";
    SecondaryType, "secondarytype";
    Tag, "tag"
);
<commit_msg>Setup search fields documentation.<commit_after>//! The fields that can be used in queries.
//!
//! Some field types can be used for multiple entities, to make it more user
//! friendly the types are reexported to the submodules corresponding to the
//! names of the entities which they can be used to query.
//!
//! Link to [MusicBrainz
//! documentation](https://musicbrainz.org/doc/Indexed_Search_Syntax).

// TODO: To verify we actually implemented all fields it might make more sense to swap our type
// and the string value in the following declarations and then sort the entries alphabetically
// again.

use super::{full_entities};
// use super::query::QueryExpression;
use super::full_entities::{Mbid, PartialDate};
// use super::entities;

pub trait SearchField {
    type Value: ToString;

    fn to_string(&self) -> String;
}

macro_rules! define_fields {
    ( $( $(#[$attr:meta])* f, $type:ident, $value:ty );* ) => {
        $(
            $(#[$attr])*
            pub struct $type ( pub $value );

            impl SearchField for $type {
                type Value = $value;

                fn to_string(&self) -> String {
                    self.0.to_string()
                }
            }
        )*
    }
}

// TODO consider whether we should rename `Comment` to `Disambiguation` or something like that to
// be more consistent with the rest of the crate.
//
// TODO: enums for quality, lang, script, etc
// TODO it's a bit ugly we have f, at the beginning of every line but its a workaround around the
// parsing ambiguity we'd have if we don't.
define_fields!(
    /// Alias of the searched entity's name.
    f, Alias, String;
    f, AreaId, Mbid;
    f, AreaIso, String;
    f, AreaIso1, String;
    f, AreaIso2, String;
    f, AreaIso3, String;
    f, AreaName, String;
    f, AreaType, full_entities::AreaType;
    f, ArtistCredit, String;
    f, ArtistId, Mbid;
    f, ArtistName, String;
    f, ArtistNameAccent, String;
    f, ArtistType, full_entities::ArtistType;
    f, Asin, String;
    /// The barcode of a `Release`.
    f, Barcode, String;
    f, BeginArea, String;
    f, BeginDate, PartialDate;
    f, CatalogNumber, String;
    f, Comment, String;
    f, Country, String;
    f, CreditName, String;
    f, DataQuality, String;
    f, EndArea, String;
    f, EndDate, PartialDate;
    f, Ended, bool;
    /// The gender of an `Artist`.
    f, Gender, String;
    f, IpiCode, String;
    f, LabelId, String;
    f, Language, String;
    f, MediumCount, u32;
    f, MediumFormat, String;
    /// The searched entity's name. (TODO implement for all relevant searches)
    f, Name, String;
    f, NumDiscIds, u32;
    f, NumDiscIdsMedium, u32;
    f, NumTracks, u32;
    f, NumTracksMedium, u32;
    f, PrimaryType, full_entities::ReleaseGroupPrimaryType;
    f, ReleaseDate, full_entities::PartialDate;
    f, ReleaseGroupId, Mbid;
    f, ReleaseGroupName, String;
    f, ReleaseGroupNameAccent, String;
    f, ReleaseId, Mbid;
    /// The name of the `Release`, without special accent characters.
    f, ReleaseName, String;
    /// The name of the `Release`, including special accent characters.
    f, ReleaseNameAccent, String;
    f, ReleaseNumber, u16;
    f, ReleaseStatus, full_entities::ReleaseStatus;
    f, Script, String;
    f, SecondaryType, String;
    f, SortName, String;
    f, Tag, String
);

macro_rules! define_entity_fields {
    (
        $field_trait:ident, $modname:ident;
        $(
            $field_type:ident, $strname:expr
        );*
    )
        =>
    {
        /// Acceptable fields searching for instances of the entity.
        pub trait $field_trait : SearchField {
            fn name() -> &'static str;
        }

        pub mod $modname {
            pub use super::$field_trait;

            $(
                pub use super::$field_type;

                impl $field_trait for $field_type {
                    fn name() -> &'static str { $strname }
                }
            )*
        }

    }
}

define_entity_fields!(
    AreaSearchField, area;

    Alias, "alias";
    AreaId, "aid";
    AreaIso, "iso";
    AreaIso1, "iso1";
    AreaIso2, "iso2";
    AreaIso3, "iso3";
    AreaName, "area";
    AreaType, "type";
    BeginDate, "begin";
    Comment, "comment";
    EndDate, "end";
    Ended, "ended";
    SortName, "sortname"
);

define_entity_fields!(
    ArtistSearchField, artist;

    Alias, "alias";
    AreaName, "area";
    ArtistId, "arid";
    ArtistName, "artist";
    ArtistNameAccent, "artistaccent";
    ArtistType, "type";
    BeginArea, "beginarea";
    BeginDate, "begin";
    Comment, "comment";
    Country, "country";
    EndArea, "endarea";
    EndDate, "end";
    Ended, "ended";
    Gender, "gender";
    IpiCode, "ipi";
    SortName, "sortname";
    Tag, "tag"
);

// TODO what are puids?
define_entity_fields!(
    ReleaseSearchField, release;

    ArtistId, "arid";
    ArtistName, "artist";
    Asin, "asin";
    Barcode, "barcode";
    CatalogNumber, "catno";
    Comment, "comment";
    Country, "country";
    CreditName, "creditname";
    ReleaseDate, "date";
    NumDiscIds, "discids";
    NumDiscIdsMedium, "discidsmedium";
    MediumFormat, "format";
    LabelId, "laid";
    Language, "lang";
    MediumCount, "mediums";
    PrimaryType, "primarytype";
    DataQuality, "quality";
    ReleaseId, "reid";
    ReleaseName, "release";
    ReleaseNameAccent, "releaseaccent";
    ReleaseGroupId, "rgid";
    Script, "script";
    SecondaryType, "secondarytype";
    ReleaseStatus, "status";
    Tag, "tag";
    NumTracks, "tracks";
    NumTracksMedium, "tracksmedium"
);

define_entity_fields!(
    ReleaseGroupSearchField, release_group;

    ArtistCredit, "artist";
    ArtistId, "arid";
    ArtistName, "artistname";
    Comment, "comment";
    CreditName, "creditname";
    PrimaryType, "primarytype";
    ReleaseGroupId, "rgid";
    ReleaseGroupName, "releasegroup";
    ReleaseGroupNameAccent, "releasegroupaccent";
    ReleaseId, "reid";
    ReleaseName, "release";
    ReleaseNumber, "releases";
    ReleaseStatus, "status";
    SecondaryType, "secondarytype";
    Tag, "tag"
);
<|endoftext|>


<commit_before>use cdchunking::{Chunker, ChunkInput, ZPAQ};
use rusqlite;
use rusqlite::{Connection, Transaction};
use rusqlite::types::ToSql;
use sha1::Sha1;
use std::fs::File;
use std::path::{Path, PathBuf};

use crate::{Error, HashDigest};

const SCHEMA: &'static str = "
    CREATE TABLE version(
        name VARCHAR(8) NOT NULL,
        version VARCHAR(16) NOT NULL
    );
    INSERT INTO version(name, version) VALUES('rs-sync', '0.1');

    CREATE TABLE files(
        file_id INTEGER NOT NULL PRIMARY KEY,
        name VARCHAR(512) NOT NULL,
        modified DATETIME NOT NULL
    );
    CREATE INDEX idx_files_name ON files(name);

    CREATE TABLE blocks(
        hash VARCHAR(40) NOT NULL,
        file_id INTEGER NOT NULL,
        offset INTEGER NOT NULL,
        PRIMARY KEY(file_id, offset)
    );
    CREATE INDEX idx_blocks_hash ON blocks(hash);
    CREATE INDEX idx_blocks_file ON blocks(file_id);
    CREATE INDEX idx_blocks_file_offset ON blocks(file_id, offset);
";

/// Index of files and blocks
pub struct Index {
    db: Connection,
}

impl Index {
    /// Open an index from a file
    pub fn open(filename: &Path) -> Result<Index, Error> {
        let exists = filename.exists();
        let db = Connection::open(filename)?;
        if !exists {
            warn!("Database doesn't exist, creating tables...");
            db.execute_batch(SCHEMA)?;
        }
        Ok(Index { db })
    }

    /// Open an in-memory index
    pub fn open_in_memory() -> Result<Index, Error> {
        let db = Connection::open_in_memory()?;
        db.execute_batch(SCHEMA)?;
        Ok(Index { db })
    }

    /// Try to find a block in the indexed files
    pub fn get_block(
        &self,
        hash: HashDigest,
    ) -> Result<Option<(PathBuf, usize)>, Error>
    {
        let mut stmt = self.db.prepare(
            "
            SELECT files.name, blocks.offset
            FROM blocks
            INNER JOIN files ON blocks.file_id = files.file_id
            WHERE blocks.hash = ?;
            ",
        )?;
        let mut rows = stmt.query(&[&hash as &dyn ToSql])?;
        if let Some(row) = rows.next() {
            let row = row?;
            let path: String = row.get(0);
            let path: PathBuf = path.into();
            let offset: i64 = row.get(1);
            let offset = offset as usize;
            Ok(Some((path, offset)))
        } else {
            Ok(None)
        }
    }

    /// Start a transaction to update the index
    pub fn transaction<'a>(
        &'a mut self
    ) -> Result<IndexTransaction<'a>, rusqlite::Error>
    {
        let tx = self.db.transaction()?;
        Ok(IndexTransaction { tx })
    }
}

/// A transaction on the index, for safety and performance
pub struct IndexTransaction<'a> {
    tx: Transaction<'a>,
}

const ZPAQ_BITS: usize = 13; // 13 bits = 8 KiB block average
const MAX_BLOCK_SIZE: usize = 1 << 15; // 32 KiB

impl<'a> IndexTransaction<'a> {
    /// Add a file to the index
    ///
    /// This returns a tuple `(file_id, up_to_date)` where `file_id` can be
    /// used to insert blocks, and `up_to_date` indicates whether the file's
    /// modification date has changed and it should be re-indexed.
    pub fn add_file(
        &mut self,
        name: &Path,
        modified: chrono::DateTime<chrono::Utc>,
    ) -> Result<(u32, bool), Error>
    {
        let mut stmt = self.tx.prepare(
            "
            SELECT file_id, modified FROM files
            WHERE name = ?;
            ",
        )?;
        let mut rows = stmt.query(&[name.to_str().expect("encoding")])?;
        if let Some(row) = rows.next() {
            let row = row?;
            let file_id: u32 = row.get(0);
            let old_modified: chrono::DateTime<chrono::Utc> = row.get(1);
            if old_modified != modified {
                info!("Resetting file {:?}, modified", name);
                // Delete blocks
                self.tx.execute(
                    "
                    DELETE FROM blocks WHERE file_id = ?;
                    ",
                    &[&file_id],
                )?;
                // Update modification time
                self.tx.execute(
                    "
                    UPDATE files SET modified = ? WHERE file_id = ?;
                    ",
                    &[&modified as &dyn ToSql, &file_id],
                )?;
                Ok((file_id, false))
            } else {
                debug!("File {:?} up to date", name);
                Ok((file_id, true))
            }
        } else {
            info!("Inserting new file {:?}", name);
            self.tx.execute(
                "
                INSERT INTO files(name, modified)
                VALUES(?, ?);
                ",
                &[&name.to_str().expect("encoding") as &dyn ToSql, &modified],
            )?;
            let file_id = self.tx.last_insert_rowid();
            Ok((file_id as u32, false))
        }
    }

    /// Remove a file and all its blocks from the index
    pub fn remove_file(
        &mut self,
        file_id: u32,
    ) -> Result<(), Error>
    {
        self.tx.execute(
            "
            DELETE FROM blocks WHERE file_id = ?;
            ",
            &[&file_id],
        )?;
        self.tx.execute(
            "
            DELETE FROM files WHERE file_id = ?;
            ",
            &[&file_id],
        )?;
        Ok(())
    }

    /// Get a list of all the files in the index
    pub fn list_files(&self) -> Result<Vec<(u32, PathBuf)>, Error> {
        let mut stmt = self.tx.prepare(
            "
            SELECT file_id, name FROM files;
            ",
        )?;
        let mut rows = stmt.query(rusqlite::NO_PARAMS)?;
        let mut results = Vec::new();
        loop {
            match rows.next() {
                Some(Ok(row)) => {
                    let path: String = row.get(1);
                    results.push((row.get(0), path.into()))
                }
                Some(Err(e)) => return Err(e.into()),
                None => break,
            }
        }
        Ok(results)
    }

    /// Add a block to the index
    pub fn add_block(
        &mut self,
        hash: HashDigest,
        file_id: u32,
        offset: usize,
    ) -> Result<(), Error>
    {
        self.tx.execute(
            "
            INSERT INTO blocks(hash, file_id, offset)
            VALUES(?, ?, ?);
            ",
            &[&hash as &dyn ToSql, &file_id, &(offset as i64)],
        )?;
        Ok(())
    }

    /// Cut up a file into blocks and add them to the index
    pub fn index_file(
        &mut self,
        name: &Path,
    ) -> Result<(), Error>
    {
        let file = File::open(name)?;
        let (file_id, up_to_date) = self.add_file(
            name,
            file.metadata()?.modified()?.into(),
        )?;
        if !up_to_date {
            // Use ZPAQ to cut the stream into blocks
            let chunker = Chunker::new(
                ZPAQ::new(ZPAQ_BITS) // 13 bits = 8 KiB block average
            );
            let mut chunk_iterator = chunker.stream(file);
            let mut start_offset = 0;
            let mut offset = 0;
            let mut sha1 = Sha1::new();
            while let Some(chunk) = chunk_iterator.read() {
                match chunk? {
                    ChunkInput::Data(mut d) => {
                        while offset - start_offset + d.len()
                            >= MAX_BLOCK_SIZE
                        {
                            let end = MAX_BLOCK_SIZE
                                + start_offset - offset;
                            sha1.update(&d[0..end]);
                            let digest = HashDigest(sha1.digest().bytes());
                            debug!(
                                "Max block size reached, adding block, \
                                 offset={}, size={}, sha1={}",
                                start_offset, offset + end - start_offset, sha1.digest(),
                            );
                            self.add_block(digest, file_id, start_offset)?;
                            offset += end;
                            start_offset = offset;
                            d = &d[end..];
                        }
                        sha1.update(d);
                        offset += d.len();
                    }
                    ChunkInput::End => {
                        let digest = HashDigest(sha1.digest().bytes());
                        debug!(
                            "Adding block, offset={}, size={}, sha1={}",
                            start_offset, offset - start_offset, sha1.digest(),
                        );
                        self.add_block(digest, file_id, start_offset)?;
                        start_offset = offset;
                        sha1.reset();
                    }
                }
            }
        }
        Ok(())
    }

    /// Commit the transaction
    pub fn commit(self) -> Result<(), rusqlite::Error> {
        self.tx.commit()
    }
}

#[cfg(test)]
mod tests {
    use std::io::Write;
    use tempfile::NamedTempFile;

    use crate::HashDigest;
    use super::{Index, MAX_BLOCK_SIZE};

    #[test]
    fn test() {
        let mut file = NamedTempFile::new().expect("tempfile");
        for i in 0..2000 {
            write!(file, "Line {}\n", i + 1).expect("tempfile");
        }
        for _ in 0..2000 {
            write!(file, "Test content\n").expect("tempfile");
        }
        file.flush().expect("tempfile");
        let mut index = Index::open_in_memory().expect("db");
        {
            let mut tx = index.transaction().expect("db");
            tx.index_file(file.path()).expect("index");
            tx.commit().expect("db");
        }
        assert!(
            index.get_block(HashDigest(*b"12345678901234567890")).expect("get")
                .is_none()
        );
        let block1 = index.get_block(HashDigest(
            *b"\xfb\x5e\xf7\xeb\xad\xd8\x2c\x80\x85\xc5\
               \xff\x63\x82\x36\x22\xba\xe0\xe2\x63\xf6"
        )).expect("get");
        assert_eq!(
            block1,
            Some((file.path().into(), 0)),
        );
        let block2 = index.get_block(HashDigest(
            *b"\x57\x0d\x8b\x30\xfc\xfd\x58\x5e\x41\x27\
               \xb5\x61\xf5\xec\xd3\x76\xff\x4d\x01\x01"
        )).expect("get");
        assert_eq!(
            block2,
            Some((file.path().into(), 11579)),
        );
        let block3 = index.get_block(HashDigest(
            *b"\xa6\xe1\x6d\x65\x26\xbd\x1d\x94\xec\x47\
               \xd2\xeb\x3f\xa4\x6f\x34\xb9\x71\x70\xa7"
        )).expect("get");
        assert_eq!(
            block3,
            Some((file.path().into(), 44347)),
        );
        assert_eq!(block3.unwrap().1 - block2.unwrap().1, MAX_BLOCK_SIZE);
    }
}
<commit_msg>Fix bad block hashes after reaching max size<commit_after>use cdchunking::{Chunker, ChunkInput, ZPAQ};
use rusqlite;
use rusqlite::{Connection, Transaction};
use rusqlite::types::ToSql;
use sha1::Sha1;
use std::fs::File;
use std::path::{Path, PathBuf};

use crate::{Error, HashDigest};

const SCHEMA: &'static str = "
    CREATE TABLE version(
        name VARCHAR(8) NOT NULL,
        version VARCHAR(16) NOT NULL
    );
    INSERT INTO version(name, version) VALUES('rs-sync', '0.1');

    CREATE TABLE files(
        file_id INTEGER NOT NULL PRIMARY KEY,
        name VARCHAR(512) NOT NULL,
        modified DATETIME NOT NULL
    );
    CREATE INDEX idx_files_name ON files(name);

    CREATE TABLE blocks(
        hash VARCHAR(40) NOT NULL,
        file_id INTEGER NOT NULL,
        offset INTEGER NOT NULL,
        PRIMARY KEY(file_id, offset)
    );
    CREATE INDEX idx_blocks_hash ON blocks(hash);
    CREATE INDEX idx_blocks_file ON blocks(file_id);
    CREATE INDEX idx_blocks_file_offset ON blocks(file_id, offset);
";

/// Index of files and blocks
pub struct Index {
    db: Connection,
}

impl Index {
    /// Open an index from a file
    pub fn open(filename: &Path) -> Result<Index, Error> {
        let exists = filename.exists();
        let db = Connection::open(filename)?;
        if !exists {
            warn!("Database doesn't exist, creating tables...");
            db.execute_batch(SCHEMA)?;
        }
        Ok(Index { db })
    }

    /// Open an in-memory index
    pub fn open_in_memory() -> Result<Index, Error> {
        let db = Connection::open_in_memory()?;
        db.execute_batch(SCHEMA)?;
        Ok(Index { db })
    }

    /// Try to find a block in the indexed files
    pub fn get_block(
        &self,
        hash: HashDigest,
    ) -> Result<Option<(PathBuf, usize)>, Error>
    {
        let mut stmt = self.db.prepare(
            "
            SELECT files.name, blocks.offset
            FROM blocks
            INNER JOIN files ON blocks.file_id = files.file_id
            WHERE blocks.hash = ?;
            ",
        )?;
        let mut rows = stmt.query(&[&hash as &dyn ToSql])?;
        if let Some(row) = rows.next() {
            let row = row?;
            let path: String = row.get(0);
            let path: PathBuf = path.into();
            let offset: i64 = row.get(1);
            let offset = offset as usize;
            Ok(Some((path, offset)))
        } else {
            Ok(None)
        }
    }

    /// Start a transaction to update the index
    pub fn transaction<'a>(
        &'a mut self
    ) -> Result<IndexTransaction<'a>, rusqlite::Error>
    {
        let tx = self.db.transaction()?;
        Ok(IndexTransaction { tx })
    }
}

/// A transaction on the index, for safety and performance
pub struct IndexTransaction<'a> {
    tx: Transaction<'a>,
}

const ZPAQ_BITS: usize = 13; // 13 bits = 8 KiB block average
const MAX_BLOCK_SIZE: usize = 1 << 15; // 32 KiB

impl<'a> IndexTransaction<'a> {
    /// Add a file to the index
    ///
    /// This returns a tuple `(file_id, up_to_date)` where `file_id` can be
    /// used to insert blocks, and `up_to_date` indicates whether the file's
    /// modification date has changed and it should be re-indexed.
    pub fn add_file(
        &mut self,
        name: &Path,
        modified: chrono::DateTime<chrono::Utc>,
    ) -> Result<(u32, bool), Error>
    {
        let mut stmt = self.tx.prepare(
            "
            SELECT file_id, modified FROM files
            WHERE name = ?;
            ",
        )?;
        let mut rows = stmt.query(&[name.to_str().expect("encoding")])?;
        if let Some(row) = rows.next() {
            let row = row?;
            let file_id: u32 = row.get(0);
            let old_modified: chrono::DateTime<chrono::Utc> = row.get(1);
            if old_modified != modified {
                info!("Resetting file {:?}, modified", name);
                // Delete blocks
                self.tx.execute(
                    "
                    DELETE FROM blocks WHERE file_id = ?;
                    ",
                    &[&file_id],
                )?;
                // Update modification time
                self.tx.execute(
                    "
                    UPDATE files SET modified = ? WHERE file_id = ?;
                    ",
                    &[&modified as &dyn ToSql, &file_id],
                )?;
                Ok((file_id, false))
            } else {
                debug!("File {:?} up to date", name);
                Ok((file_id, true))
            }
        } else {
            info!("Inserting new file {:?}", name);
            self.tx.execute(
                "
                INSERT INTO files(name, modified)
                VALUES(?, ?);
                ",
                &[&name.to_str().expect("encoding") as &dyn ToSql, &modified],
            )?;
            let file_id = self.tx.last_insert_rowid();
            Ok((file_id as u32, false))
        }
    }

    /// Remove a file and all its blocks from the index
    pub fn remove_file(
        &mut self,
        file_id: u32,
    ) -> Result<(), Error>
    {
        self.tx.execute(
            "
            DELETE FROM blocks WHERE file_id = ?;
            ",
            &[&file_id],
        )?;
        self.tx.execute(
            "
            DELETE FROM files WHERE file_id = ?;
            ",
            &[&file_id],
        )?;
        Ok(())
    }

    /// Get a list of all the files in the index
    pub fn list_files(&self) -> Result<Vec<(u32, PathBuf)>, Error> {
        let mut stmt = self.tx.prepare(
            "
            SELECT file_id, name FROM files;
            ",
        )?;
        let mut rows = stmt.query(rusqlite::NO_PARAMS)?;
        let mut results = Vec::new();
        loop {
            match rows.next() {
                Some(Ok(row)) => {
                    let path: String = row.get(1);
                    results.push((row.get(0), path.into()))
                }
                Some(Err(e)) => return Err(e.into()),
                None => break,
            }
        }
        Ok(results)
    }

    /// Add a block to the index
    pub fn add_block(
        &mut self,
        hash: HashDigest,
        file_id: u32,
        offset: usize,
    ) -> Result<(), Error>
    {
        self.tx.execute(
            "
            INSERT INTO blocks(hash, file_id, offset)
            VALUES(?, ?, ?);
            ",
            &[&hash as &dyn ToSql, &file_id, &(offset as i64)],
        )?;
        Ok(())
    }

    /// Cut up a file into blocks and add them to the index
    pub fn index_file(
        &mut self,
        name: &Path,
    ) -> Result<(), Error>
    {
        let file = File::open(name)?;
        let (file_id, up_to_date) = self.add_file(
            name,
            file.metadata()?.modified()?.into(),
        )?;
        if !up_to_date {
            // Use ZPAQ to cut the stream into blocks
            let chunker = Chunker::new(
                ZPAQ::new(ZPAQ_BITS) // 13 bits = 8 KiB block average
            );
            let mut chunk_iterator = chunker.stream(file);
            let mut start_offset = 0;
            let mut offset = 0;
            let mut sha1 = Sha1::new();
            while let Some(chunk) = chunk_iterator.read() {
                match chunk? {
                    ChunkInput::Data(mut d) => {
                        while offset - start_offset + d.len()
                            >= MAX_BLOCK_SIZE
                        {
                            let end = MAX_BLOCK_SIZE
                                + start_offset - offset;
                            sha1.update(&d[0..end]);
                            let digest = HashDigest(sha1.digest().bytes());
                            debug!(
                                "Max block size reached, adding block, \
                                 offset={}, size={}, sha1={}",
                                start_offset, offset + end - start_offset, sha1.digest(),
                            );
                            self.add_block(digest, file_id, start_offset)?;
                            offset += end;
                            start_offset = offset;
                            d = &d[end..];
                            sha1.reset();
                        }
                        sha1.update(d);
                        offset += d.len();
                    }
                    ChunkInput::End => {
                        let digest = HashDigest(sha1.digest().bytes());
                        debug!(
                            "Adding block, offset={}, size={}, sha1={}",
                            start_offset, offset - start_offset, sha1.digest(),
                        );
                        self.add_block(digest, file_id, start_offset)?;
                        start_offset = offset;
                        sha1.reset();
                    }
                }
            }
        }
        Ok(())
    }

    /// Commit the transaction
    pub fn commit(self) -> Result<(), rusqlite::Error> {
        self.tx.commit()
    }
}

#[cfg(test)]
mod tests {
    use std::io::Write;
    use tempfile::NamedTempFile;

    use crate::HashDigest;
    use super::{Index, MAX_BLOCK_SIZE};

    #[test]
    fn test() {
        let mut file = NamedTempFile::new().expect("tempfile");
        for i in 0..2000 {
            write!(file, "Line {}\n", i + 1).expect("tempfile");
        }
        for _ in 0..2000 {
            write!(file, "Test content\n").expect("tempfile");
        }
        file.flush().expect("tempfile");
        let mut index = Index::open_in_memory().expect("db");
        {
            let mut tx = index.transaction().expect("db");
            tx.index_file(file.path()).expect("index");
            tx.commit().expect("db");
        }
        assert!(
            index.get_block(HashDigest(*b"12345678901234567890")).expect("get")
                .is_none()
        );
        let block1 = index.get_block(HashDigest(
            *b"\xfb\x5e\xf7\xeb\xad\xd8\x2c\x80\x85\xc5\
               \xff\x63\x82\x36\x22\xba\xe0\xe2\x63\xf6"
        )).expect("get");
        assert_eq!(
            block1,
            Some((file.path().into(), 0)),
        );
        let block2 = index.get_block(HashDigest(
            *b"\x57\x0d\x8b\x30\xfc\xfd\x58\x5e\x41\x27\
               \xb5\x61\xf5\xec\xd3\x76\xff\x4d\x01\x01"
        )).expect("get");
        assert_eq!(
            block2,
            Some((file.path().into(), 11579)),
        );
        let block3 = index.get_block(HashDigest(
            *b"\xb9\xa8\xc2\x64\x1a\xf2\xcf\x8f\xd8\xf3\
               \x6a\x24\x56\xa3\xea\xa9\x5c\x02\x91\x27"
        )).expect("get");
        assert_eq!(
            block3,
            Some((file.path().into(), 44347)),
        );
        assert_eq!(block3.unwrap().1 - block2.unwrap().1, MAX_BLOCK_SIZE);
    }
}
<|endoftext|>


<commit_before>use byteorder::{BigEndian, ByteOrder};
use memmap::{Mmap, Protection};
use std::path::{Path, PathBuf};
use std::io::{self, Write};
use std::fs::{OpenOptions, File};
use std::{u64, usize};
use std::cmp::Ordering;

/// Number of byes in each entry pair
pub static INDEX_ENTRY_BYTES: usize = 8;
/// Number of bytes contained in the base name of the file.
pub static INDEX_FILE_NAME_LEN: usize = 20;
/// File extension for the index file.
pub static INDEX_FILE_NAME_EXTENSION: &'static str = "index";

fn binary_search<F>(index: &[u8], f: F) -> Result<usize, usize>
    where F: Fn(usize, u32) -> Ordering
{
    assert!(index.len() % INDEX_ENTRY_BYTES == 0);

    let mut i = 0usize;
    let mut j = (index.len() / INDEX_ENTRY_BYTES) - 1;

    while i < j {
        // grab midpoint
        let m = i + ((j - i) / 2);

        // read the relative offset at the midpoint
        let mi = m * INDEX_ENTRY_BYTES;
        let rel_off = BigEndian::read_u32(&index[mi..mi + 4]);

        match f(m, rel_off) {
            Ordering::Equal => return Ok(m),
            Ordering::Less => {
                i = m + 1;
            }
            Ordering::Greater => {
                j = m;
            }
        }
    }

    Err(i)
}

/// An index is a file with pairs of relative offset to file position offset
/// of messages at the relative offset messages. The index is Memory Mapped.
pub struct Index {
    file: File,
    mmap: Mmap,
    mode: AccessMode,

    /// next starting byte in index file offset to write
    next_write_pos: usize,
    base_offset: u64,
}

/// Describes the access mode of the index
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum AccessMode {
    /// Only reads are permitted.
    Read,
    /// This is the active index and can be read or written to.
    ReadWrite,
}

#[derive(Debug, Copy, Clone, PartialEq)]
pub struct IndexEntry {
    rel_offset: u32,
    base_offset: u64,
    file_pos: u32,
}

impl IndexEntry {
    #[inline]
    #[allow(dead_code)]
    pub fn relative_offset(&self) -> u32 {
        self.rel_offset
    }

    #[inline]
    #[allow(dead_code)]
    pub fn offset(&self) -> u64 {
        self.base_offset + (self.rel_offset as u64)
    }

    #[inline]
    pub fn file_position(&self) -> u32 {
        self.file_pos
    }
}

#[derive(Debug, Copy, Clone, PartialEq)]
pub enum IndexWriteError {
    IndexFull,
    OffsetLessThanBase,
}

impl Index {
    pub fn new<P>(log_dir: P, base_offset: u64, file_bytes: usize) -> io::Result<Index>
        where P: AsRef<Path>
    {
        // open the file, expecting to create it
        let index_path = {
            let mut path_buf = PathBuf::new();
            path_buf.push(&log_dir);
            path_buf.push(format!("{:020}", base_offset));
            path_buf.set_extension(INDEX_FILE_NAME_EXTENSION);
            path_buf
        };

        let index_file = OpenOptions::new().read(true)
            .write(true)
            .append(true)
            .create_new(true)
            .open(&index_path)?;

        // read the metadata and truncate
        let meta = index_file.metadata()?;
        let len = meta.len();
        if len == 0 {
            index_file.set_len(file_bytes as u64)?;
        }

        let mmap = Mmap::open(&index_file, Protection::ReadWrite)?;

        Ok(Index {
            file: index_file,
            mmap: mmap,
            mode: AccessMode::ReadWrite,
            next_write_pos: 0,
            base_offset: base_offset,
        })
    }

    #[allow(dead_code)]
    pub fn open<P>(index_path: P) -> io::Result<Index>
        where P: AsRef<Path>
    {
        let index_file = OpenOptions::new().read(true)
            .write(false)
            .append(false)
            .open(&index_path)?;

        let filename = index_path.as_ref().file_name().unwrap().to_str().unwrap();
        let base_offset = match u64::from_str_radix(&filename[0..INDEX_FILE_NAME_LEN], 10) {
            Ok(v) => v,
            Err(_) => {
                return Err(io::Error::new(io::ErrorKind::InvalidData,
                                          "Index file name does not parse as u64"))
            }
        };

        let mmap = Mmap::open(&index_file, Protection::Read)?;
        let next_write_pos = mmap.len();
        // TODO: determine if there are empty holes in the file

        Ok(Index {
            file: index_file,
            mmap: mmap,
            mode: AccessMode::Read,
            next_write_pos: next_write_pos,
            base_offset: base_offset,
        })
    }

    pub fn can_write(&self) -> bool {
        self.mode == AccessMode::ReadWrite && self.size() >= (self.next_write_pos + 8)
    }

    #[inline]
    pub fn starting_offset(&self) -> u64 {
        self.base_offset
    }

    #[inline]
    pub fn size(&self) -> usize {
        self.mmap.len()
    }

    pub fn append(&mut self, abs_offset: u64, position: u32) -> Result<(), IndexWriteError> {
        trace!("Index append {} => {}", abs_offset, position);

        assert!(abs_offset >= self.base_offset);

        if !self.can_write() {
            return Err(IndexWriteError::IndexFull);
        }

        if abs_offset < self.base_offset {
            return Err(IndexWriteError::OffsetLessThanBase);
        }

        unsafe {
            let mem_slice: &mut [u8] = self.mmap.as_mut_slice();
            let offset = (abs_offset - self.base_offset) as u32;
            let buf_pos = self.next_write_pos;

            BigEndian::write_u32(&mut mem_slice[buf_pos..buf_pos + 4], offset);
            BigEndian::write_u32(&mut mem_slice[buf_pos + 4..buf_pos + 8], position);

            self.next_write_pos += 8;

            Ok(())
        }
    }

    pub fn set_readonly(&mut self) -> io::Result<()> {
        if self.mode != AccessMode::Read {
            self.mode = AccessMode::Read;
            self.flush_sync()
        } else {
            Ok(())
        }
    }

    pub fn flush_sync(&mut self) -> io::Result<()> {
        self.mmap.flush()?;
        self.file.flush()
    }

    #[cfg(test)]
    pub fn read_entry(&self, i: usize) -> Result<Option<IndexEntry>, ()> {
        if self.size() < (i + 1) * 8 {
            return Err(());
        }

        unsafe {
            let mem_slice = self.mmap.as_slice();
            let start = i * 8;
            let offset = BigEndian::read_u32(&mem_slice[start..start + 4]);
            if offset == 0 && i > 0 {
                Ok(None)
            } else {
                let pos = BigEndian::read_u32(&mem_slice[start + 4..start + 8]);
                Ok(Some(IndexEntry {
                    rel_offset: offset,
                    base_offset: self.base_offset,
                    file_pos: pos,
                }))
            }
        }
    }

    pub fn find(&self, offset: u64) -> Option<IndexEntry> {
        if offset < self.base_offset {
            // pathological case... not worth exposing Result
            return None;
        }

        let rel_offset = (offset - self.base_offset) as u32;

        unsafe {
            let mem_slice = self.mmap.as_slice();
            match binary_search(&mem_slice[0..self.next_write_pos],
                                |_, v| v.cmp(&rel_offset)) {
                Ok(i) => {
                    let p = (i * 8) + 4;
                    Some(IndexEntry {
                        rel_offset: rel_offset,
                        base_offset: self.base_offset,
                        file_pos: BigEndian::read_u32(&mem_slice[p..p + 4]),
                    })
                }
                _ => None,
            }
        }

    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use super::super::testutil::*;
    use std::fs;
    use std::path::PathBuf;

    #[test]
    pub fn index() {
        let path = TestDir::new();
        let mut index = Index::new(&path, 9u64, 1000usize).unwrap();

        assert_eq!(1000, index.size());
        index.append(11u64, 0xffff).unwrap();
        index.append(12u64, 0xeeee).unwrap();
        index.flush_sync().unwrap();

        let e0 = index.read_entry(0).unwrap().unwrap();
        assert_eq!(2u32, e0.relative_offset());
        assert_eq!(11u64, e0.offset());
        assert_eq!(0xffff, e0.file_position());

        let e1 = index.read_entry(1).unwrap().unwrap();
        assert_eq!(3u32, e1.relative_offset());
        assert_eq!(12u64, e1.offset());
        assert_eq!(0xeeee, e1.file_position());

        // read an entry that does not exist
        let e2 = index.read_entry(2).unwrap();
        assert_eq!(None, e2);
    }

    #[test]
    pub fn index_set_readonly() {
        let path = TestDir::new();
        let mut index = Index::new(&path, 10u64, 1000usize).unwrap();

        index.append(11u64, 0xffff).unwrap();
        index.append(12u64, 0xeeee).unwrap();

        // set_readonly it
        index.set_readonly().expect("Unable to set readonly");

        // append should fail with insertion error
        assert_eq!(index.append(13u64, 0xeeeeee),
                   Err(IndexWriteError::IndexFull));

        let e1 = index.read_entry(1).unwrap().unwrap();
        assert_eq!(2u32, e1.relative_offset());
        assert_eq!(12u64, e1.offset());
        assert_eq!(0xeeee, e1.file_position());

        // read an entry that does not exist
        let e2 = index.read_entry(2).unwrap();
        assert_eq!(None, e2);
    }

    #[test]
    pub fn open_index() {
        let dir = TestDir::new();
        // issue some writes
        {
            let mut index = Index::new(&dir, 10u64, 1000usize).unwrap();
            index.append(10, 0).unwrap();
            index.append(11, 10).unwrap();
            index.append(12, 20).unwrap();
            index.append(13, 30).unwrap();
            index.append(14, 40).unwrap();
            index.set_readonly().unwrap();
        }

        // now open it
        {
            let mut index_path = PathBuf::new();
            index_path.push(&dir);
            index_path.push("00000000000000000010.index");

            let meta = fs::metadata(&index_path).unwrap();
            assert!(meta.is_file());

            let index = Index::open(&index_path).unwrap();

            for i in 0..5usize {
                let e = index.read_entry(i).unwrap();
                assert!(e.is_some());
                assert_eq!(e.unwrap().relative_offset(), i as u32);
                assert_eq!(e.unwrap().offset(), (i + 10) as u64);
                assert_eq!(e.unwrap().file_position(), (i * 10) as u32);
            }
        }
    }

    #[test]
    pub fn find() {
        let dir = TestDir::new();
        let mut index = Index::new(&dir, 10u64, 1000usize).unwrap();
        index.append(10, 1).unwrap();
        index.append(11, 2).unwrap();
        index.append(12, 3).unwrap();
        index.append(15, 4).unwrap();
        index.append(16, 5).unwrap();
        index.append(17, 6).unwrap();
        index.append(18, 7).unwrap();
        index.append(20, 8).unwrap();

        let res = index.find(16).unwrap();
        assert_eq!(6, res.relative_offset());
        assert_eq!(16, res.offset());
        assert_eq!(5, res.file_position());
    }

    #[test]
    pub fn find_nonexistant_value() {
        let dir = TestDir::new();
        let mut index = Index::new(&dir, 10u64, 1000usize).unwrap();
        index.append(10, 1).unwrap();
        index.append(11, 2).unwrap();
        index.append(12, 3).unwrap();
        index.append(15, 4).unwrap();
        index.append(16, 5).unwrap();
        index.append(17, 6).unwrap();
        index.append(18, 7).unwrap();
        index.append(20, 8).unwrap();

        let res = index.find(14);
        assert!(res.is_none());
    }

    #[test]
    pub fn find_out_of_bounds() {
        let dir = TestDir::new();
        let mut index = Index::new(&dir, 10u64, 1000usize).unwrap();
        index.append(10, 1).unwrap();
        index.append(11, 2).unwrap();
        index.append(12, 3).unwrap();
        index.append(15, 4).unwrap();
        index.append(16, 5).unwrap();
        index.append(17, 6).unwrap();
        index.append(18, 7).unwrap();
        index.append(20, 8).unwrap();

        let res = index.find(2);
        assert!(res.is_none());
    }
}
<commit_msg>Parsing last position in the index on reopen<commit_after>use byteorder::{BigEndian, ByteOrder};
use memmap::{Mmap, Protection};
use std::path::{Path, PathBuf};
use std::io::{self, Write};
use std::fs::{OpenOptions, File};
use std::{u64, usize};
use std::cmp::Ordering;

/// Number of byes in each entry pair
pub static INDEX_ENTRY_BYTES: usize = 8;
/// Number of bytes contained in the base name of the file.
pub static INDEX_FILE_NAME_LEN: usize = 20;
/// File extension for the index file.
pub static INDEX_FILE_NAME_EXTENSION: &'static str = "index";

fn binary_search<F>(index: &[u8], f: F) -> Result<usize, usize>
    where F: Fn(usize, u32) -> Ordering
{
    assert!(index.len() % INDEX_ENTRY_BYTES == 0);

    let mut i = 0usize;
    let mut j = (index.len() / INDEX_ENTRY_BYTES) - 1;

    while i < j {
        // grab midpoint
        let m = i + ((j - i) / 2);

        // read the relative offset at the midpoint
        let mi = m * INDEX_ENTRY_BYTES;
        let rel_off = BigEndian::read_u32(&index[mi..mi + 4]);

        match f(m, rel_off) {
            Ordering::Equal => return Ok(m),
            Ordering::Less => {
                i = m + 1;
            }
            Ordering::Greater => {
                j = m;
            }
        }
    }

    // HACK: we probably do not want result
    if i == j {
        let ii = i * INDEX_ENTRY_BYTES;
        let rel_off = BigEndian::read_u32(&index[ii..ii + 4]);
        if let Ordering::Equal = f(i, rel_off) {
            return Ok(i);
        }
    }

    Err(i)
}

/// An index is a file with pairs of relative offset to file position offset
/// of messages at the relative offset messages. The index is Memory Mapped.
pub struct Index {
    file: File,
    mmap: Mmap,
    mode: AccessMode,

    /// next starting byte in index file offset to write
    next_write_pos: usize,
    base_offset: u64,
}

/// Describes the access mode of the index
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum AccessMode {
    /// Only reads are permitted.
    Read,
    /// This is the active index and can be read or written to.
    ReadWrite,
}

#[derive(Debug, Copy, Clone, PartialEq)]
pub struct IndexEntry {
    rel_offset: u32,
    base_offset: u64,
    file_pos: u32,
}

impl IndexEntry {
    #[inline]
    #[allow(dead_code)]
    pub fn relative_offset(&self) -> u32 {
        self.rel_offset
    }

    #[inline]
    #[allow(dead_code)]
    pub fn offset(&self) -> u64 {
        self.base_offset + (self.rel_offset as u64)
    }

    #[inline]
    pub fn file_position(&self) -> u32 {
        self.file_pos
    }
}

#[derive(Debug, Copy, Clone, PartialEq)]
pub enum IndexWriteError {
    IndexFull,
    OffsetLessThanBase,
}

impl Index {
    pub fn new<P>(log_dir: P, base_offset: u64, file_bytes: usize) -> io::Result<Index>
        where P: AsRef<Path>
    {
        // open the file, expecting to create it
        let index_path = {
            let mut path_buf = PathBuf::new();
            path_buf.push(&log_dir);
            path_buf.push(format!("{:020}", base_offset));
            path_buf.set_extension(INDEX_FILE_NAME_EXTENSION);
            path_buf
        };

        let index_file = OpenOptions::new().read(true)
            .write(true)
            .append(true)
            .create_new(true)
            .open(&index_path)?;

        // read the metadata and truncate
        let meta = index_file.metadata()?;
        let len = meta.len();
        if len == 0 {
            index_file.set_len(file_bytes as u64)?;
        }

        let mmap = Mmap::open(&index_file, Protection::ReadWrite)?;

        Ok(Index {
            file: index_file,
            mmap: mmap,
            mode: AccessMode::ReadWrite,
            next_write_pos: 0,
            base_offset: base_offset,
        })
    }

    #[allow(dead_code)]
    pub fn open<P>(index_path: P) -> io::Result<Index>
        where P: AsRef<Path>
    {
        let index_file = OpenOptions::new().read(true)
            .write(false)
            .append(false)
            .open(&index_path)?;

        let filename = index_path.as_ref().file_name().unwrap().to_str().unwrap();
        let base_offset = match u64::from_str_radix(&filename[0..INDEX_FILE_NAME_LEN], 10) {
            Ok(v) => v,
            Err(_) => {
                return Err(io::Error::new(io::ErrorKind::InvalidData,
                                          "Index file name does not parse as u64"))
            }
        };

        let mut mmap = Mmap::open(&index_file, Protection::Read)?;

        // TODO: truncate index file from 0s
        let next_entry = unsafe {
            binary_search(mmap.as_mut_slice(), |x, y| {
                // we're at the first position, its assumed that the
                // file entry is 0 (go right)
                if x == 0 {
                    Ordering::Less
                // if the relative offset is 0, there is potentially
                // another 0 offset less than this position (go left)
                } else if y == 0 {
                    Ordering::Greater
                // else, it's non-zero (go right)
                } else {
                    Ordering::Less
                }
            // always error (nothing equal) so unwrap
            }).err().unwrap()
        };

        info!("Opening index {}, next relative entry {}", filename, next_entry);

        // TODO: truncate file, if necessary

        Ok(Index {
            file: index_file,
            mmap: mmap,
            mode: AccessMode::Read,
            next_write_pos: next_entry * INDEX_ENTRY_BYTES,
            base_offset: base_offset,
        })
    }

    pub fn can_write(&self) -> bool {
        self.mode == AccessMode::ReadWrite && self.size() >= (self.next_write_pos + INDEX_ENTRY_BYTES)
    }

    #[inline]
    pub fn starting_offset(&self) -> u64 {
        self.base_offset
    }

    #[inline]
    pub fn size(&self) -> usize {
        self.mmap.len()
    }

    pub fn append(&mut self, abs_offset: u64, position: u32) -> Result<(), IndexWriteError> {
        trace!("Index append {} => {}", abs_offset, position);

        assert!(abs_offset >= self.base_offset);

        if !self.can_write() {
            return Err(IndexWriteError::IndexFull);
        }

        if abs_offset < self.base_offset {
            return Err(IndexWriteError::OffsetLessThanBase);
        }

        unsafe {
            let mem_slice: &mut [u8] = self.mmap.as_mut_slice();
            let offset = (abs_offset - self.base_offset) as u32;
            let buf_pos = self.next_write_pos;

            BigEndian::write_u32(&mut mem_slice[buf_pos..buf_pos + 4], offset);
            BigEndian::write_u32(&mut mem_slice[buf_pos + 4..buf_pos + 8], position);

            self.next_write_pos += 8;

            Ok(())
        }
    }

    pub fn set_readonly(&mut self) -> io::Result<()> {
        if self.mode != AccessMode::Read {
            self.mode = AccessMode::Read;
            self.flush_sync()
        } else {
            Ok(())
        }
    }

    pub fn flush_sync(&mut self) -> io::Result<()> {
        self.mmap.flush()?;
        self.file.flush()
    }

    #[cfg(test)]
    pub fn read_entry(&self, i: usize) -> Result<Option<IndexEntry>, ()> {
        if self.size() < (i + 1) * 8 {
            return Err(());
        }

        unsafe {
            let mem_slice = self.mmap.as_slice();
            let start = i * 8;
            let offset = BigEndian::read_u32(&mem_slice[start..start + 4]);
            if offset == 0 && i > 0 {
                Ok(None)
            } else {
                let pos = BigEndian::read_u32(&mem_slice[start + 4..start + 8]);
                Ok(Some(IndexEntry {
                    rel_offset: offset,
                    base_offset: self.base_offset,
                    file_pos: pos,
                }))
            }
        }
    }

    pub fn find(&self, offset: u64) -> Option<IndexEntry> {
        if offset < self.base_offset {
            // pathological case... not worth exposing Result
            return None;
        }

        let rel_offset = (offset - self.base_offset) as u32;

        unsafe {
            let mem_slice = self.mmap.as_slice();
            info!("offset={} Next write pos = {}", offset, self.next_write_pos);
            match binary_search(&mem_slice[0..self.next_write_pos],
                                |_, v| v.cmp(&rel_offset)) {
                Ok(i) => {
                    let p = (i * 8) + 4;
                    Some(IndexEntry {
                        rel_offset: rel_offset,
                        base_offset: self.base_offset,
                        file_pos: BigEndian::read_u32(&mem_slice[p..p + 4]),
                    })
                }
                _ => None,
            }
        }

    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use super::super::testutil::*;
    use std::fs;
    use std::path::PathBuf;
    use env_logger;

    #[test]
    pub fn index() {
        let path = TestDir::new();
        let mut index = Index::new(&path, 9u64, 1000usize).unwrap();

        assert_eq!(1000, index.size());
        index.append(11u64, 0xffff).unwrap();
        index.append(12u64, 0xeeee).unwrap();
        index.flush_sync().unwrap();

        let e0 = index.read_entry(0).unwrap().unwrap();
        assert_eq!(2u32, e0.relative_offset());
        assert_eq!(11u64, e0.offset());
        assert_eq!(0xffff, e0.file_position());

        let e1 = index.read_entry(1).unwrap().unwrap();
        assert_eq!(3u32, e1.relative_offset());
        assert_eq!(12u64, e1.offset());
        assert_eq!(0xeeee, e1.file_position());

        // read an entry that does not exist
        let e2 = index.read_entry(2).unwrap();
        assert_eq!(None, e2);
    }

    #[test]
    pub fn index_set_readonly() {
        let path = TestDir::new();
        let mut index = Index::new(&path, 10u64, 1000usize).unwrap();

        index.append(11u64, 0xffff).unwrap();
        index.append(12u64, 0xeeee).unwrap();

        // set_readonly it
        index.set_readonly().expect("Unable to set readonly");

        // append should fail with insertion error
        assert_eq!(index.append(13u64, 0xeeeeee),
                   Err(IndexWriteError::IndexFull));

        let e1 = index.read_entry(1).unwrap().unwrap();
        assert_eq!(2u32, e1.relative_offset());
        assert_eq!(12u64, e1.offset());
        assert_eq!(0xeeee, e1.file_position());

        // read an entry that does not exist
        let e2 = index.read_entry(2).unwrap();
        assert_eq!(None, e2);
    }

    #[test]
    pub fn open_index() {
        let dir = TestDir::new();
        // issue some writes
        {
            let mut index = Index::new(&dir, 10u64, 1000usize).unwrap();
            index.append(10, 0).unwrap();
            index.append(11, 10).unwrap();
            index.append(12, 20).unwrap();
            index.append(13, 30).unwrap();
            index.append(14, 40).unwrap();
            index.set_readonly().unwrap();
        }

        // now open it
        {
            let mut index_path = PathBuf::new();
            index_path.push(&dir);
            index_path.push("00000000000000000010.index");

            let meta = fs::metadata(&index_path).unwrap();
            assert!(meta.is_file());

            let index = Index::open(&index_path).unwrap();

            for i in 0..5usize {
                let e = index.read_entry(i).unwrap();
                assert!(e.is_some());
                assert_eq!(e.unwrap().relative_offset(), i as u32);
                assert_eq!(e.unwrap().offset(), (i + 10) as u64);
                assert_eq!(e.unwrap().file_position(), (i * 10) as u32);
            }
        }
    }

    #[test]
    pub fn find() {
        let dir = TestDir::new();
        let mut index = Index::new(&dir, 10u64, 1000usize).unwrap();
        index.append(10, 1).unwrap();
        index.append(11, 2).unwrap();
        index.append(12, 3).unwrap();
        index.append(15, 4).unwrap();
        index.append(16, 5).unwrap();
        index.append(17, 6).unwrap();
        index.append(18, 7).unwrap();
        index.append(20, 8).unwrap();

        let res = index.find(16).unwrap();
        assert_eq!(6, res.relative_offset());
        assert_eq!(16, res.offset());
        assert_eq!(5, res.file_position());
    }

    #[test]
    pub fn find_nonexistant_value() {
        let dir = TestDir::new();
        let mut index = Index::new(&dir, 10u64, 1000usize).unwrap();
        index.append(10, 1).unwrap();
        index.append(11, 2).unwrap();
        index.append(12, 3).unwrap();
        index.append(15, 4).unwrap();
        index.append(16, 5).unwrap();
        index.append(17, 6).unwrap();
        index.append(18, 7).unwrap();
        index.append(20, 8).unwrap();

        let res = index.find(14);
        assert!(res.is_none());
    }

    #[test]
    pub fn find_out_of_bounds() {
        let dir = TestDir::new();
        let mut index = Index::new(&dir, 10u64, 1000usize).unwrap();
        index.append(10, 1).unwrap();
        index.append(11, 2).unwrap();
        index.append(12, 3).unwrap();
        index.append(15, 4).unwrap();
        index.append(16, 5).unwrap();
        index.append(17, 6).unwrap();
        index.append(18, 7).unwrap();
        index.append(20, 8).unwrap();

        let res = index.find(2);
        assert!(res.is_none());
    }

    #[test]
    pub fn reopen_index() {
        env_logger::init().unwrap_or(());
        let dir = TestDir::new();
        {
            let mut index = Index::new(&dir, 10u64, 1000usize).unwrap();
            index.append(10, 1).unwrap();
            index.append(11, 2).unwrap();
            index.flush_sync().unwrap();
        }

        {
            let mut index_path = PathBuf::new();
            index_path.push(&dir);
            index_path.push("00000000000000000010.index");
            let index = Index::open(&index_path).unwrap();

            let e0 = index.find(10);
            assert!(e0.is_some());
            assert_eq!(0, e0.unwrap().relative_offset());

            let e1 = index.find(11);
            assert!(e1.is_some());
            assert_eq!(1, e1.unwrap().relative_offset());

            let e2 = index.find(12);
            assert!(e2.is_none());
        }
    }
}
<|endoftext|>


<commit_before>use std::fs::{File, OpenOptions, metadata};
use std::path::{Path, PathBuf};

use bincode::SizeLimit;
use bincode::rustc_serialize::{encode_into, decode_from};

use super::Error;
use super::database::Database;
use super::column::Column;
use super::rows::Rows;

use super::super::identifier::Identifier;

pub struct Table<'a> {
    database: &'a Database,
    name: Identifier,
    meta_data: TableMetadata
}

#[derive(Clone, RustcDecodable, RustcEncodable)]
struct TableMetadata {
    engine_id: u8,
    columns: Vec<Column>
}

/// Table representation
impl<'a> Table<'a> {
    /// Create table in database
    pub fn create<'b>(name: &str, engine_id: u8, columns: Vec<Column>, db: &'b Database) -> Result<Table<'b>, Error> {
        info!("creating table: {}", name);

        // TODO: verify if name is a valid identifier

        let metadata = TableMetadata {
            engine_id: engine_id,
            columns: columns
        };

        let mut buf = try!(File::create(db.get_path().clone().join(&name).with_extension("tbl")));

        encode_into(&metadata, &mut buf, SizeLimit::Infinite).unwrap();

        Ok(Table {
            database: &db,
            name: try!(Identifier::new(name)),
            meta_data: metadata.clone()
        })
    }

    /// Load table from database
    pub fn load<'b>(name: &str, db: &'b Database) -> Result<Table<'b>, Error> {
        info!("loading table: {}", &name);
        let path = db.get_path().clone().join(&name).with_extension("tbl");
        if !try!(metadata(&path)).is_dir()
        {
            let mut file = try!(OpenOptions::new()
                        .read(true)
                        .open(&path));

            let metadata: TableMetadata = try!(decode_from(&mut file, SizeLimit::Infinite));
            info!("loaded table: {}", &name);

            Ok(Table {
                database: db,
                name: try!(Identifier::new(name)),
                meta_data: metadata
            })
        }else {
            error!("could not load table: {} at {}", &name, &path.to_str().unwrap());
            Err(Error::LoadTable)
        }
    }

    pub fn get_cols_sizes(&self) -> Vec<u32> {
        let mut column_sizes = Vec::<u32>::new();

        for v in &self.meta_data.columns {
            column_sizes.push(v.size());
        }

        column_sizes
    }

    /// Get full file path including filename and ext
    pub fn get_path(&self) -> PathBuf {
        self.database.get_path().clone().join(&self.name.get_name()).with_extension("tbl")
    }

    pub fn get_engine_id(&self) -> u8 {
        self.meta_data.engine_id
    }

}

#[cfg(test)]
mod test {
    use super::Table;
    use super::super::database::*;
    use std::fs::metadata;
    use super::super::column::Column;
    use super::super::types::Types;

    #[test]
    fn create_table() {
        let path = "/tmp/test1/";
        let name = "test_db3";
        let mut columns = Vec::<Column>::new();
        columns.push(Column::new("Name", Types::Char(10)));
        columns.push(Column::new("Age", Types::Int));

        let db = Database::create(&name, DatabaseConfig::new(&path)).unwrap();
        let table = Table::create("test_table1", 0, columns, &db).unwrap();

        assert!(!metadata("/tmp/test1/test_db3/test_table1.tbl").unwrap().is_dir());
    }

    #[test]
    fn load_table() {
        let path = "/tmp/test1/";
        let name = "test_db4";

        let mut columns = Vec::<Column>::new();
        columns.push(Column::new("Name", Types::Char(10)));
        columns.push(Column::new("Age", Types::Int));

        let db = Database::create(&name, DatabaseConfig::new(&path)).unwrap();
        let table = Table::create("test_table1", 34, columns, &db).unwrap();

        assert!(!metadata("/tmp/test1/test_db3/test_table1.tbl").unwrap().is_dir());

        let table2 = Table::load("test_table1", &db).unwrap();

        assert_eq!(table.get_engine_id(), table2.get_engine_id());
        assert_eq!(&table.meta_data.columns, &table2.meta_data.columns);
    }
}
<commit_msg>added table header offset<commit_after>use std::fs::{File, OpenOptions, metadata};
use std::path::{Path, PathBuf};

use bincode::SizeLimit;
use bincode::rustc_serialize::{encode, encode_into, decode_from};

use super::Error;
use super::database::Database;
use super::column::Column;
use super::rows::Rows;

use super::super::identifier::Identifier;

pub struct Table<'a> {
    database: &'a Database,
    name: Identifier,
    meta_data: TableMetadata
}

#[derive(Clone, RustcDecodable, RustcEncodable)]
struct TableMetadata {
    engine_id: u8,
    columns: Vec<Column>
}

/// Table representation
impl<'a> Table<'a> {
    /// Create table in database
    pub fn create<'b>(name: &str, engine_id: u8, columns: Vec<Column>, db: &'b Database) -> Result<Table<'b>, Error> {
        info!("creating table: {}", name);

        // TODO: verify if name is a valid identifier

        let metadata = TableMetadata {
            engine_id: engine_id,
            columns: columns
        };

        let mut buf = try!(File::create(db.get_path().clone().join(&name).with_extension("tbl")));

        encode_into(&metadata, &mut buf, SizeLimit::Infinite).unwrap();

        Ok(Table {
            database: &db,
            name: try!(Identifier::new(name)),
            meta_data: metadata.clone()
        })
    }

    /// Load table from database
    pub fn load<'b>(name: &str, db: &'b Database) -> Result<Table<'b>, Error> {
        info!("loading table: {}", &name);
        let path = db.get_path().clone().join(&name).with_extension("tbl");
        if !try!(metadata(&path)).is_dir()
        {
            let mut file = try!(OpenOptions::new()
                        .read(true)
                        .open(&path));

            let metadata: TableMetadata = try!(decode_from(&mut file, SizeLimit::Infinite));
            info!("loaded table: {}", &name);

            Ok(Table {
                database: db,
                name: try!(Identifier::new(name)),
                meta_data: metadata
            })
        }else {
            error!("could not load table: {} at {}", &name, &path.to_str().unwrap());
            Err(Error::LoadTable)
        }
    }

    pub fn get_cols_sizes(&self) -> Vec<u32> {
        let mut column_sizes = Vec::<u32>::new();

        for v in &self.meta_data.columns {
            column_sizes.push(v.size());
        }

        column_sizes
    }

    pub fn get_table_header_offset(&self) -> u64 {
        let bytes: Vec<u8> = encode(&self.meta_data, SizeLimit::Infinite).unwrap();
        bytes.len() as u64
    }

    /// Get full file path including filename and ext
    pub fn get_path(&self) -> PathBuf {
        self.database.get_path().clone().join(&self.name.get_name()).with_extension("tbl")
    }

    pub fn get_engine_id(&self) -> u8 {
        self.meta_data.engine_id
    }

}

#[cfg(test)]
mod test {
    use super::Table;
    use super::super::database::*;
    use std::fs::metadata;
    use super::super::column::Column;
    use super::super::types::Types;

    #[test]
    fn create_table() {
        let path = "/tmp/test1/";
        let name = "test_db3";
        let mut columns = Vec::<Column>::new();
        columns.push(Column::new("Name", Types::Char(10)));
        columns.push(Column::new("Age", Types::Int));

        let db = Database::create(&name, DatabaseConfig::new(&path)).unwrap();
        let table = Table::create("test_table1", 0, columns, &db).unwrap();

        assert!(!metadata("/tmp/test1/test_db3/test_table1.tbl").unwrap().is_dir());
    }

    #[test]
    fn load_table() {
        let path = "/tmp/test1/";
        let name = "test_db4";

        let mut columns = Vec::<Column>::new();
        columns.push(Column::new("Name", Types::Char(10)));
        columns.push(Column::new("Age", Types::Int));

        let db = Database::create(&name, DatabaseConfig::new(&path)).unwrap();
        let table = Table::create("test_table1", 34, columns, &db).unwrap();

        assert!(!metadata("/tmp/test1/test_db3/test_table1.tbl").unwrap().is_dir());

        let table2 = Table::load("test_table1", &db).unwrap();

        assert_eq!(table.get_engine_id(), table2.get_engine_id());
        assert_eq!(&table.meta_data.columns, &table2.meta_data.columns);
    }
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::time::Duration;

#[cfg(feature = "dbus_enabled")]
use tokio::sync::mpsc::UnboundedSender;
use tokio::{task::spawn, time::sleep};

#[cfg(feature = "dbus_enabled")]
use crate::dbus_api::DbusAction;
use crate::{
    engine::{Engine, LockableEngine},
    stratis::errors::StratisResult,
};

/// Runs checks on thin pool usage and filesystem usage to determine whether either
/// need to be extended.
async fn check_pool_and_fs<E>(
    engine: LockableEngine<E>,
    #[cfg(feature = "dbus_enabled")] sender: UnboundedSender<DbusAction<E>>,
) -> StratisResult<()>
where
    E: Engine,
{
    loop {
        sleep(Duration::from_secs(10)).await;
        let mut lock = engine.lock().await;
        #[cfg(feature = "min")]
        {
            let _ = lock.pool_evented(None)?;
            let _ = lock.fs_evented(None)?;
        }
        #[cfg(feature = "dbus_enabled")]
        {
            let pool_diffs = lock.pool_evented(None)?;
            for action in DbusAction::from_pool_diffs(pool_diffs) {
                if let Err(e) = sender.send(action) {
                    warn!(
                        "Failed to update D-Bus API with information on changed properties: {}",
                        e
                    );
                }
            }
            let fs_diffs = lock.fs_evented(None)?;
            for action in DbusAction::from_fs_diffs(fs_diffs) {
                if let Err(e) = sender.send(action) {
                    warn!(
                        "Failed to update D-Bus API with information on changed properties: {}",
                        e
                    );
                }
            }
        }
    }
}

/// Run all timed background tasks.
///
/// Currently runs a timer to check thin pool and filesystem usage.
pub async fn run_timers<E>(
    engine: LockableEngine<E>,
    #[cfg(feature = "dbus_enabled")] sender: UnboundedSender<DbusAction<E>>,
) -> StratisResult<()>
where
    E: 'static + Engine,
{
    spawn(check_pool_and_fs(
        engine,
        #[cfg(feature = "dbus_enabled")]
        sender,
    ))
    .await??;
    Ok(())
}
<commit_msg>Fix behavior on timed pool and filesystem check failure<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::time::Duration;

#[cfg(feature = "dbus_enabled")]
use tokio::sync::mpsc::UnboundedSender;
use tokio::{task::spawn, time::sleep};

#[cfg(feature = "dbus_enabled")]
use crate::dbus_api::DbusAction;
use crate::{
    engine::{Engine, LockableEngine},
    stratis::errors::StratisResult,
};

/// Runs checks on thin pool usage and filesystem usage to determine whether either
/// need to be extended.
async fn check_pool_and_fs<E>(
    engine: LockableEngine<E>,
    #[cfg(feature = "dbus_enabled")] sender: UnboundedSender<DbusAction<E>>,
) where
    E: Engine,
{
    async fn process_checks<E>(
        engine: &LockableEngine<E>,
        #[cfg(feature = "dbus_enabled")] sender: &UnboundedSender<DbusAction<E>>,
    ) -> StratisResult<()>
    where
        E: Engine,
    {
        let mut lock = engine.lock().await;
        #[cfg(feature = "min")]
        {
            let _ = lock.pool_evented(None)?;
            let _ = lock.fs_evented(None)?;
        }
        #[cfg(feature = "dbus_enabled")]
        {
            let pool_diffs = lock.pool_evented(None)?;
            for action in DbusAction::from_pool_diffs(pool_diffs) {
                if let Err(e) = sender.send(action) {
                    warn!(
                        "Failed to update D-Bus API with information on changed properties: {}",
                        e
                    );
                }
            }
            let fs_diffs = lock.fs_evented(None)?;
            for action in DbusAction::from_fs_diffs(fs_diffs) {
                if let Err(e) = sender.send(action) {
                    warn!(
                        "Failed to update D-Bus API with information on changed properties: {}",
                        e
                    );
                }
            }
        }
        Ok(())
    }

    loop {
        if let Err(e) = process_checks(
            &engine,
            #[cfg(feature = "dbus_enabled")]
            &sender,
        )
        .await
        {
            warn!("Failed to handle timed pool and filesystem checks: {}", e);
        }
        sleep(Duration::from_secs(10)).await;
    }
}

/// Run all timed background tasks.
///
/// Currently runs a timer to check thin pool and filesystem usage.
pub async fn run_timers<E>(
    engine: LockableEngine<E>,
    #[cfg(feature = "dbus_enabled")] sender: UnboundedSender<DbusAction<E>>,
) -> StratisResult<()>
where
    E: 'static + Engine,
{
    spawn(check_pool_and_fs(
        engine,
        #[cfg(feature = "dbus_enabled")]
        sender,
    ))
    .await?;
    Ok(())
}
<|endoftext|>


<commit_before>#![deny(warnings)]
#![feature(asm)]
#![feature(const_fn)]
#![feature(process_exec)]

extern crate orbclient;

#[cfg(not(target_os = "redox"))]
extern crate libc;

#[cfg(target_os = "redox")]
extern crate syscall;

use orbclient::event;
use std::{env, str};
use std::error::Error;
use std::fs::{File, OpenOptions};
use std::io::{self, Result, Read, Write};
use std::os::unix::io::{FromRawFd, IntoRawFd, RawFd};
use std::os::unix::process::CommandExt;
use std::process::{Command, Stdio};

use console::Console;
use getpty::getpty;

mod console;
mod getpty;

#[cfg(not(target_os="redox"))]
pub fn before_exec() -> Result<()> {
    use libc;
    unsafe {
        libc::setsid();
        libc::ioctl(0, libc::TIOCSCTTY, 1);
    }
    Ok(())
}

#[cfg(target_os="redox")]
pub fn before_exec() -> Result<()> {
    Ok(())
}

#[cfg(target_os = "redox")]
fn handle(console: &mut Console, master_fd: RawFd) {
    extern crate syscall;

    use std::os::unix::io::AsRawFd;

    let mut event_file = File::open("event:").expect("terminal: failed to open event file");

    let window_fd = console.window.as_raw_fd();
    syscall::fevent(window_fd, syscall::flag::EVENT_READ).expect("terminal: failed to fevent console window");

    let mut master = unsafe { File::from_raw_fd(master_fd) };
    syscall::fevent(master_fd, syscall::flag::EVENT_READ).expect("terminal: failed to fevent master PTY");

    let mut handle_event = |event_id: usize, event_count: usize| -> bool {
        if event_id == window_fd {
            for event in console.window.events() {
                if event.code == event::EVENT_QUIT {
                    return false;
                }

                console.input(&event);
            }

            if ! console.input.is_empty()  {
                if let Err(err) = master.write(&console.input) {
                    let term_stderr = io::stderr();
                    let mut term_stderr = term_stderr.lock();

                    let _ = term_stderr.write(b"failed to write stdin: ");
                    let _ = term_stderr.write(err.description().as_bytes());
                    let _ = term_stderr.write(b"\n");
                    return false;
                }
                console.input.clear();
            }
        } else if event_id == master_fd {
            let mut packet = [0; 4096];
            let count = master.read(&mut packet).expect("terminal: failed to read master PTY");
            if count == 0 {
                if event_count == 0 {
                    return false;
                }
            } else {
                console.write(&packet[1..count], true).expect("terminal: failed to write to console");

                if packet[0] & 1 == 1 {
                    console.redraw();
                }
            }
        } else {
            println!("Unknown event {}", event_id);
        }

        true
    };

    handle_event(window_fd, 0);
    handle_event(master_fd, 0);

    'events: loop {
        let mut sys_event = syscall::Event::default();
        event_file.read(&mut sys_event).expect("terminal: failed to read event file");
        if ! handle_event(sys_event.id, sys_event.data) {
            break 'events;
        }
    }
}

#[cfg(not(target_os = "redox"))]
fn handle(console: &mut Console, master_fd: RawFd) {
    use libc;
    use std::io::ErrorKind;
    use std::thread;
    use std::time::Duration;

    unsafe {
        let size = libc::winsize {
            ws_row: console.console.h as libc::c_ushort,
            ws_col: console.console.w as libc::c_ushort,
            ws_xpixel: 0,
            ws_ypixel: 0
        };
        libc::ioctl(master_fd, libc::TIOCSWINSZ, &size as *const libc::winsize);
    }

    console.console.raw_mode = true;

    let mut master = unsafe { File::from_raw_fd(master_fd) };

    loop {
        for event in console.window.events() {
            if event.code == event::EVENT_QUIT {
                return;
            }

            console.input(&event);
        }

        if ! console.input.is_empty()  {
            if let Err(err) = master.write(&console.input) {
                let term_stderr = io::stderr();
                let mut term_stderr = term_stderr.lock();

                let _ = term_stderr.write(b"failed to write stdin: ");
                let _ = term_stderr.write(err.description().as_bytes());
                let _ = term_stderr.write(b"\n");
                return;
            }
            console.input.clear();
        }

        let mut packet = [0; 4096];
        match master.read(&mut packet) {
            Ok(count) => if count == 0 {
                return;
            } else {
                console.write(&packet[1..count], true).expect("terminal: failed to write to console");

                //if packet[0] & 1 == 1
                {
                    console.redraw();
                }
            },
            Err(err) => match err.kind() {
                ErrorKind::WouldBlock => (),
                _ => panic!("terminal: failed to read master PTY: {:?}", err)
            }
        }

        thread::sleep(Duration::new(0, 100));
    }
}

fn main() {
    let shell = env::args().nth(1).unwrap_or("sh".to_string());

    let (master_fd, tty_path) = getpty();

    let slave_stdin = OpenOptions::new().read(true).write(true).open(&tty_path).unwrap();
    let slave_stdout = OpenOptions::new().read(true).write(true).open(&tty_path).unwrap();
    let slave_stderr = OpenOptions::new().read(true).write(true).open(&tty_path).unwrap();

    let width = 800;
    let height = 600;

    env::set_var("COLUMNS", format!("{}", width / 8));
    env::set_var("LINES", format!("{}", height / 16));
    env::set_var("TERM", "linux");
    env::set_var("TTY", format!("{}", tty_path.display()));

    match unsafe {
        Command::new(&shell)
            .stdin(Stdio::from_raw_fd(slave_stdin.into_raw_fd()))
            .stdout(Stdio::from_raw_fd(slave_stdout.into_raw_fd()))
            .stderr(Stdio::from_raw_fd(slave_stderr.into_raw_fd()))
            .before_exec(|| {
                before_exec()
            })
            .spawn()
    } {
        Ok(mut process) => {
            let mut console = Console::new(width, height);
            handle(&mut console, master_fd);

            let _ = process.kill();
            process.wait().expect("terminal: failed to wait on shell");
        },
        Err(err) => {
            let term_stderr = io::stderr();
            let mut term_stderr = term_stderr.lock();
            let _ = term_stderr.write(b"failed to execute '");
            let _ = term_stderr.write(shell.as_bytes());
            let _ = term_stderr.write(b"': ");
            let _ = term_stderr.write(err.description().as_bytes());
            let _ = term_stderr.write(b"\n");
        }
    }
}
<commit_msg>Remove unnecessary feature process_exec<commit_after>#![deny(warnings)]
#![feature(asm)]
#![feature(const_fn)]

extern crate orbclient;

#[cfg(not(target_os = "redox"))]
extern crate libc;

#[cfg(target_os = "redox")]
extern crate syscall;

use orbclient::event;
use std::{env, str};
use std::error::Error;
use std::fs::{File, OpenOptions};
use std::io::{self, Result, Read, Write};
use std::os::unix::io::{FromRawFd, IntoRawFd, RawFd};
use std::os::unix::process::CommandExt;
use std::process::{Command, Stdio};

use console::Console;
use getpty::getpty;

mod console;
mod getpty;

#[cfg(not(target_os="redox"))]
pub fn before_exec() -> Result<()> {
    use libc;
    unsafe {
        libc::setsid();
        libc::ioctl(0, libc::TIOCSCTTY, 1);
    }
    Ok(())
}

#[cfg(target_os="redox")]
pub fn before_exec() -> Result<()> {
    Ok(())
}

#[cfg(target_os = "redox")]
fn handle(console: &mut Console, master_fd: RawFd) {
    extern crate syscall;

    use std::os::unix::io::AsRawFd;

    let mut event_file = File::open("event:").expect("terminal: failed to open event file");

    let window_fd = console.window.as_raw_fd();
    syscall::fevent(window_fd, syscall::flag::EVENT_READ).expect("terminal: failed to fevent console window");

    let mut master = unsafe { File::from_raw_fd(master_fd) };
    syscall::fevent(master_fd, syscall::flag::EVENT_READ).expect("terminal: failed to fevent master PTY");

    let mut handle_event = |event_id: usize, event_count: usize| -> bool {
        if event_id == window_fd {
            for event in console.window.events() {
                if event.code == event::EVENT_QUIT {
                    return false;
                }

                console.input(&event);
            }

            if ! console.input.is_empty()  {
                if let Err(err) = master.write(&console.input) {
                    let term_stderr = io::stderr();
                    let mut term_stderr = term_stderr.lock();

                    let _ = term_stderr.write(b"failed to write stdin: ");
                    let _ = term_stderr.write(err.description().as_bytes());
                    let _ = term_stderr.write(b"\n");
                    return false;
                }
                console.input.clear();
            }
        } else if event_id == master_fd {
            let mut packet = [0; 4096];
            let count = master.read(&mut packet).expect("terminal: failed to read master PTY");
            if count == 0 {
                if event_count == 0 {
                    return false;
                }
            } else {
                console.write(&packet[1..count], true).expect("terminal: failed to write to console");

                if packet[0] & 1 == 1 {
                    console.redraw();
                }
            }
        } else {
            println!("Unknown event {}", event_id);
        }

        true
    };

    handle_event(window_fd, 0);
    handle_event(master_fd, 0);

    'events: loop {
        let mut sys_event = syscall::Event::default();
        event_file.read(&mut sys_event).expect("terminal: failed to read event file");
        if ! handle_event(sys_event.id, sys_event.data) {
            break 'events;
        }
    }
}

#[cfg(not(target_os = "redox"))]
fn handle(console: &mut Console, master_fd: RawFd) {
    use libc;
    use std::io::ErrorKind;
    use std::thread;
    use std::time::Duration;

    unsafe {
        let size = libc::winsize {
            ws_row: console.console.h as libc::c_ushort,
            ws_col: console.console.w as libc::c_ushort,
            ws_xpixel: 0,
            ws_ypixel: 0
        };
        libc::ioctl(master_fd, libc::TIOCSWINSZ, &size as *const libc::winsize);
    }

    console.console.raw_mode = true;

    let mut master = unsafe { File::from_raw_fd(master_fd) };

    loop {
        for event in console.window.events() {
            if event.code == event::EVENT_QUIT {
                return;
            }

            console.input(&event);
        }

        if ! console.input.is_empty()  {
            if let Err(err) = master.write(&console.input) {
                let term_stderr = io::stderr();
                let mut term_stderr = term_stderr.lock();

                let _ = term_stderr.write(b"failed to write stdin: ");
                let _ = term_stderr.write(err.description().as_bytes());
                let _ = term_stderr.write(b"\n");
                return;
            }
            console.input.clear();
        }

        let mut packet = [0; 4096];
        match master.read(&mut packet) {
            Ok(count) => if count == 0 {
                return;
            } else {
                console.write(&packet[1..count], true).expect("terminal: failed to write to console");

                //if packet[0] & 1 == 1
                {
                    console.redraw();
                }
            },
            Err(err) => match err.kind() {
                ErrorKind::WouldBlock => (),
                _ => panic!("terminal: failed to read master PTY: {:?}", err)
            }
        }

        thread::sleep(Duration::new(0, 100));
    }
}

fn main() {
    let shell = env::args().nth(1).unwrap_or("sh".to_string());

    let (master_fd, tty_path) = getpty();

    let slave_stdin = OpenOptions::new().read(true).write(true).open(&tty_path).unwrap();
    let slave_stdout = OpenOptions::new().read(true).write(true).open(&tty_path).unwrap();
    let slave_stderr = OpenOptions::new().read(true).write(true).open(&tty_path).unwrap();

    let width = 800;
    let height = 600;

    env::set_var("COLUMNS", format!("{}", width / 8));
    env::set_var("LINES", format!("{}", height / 16));
    env::set_var("TERM", "linux");
    env::set_var("TTY", format!("{}", tty_path.display()));

    match unsafe {
        Command::new(&shell)
            .stdin(Stdio::from_raw_fd(slave_stdin.into_raw_fd()))
            .stdout(Stdio::from_raw_fd(slave_stdout.into_raw_fd()))
            .stderr(Stdio::from_raw_fd(slave_stderr.into_raw_fd()))
            .before_exec(|| {
                before_exec()
            })
            .spawn()
    } {
        Ok(mut process) => {
            let mut console = Console::new(width, height);
            handle(&mut console, master_fd);

            let _ = process.kill();
            process.wait().expect("terminal: failed to wait on shell");
        },
        Err(err) => {
            let term_stderr = io::stderr();
            let mut term_stderr = term_stderr.lock();
            let _ = term_stderr.write(b"failed to execute '");
            let _ = term_stderr.write(shell.as_bytes());
            let _ = term_stderr.write(b"': ");
            let _ = term_stderr.write(err.description().as_bytes());
            let _ = term_stderr.write(b"\n");
        }
    }
}
<|endoftext|>


<commit_before>use std::sync::Arc;
use std::path::Path;

use img;

use ::Point2i;
use interaction::SurfaceInteraction;
use mipmap::{MIPMap, WrapMode};
use spectrum::Spectrum;
use texture::{Texture, TextureMapping2D, UVMapping2D};

pub struct ImageTexture {
    mapping: Box<TextureMapping2D + Send + Sync>,
    mipmap: Arc<MIPMap<Spectrum>>,
}

impl ImageTexture {
    pub fn new(path: &Path) -> ImageTexture {
        info!("Loading texture {}", path.display());
        // TODO log warning and use constant texture if cannot open texture file
        let buf = img::open(path).unwrap();
        let rgb = buf.to_rgb();
        let res = Point2i::new(rgb.width(), rgb.height());
        let pixels: Vec<Spectrum> = rgb.pixels()
            .map(|p| Spectrum::from_srgb(&p.data))
            .collect();

        ImageTexture {
            mapping: Box::new(UVMapping2D::new(2.0, 2.0, 0.0, 0.0)),
            mipmap: Arc::new(MIPMap::new(&res, &pixels[..], false, 0.0, WrapMode::Repeat)),
        }
    }
}

impl Texture<Spectrum> for ImageTexture {
    fn evaluate(&self, si: &SurfaceInteraction) -> Spectrum {
        let st = self.mapping.map(si);
        // TODO Call correct lookup method once we have ray differentials
        self.mipmap.lookup(&st, 0.0)
    }
}

#[test]
fn load_texture() {
    ImageTexture::new(&Path::new("lines.png"));
    assert!(true);
}
<commit_msg>Don't gamma-correct data read from texture files.<commit_after>use std::sync::Arc;
use std::path::Path;

use img;

use ::Point2i;
use interaction::SurfaceInteraction;
use mipmap::{MIPMap, WrapMode};
use spectrum::Spectrum;
use texture::{Texture, TextureMapping2D, UVMapping2D};

pub struct ImageTexture {
    mapping: Box<TextureMapping2D + Send + Sync>,
    mipmap: Arc<MIPMap<Spectrum>>,
}

impl ImageTexture {
    pub fn new(path: &Path) -> ImageTexture {
        info!("Loading texture {}", path.display());
        // TODO log warning and use constant texture if cannot open texture file
        let buf = img::open(path).unwrap();
        let rgb = buf.to_rgb();
        let res = Point2i::new(rgb.width(), rgb.height());
        let pixels: Vec<Spectrum> = rgb.pixels()
            .map(|p| {
                let r = p.data[0] as f32 / 255.0;
                let g = p.data[1] as f32 / 255.0;
                let b = p.data[2] as f32 / 255.0;
                Spectrum::rgb(r, g, b)
            })
            .collect();

        ImageTexture {
            mapping: Box::new(UVMapping2D::new(2.0, 2.0, 0.0, 0.0)),
            mipmap: Arc::new(MIPMap::new(&res, &pixels[..], false, 0.0, WrapMode::Repeat)),
        }
    }
}

impl Texture<Spectrum> for ImageTexture {
    fn evaluate(&self, si: &SurfaceInteraction) -> Spectrum {
        let st = self.mapping.map(si);
        // TODO Call correct lookup method once we have ray differentials
        self.mipmap.lookup(&st, 0.0)
    }
}

#[test]
fn load_texture() {
    ImageTexture::new(&Path::new("lines.png"));
    assert!(true);
}
<|endoftext|>