<commit_before>use std::sync::RwLock;
use std::collections::{BTreeMap, HashMap};

use rocksdb::{DB, Writable, WriteBatch, IteratorMode, Direction};
use byteorder::{ByteOrder, BigEndian, WriteBytesExt};

use errors::{RocksDBReadError, RocksDBWriteError};
use key_builder::KeyBuilder;
use segment_ops::SegmentMergeError;
use search::doc_id_set::DocIdSet;

#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
pub struct DocRef(u32, u16);

impl DocRef {
    pub fn segment(&self) -> u32 {
        self.0
    }

    pub fn ord(&self) -> u16 {
        self.1
    }

    pub fn as_u64(&self) -> u64 {
        (self.0 as u64) << 16 | (self.1 as u64)
    }

    pub fn from_segment_ord(segment: u32, ord: u16) -> DocRef {
        DocRef(segment, ord)
    }

    pub fn from_u64(val: u64) -> DocRef {
        let segment = (val >> 16) & 0xFFFFFFFF;
        let ord = val & 0xFFFF;
        DocRef(segment as u32, ord as u16)
    }
}

/// Manages the index's "document index"
pub struct DocumentIndexManager {
    primary_key_index: RwLock<BTreeMap<Vec<u8>, DocRef>>,
}

impl DocumentIndexManager {
    /// Generates a new document index
    pub fn new(_db: &DB) -> Result<DocumentIndexManager, RocksDBWriteError> {
        Ok(DocumentIndexManager {
            primary_key_index: RwLock::new(BTreeMap::new()),
        })
    }

    /// Loads the document index from an index
    pub fn open(db: &DB) -> Result<DocumentIndexManager, RocksDBReadError> {
        // Read primary key index
        let mut primary_key_index = BTreeMap::new();
        for (k, v) in db.iterator(IteratorMode::From(b"k", Direction::Forward)) {
            if k[0] != b'k' {
                break;
            }

            let segment = BigEndian::read_u32(&v[0..4]);
            let ord = BigEndian::read_u16(&v[4..6]);
            let doc_ref = DocRef::from_segment_ord(segment, ord);

            primary_key_index.insert(k[1..].to_vec(), doc_ref);
        }

        Ok(DocumentIndexManager {
            primary_key_index: RwLock::new(primary_key_index),
        })
    }

    fn delete_document_by_ref_unchecked(&self, write_batch: &WriteBatch, doc_ref: DocRef) -> Result<(), RocksDBWriteError> {
        let kb = KeyBuilder::segment_del_list(doc_ref.segment());
        let mut previous_doc_id_bytes = [0; 2];
        BigEndian::write_u16(&mut previous_doc_id_bytes, doc_ref.ord());
        if let Err(e) = write_batch.merge(&kb.key(), &previous_doc_id_bytes) {
            return Err(RocksDBWriteError::new_merge(kb.key().to_vec(), e));
        }

        // Increment deleted docs
        let kb = KeyBuilder::segment_stat(doc_ref.segment(), b"deleted_docs");
        let mut inc_bytes = [0; 8];
        BigEndian::write_i64(&mut inc_bytes, 1);
        if let Err(e) = write_batch.merge(&kb.key(), &inc_bytes) {
            return Err(RocksDBWriteError::new_merge(kb.key().to_vec(), e));
        }

        Ok(())
    }

    pub fn insert_or_replace_key(&self, db: &DB, key: &Vec<u8>, doc_ref: DocRef) -> Result<Option<DocRef>, RocksDBWriteError> {
        // Update primary_key_index
        let write_batch = WriteBatch::new();
        let previous_doc_ref = self.primary_key_index.write().unwrap().insert(key.clone(), doc_ref);

        let kb = KeyBuilder::primary_key_index(key);
        let mut doc_ref_bytes = [0; 6];
        BigEndian::write_u32(&mut doc_ref_bytes, doc_ref.segment());
        BigEndian::write_u16(&mut doc_ref_bytes[4..], doc_ref.ord());
        if let Err(e) = write_batch.put(&kb.key(), &doc_ref_bytes) {
            return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e));
        }

        // If there was a document there previously, delete it
        if let Some(previous_doc_ref) = previous_doc_ref {
            try!(self.delete_document_by_ref_unchecked(&write_batch, previous_doc_ref));
        }

        // Write document data
        if let Err(e) = db.write(write_batch) {
            return Err(RocksDBWriteError::new_commit_write_batch(e));
        }

        Ok(previous_doc_ref)
    }

    pub fn delete_document_by_key(&self, db: &DB, key: &Vec<u8>) -> Result<Option<DocRef>, RocksDBWriteError> {
        // Remove document from index
        let doc_ref = self.primary_key_index.write().unwrap().remove(key);

        if let Some(doc_ref) = doc_ref {
            let mut write_batch = WriteBatch::new();

            try!(self.delete_document_by_ref_unchecked(&mut write_batch, doc_ref));

            if let Err(e) = db.write(write_batch) {
                return Err(RocksDBWriteError::new_commit_write_batch(e));
            }
        }

        Ok(doc_ref)
    }

    pub fn contains_document_key(&self, key: &Vec<u8>) -> bool {
        self.primary_key_index.read().unwrap().contains_key(key)
    }

    pub fn commit_segment_merge(&self, db: &DB, write_batch: WriteBatch, source_segments: &Vec<u32>, dest_segment: u32, doc_ref_mapping: &HashMap<DocRef, u16>) -> Result<(), SegmentMergeError> {
        // Lock the primary key index
        let mut primary_key_index = self.primary_key_index.write().unwrap();

        // Update primary keys to point to their new locations
        let mut keys_to_update: HashMap<Vec<u8>, DocRef> = HashMap::with_capacity(doc_ref_mapping.len());
        for (key, doc_ref) in primary_key_index.iter() {
            if doc_ref_mapping.contains_key(&doc_ref) {
                keys_to_update.insert(key.clone(), *doc_ref);
            }
        }

        for (key, doc_ref) in keys_to_update {
            let new_doc_ord = doc_ref_mapping.get(&doc_ref).unwrap();
            let new_doc_ref = DocRef::from_segment_ord(dest_segment, *new_doc_ord);

            primary_key_index.insert(key, new_doc_ref);
        }

        // Merge deletion lists
        // Must be done while the primary_key_index is locked as this prevents any more documents being deleted
        let mut deletion_list = Vec::new();
        for source_segment in source_segments {
            let kb = KeyBuilder::segment_del_list(*source_segment);
            match db.get(&kb.key()) {
                Ok(Some(docid_set)) => {
                    for doc_id in DocIdSet::FromRDB(docid_set).iter() {
                        let doc_ref = DocRef::from_segment_ord(*source_segment, doc_id);
                        let new_doc_id = doc_ref_mapping.get(&doc_ref).unwrap();
                        deletion_list.write_u16::<BigEndian>(*new_doc_id).unwrap();
                    }
                }
                Ok(None) => {},
                Err(e) => return Err(RocksDBReadError::new(kb.key().to_vec(), e).into()),
            }
        }

        let kb = KeyBuilder::segment_del_list(dest_segment);
        if let Err(e) = db.put(&kb.key(), &deletion_list) {
            return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e).into());
        }

        // Commit!
        if let Err(e) = db.write(write_batch) {
            return Err(RocksDBWriteError::new_commit_write_batch(e).into());
        }

        Ok(())
    }
}
<commit_msg>Flush PK index change to disk after segment merge<commit_after>use std::sync::RwLock;
use std::collections::{BTreeMap, HashMap};

use rocksdb::{DB, Writable, WriteBatch, IteratorMode, Direction};
use byteorder::{ByteOrder, BigEndian, WriteBytesExt};

use errors::{RocksDBReadError, RocksDBWriteError};
use key_builder::KeyBuilder;
use segment_ops::SegmentMergeError;
use search::doc_id_set::DocIdSet;

#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
pub struct DocRef(u32, u16);

impl DocRef {
    pub fn segment(&self) -> u32 {
        self.0
    }

    pub fn ord(&self) -> u16 {
        self.1
    }

    pub fn as_u64(&self) -> u64 {
        (self.0 as u64) << 16 | (self.1 as u64)
    }

    pub fn from_segment_ord(segment: u32, ord: u16) -> DocRef {
        DocRef(segment, ord)
    }

    pub fn from_u64(val: u64) -> DocRef {
        let segment = (val >> 16) & 0xFFFFFFFF;
        let ord = val & 0xFFFF;
        DocRef(segment as u32, ord as u16)
    }
}

/// Manages the index's "document index"
pub struct DocumentIndexManager {
    primary_key_index: RwLock<BTreeMap<Vec<u8>, DocRef>>,
}

impl DocumentIndexManager {
    /// Generates a new document index
    pub fn new(_db: &DB) -> Result<DocumentIndexManager, RocksDBWriteError> {
        Ok(DocumentIndexManager {
            primary_key_index: RwLock::new(BTreeMap::new()),
        })
    }

    /// Loads the document index from an index
    pub fn open(db: &DB) -> Result<DocumentIndexManager, RocksDBReadError> {
        // Read primary key index
        let mut primary_key_index = BTreeMap::new();
        for (k, v) in db.iterator(IteratorMode::From(b"k", Direction::Forward)) {
            if k[0] != b'k' {
                break;
            }

            let segment = BigEndian::read_u32(&v[0..4]);
            let ord = BigEndian::read_u16(&v[4..6]);
            let doc_ref = DocRef::from_segment_ord(segment, ord);

            primary_key_index.insert(k[1..].to_vec(), doc_ref);
        }

        Ok(DocumentIndexManager {
            primary_key_index: RwLock::new(primary_key_index),
        })
    }

    fn delete_document_by_ref_unchecked(&self, write_batch: &WriteBatch, doc_ref: DocRef) -> Result<(), RocksDBWriteError> {
        let kb = KeyBuilder::segment_del_list(doc_ref.segment());
        let mut previous_doc_id_bytes = [0; 2];
        BigEndian::write_u16(&mut previous_doc_id_bytes, doc_ref.ord());
        if let Err(e) = write_batch.merge(&kb.key(), &previous_doc_id_bytes) {
            return Err(RocksDBWriteError::new_merge(kb.key().to_vec(), e));
        }

        // Increment deleted docs
        let kb = KeyBuilder::segment_stat(doc_ref.segment(), b"deleted_docs");
        let mut inc_bytes = [0; 8];
        BigEndian::write_i64(&mut inc_bytes, 1);
        if let Err(e) = write_batch.merge(&kb.key(), &inc_bytes) {
            return Err(RocksDBWriteError::new_merge(kb.key().to_vec(), e));
        }

        Ok(())
    }

    pub fn insert_or_replace_key(&self, db: &DB, key: &Vec<u8>, doc_ref: DocRef) -> Result<Option<DocRef>, RocksDBWriteError> {
        // Update primary_key_index
        let write_batch = WriteBatch::new();
        let previous_doc_ref = self.primary_key_index.write().unwrap().insert(key.clone(), doc_ref);

        let kb = KeyBuilder::primary_key_index(key);
        let mut doc_ref_bytes = [0; 6];
        BigEndian::write_u32(&mut doc_ref_bytes, doc_ref.segment());
        BigEndian::write_u16(&mut doc_ref_bytes[4..], doc_ref.ord());
        if let Err(e) = write_batch.put(&kb.key(), &doc_ref_bytes) {
            return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e));
        }

        // If there was a document there previously, delete it
        if let Some(previous_doc_ref) = previous_doc_ref {
            try!(self.delete_document_by_ref_unchecked(&write_batch, previous_doc_ref));
        }

        // Write document data
        if let Err(e) = db.write(write_batch) {
            return Err(RocksDBWriteError::new_commit_write_batch(e));
        }

        Ok(previous_doc_ref)
    }

    pub fn delete_document_by_key(&self, db: &DB, key: &Vec<u8>) -> Result<Option<DocRef>, RocksDBWriteError> {
        // Remove document from index
        let doc_ref = self.primary_key_index.write().unwrap().remove(key);

        if let Some(doc_ref) = doc_ref {
            let mut write_batch = WriteBatch::new();

            try!(self.delete_document_by_ref_unchecked(&mut write_batch, doc_ref));

            if let Err(e) = db.write(write_batch) {
                return Err(RocksDBWriteError::new_commit_write_batch(e));
            }
        }

        Ok(doc_ref)
    }

    pub fn contains_document_key(&self, key: &Vec<u8>) -> bool {
        self.primary_key_index.read().unwrap().contains_key(key)
    }

    pub fn commit_segment_merge(&self, db: &DB, write_batch: WriteBatch, source_segments: &Vec<u32>, dest_segment: u32, doc_ref_mapping: &HashMap<DocRef, u16>) -> Result<(), SegmentMergeError> {
        // Lock the primary key index
        let mut primary_key_index = self.primary_key_index.write().unwrap();

        // Update primary keys to point to their new locations
        let mut keys_to_update: HashMap<Vec<u8>, DocRef> = HashMap::with_capacity(doc_ref_mapping.len());
        for (key, doc_ref) in primary_key_index.iter() {
            if doc_ref_mapping.contains_key(&doc_ref) {
                keys_to_update.insert(key.clone(), *doc_ref);
            }
        }

        for (key, doc_ref) in keys_to_update {
            let new_doc_ord = doc_ref_mapping.get(&doc_ref).unwrap();
            let new_doc_ref = DocRef::from_segment_ord(dest_segment, *new_doc_ord);

            let kb = KeyBuilder::primary_key_index(&key);
            let mut doc_ref_bytes = [0; 6];
            BigEndian::write_u32(&mut doc_ref_bytes, new_doc_ref.segment());
            BigEndian::write_u16(&mut doc_ref_bytes[4..], new_doc_ref.ord());
            if let Err(e) = write_batch.put(&kb.key(), &doc_ref_bytes) {
                return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e).into());
            }

            primary_key_index.insert(key, new_doc_ref);
        }

        // Merge deletion lists
        // Must be done while the primary_key_index is locked as this prevents any more documents being deleted
        let mut deletion_list = Vec::new();
        for source_segment in source_segments {
            let kb = KeyBuilder::segment_del_list(*source_segment);
            match db.get(&kb.key()) {
                Ok(Some(docid_set)) => {
                    for doc_id in DocIdSet::FromRDB(docid_set).iter() {
                        let doc_ref = DocRef::from_segment_ord(*source_segment, doc_id);
                        let new_doc_id = doc_ref_mapping.get(&doc_ref).unwrap();
                        deletion_list.write_u16::<BigEndian>(*new_doc_id).unwrap();
                    }
                }
                Ok(None) => {},
                Err(e) => return Err(RocksDBReadError::new(kb.key().to_vec(), e).into()),
            }
        }

        let kb = KeyBuilder::segment_del_list(dest_segment);
        if let Err(e) = db.put(&kb.key(), &deletion_list) {
            return Err(RocksDBWriteError::new_put(kb.key().to_vec(), e).into());
        }

        // Commit!
        if let Err(e) = db.write(write_batch) {
            return Err(RocksDBWriteError::new_commit_write_batch(e).into());
        }

        Ok(())
    }
}
<|endoftext|>


<commit_before>use intern::InternedString;
use grammar::parse_tree::{ActionKind, Alternative, ExprSymbol, Symbol, SymbolKind};

#[derive(Debug)]
pub enum AlternativeAction<'a> {
    User(&'a ActionKind),
    Default(Symbols<'a>),
}

#[derive(Debug)]
pub enum Symbols<'a> {
    Named(Vec<(usize, InternedString, &'a Symbol)>),
    Anon(Vec<(usize, &'a Symbol)>),
}

pub fn analyze_action<'a>(alt: &'a Alternative) -> AlternativeAction<'a> {
    // We can't infer types for alternatives with actions
    if let Some(ref code) = alt.action {
        return AlternativeAction::User(code);
    }

    AlternativeAction::Default(analyze_expr(&alt.expr))
}

pub fn analyze_expr<'a>(expr: &'a ExprSymbol) -> Symbols<'a> {
    // First look for named symbols.
    let named_symbols: Vec<_> =
        expr.symbols
            .iter()
            .enumerate()
            .filter_map(|(idx, sym)| match sym.kind {
                SymbolKind::Name(id, ref sub) => Some((idx, id, &**sub)),
                _ => None,
            })
            .collect();
    if !named_symbols.is_empty() {
        return Symbols::Named(named_symbols);
    }

    // Otherwise, make a tuple of the items they chose with `<>`.
    let chosen_symbol_types: Vec<_> =
        expr.symbols
            .iter()
            .enumerate()
            .filter_map(|(idx, sym)| match sym.kind {
                SymbolKind::Choose(ref sub) => Some((idx, &**sub)),
                _ => None,
            })
            .collect();
    if !chosen_symbol_types.is_empty() {
        return Symbols::Anon(chosen_symbol_types);
    }

    // If they didn't choose anything with `<>`, make a tuple of everything.
    Symbols::Anon(expr.symbols.iter().enumerate().collect())
}

#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub enum Presence
{
    None,
    InCurlyBrackets,
    Normal
}

impl Presence {
    pub fn is_in_curly_brackets(&self) -> bool {
        *self == Presence::InCurlyBrackets
    }
}

pub fn check_between_braces(action: &str) -> Presence
{
    if let Some(funky_index) = action.find("<>") {
        let (before, after) = {
            let (before, after) = action.split_at(funky_index);
            (before.trim(), after[2..].trim())
        };

        let last_before = before.chars().last();
        let next_after = after.chars().next();
        if let (Some('{'), Some('}')) = (last_before, next_after) {
            Presence::InCurlyBrackets
        } else {
            Presence::Normal
        }
    } else {
        Presence::None
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn detecting_normal_funky_expression() {
        assert_eq!(Presence::Normal, check_between_braces("<>"));
        assert_eq!(Presence::Normal, check_between_braces("ble <> blaa"));
        assert_eq!(Presence::Normal, check_between_braces("ble <> } b"));
        assert_eq!(Presence::Normal, check_between_braces("bl{ e <> } b"));
        assert_eq!(Presence::Normal, check_between_braces("bl{ e <>} b"));
        assert_eq!(Presence::Normal, check_between_braces("bl{ e <> e } b"));
        assert_eq!(Presence::Normal, check_between_braces("bl{ <> e } b"));
        assert_eq!(Presence::Normal, check_between_braces("bl{<> e } b"));
        assert_eq!(Presence::Normal, check_between_braces("bl{<>"));
        assert_eq!(Presence::Normal, check_between_braces("<>}"));
    }

    #[test]
    fn detecting_nopresence_of_funky_expression() {
        assert_eq!(Presence::None, check_between_braces("< >"));
        assert_eq!(Presence::None, check_between_braces("ble <b> blaa"));
    }

    #[test]
    fn detecting_incurlybrackets_funky_expression() {
        assert_eq!(Presence::InCurlyBrackets, check_between_braces("{<>}"));
        assert_eq!(Presence::InCurlyBrackets, check_between_braces("ble{<> }blaa"));
        assert_eq!(Presence::InCurlyBrackets, check_between_braces("ble{ <> } b"));
        assert_eq!(Presence::InCurlyBrackets, check_between_braces("bl{         <>} b"));
        assert_eq!(Presence::InCurlyBrackets, check_between_braces("bl{<>} b"));
        assert_eq!(Presence::InCurlyBrackets, check_between_braces("bl{<>         } b"));
    }

}<commit_msg>nit: fix braces placement<commit_after>use intern::InternedString;
use grammar::parse_tree::{ActionKind, Alternative, ExprSymbol, Symbol, SymbolKind};

#[derive(Debug)]
pub enum AlternativeAction<'a> {
    User(&'a ActionKind),
    Default(Symbols<'a>),
}

#[derive(Debug)]
pub enum Symbols<'a> {
    Named(Vec<(usize, InternedString, &'a Symbol)>),
    Anon(Vec<(usize, &'a Symbol)>),
}

pub fn analyze_action<'a>(alt: &'a Alternative) -> AlternativeAction<'a> {
    // We can't infer types for alternatives with actions
    if let Some(ref code) = alt.action {
        return AlternativeAction::User(code);
    }

    AlternativeAction::Default(analyze_expr(&alt.expr))
}

pub fn analyze_expr<'a>(expr: &'a ExprSymbol) -> Symbols<'a> {
    // First look for named symbols.
    let named_symbols: Vec<_> =
        expr.symbols
            .iter()
            .enumerate()
            .filter_map(|(idx, sym)| match sym.kind {
                SymbolKind::Name(id, ref sub) => Some((idx, id, &**sub)),
                _ => None,
            })
            .collect();
    if !named_symbols.is_empty() {
        return Symbols::Named(named_symbols);
    }

    // Otherwise, make a tuple of the items they chose with `<>`.
    let chosen_symbol_types: Vec<_> =
        expr.symbols
            .iter()
            .enumerate()
            .filter_map(|(idx, sym)| match sym.kind {
                SymbolKind::Choose(ref sub) => Some((idx, &**sub)),
                _ => None,
            })
            .collect();
    if !chosen_symbol_types.is_empty() {
        return Symbols::Anon(chosen_symbol_types);
    }

    // If they didn't choose anything with `<>`, make a tuple of everything.
    Symbols::Anon(expr.symbols.iter().enumerate().collect())
}

#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub enum Presence {
    None,
    InCurlyBrackets,
    Normal
}

impl Presence {
    pub fn is_in_curly_brackets(&self) -> bool {
        *self == Presence::InCurlyBrackets
    }
}

pub fn check_between_braces(action: &str) -> Presence {
    if let Some(funky_index) = action.find("<>") {
        let (before, after) = {
            let (before, after) = action.split_at(funky_index);
            (before.trim(), after[2..].trim())
        };

        let last_before = before.chars().last();
        let next_after = after.chars().next();
        if let (Some('{'), Some('}')) = (last_before, next_after) {
            Presence::InCurlyBrackets
        } else {
            Presence::Normal
        }
    } else {
        Presence::None
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn detecting_normal_funky_expression() {
        assert_eq!(Presence::Normal, check_between_braces("<>"));
        assert_eq!(Presence::Normal, check_between_braces("ble <> blaa"));
        assert_eq!(Presence::Normal, check_between_braces("ble <> } b"));
        assert_eq!(Presence::Normal, check_between_braces("bl{ e <> } b"));
        assert_eq!(Presence::Normal, check_between_braces("bl{ e <>} b"));
        assert_eq!(Presence::Normal, check_between_braces("bl{ e <> e } b"));
        assert_eq!(Presence::Normal, check_between_braces("bl{ <> e } b"));
        assert_eq!(Presence::Normal, check_between_braces("bl{<> e } b"));
        assert_eq!(Presence::Normal, check_between_braces("bl{<>"));
        assert_eq!(Presence::Normal, check_between_braces("<>}"));
    }

    #[test]
    fn detecting_nopresence_of_funky_expression() {
        assert_eq!(Presence::None, check_between_braces("< >"));
        assert_eq!(Presence::None, check_between_braces("ble <b> blaa"));
    }

    #[test]
    fn detecting_incurlybrackets_funky_expression() {
        assert_eq!(Presence::InCurlyBrackets, check_between_braces("{<>}"));
        assert_eq!(Presence::InCurlyBrackets, check_between_braces("ble{<> }blaa"));
        assert_eq!(Presence::InCurlyBrackets, check_between_braces("ble{ <> } b"));
        assert_eq!(Presence::InCurlyBrackets, check_between_braces("bl{         <>} b"));
        assert_eq!(Presence::InCurlyBrackets, check_between_braces("bl{<>} b"));
        assert_eq!(Presence::InCurlyBrackets, check_between_braces("bl{<>         } b"));
    }

}
<|endoftext|>


<commit_before>extern crate env_logger;
extern crate glob;
extern crate log;

extern crate protobuf_codegen_pure;

extern crate protobuf_test_common;

use std::io::Read;
use std::io::Write;

use std::fs;
use std::path::Path;

use protobuf_test_common::build::*;

fn copy_test<P1: AsRef<Path>, P2: AsRef<Path>>(src: P1, dst: P2) {
    eprintln!("copy {:?} to {:?}", src.as_ref(), dst.as_ref());
    let mut content = Vec::new();
    fs::File::open(src.as_ref())
        .expect(&format!("open {}", src.as_ref().display()))
        .read_to_end(&mut content)
        .expect("read_to_end");

    let mut write = fs::File::create(dst).expect("create");
    writeln!(write, "// generated").expect("write");
    writeln!(write, "// copied from {}", src.as_ref().display()).expect("write");
    writeln!(write, "").expect("write");
    write.write_all(&content).expect("write_all");
    writeln!(write, "// generated").expect("write");
    write.flush().expect("flush");
}

fn copy_from_protobuf_test(path: &str) {
    copy_test(&format!("../protobuf-test/{}", path), &format!("{}", path))
}

enum FileNameClass {
    ModRs,
    TestRs,
    Proto,
    GeneratedRs,
    Ignore,
}

fn classify_file_name(name: &str) -> FileNameClass {
    if name.starts_with(".") || name.ends_with(".md") || name.ends_with(".sh") {
        FileNameClass::Ignore
    } else if name.ends_with("_pb.rs") || name.ends_with("_pb_proto3.rs") {
        FileNameClass::GeneratedRs
    } else if name == "mod.rs" {
        FileNameClass::ModRs
    } else if name.ends_with(".proto") || name.ends_with(".proto3") {
        FileNameClass::Proto
    } else if name.ends_with(".rs") {
        FileNameClass::TestRs
    } else {
        panic!("unknown test file: {}", name);
    }
}

// Copy tests from `protobuf-test` directory to the same directory here
fn copy_tests(dir: &str) {
    let src_dir = format!("../protobuf-test/{}", dir);
    for entry in fs::read_dir(&src_dir).expect(&format!("read_dir {}", src_dir)) {
        let file_name = entry.expect("entry").file_name().into_string().unwrap();

        match classify_file_name(&file_name) {
            FileNameClass::ModRs | FileNameClass::Ignore | FileNameClass::GeneratedRs => {}
            FileNameClass::TestRs | FileNameClass::Proto => {
                copy_from_protobuf_test(&format!("{}/{}", dir, file_name))
            }
        }
    }
}

fn gen_in_dir(dir: &str, include_dir: &str) {
    gen_in_dir_impl(
        dir,
        include_dir,
        |GenInDirArgs {
             out_dir,
             input,
             includes,
             customize,
         }| {
            protobuf_codegen_pure::Args::new()
                .out_dir(out_dir)
                .inputs(input)
                .includes(includes)
                .customize(customize)
                .run()
        },
    );
}

fn print_rerun_if_changed<P: AsRef<Path>>(path: P) {
    let path = path.as_ref();
    // Doesn't seem to do anything
    println!("rerun-if-changed={}", path.to_str().expect("to_str"));
    if path.is_dir() {
        for child in fs::read_dir(path).expect("read_dir") {
            let child = child.expect("child").path();
            print_rerun_if_changed(child);
        }
    }
}

fn generate_interop() {
    copy_from_protobuf_test("src/interop/mod.rs");
    copy_from_protobuf_test("src/interop/json.rs");

    protobuf_codegen_pure::Args::new()
        .out_dir("src/interop")
        .includes(&["../interop/cxx", "../proto"])
        .input("../interop/cxx/interop_pb.proto")
        .run()
        .unwrap();
}

fn generate_pb_rs() {
    print_rerun_if_changed("../protobuf-test");

    copy_tests("src/v2");
    gen_in_dir("src/v2", "src/v2");

    copy_tests("src/v3");
    gen_in_dir("src/v3", "src/v3");

    copy_tests("src/common/v2");
    gen_in_dir("src/common/v2", "src/common/v2");

    copy_tests_v2_v3("src/common/v2", "src/common/v3");
    gen_in_dir("src/common/v3", "src/common/v3");

    copy_tests("src/google/protobuf");
    gen_in_dir("src/google/protobuf", "src");

    generate_interop();
}

fn main() {
    env_logger::init();

    cfg_serde();

    clean_old_files();
    generate_pb_rs();
}
<commit_msg>Fix test file classification<commit_after>extern crate env_logger;
extern crate glob;
extern crate log;

extern crate protobuf_codegen_pure;

extern crate protobuf_test_common;

use std::io::Read;
use std::io::Write;

use std::fs;
use std::path::Path;

use protobuf_test_common::build::*;

fn copy_test<P1: AsRef<Path>, P2: AsRef<Path>>(src: P1, dst: P2) {
    eprintln!("copy {:?} to {:?}", src.as_ref(), dst.as_ref());
    let mut content = Vec::new();
    fs::File::open(src.as_ref())
        .expect(&format!("open {}", src.as_ref().display()))
        .read_to_end(&mut content)
        .expect("read_to_end");

    let mut write = fs::File::create(dst).expect("create");
    writeln!(write, "// generated").expect("write");
    writeln!(write, "// copied from {}", src.as_ref().display()).expect("write");
    writeln!(write, "").expect("write");
    write.write_all(&content).expect("write_all");
    writeln!(write, "// generated").expect("write");
    write.flush().expect("flush");
}

fn copy_from_protobuf_test(path: &str) {
    copy_test(&format!("../protobuf-test/{}", path), &format!("{}", path))
}

enum FileNameClass {
    ModRs,
    TestRs,
    Proto,
    GeneratedRs,
    Ignore,
}

fn classify_file_name(dir: &str, name: &str) -> FileNameClass {
    if name.starts_with(".") || name.ends_with(".md") || name.ends_with(".sh") {
        FileNameClass::Ignore
    } else if name.ends_with("_pb.rs") || name.ends_with("_pb_proto3.rs") {
        FileNameClass::GeneratedRs
    } else if name == "mod.rs" {
        FileNameClass::ModRs
    } else if name.ends_with(".proto") || name.ends_with(".proto3") {
        FileNameClass::Proto
    } else if name.ends_with(".rs") {
        if dir == "src/google/protobuf" {
            FileNameClass::GeneratedRs
        } else {
            FileNameClass::TestRs
        }
    } else {
        panic!("unknown test file: {}", name);
    }
}

// Copy tests from `protobuf-test` directory to the same directory here
fn copy_tests(dir: &str) {
    let src_dir = format!("../protobuf-test/{}", dir);
    for entry in fs::read_dir(&src_dir).expect(&format!("read_dir {}", src_dir)) {
        let file_name = entry.expect("entry").file_name().into_string().unwrap();

        match classify_file_name(dir, &file_name) {
            FileNameClass::ModRs | FileNameClass::Ignore | FileNameClass::GeneratedRs => {}
            FileNameClass::TestRs | FileNameClass::Proto => {
                copy_from_protobuf_test(&format!("{}/{}", dir, file_name))
            }
        }
    }
}

fn gen_in_dir(dir: &str, include_dir: &str) {
    gen_in_dir_impl(
        dir,
        include_dir,
        |GenInDirArgs {
             out_dir,
             input,
             includes,
             customize,
         }| {
            protobuf_codegen_pure::Args::new()
                .out_dir(out_dir)
                .inputs(input)
                .includes(includes)
                .customize(customize)
                .run()
        },
    );
}

fn print_rerun_if_changed<P: AsRef<Path>>(path: P) {
    let path = path.as_ref();
    // Doesn't seem to do anything
    println!("rerun-if-changed={}", path.to_str().expect("to_str"));
    if path.is_dir() {
        for child in fs::read_dir(path).expect("read_dir") {
            let child = child.expect("child").path();
            print_rerun_if_changed(child);
        }
    }
}

fn generate_interop() {
    copy_from_protobuf_test("src/interop/mod.rs");
    copy_from_protobuf_test("src/interop/json.rs");

    protobuf_codegen_pure::Args::new()
        .out_dir("src/interop")
        .includes(&["../interop/cxx", "../proto"])
        .input("../interop/cxx/interop_pb.proto")
        .run()
        .unwrap();
}

fn generate_pb_rs() {
    print_rerun_if_changed("../protobuf-test");

    copy_tests("src/v2");
    gen_in_dir("src/v2", "src/v2");

    copy_tests("src/v3");
    gen_in_dir("src/v3", "src/v3");

    copy_tests("src/common/v2");
    gen_in_dir("src/common/v2", "src/common/v2");

    copy_tests_v2_v3("src/common/v2", "src/common/v3");
    gen_in_dir("src/common/v3", "src/common/v3");

    copy_tests("src/google/protobuf");
    gen_in_dir("src/google/protobuf", "src");

    generate_interop();
}

fn main() {
    env_logger::init();

    cfg_serde();

    clean_old_files();
    generate_pb_rs();
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate clap;

use clap::{App, Arg, ArgMatches, SubCommand, AppSettings};
use rusqlite as rs;
use std::path::Path;

use std::error::Error;
use std::vec::Vec;
use std::ffi::OsString;
use std::collections::HashSet;

use chrono::Local;
use chrono::DateTime;
use chrono::Datelike;
use glob::glob;

fn main() {
    let matches = App::new("Karmator maintance batch")
        .version(crate_version!())
        .author(crate_authors!("\n"))
        .about("Handles the maintance work for karmator")
        .setting(AppSettings::ColoredHelp)
        .subcommand(
            SubCommand::with_name("runs")
                .about("Detect runs of votes")
                .arg(
                    Arg::with_name("min")
                        .short("m")
                        .help("Min count of runs before outputting")
                        .takes_value(true),
                )
                .arg(
                    Arg::with_name("delete")
                        .long("delete")
                        .help("Delete the runs detected"),
                )
                .arg(
                    Arg::with_name("FILE")
                        .help("Database file to operate on")
                        .required(true),
                ),
        )
        .subcommand(
            SubCommand::with_name("prune")
                .about("Prune and pack up old backups")
                .arg(
                    Arg::with_name("delete")
                        .long("delete")
                        .help("Delete the old files"),
                )
                .arg(
                    Arg::with_name("BACKUPS")
                        .help("Backup directory to prune")
                        .required(true),
                ),
        )
        .get_matches();

    match matches.subcommand() {
        ("runs", Some(m))  => {
            let filename = m.value_of("FILE").unwrap();
            let min = value_t!(m, "min", u32).unwrap_or(20);
            let delete = m.is_present("delete");

            run(filename, min, delete)
        },
        ("prune", Some(m)) => {
            let directory = m.value_of("BACKUPS").unwrap();
            let delete = m.is_present("delete");

            prune(directory, delete)
        }
        _ => {
            println!("{}", matches.usage());
            Ok(())
        },
    };
}

#[derive(Debug, Clone)]
struct Vote {
    id: i32,
    by_whom_name: String,
    for_what_name: String,
    amount: i8,
}
impl PartialEq for Vote {
    fn eq(&self, other: &Self) -> bool {
        (self.by_whom_name == other.by_whom_name)
            && (self.for_what_name == other.for_what_name)
            && (self.amount == other.amount)
    }
}

#[derive(Debug)]
struct RunVal {
    oldest_id: i32,
    newest_id: i32,
    by_whom_name: String,
    for_what_name: String,
    amount: i8,
    count: u32,
}

fn get_run_val(srv: &Vote, pv: &Vote, count: u32) -> RunVal {
    RunVal {
        oldest_id: srv.id,
        newest_id: pv.id,
        by_whom_name: srv.by_whom_name.clone(),
        for_what_name: srv.for_what_name.clone(),
        amount: srv.amount,
        count: count,
    }
}

fn str_amount(amount: i8) -> &'static str {
    match amount {
        -1 => "Down",
        0 => "Side",
        1 => "Up",
        _ => panic!("invalid amount"),
    }
}

fn run(filename: &str, min: u32, delete: bool) -> Result<(), Box<dyn Error>> {
    let conn =
        rs::Connection::open_with_flags(Path::new(filename), rs::OpenFlags::SQLITE_OPEN_READ_WRITE)
            .expect(&format!("Connection error: {}", filename));

    let mut stmt = conn.prepare("SELECT id, by_whom_name, for_what_name, amount FROM votes")?;
    let vote_iter = stmt.query_map(rs::params![], |row| {
        Ok(Vote {
            id: row.get(0)?,
            by_whom_name: row.get(1)?,
            for_what_name: row.get(2)?,
            amount: row.get(3)?,
        })
    })?;

    // Time to compute the run
    let mut runs = Vec::new();

    let mut start_run_vote = None;
    let mut prev_vote = None;
    let mut count = 0;

    for rvote in vote_iter {
        let vote = rvote?;

        match (&start_run_vote, &prev_vote) {
            (None, None) => {
                start_run_vote = Some(vote.clone());
                prev_vote = Some(vote);
                count = 1; // Run of 1
            }
            (Some(srv), Some(pv)) => {
                if pv == &vote {
                    // Current vote + prev vote are the same, inc prev vote
                    prev_vote = Some(vote);
                    count += 1;
                } else {
                    // Current vote != prev vote, record the run, and reset
                    runs.push(get_run_val(srv, pv, count));

                    start_run_vote = Some(vote.clone());
                    prev_vote = Some(vote);
                    count = 1; // Run of 1
                }
            }
            (_, _) => panic!("Shouldn't happen"),
        };
    }

    // Record the last run
    runs.push(get_run_val(
        &start_run_vote.unwrap(),
        &prev_vote.unwrap(),
        count,
    ));

    if delete {
        // Scan and delete the offenders
        let mut stmt = conn.prepare("DELETE FROM votes WHERE id >= ? and id <= ?")?;
        for r in &runs {
            if r.count > min {
                let deleted = stmt.execute(rs::params![r.oldest_id, r.newest_id])?;

                if (r.count as usize) != deleted {
                    panic!("Expected: {} to be deleted, got {}", r.count, deleted);
                }
            }
        }
    } else {
        // Now we can scan for anything that > min and print them
        println!(
            "{: >8}, {: >8}, {: >14.14}, {: >14.14}, {: >6}, {: >5}",
            "start_id", "end_id", "by_whom_name", "for_what_name", "amount", "count"
        );
        for r in &runs {
            if r.count > min {
                println!(
                    "{: >8}, {: >8}, {: >14.14}, {: >14.14}, {: >6}, {: >5}",
                    r.oldest_id,
                    r.newest_id,
                    r.by_whom_name,
                    r.for_what_name,
                    str_amount(r.amount),
                    r.count
                );
            }
        }
    }

    Ok(())
}

fn prune(directory: &str, delete: bool) -> Result<(), Box<dyn Error>> {
    let now: DateTime<Local> = Local::now();
    let year = now.year();
    let month = now.month();

    // Fetch a set of all of the file in the current month+year
    let current_month_year = collect_glob(directory, &format!("/db-backup-{}-{:02}-??.sqlite.zst", year, month));

    // Fetch a set of all of the file that is in previous year + first of the month
    // If == 12, pack it up
    let previous_first_month = collect_glob(directory, &format!("/db-backup-{}-??-01.sqlite.zst", year - 1));

    // Fetch a set of all of the file that is in current year + first of the month
    // if == 12 pack it up
    let current_first_month = collect_glob(directory, &format!("/db-backup-{}-??-01.sqlite.zst", year));

    // Fetch a set of all of the files
    let all_files = collect_glob(directory, "/db-backup-????-??-??.sqlite.zst");

    println!("cmy: {:?}\n", current_month_year);
    println!("pfm: {:?}\n", previous_first_month);
    println!("cfm: {:?}\n", current_first_month);
    println!("All: {:?}\n", all_files);

    // Calculate what to keep and what to delete
    let delete = delete_set(&all_files, vec![&current_month_year, &previous_first_month, &current_first_month]);

    println!("DELETE: {:?}\n", delete);

    println!("cmy: {:?}", current_month_year.len());
    println!("pfm: {:?}", previous_first_month.len());
    println!("cfm: {:?}", current_first_month.len());
    println!("keeping: {:?}\n", current_month_year.len() + previous_first_month.len() + current_first_month.len());

    println!("All: {:?}", all_files.len());
    println!("DELETE: {:?}", delete.len());
    println!("keeping: {:?}", all_files.len() - delete.len());
    println!("Difference is cos of double counting the current_first_month + current_month_year's first day file");

    // Rules:
    //  - Keep current month of daily snapshot
    //  - Keep first of each month in a year
    //  - When there is 12 first month snapshot of a year, compact it into a tarfile

    Ok(())
}

fn collect_glob(directory: &str, glob_str: &str) -> HashSet<OsString> {
    glob(&(directory.to_string() + glob_str)).unwrap()
        .flatten()
        .flat_map(|e| e.file_name().map(|s| s.to_os_string()))
        .collect::<HashSet<OsString>>()
}

fn delete_set(all: &HashSet<OsString>, keep: Vec<&HashSet<OsString>>) -> HashSet<OsString> {
    let mut delete = all.clone();

    for hs in keep {
        let out = delete.difference(&hs).map(|e| e.clone()).collect();
        delete = out;
    }

    delete
}
<commit_msg>Build the yearly tarball<commit_after>#[macro_use]
extern crate clap;

use clap::{App, Arg, SubCommand, AppSettings};
use rusqlite as rs;
use std::path::Path;

use std::error::Error;
use std::vec::Vec;
use std::ffi::OsString;
use std::collections::HashSet;

use chrono::Local;
use chrono::DateTime;
use chrono::Datelike;
use glob::glob;

use count_write::CountWrite;
use zstd::stream::copy_decode;
use zstd::stream::write::Encoder;
use zstd::stream::read::Decoder;
use tar::Builder;
use tar::Header;
use std::io::{SeekFrom, Seek};
use std::fs::File;

fn main() {
    let matches = App::new("Karmator maintance batch")
        .version(crate_version!())
        .author(crate_authors!("\n"))
        .about("Handles the maintance work for karmator")
        .setting(AppSettings::ColoredHelp)
        .subcommand(
            SubCommand::with_name("runs")
                .about("Detect runs of votes")
                .arg(
                    Arg::with_name("min")
                        .short("m")
                        .help("Min count of runs before outputting")
                        .takes_value(true),
                )
                .arg(
                    Arg::with_name("delete")
                        .long("delete")
                        .help("Delete the runs detected"),
                )
                .arg(
                    Arg::with_name("FILE")
                        .help("Database file to operate on")
                        .required(true),
                ),
        )
        .subcommand(
            SubCommand::with_name("prune")
                .about("Prune and pack up old backups")
                .arg(
                    Arg::with_name("delete")
                        .long("delete")
                        .help("Delete the old files"),
                )
                .arg(
                    Arg::with_name("BACKUPS")
                        .help("Backup directory to prune")
                        .required(true),
                ),
        )
        .get_matches();

    match matches.subcommand() {
        ("runs", Some(m))  => {
            let filename = m.value_of("FILE").unwrap();
            let min = value_t!(m, "min", u32).unwrap_or(20);
            let delete = m.is_present("delete");

            run(filename, min, delete)
        },
        ("prune", Some(m)) => {
            let directory = m.value_of("BACKUPS").unwrap();
            let delete = m.is_present("delete");

            prune(directory, delete)
        }
        _ => {
            println!("{}", matches.usage());
            Ok(())
        },
    }.unwrap();
}

#[derive(Debug, Clone)]
struct Vote {
    id: i32,
    by_whom_name: String,
    for_what_name: String,
    amount: i8,
}
impl PartialEq for Vote {
    fn eq(&self, other: &Self) -> bool {
        (self.by_whom_name == other.by_whom_name)
            && (self.for_what_name == other.for_what_name)
            && (self.amount == other.amount)
    }
}

#[derive(Debug)]
struct RunVal {
    oldest_id: i32,
    newest_id: i32,
    by_whom_name: String,
    for_what_name: String,
    amount: i8,
    count: u32,
}

fn get_run_val(srv: &Vote, pv: &Vote, count: u32) -> RunVal {
    RunVal {
        oldest_id: srv.id,
        newest_id: pv.id,
        by_whom_name: srv.by_whom_name.clone(),
        for_what_name: srv.for_what_name.clone(),
        amount: srv.amount,
        count: count,
    }
}

fn str_amount(amount: i8) -> &'static str {
    match amount {
        -1 => "Down",
        0 => "Side",
        1 => "Up",
        _ => panic!("invalid amount"),
    }
}

fn run(filename: &str, min: u32, delete: bool) -> Result<(), Box<dyn Error>> {
    let conn =
        rs::Connection::open_with_flags(Path::new(filename), rs::OpenFlags::SQLITE_OPEN_READ_WRITE)
            .expect(&format!("Connection error: {}", filename));

    let mut stmt = conn.prepare("SELECT id, by_whom_name, for_what_name, amount FROM votes")?;
    let vote_iter = stmt.query_map(rs::params![], |row| {
        Ok(Vote {
            id: row.get(0)?,
            by_whom_name: row.get(1)?,
            for_what_name: row.get(2)?,
            amount: row.get(3)?,
        })
    })?;

    // Time to compute the run
    let mut runs = Vec::new();

    let mut start_run_vote = None;
    let mut prev_vote = None;
    let mut count = 0;

    for rvote in vote_iter {
        let vote = rvote?;

        match (&start_run_vote, &prev_vote) {
            (None, None) => {
                start_run_vote = Some(vote.clone());
                prev_vote = Some(vote);
                count = 1; // Run of 1
            }
            (Some(srv), Some(pv)) => {
                if pv == &vote {
                    // Current vote + prev vote are the same, inc prev vote
                    prev_vote = Some(vote);
                    count += 1;
                } else {
                    // Current vote != prev vote, record the run, and reset
                    runs.push(get_run_val(srv, pv, count));

                    start_run_vote = Some(vote.clone());
                    prev_vote = Some(vote);
                    count = 1; // Run of 1
                }
            }
            (_, _) => panic!("Shouldn't happen"),
        };
    }

    // Record the last run
    runs.push(get_run_val(
        &start_run_vote.unwrap(),
        &prev_vote.unwrap(),
        count,
    ));

    if delete {
        // Scan and delete the offenders
        let mut stmt = conn.prepare("DELETE FROM votes WHERE id >= ? and id <= ?")?;
        for r in &runs {
            if r.count > min {
                let deleted = stmt.execute(rs::params![r.oldest_id, r.newest_id])?;

                if (r.count as usize) != deleted {
                    panic!("Expected: {} to be deleted, got {}", r.count, deleted);
                }
            }
        }
    } else {
        // Now we can scan for anything that > min and print them
        println!(
            "{: >8}, {: >8}, {: >14.14}, {: >14.14}, {: >6}, {: >5}",
            "start_id", "end_id", "by_whom_name", "for_what_name", "amount", "count"
        );
        for r in &runs {
            if r.count > min {
                println!(
                    "{: >8}, {: >8}, {: >14.14}, {: >14.14}, {: >6}, {: >5}",
                    r.oldest_id,
                    r.newest_id,
                    r.by_whom_name,
                    r.for_what_name,
                    str_amount(r.amount),
                    r.count
                );
            }
        }
    }

    Ok(())
}

fn prune(directory: &str, delete: bool) -> Result<(), Box<dyn Error>> {
    let now: DateTime<Local> = Local::now();
    let year = now.year();
    let month = now.month();

    // Fetch a set of all of the files
    let all_files = collect_glob(directory, "/db-backup-????-??-??.sqlite.zst");

    // Fetch a set of all of the file in the current month+year
    let current_month_year = collect_glob(directory, &format!("/db-backup-{}-{:02}-??.sqlite.zst", year, month));

    // Fetch a set of all of the file that is in previous year + first of the month
    let previous_first_month = collect_glob(directory, &format!("/db-backup-{}-??-01.sqlite.zst", year - 1));

    // Fetch a set of all of the file that is in current year + first of the month
    let current_first_month = collect_glob(directory, &format!("/db-backup-{}-??-01.sqlite.zst", year));

    // Calculate the initial set of files to prune
    let delete = delete_set(&all_files, vec![&current_month_year, &previous_first_month, &current_first_month]);

    println!("cmy: {:?}", current_month_year.len());
    println!("pfm: {:?}", previous_first_month.len());
    println!("cfm: {:?}", current_first_month.len());
    println!("keeping: {:?}\n", current_month_year.len() + previous_first_month.len() + current_first_month.len());

    println!("All: {:?}", all_files.len());
    println!("DELETE: {:?}", delete.len());
    println!("keeping: {:?}", all_files.len() - delete.len());
    println!("Difference is cos of double counting the current_first_month + current_month_year's first day file");

    // Compact pfm + cfm into their years
    let mut count = CountWrite::from(std::io::sink());
    {
        let mut tar = Builder::new(Encoder::new(&mut count, 21)?.auto_finish());

        for f in current_first_month.iter() {
            let mut file = File::open(f)?;
            let filename = Path::new(f).file_name().unwrap();
            let filesize = {
                let mut count = CountWrite::from(std::io::sink());
                copy_decode(&file, &mut count)?;
                count.count()
            };

            let mut header = Header::new_gnu();
            header.set_path(filename)?;
            header.set_size(filesize);
            header.set_cksum();

            file.seek(SeekFrom::Start(0))?;
            tar.append(
                &header,
                Decoder::new(std::fs::File::open(f)?)?
            )?;
        }

        tar.finish()?;
    }
    println!("tarball: {:?}", &count.count());

    // Rules:
    //  - Keep current month of daily snapshot
    //  - Keep first of each month in a year
    //  - When there is 12 first month snapshot of a year, compact it into a tarfile

    Ok(())
}

fn collect_glob(directory: &str, glob_str: &str) -> HashSet<OsString> {
    glob(&(directory.to_string() + glob_str)).unwrap()
        .flatten()
        .map(|e| e.into_os_string())
        .collect::<HashSet<OsString>>()
}

fn delete_set(all: &HashSet<OsString>, keep: Vec<&HashSet<OsString>>) -> HashSet<OsString> {
    let mut delete = all.clone();

    for hs in keep {
        let out = delete.difference(&hs).map(|e| e.clone()).collect();
        delete = out;
    }

    delete
}
<|endoftext|>


<commit_before>#![deny(warnings)]

extern crate binutils;

use std::env;
use std::fs;
use std::io::{self, Write, Read, Stderr};
use std::mem;

use binutils::extra::option::OptionalExt;
use binutils::extra::io::{WriteExt, fail};
use binutils::convert::{u8_to_hex, hex_to_u8, u32_byte_array, hex_to_ascii, ascii_to_hex};
use binutils::strings::IsPrintable;

const HELP: &'static [u8] = br#"
    NAME
        hexdump - dump the hexidecimal representation of a byte stream.
    SYNOPSIS
        hexdump [-h | --help] [-r | --reverse] [FILE]
    DESCRIPTION
        This utility will dump the hexidecimal representation of a file or the standard input, in a stylized way. Hexdump utility behaves like 'xxd'.

        The first column signifies the address of the first byte on the line. Each line contains 16 bytes, grouped in groups of two bytes, sepereated by space. The last column contains the printable characters in the last 16 bytes. The non-printable characters are replaced by a '.'.
    OPTIONS
        -h
        --help
            Print this manual page.
        -r
        --reverse
            Do the reverse dump (consume the dump and output the bytes it defines). This is useful for usage within editors.
    AUTHOR
        This program was written by Ticki. Bugs should be reported in the Github repository, 'redox-os/binutils'.
    COPYRIGHT
        Copyright (c) 2016 Ticki

        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

        Someone once read this. True story, bruh.

        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"#;

/// Encode a single byte to the output stream
fn encode_byte<R: Read, W: Write>(stdin: &mut R, mut stdout: &mut W, stderr: &mut Stderr) -> Option<u8> {
    let byte = if let Some(x) = stdin.bytes().next() {
        x.try(&mut *stderr)
    } else {
        return None;
    };

    // Convert the raw byte to hexadecimal
    let hex = u8_to_hex(byte);

    // Write it to stdout.
    stdout.write(&[hex_to_ascii(hex.0), hex_to_ascii(hex.1)]).try(stderr);

    Some(if byte.is_printable() {
        byte
    } else {
        // If it is non-printable, write `.` to char-buffer instead.
        b'.'
    })
}

fn encode<R: Read, W: Write>(mut stdin: R, mut stdout: W, mut stderr: Stderr) {
    let rem;
    // This is the char buffer. The last 16 byte column, which prints the printable characters.
    // The non-printable ones are replaced with `.`.
    let mut ascii: [u8; 16] = unsafe { mem::uninitialized() };

    let mut line = 0;

    'a: loop {
        // Iterate over the bytes in the line number times 16.
        for &b in u32_byte_array(line * 16).iter() {
            let hex = u8_to_hex(b);
            // Print this value to the first column (denoting the address of the first byte of the
            // line.)
            stdout.write(&[hex_to_ascii(hex.0), hex_to_ascii(hex.1)]).try(&mut stderr);
        }
        stdout.write(b": ").try(&mut stderr);

        // Now, we go over the actual data, printing it in hexadecimal.
        for n in 0..8 {
            // We add the char to the char buffer, and print it in two hex digits.
            ascii[n * 2] = if let Some(x) = encode_byte(&mut stdin, &mut stdout, &mut stderr) {
                x
            } else {
                // The end of the file is reached, set the remainder, which will later be used for
                // alignment of the last column.
                rem = n;
                break 'a;
            };
            ascii[n * 2 + 1] = if let Some(x) = encode_byte(&mut stdin, &mut stdout, &mut stderr) {
                x
            } else {
                rem = n;
                break 'a;
            };
            // Seperate every two hex digits by a space.
            stdout.write(b" ").try(&mut stderr);
        }

        stdout.write(b" ").try(&mut stderr);
        // Print the ASCII buffer at the end of the line.
        stdout.writeln(&ascii).try(&mut stderr);

        // Increment the line number.
        line += 1;
    }

    if rem != 0 {
        // We now align the last column using the remainder set before.
        for _ in 0..41 - rem * 5 {
            stdout.write(b" ").try(&mut stderr);
        }
        stdout.write(&ascii[..rem * 2]).try(&mut stderr);
    }

    stdout.write(b"\n").try(&mut stderr);
}

fn decode<R: Read, W: Write>(stdin: R, mut stdout: W, mut stderr: Stderr) {
    let mut stdin = stdin.bytes().filter(|x| x.as_ref().ok() != Some(&b' '));

    loop {
        // Skip the first column
        stdin.nth(8);
        // Process the inner 8 columns
        for _ in 0..16 {
            // The first hex digit to decode.
            let h1 = ascii_to_hex(
                if let Some(x) = stdin.next() {
                    x.try(&mut stderr)
                } else {
                    return;
                }
            );
            // The second hex digit to decode.
            let h2 = ascii_to_hex(
                if let Some(x) = stdin.next() {
                    x.try(&mut stderr)
                } else {
                    return;
                }
            );

            // Write the decoded, joined hex digits in binary form.
            stdout.write(&[hex_to_u8((h1, h2))]).try(&mut stderr);
        }

        // Skip the rest until newline.
        loop {
            if let Some(x) = stdin.next() {
                if x.try(&mut stderr) == b'\n' {
                    break;
                }
            } else {
                return;
            }
        }
    }
}

fn main() {
    let stdout = io::stdout();
    let mut stdout = stdout.lock();
    let mut stderr = io::stderr();

    let mut args = env::args();

    // Arguments should be <= 2
    if args.len() > 2 {
        fail("too many arguments.", &mut stderr);
    }

    match args.nth(1) {
        None => encode(io::stdin(), stdout, stderr),
        Some(a) => match a.as_ref() { // MIR plz
            "-h" | "--help" => {
                // HEEEEEELP.
                stdout.writeln(HELP).try(&mut stderr);
            },
            "-r" | "--reverse" => {
                match args.next() {
                    // Decode.
                    None => {
                        let stdin = io::stdin();
                        decode(stdin.lock(), stdout, stderr);
                    }
                    // Encode.
                    Some(f) => {
                        let file = fs::File::open(f).try(&mut stderr);
                        decode(file, stdout, stderr);
                    }
                }
            },
            // Read from a file, instead of standard input.
            f => {
                let file = fs::File::open(f).try(&mut stderr);
                encode(file, stdout, stderr);
            },
        },
    }
}
<commit_msg>Fix unused mut error<commit_after>#![deny(warnings)]

extern crate binutils;

use std::env;
use std::fs;
use std::io::{self, Write, Read, Stderr};
use std::mem;

use binutils::extra::option::OptionalExt;
use binutils::extra::io::{WriteExt, fail};
use binutils::convert::{u8_to_hex, hex_to_u8, u32_byte_array, hex_to_ascii, ascii_to_hex};
use binutils::strings::IsPrintable;

const HELP: &'static [u8] = br#"
    NAME
        hexdump - dump the hexidecimal representation of a byte stream.
    SYNOPSIS
        hexdump [-h | --help] [-r | --reverse] [FILE]
    DESCRIPTION
        This utility will dump the hexidecimal representation of a file or the standard input, in a stylized way. Hexdump utility behaves like 'xxd'.

        The first column signifies the address of the first byte on the line. Each line contains 16 bytes, grouped in groups of two bytes, sepereated by space. The last column contains the printable characters in the last 16 bytes. The non-printable characters are replaced by a '.'.
    OPTIONS
        -h
        --help
            Print this manual page.
        -r
        --reverse
            Do the reverse dump (consume the dump and output the bytes it defines). This is useful for usage within editors.
    AUTHOR
        This program was written by Ticki. Bugs should be reported in the Github repository, 'redox-os/binutils'.
    COPYRIGHT
        Copyright (c) 2016 Ticki

        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

        Someone once read this. True story, bruh.

        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"#;

/// Encode a single byte to the output stream
fn encode_byte<R: Read, W: Write>(stdin: &mut R, stdout: &mut W, stderr: &mut Stderr) -> Option<u8> {
    let byte = if let Some(x) = stdin.bytes().next() {
        x.try(&mut *stderr)
    } else {
        return None;
    };

    // Convert the raw byte to hexadecimal
    let hex = u8_to_hex(byte);

    // Write it to stdout.
    stdout.write(&[hex_to_ascii(hex.0), hex_to_ascii(hex.1)]).try(stderr);

    Some(if byte.is_printable() {
        byte
    } else {
        // If it is non-printable, write `.` to char-buffer instead.
        b'.'
    })
}

fn encode<R: Read, W: Write>(mut stdin: R, mut stdout: W, mut stderr: Stderr) {
    let rem;
    // This is the char buffer. The last 16 byte column, which prints the printable characters.
    // The non-printable ones are replaced with `.`.
    let mut ascii: [u8; 16] = unsafe { mem::uninitialized() };

    let mut line = 0;

    'a: loop {
        // Iterate over the bytes in the line number times 16.
        for &b in u32_byte_array(line * 16).iter() {
            let hex = u8_to_hex(b);
            // Print this value to the first column (denoting the address of the first byte of the
            // line.)
            stdout.write(&[hex_to_ascii(hex.0), hex_to_ascii(hex.1)]).try(&mut stderr);
        }
        stdout.write(b": ").try(&mut stderr);

        // Now, we go over the actual data, printing it in hexadecimal.
        for n in 0..8 {
            // We add the char to the char buffer, and print it in two hex digits.
            ascii[n * 2] = if let Some(x) = encode_byte(&mut stdin, &mut stdout, &mut stderr) {
                x
            } else {
                // The end of the file is reached, set the remainder, which will later be used for
                // alignment of the last column.
                rem = n;
                break 'a;
            };
            ascii[n * 2 + 1] = if let Some(x) = encode_byte(&mut stdin, &mut stdout, &mut stderr) {
                x
            } else {
                rem = n;
                break 'a;
            };
            // Seperate every two hex digits by a space.
            stdout.write(b" ").try(&mut stderr);
        }

        stdout.write(b" ").try(&mut stderr);
        // Print the ASCII buffer at the end of the line.
        stdout.writeln(&ascii).try(&mut stderr);

        // Increment the line number.
        line += 1;
    }

    if rem != 0 {
        // We now align the last column using the remainder set before.
        for _ in 0..41 - rem * 5 {
            stdout.write(b" ").try(&mut stderr);
        }
        stdout.write(&ascii[..rem * 2]).try(&mut stderr);
    }

    stdout.write(b"\n").try(&mut stderr);
}

fn decode<R: Read, W: Write>(stdin: R, mut stdout: W, mut stderr: Stderr) {
    let mut stdin = stdin.bytes().filter(|x| x.as_ref().ok() != Some(&b' '));

    loop {
        // Skip the first column
        stdin.nth(8);
        // Process the inner 8 columns
        for _ in 0..16 {
            // The first hex digit to decode.
            let h1 = ascii_to_hex(
                if let Some(x) = stdin.next() {
                    x.try(&mut stderr)
                } else {
                    return;
                }
            );
            // The second hex digit to decode.
            let h2 = ascii_to_hex(
                if let Some(x) = stdin.next() {
                    x.try(&mut stderr)
                } else {
                    return;
                }
            );

            // Write the decoded, joined hex digits in binary form.
            stdout.write(&[hex_to_u8((h1, h2))]).try(&mut stderr);
        }

        // Skip the rest until newline.
        loop {
            if let Some(x) = stdin.next() {
                if x.try(&mut stderr) == b'\n' {
                    break;
                }
            } else {
                return;
            }
        }
    }
}

fn main() {
    let stdout = io::stdout();
    let mut stdout = stdout.lock();
    let mut stderr = io::stderr();

    let mut args = env::args();

    // Arguments should be <= 2
    if args.len() > 2 {
        fail("too many arguments.", &mut stderr);
    }

    match args.nth(1) {
        None => encode(io::stdin(), stdout, stderr),
        Some(a) => match a.as_ref() { // MIR plz
            "-h" | "--help" => {
                // HEEEEEELP.
                stdout.writeln(HELP).try(&mut stderr);
            },
            "-r" | "--reverse" => {
                match args.next() {
                    // Decode.
                    None => {
                        let stdin = io::stdin();
                        decode(stdin.lock(), stdout, stderr);
                    }
                    // Encode.
                    Some(f) => {
                        let file = fs::File::open(f).try(&mut stderr);
                        decode(file, stdout, stderr);
                    }
                }
            },
            // Read from a file, instead of standard input.
            f => {
                let file = fs::File::open(f).try(&mut stderr);
                encode(file, stdout, stderr);
            },
        },
    }
}
<|endoftext|>


<commit_before>// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#![cfg(not(test))]

extern crate env_logger;
extern crate getopts;
extern crate rustfmt_nightly as rustfmt;

use std::{env, error};
use std::fs::File;
use std::io::{self, Read, Write};
use std::path::{Path, PathBuf};
use std::str::FromStr;

use getopts::{Matches, Options};

use rustfmt::{run, Input, Summary};
use rustfmt::file_lines::FileLines;
use rustfmt::config::{get_toml_path, Color, Config, WriteMode};

type FmtError = Box<error::Error + Send + Sync>;
type FmtResult<T> = std::result::Result<T, FmtError>;

/// Rustfmt operations.
enum Operation {
    /// Format files and their child modules.
    Format {
        files: Vec<PathBuf>,
        config_path: Option<PathBuf>,
        minimal_config_path: Option<String>,
    },
    /// Print the help message.
    Help,
    // Print version information
    Version,
    /// Print detailed configuration help.
    ConfigHelp,
    /// Output default config to a file, or stdout if None
    ConfigOutputDefault {
        path: Option<String>,
    },
    /// No file specified, read from stdin
    Stdin {
        input: String,
        config_path: Option<PathBuf>,
    },
}

/// Parsed command line options.
#[derive(Clone, Debug, Default)]
struct CliOptions {
    skip_children: bool,
    verbose: bool,
    write_mode: Option<WriteMode>,
    color: Option<Color>,
    file_lines: FileLines, // Default is all lines in all files.
    unstable_features: bool,
}

impl CliOptions {
    fn from_matches(matches: &Matches) -> FmtResult<CliOptions> {
        let mut options = CliOptions::default();
        options.skip_children = matches.opt_present("skip-children");
        options.verbose = matches.opt_present("verbose");
        let unstable_features = matches.opt_present("unstable-features");
        let rust_nightly = option_env!("CFG_RELEASE_CHANNEL")
            .map(|c| c == "nightly")
            .unwrap_or(false);
        if unstable_features && !rust_nightly {
            return Err(FmtError::from(
                "Unstable features are only available on Nightly channel",
            ));
        } else {
            options.unstable_features = unstable_features;
        }

        if let Some(ref write_mode) = matches.opt_str("write-mode") {
            if let Ok(write_mode) = WriteMode::from_str(write_mode) {
                options.write_mode = Some(write_mode);
            } else {
                return Err(FmtError::from(format!(
                    "Invalid write-mode: {}",
                    write_mode
                )));
            }
        }

        if let Some(ref color) = matches.opt_str("color") {
            match Color::from_str(color) {
                Ok(color) => options.color = Some(color),
                _ => return Err(FmtError::from(format!("Invalid color: {}", color))),
            }
        }

        if let Some(ref file_lines) = matches.opt_str("file-lines") {
            options.file_lines = file_lines.parse()?;
        }

        Ok(options)
    }

    fn apply_to(self, config: &mut Config) {
        config.set().skip_children(self.skip_children);
        config.set().verbose(self.verbose);
        config.set().file_lines(self.file_lines);
        config.set().unstable_features(self.unstable_features);
        if let Some(write_mode) = self.write_mode {
            config.set().write_mode(write_mode);
        }
        if let Some(color) = self.color {
            config.set().color(color);
        }
    }
}

/// read the given config file path recursively if present else read the project file path
fn match_cli_path_or_file(
    config_path: Option<PathBuf>,
    input_file: &Path,
) -> FmtResult<(Config, Option<PathBuf>)> {
    if let Some(config_file) = config_path {
        let toml = Config::from_toml_path(config_file.as_ref())?;
        return Ok((toml, Some(config_file)));
    }
    Config::from_resolved_toml_path(input_file).map_err(FmtError::from)
}

fn make_opts() -> Options {
    let mut opts = Options::new();

    opts.optflag("h", "help", "Show this message");
    opts.optflag("V", "version", "Show version information");
    opts.optflag("v", "verbose", "Print verbose output");
    opts.optflag("", "skip-children", "Don't reformat child modules");
    opts.optflag(
        "",
        "unstable-features",
        "Enables unstable features. Only available on nightly channel",
    );
    opts.optflag(
        "",
        "config-help",
        "Show details of rustfmt configuration options",
    );
    opts.optflag(
        "",
        "error-on-unformatted",
        "Error if unable to get comments or string literals within max_width, \
         or they are left with trailing whitespaces",
    );

    opts.optopt(
        "",
        "write-mode",
        "How to write output (not usable when piping from stdin)",
        "[replace|overwrite|display|plain|diff|coverage|checkstyle]",
    );
    opts.optopt(
        "",
        "color",
        "Use colored output (if supported)",
        "[always|never|auto]",
    );
    opts.optopt(
        "",
        "dump-default-config",
        "Dumps default configuration to PATH. PATH defaults to stdout, if omitted.",
        "PATH",
    );
    opts.optopt(
        "",
        "dump-minimal-config",
        "Dumps configuration options that were checked during formatting to a file.",
        "PATH",
    );
    opts.optopt(
        "",
        "config-path",
        "Recursively searches the given path for the rustfmt.toml config file. If not \
         found reverts to the input file path",
        "[Path for the configuration file]",
    );
    opts.optopt(
        "",
        "file-lines",
        "Format specified line ranges. See README for more detail on the JSON format.",
        "JSON",
    );

    opts
}

fn execute(opts: &Options) -> FmtResult<Summary> {
    let matches = opts.parse(env::args().skip(1))?;

    match determine_operation(&matches)? {
        Operation::Help => {
            print_usage_to_stdout(opts, "");
            Summary::print_exit_codes();
            Ok(Summary::default())
        }
        Operation::Version => {
            print_version();
            Ok(Summary::default())
        }
        Operation::ConfigHelp => {
            Config::print_docs();
            Ok(Summary::default())
        }
        Operation::ConfigOutputDefault { path } => {
            let toml = Config::default().all_options().to_toml()?;
            if let Some(path) = path {
                let mut file = File::create(path)?;
                file.write_all(toml.as_bytes())?;
            } else {
                io::stdout().write_all(toml.as_bytes())?;
            }
            Ok(Summary::default())
        }
        Operation::Stdin { input, config_path } => {
            // try to read config from local directory
            let (mut config, _) =
                match_cli_path_or_file(config_path, &env::current_dir().unwrap())?;

            // write_mode is always Plain for Stdin.
            config.set().write_mode(WriteMode::Plain);

            // parse file_lines
            if let Some(ref file_lines) = matches.opt_str("file-lines") {
                config.set().file_lines(file_lines.parse()?);
                for f in config.file_lines().files() {
                    if f != "stdin" {
                        eprintln!("Warning: Extra file listed in file_lines option '{}'", f);
                    }
                }
            }

            let mut error_summary = Summary::default();
            if config.version_meets_requirement(&mut error_summary) {
                error_summary.add(run(Input::Text(input), &config));
            }

            Ok(error_summary)
        }
        Operation::Format {
            files,
            config_path,
            minimal_config_path,
        } => {
            let options = CliOptions::from_matches(&matches)?;

            for f in options.file_lines.files() {
                if !files.contains(&PathBuf::from(f)) {
                    eprintln!("Warning: Extra file listed in file_lines option '{}'", f);
                }
            }

            let mut config = Config::default();
            // Load the config path file if provided
            if let Some(config_file) = config_path.as_ref() {
                config = Config::from_toml_path(config_file.as_ref())?;
            };

            if options.verbose {
                if let Some(path) = config_path.as_ref() {
                    println!("Using rustfmt config file {}", path.display());
                }
            }

            let mut error_summary = Summary::default();
            for file in files {
                if !file.exists() {
                    eprintln!("Error: file `{}` does not exist", file.to_str().unwrap());
                    error_summary.add_operational_error();
                } else if file.is_dir() {
                    eprintln!("Error: `{}` is a directory", file.to_str().unwrap());
                    error_summary.add_operational_error();
                } else {
                    // Check the file directory if the config-path could not be read or not provided
                    if config_path.is_none() {
                        let (config_tmp, path_tmp) =
                            Config::from_resolved_toml_path(file.parent().unwrap())?;
                        if options.verbose {
                            if let Some(path) = path_tmp.as_ref() {
                                println!(
                                    "Using rustfmt config file {} for {}",
                                    path.display(),
                                    file.display()
                                );
                            }
                        }
                        config = config_tmp;
                    }

                    if !config.version_meets_requirement(&mut error_summary) {
                        break;
                    }

                    options.clone().apply_to(&mut config);
                    error_summary.add(run(Input::File(file), &config));
                }
            }

            // If we were given a path via dump-minimal-config, output any options
            // that were used during formatting as TOML.
            if let Some(path) = minimal_config_path {
                let mut file = File::create(path)?;
                let toml = config.used_options().to_toml()?;
                file.write_all(toml.as_bytes())?;
            }

            Ok(error_summary)
        }
    }
}

fn main() {
    let _ = env_logger::init();

    let opts = make_opts();

    let exit_code = match execute(&opts) {
        Ok(summary) => {
            if summary.has_operational_errors() {
                1
            } else if summary.has_parsing_errors() {
                2
            } else if summary.has_formatting_errors() {
                3
            } else if summary.has_diff {
                // should only happen in diff mode
                4
            } else {
                assert!(summary.has_no_errors());
                0
            }
        }
        Err(e) => {
            print_usage_to_stderr(&opts, &e.to_string());
            1
        }
    };
    // Make sure standard output is flushed before we exit.
    std::io::stdout().flush().unwrap();

    // Exit with given exit code.
    //
    // NOTE: This immediately terminates the process without doing any cleanup,
    // so make sure to finish all necessary cleanup before this is called.
    std::process::exit(exit_code);
}

macro_rules! print_usage {
    ($print:ident, $opts:ident, $reason:expr) => ({
        let msg = format!(
            "{}\n\nusage: {} [options] <file>...",
            $reason,
            env::args_os().next().unwrap().to_string_lossy()
        );
        $print!("{}", $opts.usage(&msg));
    })
}

fn print_usage_to_stdout(opts: &Options, reason: &str) {
    print_usage!(println, opts, reason);
}

fn print_usage_to_stderr(opts: &Options, reason: &str) {
    print_usage!(eprintln, opts, reason);
}

fn print_version() {
    println!(
        "{}-nightly{}",
        env!("CARGO_PKG_VERSION"),
        include_str!(concat!(env!("OUT_DIR"), "/commit-info.txt"))
    )
}

fn determine_operation(matches: &Matches) -> FmtResult<Operation> {
    if matches.opt_present("h") {
        return Ok(Operation::Help);
    }

    if matches.opt_present("config-help") {
        return Ok(Operation::ConfigHelp);
    }

    if matches.opt_present("dump-default-config") {
        // NOTE for some reason when configured with HasArg::Maybe + Occur::Optional opt_default
        // doesn't recognize `--foo bar` as a long flag with an argument but as a long flag with no
        // argument *plus* a free argument. Thus we check for that case in this branch -- this is
        // required for backward compatibility.
        if let Some(path) = matches.free.get(0) {
            return Ok(Operation::ConfigOutputDefault {
                path: Some(path.clone()),
            });
        } else {
            return Ok(Operation::ConfigOutputDefault {
                path: matches.opt_str("dump-default-config"),
            });
        }
    }

    if matches.opt_present("version") {
        return Ok(Operation::Version);
    }

    let config_path_not_found = |path: &str| -> FmtResult<Operation> {
        Err(FmtError::from(format!(
            "Error: unable to find a config file for the given path: `{}`",
            path
        )))
    };

    // Read the config_path and convert to parent dir if a file is provided.
    // If a config file cannot be found from the given path, return error.
    let config_path: Option<PathBuf> = match matches.opt_str("config-path").map(PathBuf::from) {
        Some(ref path) if !path.exists() => return config_path_not_found(path.to_str().unwrap()),
        Some(ref path) if path.is_dir() => {
            let config_file_path = get_toml_path(path)?;
            if config_file_path.is_some() {
                config_file_path
            } else {
                return config_path_not_found(path.to_str().unwrap());
            }
        }
        path => path,
    };

    // If no path is given, we won't output a minimal config.
    let minimal_config_path = matches.opt_str("dump-minimal-config");

    // if no file argument is supplied, read from stdin
    if matches.free.is_empty() {
        let mut buffer = String::new();
        io::stdin().read_to_string(&mut buffer)?;

        return Ok(Operation::Stdin {
            input: buffer,
            config_path: config_path,
        });
    }

    let files: Vec<_> = matches
        .free
        .iter()
        .map(|s| {
            let p = PathBuf::from(s);
            // we will do comparison later, so here tries to canonicalize first
            // to get the expected behavior.
            p.canonicalize().unwrap_or(p)
        })
        .collect();

    Ok(Operation::Format {
        files: files,
        config_path: config_path,
        minimal_config_path: minimal_config_path,
    })
}
<commit_msg>Print command line options in alphabetical order<commit_after>// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#![cfg(not(test))]

extern crate env_logger;
extern crate getopts;
extern crate rustfmt_nightly as rustfmt;

use std::{env, error};
use std::fs::File;
use std::io::{self, Read, Write};
use std::path::{Path, PathBuf};
use std::str::FromStr;

use getopts::{Matches, Options};

use rustfmt::{run, Input, Summary};
use rustfmt::file_lines::FileLines;
use rustfmt::config::{get_toml_path, Color, Config, WriteMode};

type FmtError = Box<error::Error + Send + Sync>;
type FmtResult<T> = std::result::Result<T, FmtError>;

/// Rustfmt operations.
enum Operation {
    /// Format files and their child modules.
    Format {
        files: Vec<PathBuf>,
        config_path: Option<PathBuf>,
        minimal_config_path: Option<String>,
    },
    /// Print the help message.
    Help,
    // Print version information
    Version,
    /// Print detailed configuration help.
    ConfigHelp,
    /// Output default config to a file, or stdout if None
    ConfigOutputDefault {
        path: Option<String>,
    },
    /// No file specified, read from stdin
    Stdin {
        input: String,
        config_path: Option<PathBuf>,
    },
}

/// Parsed command line options.
#[derive(Clone, Debug, Default)]
struct CliOptions {
    skip_children: bool,
    verbose: bool,
    write_mode: Option<WriteMode>,
    color: Option<Color>,
    file_lines: FileLines, // Default is all lines in all files.
    unstable_features: bool,
}

impl CliOptions {
    fn from_matches(matches: &Matches) -> FmtResult<CliOptions> {
        let mut options = CliOptions::default();
        options.skip_children = matches.opt_present("skip-children");
        options.verbose = matches.opt_present("verbose");
        let unstable_features = matches.opt_present("unstable-features");
        let rust_nightly = option_env!("CFG_RELEASE_CHANNEL")
            .map(|c| c == "nightly")
            .unwrap_or(false);
        if unstable_features && !rust_nightly {
            return Err(FmtError::from(
                "Unstable features are only available on Nightly channel",
            ));
        } else {
            options.unstable_features = unstable_features;
        }

        if let Some(ref write_mode) = matches.opt_str("write-mode") {
            if let Ok(write_mode) = WriteMode::from_str(write_mode) {
                options.write_mode = Some(write_mode);
            } else {
                return Err(FmtError::from(format!(
                    "Invalid write-mode: {}",
                    write_mode
                )));
            }
        }

        if let Some(ref color) = matches.opt_str("color") {
            match Color::from_str(color) {
                Ok(color) => options.color = Some(color),
                _ => return Err(FmtError::from(format!("Invalid color: {}", color))),
            }
        }

        if let Some(ref file_lines) = matches.opt_str("file-lines") {
            options.file_lines = file_lines.parse()?;
        }

        Ok(options)
    }

    fn apply_to(self, config: &mut Config) {
        config.set().skip_children(self.skip_children);
        config.set().verbose(self.verbose);
        config.set().file_lines(self.file_lines);
        config.set().unstable_features(self.unstable_features);
        if let Some(write_mode) = self.write_mode {
            config.set().write_mode(write_mode);
        }
        if let Some(color) = self.color {
            config.set().color(color);
        }
    }
}

/// read the given config file path recursively if present else read the project file path
fn match_cli_path_or_file(
    config_path: Option<PathBuf>,
    input_file: &Path,
) -> FmtResult<(Config, Option<PathBuf>)> {
    if let Some(config_file) = config_path {
        let toml = Config::from_toml_path(config_file.as_ref())?;
        return Ok((toml, Some(config_file)));
    }
    Config::from_resolved_toml_path(input_file).map_err(FmtError::from)
}

fn make_opts() -> Options {
    let mut opts = Options::new();

    // Sorted in alphabetical order.
    opts.optopt(
        "",
        "color",
        "Use colored output (if supported)",
        "[always|never|auto]",
    );
    opts.optflag(
        "",
        "config-help",
        "Show details of rustfmt configuration options",
    );
    opts.optopt(
        "",
        "config-path",
        "Recursively searches the given path for the rustfmt.toml config file. If not \
         found reverts to the input file path",
        "[Path for the configuration file]",
    );
    opts.optopt(
        "",
        "dump-default-config",
        "Dumps default configuration to PATH. PATH defaults to stdout, if omitted.",
        "PATH",
    );
    opts.optopt(
        "",
        "dump-minimal-config",
        "Dumps configuration options that were checked during formatting to a file.",
        "PATH",
    );
    opts.optflag(
        "",
        "error-on-unformatted",
        "Error if unable to get comments or string literals within max_width, \
         or they are left with trailing whitespaces",
    );
    opts.optopt(
        "",
        "file-lines",
        "Format specified line ranges. See README for more detail on the JSON format.",
        "JSON",
    );
    opts.optflag("h", "help", "Show this message");
    opts.optflag("", "skip-children", "Don't reformat child modules");
    opts.optflag(
        "",
        "unstable-features",
        "Enables unstable features. Only available on nightly channel",
    );
    opts.optflag("v", "verbose", "Print verbose output");
    opts.optflag("V", "version", "Show version information");
    opts.optopt(
        "",
        "write-mode",
        "How to write output (not usable when piping from stdin)",
        "[replace|overwrite|display|plain|diff|coverage|checkstyle]",
    );

    opts
}

fn execute(opts: &Options) -> FmtResult<Summary> {
    let matches = opts.parse(env::args().skip(1))?;

    match determine_operation(&matches)? {
        Operation::Help => {
            print_usage_to_stdout(opts, "");
            Summary::print_exit_codes();
            Ok(Summary::default())
        }
        Operation::Version => {
            print_version();
            Ok(Summary::default())
        }
        Operation::ConfigHelp => {
            Config::print_docs();
            Ok(Summary::default())
        }
        Operation::ConfigOutputDefault { path } => {
            let toml = Config::default().all_options().to_toml()?;
            if let Some(path) = path {
                let mut file = File::create(path)?;
                file.write_all(toml.as_bytes())?;
            } else {
                io::stdout().write_all(toml.as_bytes())?;
            }
            Ok(Summary::default())
        }
        Operation::Stdin { input, config_path } => {
            // try to read config from local directory
            let (mut config, _) =
                match_cli_path_or_file(config_path, &env::current_dir().unwrap())?;

            // write_mode is always Plain for Stdin.
            config.set().write_mode(WriteMode::Plain);

            // parse file_lines
            if let Some(ref file_lines) = matches.opt_str("file-lines") {
                config.set().file_lines(file_lines.parse()?);
                for f in config.file_lines().files() {
                    if f != "stdin" {
                        eprintln!("Warning: Extra file listed in file_lines option '{}'", f);
                    }
                }
            }

            let mut error_summary = Summary::default();
            if config.version_meets_requirement(&mut error_summary) {
                error_summary.add(run(Input::Text(input), &config));
            }

            Ok(error_summary)
        }
        Operation::Format {
            files,
            config_path,
            minimal_config_path,
        } => {
            let options = CliOptions::from_matches(&matches)?;

            for f in options.file_lines.files() {
                if !files.contains(&PathBuf::from(f)) {
                    eprintln!("Warning: Extra file listed in file_lines option '{}'", f);
                }
            }

            let mut config = Config::default();
            // Load the config path file if provided
            if let Some(config_file) = config_path.as_ref() {
                config = Config::from_toml_path(config_file.as_ref())?;
            };

            if options.verbose {
                if let Some(path) = config_path.as_ref() {
                    println!("Using rustfmt config file {}", path.display());
                }
            }

            let mut error_summary = Summary::default();
            for file in files {
                if !file.exists() {
                    eprintln!("Error: file `{}` does not exist", file.to_str().unwrap());
                    error_summary.add_operational_error();
                } else if file.is_dir() {
                    eprintln!("Error: `{}` is a directory", file.to_str().unwrap());
                    error_summary.add_operational_error();
                } else {
                    // Check the file directory if the config-path could not be read or not provided
                    if config_path.is_none() {
                        let (config_tmp, path_tmp) =
                            Config::from_resolved_toml_path(file.parent().unwrap())?;
                        if options.verbose {
                            if let Some(path) = path_tmp.as_ref() {
                                println!(
                                    "Using rustfmt config file {} for {}",
                                    path.display(),
                                    file.display()
                                );
                            }
                        }
                        config = config_tmp;
                    }

                    if !config.version_meets_requirement(&mut error_summary) {
                        break;
                    }

                    options.clone().apply_to(&mut config);
                    error_summary.add(run(Input::File(file), &config));
                }
            }

            // If we were given a path via dump-minimal-config, output any options
            // that were used during formatting as TOML.
            if let Some(path) = minimal_config_path {
                let mut file = File::create(path)?;
                let toml = config.used_options().to_toml()?;
                file.write_all(toml.as_bytes())?;
            }

            Ok(error_summary)
        }
    }
}

fn main() {
    let _ = env_logger::init();

    let opts = make_opts();

    let exit_code = match execute(&opts) {
        Ok(summary) => {
            if summary.has_operational_errors() {
                1
            } else if summary.has_parsing_errors() {
                2
            } else if summary.has_formatting_errors() {
                3
            } else if summary.has_diff {
                // should only happen in diff mode
                4
            } else {
                assert!(summary.has_no_errors());
                0
            }
        }
        Err(e) => {
            print_usage_to_stderr(&opts, &e.to_string());
            1
        }
    };
    // Make sure standard output is flushed before we exit.
    std::io::stdout().flush().unwrap();

    // Exit with given exit code.
    //
    // NOTE: This immediately terminates the process without doing any cleanup,
    // so make sure to finish all necessary cleanup before this is called.
    std::process::exit(exit_code);
}

macro_rules! print_usage {
    ($print:ident, $opts:ident, $reason:expr) => ({
        let msg = format!(
            "{}\n\nusage: {} [options] <file>...",
            $reason,
            env::args_os().next().unwrap().to_string_lossy()
        );
        $print!("{}", $opts.usage(&msg));
    })
}

fn print_usage_to_stdout(opts: &Options, reason: &str) {
    print_usage!(println, opts, reason);
}

fn print_usage_to_stderr(opts: &Options, reason: &str) {
    print_usage!(eprintln, opts, reason);
}

fn print_version() {
    println!(
        "{}-nightly{}",
        env!("CARGO_PKG_VERSION"),
        include_str!(concat!(env!("OUT_DIR"), "/commit-info.txt"))
    )
}

fn determine_operation(matches: &Matches) -> FmtResult<Operation> {
    if matches.opt_present("h") {
        return Ok(Operation::Help);
    }

    if matches.opt_present("config-help") {
        return Ok(Operation::ConfigHelp);
    }

    if matches.opt_present("dump-default-config") {
        // NOTE for some reason when configured with HasArg::Maybe + Occur::Optional opt_default
        // doesn't recognize `--foo bar` as a long flag with an argument but as a long flag with no
        // argument *plus* a free argument. Thus we check for that case in this branch -- this is
        // required for backward compatibility.
        if let Some(path) = matches.free.get(0) {
            return Ok(Operation::ConfigOutputDefault {
                path: Some(path.clone()),
            });
        } else {
            return Ok(Operation::ConfigOutputDefault {
                path: matches.opt_str("dump-default-config"),
            });
        }
    }

    if matches.opt_present("version") {
        return Ok(Operation::Version);
    }

    let config_path_not_found = |path: &str| -> FmtResult<Operation> {
        Err(FmtError::from(format!(
            "Error: unable to find a config file for the given path: `{}`",
            path
        )))
    };

    // Read the config_path and convert to parent dir if a file is provided.
    // If a config file cannot be found from the given path, return error.
    let config_path: Option<PathBuf> = match matches.opt_str("config-path").map(PathBuf::from) {
        Some(ref path) if !path.exists() => return config_path_not_found(path.to_str().unwrap()),
        Some(ref path) if path.is_dir() => {
            let config_file_path = get_toml_path(path)?;
            if config_file_path.is_some() {
                config_file_path
            } else {
                return config_path_not_found(path.to_str().unwrap());
            }
        }
        path => path,
    };

    // If no path is given, we won't output a minimal config.
    let minimal_config_path = matches.opt_str("dump-minimal-config");

    // if no file argument is supplied, read from stdin
    if matches.free.is_empty() {
        let mut buffer = String::new();
        io::stdin().read_to_string(&mut buffer)?;

        return Ok(Operation::Stdin {
            input: buffer,
            config_path: config_path,
        });
    }

    let files: Vec<_> = matches
        .free
        .iter()
        .map(|s| {
            let p = PathBuf::from(s);
            // we will do comparison later, so here tries to canonicalize first
            // to get the expected behavior.
            p.canonicalize().unwrap_or(p)
        })
        .collect();

    Ok(Operation::Format {
        files: files,
        config_path: config_path,
        minimal_config_path: minimal_config_path,
    })
}
<|endoftext|>