<commit_before>extern crate crc;
extern crate ci_capnp;
extern crate chrono;

use std::io;

use crc::crc32;

use ci_capnp::FileEntry;

use std::io::Read;

use chrono::TimeZone;

struct WithCrc {
    entry: FileEntry,
    crc: u32,
}

fn main() {
    let input = io::stdin();
    let mut input = &mut input.lock();

    let mut all = Vec::new();

    loop {
        let entry: FileEntry = match ci_capnp::read_entry(input).expect("reading heder") {
            Some(x) => x,
            None => break,
        };

        let mut crc = 0;

        if entry.content_follows {
            let mut limited = input.take(entry.len);
            loop {
                let mut buf = [0u8; 4096];
                let found = limited.read(&mut buf).expect("reading data");
                if 0 == found {
                    break;
                }

                crc = crc32::update(crc, &crc32::CASTAGNOLI_TABLE, &buf[0..found]);
            }
        }

        all.push(WithCrc { entry, crc });
    }

    all.sort_by(|left, right| {
        use std::cmp::Ordering;
        let left = &left.entry.paths;
        let right = &right.entry.paths;
        match left.len().cmp(&right.len()) {
            Ordering::Equal => {}
            other => return other,
        };

        for i in 0..left.len() {
            match left[i].cmp(&right[i]) {
                Ordering::Equal => {}
                other => return other,
            }
        }

        return Ordering::Equal;
    });

    for item in all {
        let entry: FileEntry = item.entry;
        println!(" - paths:");
        for path in entry.paths {
            println!("          - {}", path);
        }

        println!("   type:  {:?}", entry.item_type);

        if 0 != entry.len {
            println!("   wrap:  {:?}", entry.container);
            println!("   data:  {:?}", entry.content_follows);
            println!("   size:  {}", entry.len);
            println!("   crc:   {:08x}", item.crc);
        }

        date("atime", entry.atime);
        date("mtime", entry.mtime);
        date("ctime", entry.ctime);
        date("btime", entry.btime);

        use ci_capnp::Ownership;
        match entry.ownership {
            Ownership::Unknown => {}
            Ownership::Posix { user, group, mode } => {
                println!("   uid:   {}", user.id);
                println!("   gid:   {}", group.id);
                if !user.name.is_empty() {
                    println!("   user:  {}", user.name);
                }
                if !group.name.is_empty() {
                    println!("   group: {}", group.name);
                }

                println!("   mode:  {:o}", mode);
            }
        }

        if !entry.xattrs.is_empty() {
            println!("   xattrs:");
            let mut keys: Vec<&String> = entry.xattrs.keys().collect();
            keys.sort();
            for key in keys {
                println!("     {}: {:?}", key, entry.xattrs[key]);
            }
        }
    }
}

fn date(whence: &str, nanos: u64) {
    if 0 != nanos {
        println!(
            "   {}: {}",
            whence,
            chrono::Utc.timestamp(
                (nanos / 1_000_000_000) as i64,
                (nanos % 1_000_000_000) as u32,
            )
        );
    }
}
<commit_msg>dump with a more traditional sorting<commit_after>extern crate crc;
extern crate ci_capnp;
extern crate chrono;

use std::io;

use crc::crc32;

use ci_capnp::FileEntry;

use std::io::Read;

use chrono::TimeZone;

struct WithCrc {
    entry: FileEntry,
    crc: u32,
}

fn main() {
    let input = io::stdin();
    let mut input = &mut input.lock();

    let mut all = Vec::new();

    loop {
        let entry: FileEntry = match ci_capnp::read_entry(input).expect("reading heder") {
            Some(x) => x,
            None => break,
        };

        let mut crc = 0;

        if entry.content_follows {
            let mut limited = input.take(entry.len);
            loop {
                let mut buf = [0u8; 4096];
                let found = limited.read(&mut buf).expect("reading data");
                if 0 == found {
                    break;
                }

                crc = crc32::update(crc, &crc32::CASTAGNOLI_TABLE, &buf[0..found]);
            }
        }

        all.push(WithCrc { entry, crc });
    }

    all.sort_by(|left, right| {
        use std::cmp::min;
        use std::cmp::Ordering;
        let left = &left.entry.paths;
        let right = &right.entry.paths;
        for i in 1..1 + min(left.len(), right.len()) {
            match left[left.len() - i].cmp(&right[right.len() - i]) {
                Ordering::Equal => {}
                other => return other,
            }
        }

        left.len().cmp(&right.len())
    });

    for item in all {
        let entry: FileEntry = item.entry;
        println!(" - paths:");
        for path in entry.paths {
            println!("          - {}", path);
        }

        println!("   type:  {:?}", entry.item_type);

        if 0 != entry.len {
            println!("   wrap:  {:?}", entry.container);
            println!("   data:  {:?}", entry.content_follows);
            println!("   size:  {}", entry.len);
            println!("   crc:   {:08x}", item.crc);
        }

        date("atime", entry.atime);
        date("mtime", entry.mtime);
        date("ctime", entry.ctime);
        date("btime", entry.btime);

        use ci_capnp::Ownership;
        match entry.ownership {
            Ownership::Unknown => {}
            Ownership::Posix { user, group, mode } => {
                println!("   uid:   {}", user.id);
                println!("   gid:   {}", group.id);
                if !user.name.is_empty() {
                    println!("   user:  {}", user.name);
                }
                if !group.name.is_empty() {
                    println!("   group: {}", group.name);
                }

                println!("   mode:  0o{:04o}", mode);
            }
        }

        if !entry.xattrs.is_empty() {
            println!("   xattrs:");
            let mut keys: Vec<&String> = entry.xattrs.keys().collect();
            keys.sort();
            for key in keys {
                println!("     {}: {:?}", key, entry.xattrs[key]);
            }
        }
    }
}

fn date(whence: &str, nanos: u64) {
    if 0 != nanos {
        println!(
            "   {}: {}",
            whence,
            chrono::Utc.timestamp(
                (nanos / 1_000_000_000) as i64,
                (nanos % 1_000_000_000) as u32,
            )
        );
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2016-2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#[macro_use]
extern crate log;
extern crate protobuf;
extern crate time;
extern crate zmq;

mod message;

use message::event::EventEnvelope;
use protobuf::parse_from_bytes;
use std::collections::HashMap;
use std::collections::HashSet;
use zmq::{Context, PULL, XPUB};

// Proxies messages coming into `frontend_port` out through
/// `backend_port`, caching recent messages for new subscribers.
///
/// Event publishers should connect a ZMQ `PUSH` socket to
/// `frontend_port` and send `EventEnvelope` protobuf messages to
/// it. Publishers can connect to multiple such proxy processes;
/// messages will be fairly-dealt to all connected proxies.
///
/// Event subscribers should connect a ZMQ `SUB` socket to
/// `backend_port` to receive `EventEnvelope` protobuf
/// messages. Subscribers should connect to all available proxy
/// processes to ensure they receive all event messages.
///
/// Upon connection, subscribers will receive, from each connected
/// proxy, the most recent messages from each service and from each
/// ring member. Subscribers are responsible for sorting the messages
/// received by timestamp.
pub fn proxy(frontend_port: i32, backend_port: i32) {
    let ctx = Context::new();

    let pull_sock = ctx.socket(PULL).unwrap();
    let pull_bind = format!("tcp://*:{}", frontend_port);
    assert!(pull_sock.bind(&pull_bind).is_ok());

    let xpub_sock = ctx.socket(XPUB).unwrap();
    let xpub_bind = format!("tcp://*:{}", backend_port);
    assert!(xpub_sock.bind(&xpub_bind).is_ok());

    // We'll cache the most recent messages from each service and each
    // ring member. When new subscribers connect, we can send them
    // this "snapshot" of current activity.
    let mut service_cache = HashMap::new();
    let mut member_cache = HashMap::new();

    let mut poll_items = [pull_sock.as_poll_item(zmq::POLLIN),
                          xpub_sock.as_poll_item(zmq::POLLIN)];

    loop {
        // A timeout of -1 says to wait indefinitely until a message comes
        if zmq::poll(&mut poll_items, -1).is_err() {
            break; // This will stop the event service
        }

        if poll_items[0].is_readable() {
            // An event was published!

            let bytes = pull_sock.recv_bytes(0).unwrap();
            let event = parse_from_bytes::<EventEnvelope>(&bytes).unwrap();
            let member_id = event.get_member_id();
            let timestamp = event.get_timestamp();
            let service = event.get_service().to_string();
            if service.is_empty() {
                warn!("missing service: {:?}", event);
                continue;
            }

            println!("EVENTSRV: Timestamp {}", timestamp);
            println!("EVENTSRV: Member ID {}", member_id);
            println!("EVENTSRV: Service {}", service);

            // Store the bytes of the message in the cache. For the
            // service cache, we also record the member ID, and vice
            // versa for the member cache; these data will be used for
            // deduplication of messages being sent to new subscribers.
            service_cache.insert(service.clone(), (member_id, bytes.clone()));
            member_cache.insert(member_id, (service, bytes.clone()));

            println!("EVENTSRV: Service Cache {:?}", service_cache.keys());
            println!("EVENTSRV: Member Cache {:?}\n", member_cache.keys());

            xpub_sock.send(&bytes, 0).unwrap();
        }

        if poll_items[1].is_readable() {
            // A subscriber connected; let's ensure they've got a
            // snapshot of what's currently happening.

            // Event is one byte 0=unsub or 1=sub, followed by topic
            let event = xpub_sock.recv_bytes(0).unwrap();
            if event[0] == 1 {
                // The subscriber has subscribed. Send all unique
                // cached messages to it.
                //
                // First we'll send all the latest messages from the
                // services, keeping track of which ring members those
                // were from. Then, we'll send the latest messages
                // from the members, but only if we didn't just send a
                // service message from them. This prevents us from
                // sending the same message twice.

                let mut members_encountered = HashSet::new();

                for (service, &(member_id, ref message)) in &service_cache {
                    members_encountered.insert(member_id);
                    println!("\tSending message for {}/{}", service, member_id);
                    xpub_sock.send(&message, 0).unwrap();
                }
                println!("\t---");
                for (member_id, &(ref service, ref message)) in &member_cache {
                    if !(members_encountered.contains(member_id)) {
                        println!("\tSending message for {}/{}", service, member_id);
                        xpub_sock.send(&message, 0).unwrap();
                    }
                }
            }
        }
    }
}
<commit_msg>Oops... forgot a documentation slash<commit_after>// Copyright (c) 2016-2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#[macro_use]
extern crate log;
extern crate protobuf;
extern crate time;
extern crate zmq;

mod message;

use message::event::EventEnvelope;
use protobuf::parse_from_bytes;
use std::collections::HashMap;
use std::collections::HashSet;
use zmq::{Context, PULL, XPUB};

/// Proxies messages coming into `frontend_port` out through
/// `backend_port`, caching recent messages for new subscribers.
///
/// Event publishers should connect a ZMQ `PUSH` socket to
/// `frontend_port` and send `EventEnvelope` protobuf messages to
/// it. Publishers can connect to multiple such proxy processes;
/// messages will be fairly-dealt to all connected proxies.
///
/// Event subscribers should connect a ZMQ `SUB` socket to
/// `backend_port` to receive `EventEnvelope` protobuf
/// messages. Subscribers should connect to all available proxy
/// processes to ensure they receive all event messages.
///
/// Upon connection, subscribers will receive, from each connected
/// proxy, the most recent messages from each service and from each
/// ring member. Subscribers are responsible for sorting the messages
/// received by timestamp.
pub fn proxy(frontend_port: i32, backend_port: i32) {
    let ctx = Context::new();

    let pull_sock = ctx.socket(PULL).unwrap();
    let pull_bind = format!("tcp://*:{}", frontend_port);
    assert!(pull_sock.bind(&pull_bind).is_ok());

    let xpub_sock = ctx.socket(XPUB).unwrap();
    let xpub_bind = format!("tcp://*:{}", backend_port);
    assert!(xpub_sock.bind(&xpub_bind).is_ok());

    // We'll cache the most recent messages from each service and each
    // ring member. When new subscribers connect, we can send them
    // this "snapshot" of current activity.
    let mut service_cache = HashMap::new();
    let mut member_cache = HashMap::new();

    let mut poll_items = [pull_sock.as_poll_item(zmq::POLLIN),
                          xpub_sock.as_poll_item(zmq::POLLIN)];

    loop {
        // A timeout of -1 says to wait indefinitely until a message comes
        if zmq::poll(&mut poll_items, -1).is_err() {
            break; // This will stop the event service
        }

        if poll_items[0].is_readable() {
            // An event was published!

            let bytes = pull_sock.recv_bytes(0).unwrap();
            let event = parse_from_bytes::<EventEnvelope>(&bytes).unwrap();
            let member_id = event.get_member_id();
            let timestamp = event.get_timestamp();
            let service = event.get_service().to_string();
            if service.is_empty() {
                warn!("missing service: {:?}", event);
                continue;
            }

            println!("EVENTSRV: Timestamp {}", timestamp);
            println!("EVENTSRV: Member ID {}", member_id);
            println!("EVENTSRV: Service {}", service);

            // Store the bytes of the message in the cache. For the
            // service cache, we also record the member ID, and vice
            // versa for the member cache; these data will be used for
            // deduplication of messages being sent to new subscribers.
            service_cache.insert(service.clone(), (member_id, bytes.clone()));
            member_cache.insert(member_id, (service, bytes.clone()));

            println!("EVENTSRV: Service Cache {:?}", service_cache.keys());
            println!("EVENTSRV: Member Cache {:?}\n", member_cache.keys());

            xpub_sock.send(&bytes, 0).unwrap();
        }

        if poll_items[1].is_readable() {
            // A subscriber connected; let's ensure they've got a
            // snapshot of what's currently happening.

            // Event is one byte 0=unsub or 1=sub, followed by topic
            let event = xpub_sock.recv_bytes(0).unwrap();
            if event[0] == 1 {
                // The subscriber has subscribed. Send all unique
                // cached messages to it.
                //
                // First we'll send all the latest messages from the
                // services, keeping track of which ring members those
                // were from. Then, we'll send the latest messages
                // from the members, but only if we didn't just send a
                // service message from them. This prevents us from
                // sending the same message twice.

                let mut members_encountered = HashSet::new();

                for (service, &(member_id, ref message)) in &service_cache {
                    members_encountered.insert(member_id);
                    println!("\tSending message for {}/{}", service, member_id);
                    xpub_sock.send(&message, 0).unwrap();
                }
                println!("\t---");
                for (member_id, &(ref service, ref message)) in &member_cache {
                    if !(members_encountered.contains(member_id)) {
                        println!("\tSending message for {}/{}", service, member_id);
                        xpub_sock.send(&message, 0).unwrap();
                    }
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>use lexer::{Token, Lexer};
use ast::{Ast, CC};

use std::rc::Rc;

pub struct Parser<'input> {
    lexer: Lexer<'input>
}

impl<'input> Parser<'input> {
    pub fn new(source: &'input str) -> Parser<'input> {
        Parser {
            lexer: Lexer::new(source)
        }
    }

    pub fn get_list(&mut self) -> Vec<Ast> {
        match self.get_expr() {
            Some(expr) => match self.lexer.next() {
                None => vec![],
                Some(Token::RParen) => vec![expr],
                Some(tok) => {
                    self.lexer.unread(tok);
                    let mut seq = self.get_list();
                    seq.insert(0, expr);

                    seq
                }
            },
            None => vec![],
        }
    }

    pub fn get_expr(&mut self) -> Option<Ast> {
        match self.lexer.next() {
            Some(Token::Symbol(s)) => Some(Ast::Symbol(Rc::new(s.to_string()))),
            Some(Token::Number(n)) => Some(Ast::Number(n)),
            Some(Token::LParen) => Some(Ast::List(self.get_list())),
            Some(Token::RParen) => panic!("line {}: {} unmatched ')'",
                                          self.lexer.line_num,
                                          self.lexer.col),
            _ => None,
        }
    }

    pub fn get_arg_names(args: &[Ast]) -> Vec<Rc<String>> {
        let arg_names : Vec<Rc<String>> =
            args.iter().map(|arg| match arg {
                Ast::Symbol(name) => name.clone(),
                _ => panic!("invalid arg"),
            }).collect();

        arg_names
    }

    pub fn get_ast(expr: &Ast) -> Ast {
        match expr {
            Ast::Symbol(sym) =>
                match &sym[..] {
                    "#f" => Ast::Bool(false),
                    "#t" => Ast::Bool(true),
                    _ => Ast::Symbol(Rc::new(sym.to_string()))
                },
            Ast::List(elts) =>
                match &elts[..] {
                    [Ast::Symbol(k), Ast::List(defelts), body]
                        if &k[..] == "define" => {
                            let name = &defelts[0];
                            let args = &defelts[1..];

                            if let Ast::Symbol(name) = name {
                                Ast::Define(name.clone(),
                                            Parser::get_arg_names(args),
                                            Box::new(Parser::get_ast(body)))
                            } else {
                                panic!("invalid function prototype");
                            }
                        },
                    [Ast::Symbol(k), cnd, thn, els]
                        if &k[..] == "if" => {
                            Ast::If(Box::new(Parser::get_ast(cnd)),
                                    Box::new(Parser::get_ast(thn)),
                                    Box::new(Parser::get_ast(els)))
                        },
                    [Ast::Symbol(k), Ast::List(bindings), body]
                        if &k[..] == "let" => {
                            let mut astified_bindings = vec![];
                            for bind_pair in bindings {
                                if let Ast::List(kv) = bind_pair {
                                    let (key, val) =
                                        if kv.len() == 2 {
                                            let astified_val = Parser::get_ast(&kv[1]);
                                            let keyname =
                                                if let Ast::Symbol(k) = kv[0].clone() {
                                                    k
                                                } else {
                                                    panic!("let binding key is not symbol");
                                                };
                                            (keyname, astified_val)
                                        } else {
                                            panic!("invalid let binding syntax");
                                        };

                                    astified_bindings.push((key, val));
                                }
                            }

                            Ast::Let(astified_bindings,
                                     Box::new(Parser::get_ast(body)))
                        },
                    [Ast::Symbol(k), Ast::List(args), body]
                        if &k[..] == "lambda" => {
                            Ast::Lambda(Parser::get_arg_names(&args),
                                        Box::new(Parser::get_ast(body)))
                        },
                    [Ast::Symbol(k), _..]
                        if &k[..] == "tuple" => {
                            let tuple_elts = elts[1..].iter()
                            // TODO: move?
                                .map(|e| Parser::get_ast(e))
                                .collect();
                            Ast::Tuple(tuple_elts)
                        },
                    [Ast::Symbol(cmp), left, right]
                        if (&cmp[..] == ">" || &cmp[..] == "<" ||
                            &cmp[..] == "<=" || &cmp[..] == ">=" ||
                            &cmp[..] == "=") => {
                            let cc = match &cmp[..] {
                                ">" => CC::G,
                                "<" => CC::L,
                                ">=" => CC::GE,
                                "<=" => CC::LE,
                                "=" => CC::E,
                                &_ => panic!("invalid cmp op"),
                            };

                            Ast::Cmp(cc,
                                     Box::new(Parser::get_ast(left)),
                                     Box::new(Parser::get_ast(right)))
                        },
                    [f, _..] => {
                        let args = elts[1..].iter().map(|e| Parser::get_ast(e)) // TODO: move
                            .collect();

                        Ast::App(Box::new(f.clone()), args)
                    }
                    _ => Ast::Nil,
                },
            _ => expr.clone(),
        }
    }

    pub fn read(&mut self) -> Option<Ast> {
        match self.get_expr() {
            Some(expr) => Some(Parser::get_ast(&expr)),
            None => None,
        }
    }
}

impl<'input> Iterator for Parser<'input> {
    type Item = Ast;

    fn next(&mut self) -> Option<Ast> {
        self.read()
    }
}

fn main() {
    let mut p = Parser::new("(if #t 42 (define (foo x y) (+ (* 1 2) 2)))");
    println!("{:?}", Parser::get_ast(&p.get_expr().unwrap()));

    let mut p2 = Parser::new("(let ((x 10) (y 2)) (+ x y))");
    println!("{:?}", Parser::get_ast(&p2.get_expr().unwrap()));

    let mut p3 = Parser::new("(lambda (x y) (+ x y))");
    println!("{:?}", Parser::get_ast(&p3.get_expr().unwrap()));

    let mut p4 = Parser::new("(tuple 1 2 3)");
    println!("{:?}", Parser::get_ast(&p4.get_expr().unwrap()));

    let mut p5 = Parser::new("(> 1 2)");
    println!("{:?}", Parser::get_ast(&p5.get_expr().unwrap()));

    let mut p6 = Parser::new("(+ 1 2)");
    println!("{:?}", Parser::get_ast(&p6.get_expr().unwrap()));

    let ps = Parser::new("(> 1 2) (+ 1 2)");
    for a in ps {
        println!("{:?}", a);
    }
}
<commit_msg>Switch to new slice pattern-matching syntax<commit_after>use lexer::{Token, Lexer};
use ast::{Ast, CC};

use std::rc::Rc;

pub struct Parser<'input> {
    lexer: Lexer<'input>
}

impl<'input> Parser<'input> {
    pub fn new(source: &'input str) -> Parser<'input> {
        Parser {
            lexer: Lexer::new(source)
        }
    }

    pub fn get_list(&mut self) -> Vec<Ast> {
        match self.get_expr() {
            Some(expr) => match self.lexer.next() {
                None => vec![],
                Some(Token::RParen) => vec![expr],
                Some(tok) => {
                    self.lexer.unread(tok);
                    let mut seq = self.get_list();
                    seq.insert(0, expr);

                    seq
                }
            },
            None => vec![],
        }
    }

    pub fn get_expr(&mut self) -> Option<Ast> {
        match self.lexer.next() {
            Some(Token::Symbol(s)) => Some(Ast::Symbol(Rc::new(s.to_string()))),
            Some(Token::Number(n)) => Some(Ast::Number(n)),
            Some(Token::LParen) => Some(Ast::List(self.get_list())),
            Some(Token::RParen) => panic!("line {}: {} unmatched ')'",
                                          self.lexer.line_num,
                                          self.lexer.col),
            _ => None,
        }
    }

    pub fn get_arg_names(args: &[Ast]) -> Vec<Rc<String>> {
        let arg_names : Vec<Rc<String>> =
            args.iter().map(|arg| match arg {
                Ast::Symbol(name) => name.clone(),
                _ => panic!("invalid arg"),
            }).collect();

        arg_names
    }

    pub fn get_ast(expr: &Ast) -> Ast {
        match expr {
            Ast::Symbol(sym) =>
                match &sym[..] {
                    "#f" => Ast::Bool(false),
                    "#t" => Ast::Bool(true),
                    _ => Ast::Symbol(Rc::new(sym.to_string()))
                },
            Ast::List(elts) =>
                match &elts[..] {
                    [Ast::Symbol(k), Ast::List(defelts), body]
                        if &k[..] == "define" => {
                            let name = &defelts[0];
                            let args = &defelts[1..];

                            if let Ast::Symbol(name) = name {
                                Ast::Define(name.clone(),
                                            Parser::get_arg_names(args),
                                            Box::new(Parser::get_ast(body)))
                            } else {
                                panic!("invalid function prototype");
                            }
                        },
                    [Ast::Symbol(k), cnd, thn, els]
                        if &k[..] == "if" => {
                            Ast::If(Box::new(Parser::get_ast(cnd)),
                                    Box::new(Parser::get_ast(thn)),
                                    Box::new(Parser::get_ast(els)))
                        },
                    [Ast::Symbol(k), Ast::List(bindings), body]
                        if &k[..] == "let" => {
                            let mut astified_bindings = vec![];
                            for bind_pair in bindings {
                                if let Ast::List(kv) = bind_pair {
                                    let (key, val) =
                                        if kv.len() == 2 {
                                            let astified_val = Parser::get_ast(&kv[1]);
                                            let keyname =
                                                if let Ast::Symbol(k) = kv[0].clone() {
                                                    k
                                                } else {
                                                    panic!("let binding key is not symbol");
                                                };
                                            (keyname, astified_val)
                                        } else {
                                            panic!("invalid let binding syntax");
                                        };

                                    astified_bindings.push((key, val));
                                }
                            }

                            Ast::Let(astified_bindings,
                                     Box::new(Parser::get_ast(body)))
                        },
                    [Ast::Symbol(k), Ast::List(args), body]
                        if &k[..] == "lambda" => {
                            Ast::Lambda(Parser::get_arg_names(&args),
                                        Box::new(Parser::get_ast(body)))
                        },
                    [Ast::Symbol(k), ..]
                        if &k[..] == "tuple" => {
                            let tuple_elts = elts[1..].iter()
                            // TODO: move?
                                .map(|e| Parser::get_ast(e))
                                .collect();
                            Ast::Tuple(tuple_elts)
                        },
                    [Ast::Symbol(cmp), left, right]
                        if (&cmp[..] == ">" || &cmp[..] == "<" ||
                            &cmp[..] == "<=" || &cmp[..] == ">=" ||
                            &cmp[..] == "=") => {
                            let cc = match &cmp[..] {
                                ">" => CC::G,
                                "<" => CC::L,
                                ">=" => CC::GE,
                                "<=" => CC::LE,
                                "=" => CC::E,
                                &_ => panic!("invalid cmp op"),
                            };

                            Ast::Cmp(cc,
                                     Box::new(Parser::get_ast(left)),
                                     Box::new(Parser::get_ast(right)))
                        },
                    [f, ..] => {
                        let args = elts[1..].iter().map(|e| Parser::get_ast(e)) // TODO: move
                            .collect();

                        Ast::App(Box::new(f.clone()), args)
                    }
                    _ => Ast::Nil,
                },
            _ => expr.clone(),
        }
    }

    pub fn read(&mut self) -> Option<Ast> {
        match self.get_expr() {
            Some(expr) => Some(Parser::get_ast(&expr)),
            None => None,
        }
    }
}

impl<'input> Iterator for Parser<'input> {
    type Item = Ast;

    fn next(&mut self) -> Option<Ast> {
        self.read()
    }
}

fn main() {
    let mut p = Parser::new("(if #t 42 (define (foo x y) (+ (* 1 2) 2)))");
    println!("{:?}", Parser::get_ast(&p.get_expr().unwrap()));

    let mut p2 = Parser::new("(let ((x 10) (y 2)) (+ x y))");
    println!("{:?}", Parser::get_ast(&p2.get_expr().unwrap()));

    let mut p3 = Parser::new("(lambda (x y) (+ x y))");
    println!("{:?}", Parser::get_ast(&p3.get_expr().unwrap()));

    let mut p4 = Parser::new("(tuple 1 2 3)");
    println!("{:?}", Parser::get_ast(&p4.get_expr().unwrap()));

    let mut p5 = Parser::new("(> 1 2)");
    println!("{:?}", Parser::get_ast(&p5.get_expr().unwrap()));

    let mut p6 = Parser::new("(+ 1 2)");
    println!("{:?}", Parser::get_ast(&p6.get_expr().unwrap()));

    let ps = Parser::new("(> 1 2) (+ 1 2)");
    for a in ps {
        println!("{:?}", a);
    }
}
<|endoftext|>


<commit_before>
use ast::ExprAst;
use ast::BoolNode;
use ast::CharNode;
use ast::IntNode;
use ast::StrNode;
use ast::PairNode;
use ast::SymbolNode;
use ast::EmptyListNode;

use ast::Ast;

#[deriving(Clone, Show)]
pub struct Parser {
    code: String,
    cur: uint,
    col: uint,
    line: uint
}

#[allow(dead_code)]
pub struct ParserError {
    line: uint,
    col: uint,
    desc: String
}

#[allow(dead_code)]
impl ParserError {
    pub fn new(line: uint, col: uint, desc: String) -> ParserError {
        ParserError {
            line: line,
            col: col,
            desc: desc
        }
    }
}

#[allow(dead_code)]
pub type ParseResult<T> = Result<T, ParserError>;

#[allow(dead_code)]
impl Parser {
    pub fn new() -> Parser {
        Parser{
            code: "".to_string(),
            line: 0,
            cur: 0,
            col: 0
        }
    }

    pub fn load(&mut self, code: String) -> ExprAst {
        self.code = code;
        self.line = 1;
        self.cur = 0;
        self.col = 0;
        self.skip_space();
        self.read_exp()
    }

    //============= private methods =================
    fn read_exp(&mut self) -> ExprAst {
        let mut cur = self.readc();
        if cur == '#' {
            let next = self.readc();
            match next {
                't' => return ExprAst::Bool(BoolNode::new(true)),
                'f' => return ExprAst::Bool(BoolNode::new(false)),
                '\\' => return self.read_char(),
                _ => panic!("error")
            }
        } else if UnicodeChar::is_numeric(cur) ||
                  (cur == '-' && (UnicodeChar::is_numeric(self.peekc()))) {
            let mut sign = 1i;
            let mut num = 0i;
            if cur == '-' {
                sign = -1;
            } else {
                self.unread();
            }
            loop {
                cur = self.readc();
                if !UnicodeChar::is_numeric(cur) {
                    break;
                }
                num = (num * 10i) + (cur as int - '0' as int);
            }
            num *= sign;
            if self.is_delimiter(cur) {
                self.unread();
                return ExprAst::Int(IntNode::new(num));
            } else {
                panic!("number not followed by delimiter");
            }
        } else if cur == '\"' {
            let mut buf = String::new();
            loop {
                cur = self.readc();
                if cur == '\"' {
                    break;
                }
                buf.push(cur);
            }
            return ExprAst::Str(StrNode::new(buf.as_slice()));
        } else if cur == '(' && cur != ')' {
            // rust-mode bug here
            return self.read_pair();
        } else if cur == '\'' {
            let quote_sym = ExprAst::Symbol(SymbolNode::new("quote"));
            let quote_exp = ExprAst::Pair(PairNode::new(self.read_exp(),
                                                        ExprAst::EmptyList(EmptyListNode::new())));
            return ExprAst::Pair(PairNode::new(quote_sym,
                                               quote_exp));
        }
        ExprAst::Int(IntNode::new(0))
    }

    fn read_pair(&mut self) -> ExprAst {
        self.skip_space();
        let mut cur = self.readc();
        // rust-mode bug here
        if cur != '(' && cur == ')' {
            return ExprAst::EmptyList(EmptyListNode::new());
        }
        self.unread();
        let car_obj = self.read_exp();
        self.skip_space();
        cur = self.readc();
        if cur != '.' {
            self.unread();
            let cdr_obj = self.read_pair();
            return ExprAst::Pair(PairNode::new(car_obj, cdr_obj));
        } else {
            return ExprAst::EmptyList(EmptyListNode::new());
        }
    }

    fn is_delimiter(&self, ch: char) -> bool {
        ch.is_whitespace() ||
            ch == '\"' || ch == '(' || ch == ')' ||  ch == ';' ||
            (ch as u32 == 0)
    }

    fn is_initial(&self, ch: char) -> bool {
        ch.is_alphabetic() ||
            ch == '*' || ch == '/' || ch == '+' || ch == '-' ||
            ch == '>' || ch == '<' || ch == '=' || ch == '?' ||
            ch == '!'
    }

    fn skip_space(&mut self) {
        while self.peekc().is_whitespace() {
            self.readc();
        }
    }

    fn peekc(&self) -> char {
        if self.cur >= self.code.len() {
            panic!("invalid position");
        }
        self.code.char_at(self.cur)
    }

    fn prevc(&self) -> char {
        if self.cur <= 0 {
            panic!("invalid position");
        }
        self.code.char_at(self.cur - 1)
    }

    fn readc(&mut self) -> char {
        if self.cur < self.code.len() {
            let res = self.peekc();
            if res == '\n' {
                self.line += 1;
            }
            self.cur += 1;
            res
        } else {
            0 as char
        }
    }

    fn unread(&mut self) {
        if self.cur == 0 {
            panic!("error current position");
        }
        if self.prevc() == '\n' {
            assert!(self.line > 1);
            self.line -= 1;
        }
        self.cur -= 1;
    }

    fn read_char(&mut self) -> ExprAst {
        ExprAst::Char(CharNode::new('a'))
    }
}

#[test]
fn test_parser() {
    let mut parser = Parser::new();
    let res = parser.load("11".to_string());
    assert!(res.as_int() == 11);
    res.print();

    let res = parser.load("-11".to_string());
    assert!(res.as_int() == -11);
    res.print();

    let res = parser.load(r#""hello""#.to_string());
    assert!(res.as_str() == "hello");
    res.print();

    let res = parser.load("()".to_string());
    assert!(res.is_empty_list());
    res.print();

    let res = parser.load("(1 2)".to_string());
    assert!(res.is_pair());
    assert!(res.car().as_int() == 1);
    assert!(res.cdr().car().as_int() == 2);
    assert!(res.cdr().cdr().is_empty_list());
    res.print();
}
<commit_msg>more test<commit_after>
use ast::ExprAst;
use ast::BoolNode;
use ast::CharNode;
use ast::IntNode;
use ast::StrNode;
use ast::PairNode;
use ast::SymbolNode;
use ast::EmptyListNode;

use ast::Ast;

#[deriving(Clone, Show)]
pub struct Parser {
    code: String,
    cur: uint,
    col: uint,
    line: uint
}

#[allow(dead_code)]
pub struct ParserError {
    line: uint,
    col: uint,
    desc: String
}

#[allow(dead_code)]
impl ParserError {
    pub fn new(line: uint, col: uint, desc: String) -> ParserError {
        ParserError {
            line: line,
            col: col,
            desc: desc
        }
    }
}

#[allow(dead_code)]
pub type ParseResult<T> = Result<T, ParserError>;

#[allow(dead_code)]
impl Parser {
    pub fn new() -> Parser {
        Parser{
            code: "".to_string(),
            line: 0,
            cur: 0,
            col: 0
        }
    }

    pub fn load(&mut self, code: String) -> ExprAst {
        self.code = code;
        self.line = 1;
        self.cur = 0;
        self.col = 0;
        self.skip_space();
        self.read_exp()
    }

    //============= private methods =================
    fn read_exp(&mut self) -> ExprAst {
        let mut cur = self.readc();
        if cur == '#' {
            let next = self.readc();
            match next {
                't' => return ExprAst::Bool(BoolNode::new(true)),
                'f' => return ExprAst::Bool(BoolNode::new(false)),
                '\\' => return self.read_char(),
                _ => panic!("error")
            }
        } else if UnicodeChar::is_numeric(cur) ||
            (cur == '-' && (UnicodeChar::is_numeric(self.peekc()))) {
                let mut sign = 1i;
                let mut num = 0i;
                if cur == '-' {
                    sign = -1;
                } else {
                    self.unread();
                }
                loop {
                    cur = self.readc();
                    if !UnicodeChar::is_numeric(cur) {
                        break;
                    }
                    num = (num * 10i) + (cur as int - '0' as int);
                }
                num *= sign;
                if self.is_delimiter(cur) {
                    self.unread();
                    return ExprAst::Int(IntNode::new(num));
                } else {
                    panic!("number not followed by delimiter");
                }
            } else if cur == '\"' {
                let mut buf = String::new();
                loop {
                    cur = self.readc();
                    if cur == '\"' {
                        break;
                    }
                    buf.push(cur);
                }
                return ExprAst::Str(StrNode::new(buf.as_slice()));
            } else if cur == '(' && cur != ')' {
                // rust-mode bug here
                return self.read_pair();
            } else if self.is_initial(cur) {
                let mut buf = String::new();
                buf.push(cur);
                loop {
                    cur = self.readc();
                    if !(self.is_initial(cur) || UnicodeChar::is_numeric(cur)) {
                        break;
                    }
                    buf.push(cur);
                }
                if self.is_delimiter(cur) {
                    self.unread();
                    return ExprAst::Symbol(SymbolNode::new(buf.as_slice()));
                }
            } else if cur == '\'' {
                let quote_sym = ExprAst::Symbol(SymbolNode::new("quote"));
                let quote_exp = ExprAst::Pair(PairNode::new(self.read_exp(),
                                                            ExprAst::EmptyList(EmptyListNode::new())));
                return ExprAst::Pair(PairNode::new(quote_sym,
                                                   quote_exp));
            }
        ExprAst::Int(IntNode::new(0))
    }

    fn read_pair(&mut self) -> ExprAst {
        self.skip_space();
        let mut cur = self.readc();
        // rust-mode bug here
        if cur != '(' && cur == ')' {
            return ExprAst::EmptyList(EmptyListNode::new());
        }
        self.unread();
        let car_obj = self.read_exp();
        self.skip_space();
        cur = self.readc();
        if cur != '.' {
            self.unread();
            let cdr_obj = self.read_pair();
            return ExprAst::Pair(PairNode::new(car_obj, cdr_obj));
        } else {
            return ExprAst::EmptyList(EmptyListNode::new());
        }
    }

    fn is_delimiter(&self, ch: char) -> bool {
        ch.is_whitespace() ||
            ch == '\"' || ch == '(' || ch == ')' ||  ch == ';' ||
            (ch as u32 == 0)
    }

    fn is_initial(&self, ch: char) -> bool {
        ch.is_alphabetic() ||
            ch == '*' || ch == '/' || ch == '+' || ch == '-' ||
            ch == '>' || ch == '<' || ch == '=' || ch == '?' ||
            ch == '!'
    }

    fn skip_space(&mut self) {
        while self.peekc().is_whitespace() {
            self.readc();
        }
    }

    fn peekc(&self) -> char {
        if self.cur >= self.code.len() {
            panic!("invalid position");
        }
        self.code.char_at(self.cur)
    }

    fn prevc(&self) -> char {
        if self.cur <= 0 {
            panic!("invalid position");
        }
        self.code.char_at(self.cur - 1)
    }

    fn readc(&mut self) -> char {
        if self.cur < self.code.len() {
            let res = self.peekc();
            if res == '\n' {
                self.line += 1;
            }
            self.cur += 1;
            res
        } else {
            0 as char
        }
    }

    fn unread(&mut self) {
        if self.cur == 0 {
            panic!("error current position");
        }
        if self.prevc() == '\n' {
            assert!(self.line > 1);
            self.line -= 1;
        }
        self.cur -= 1;
    }

    fn read_char(&mut self) -> ExprAst {
        ExprAst::Char(CharNode::new('a'))
    }
}

#[test]
fn test_parser() {
    let mut parser = Parser::new();
    let res = parser.load("11".to_string());
    assert!(res.as_int() == 11);
    res.print();

    let res = parser.load("-11".to_string());
    assert!(res.as_int() == -11);
    res.print();

    let res = parser.load(r#""hello""#.to_string());
    assert!(res.as_str() == "hello");
    res.print();

    let res = parser.load("()".to_string());
    assert!(res.is_empty_list());
    res.print();

    let res = parser.load("(1 2)".to_string());
    assert!(res.is_pair());
    assert!(res.car().as_int() == 1);
    assert!(res.cdr().car().as_int() == 2);
    assert!(res.cdr().cdr().is_empty_list());

    let res = parser.load("(+ 1 2)".to_string());
    assert!(res.is_pair());
    assert!(res.car().is_symbol());
    assert!(res.cdr().car().as_int() == 1);
    assert!(res.cdr().cdr().car().as_int() == 2);
}
<|endoftext|>


<commit_before>use nom::{IResult, alphanumeric, eof, line_ending, space};
use std::str;

#[derive(Debug, PartialEq)]
pub struct SqlQuery {
    fields: Vec<String>,
    table: String,
    cond: Option<ConditionTree>,
}

#[derive(Debug, PartialEq)]
pub struct ConditionTree {
    field: String,
    expr: String,
}

/// Parse rule for a comma-separated list.
named!(csvlist<&[u8], Vec<&str> >,
       many0!(
           map_res!(
               chain!(
                   fieldname: alphanumeric ~
                   opt!(
                       chain!(
                           tag!(",") ~
                           opt!(space),
                           ||{}
                       )
                   ),
                   ||{ fieldname }
               ),
               str::from_utf8
           )
       )
);

/// Parse list of columns/fields.
/// XXX(malte): add support for named table notation
named!(fieldlist<&[u8], Vec<&str> >,
       alt!(
           tag!("*") => { |_| vec!["ALL".into()] }
         | csvlist
       )
);

named!(where_clause<&[u8], ConditionTree>,
    chain!(
        tag!("where") ~
        space ~
        field: map_res!(alphanumeric, str::from_utf8) ~
        space? ~
        tag!("=") ~
        space? ~
        expr: map_res!(tag_s!(b"?"), str::from_utf8),
        || {
            ConditionTree {
                field: String::from(field),
                expr: String::from(expr),
            }
        }
    )
);

/// Parse rule for a SQL selection query.
named!(selection<&[u8], SqlQuery>,
    chain!(
        tag!("select") ~
        space ~
        fields: fieldlist ~
        space ~
        tag!("from") ~
        space ~
        table: map_res!(alphanumeric, str::from_utf8) ~
        space? ~
        cond: opt!(where_clause) ~
        space? ~
        alt!(eof | tag!(";") | line_ending),  // N.B.: eof must come FIRST
        || {
            SqlQuery {
                table: String::from(table),
                fields: fields.iter().map(|s| String::from(*s)).collect(),
                cond: cond
            }
        }
    )
);

pub fn parse_query(input: &str) -> Result<SqlQuery, &str> {
    // we process all queries in lowercase to avoid having to deal with capitalization in the
    // parser.
    let q_lower = input.to_lowercase();

    // TODO(malte): appropriately pass through errors from nom
    match selection(&q_lower.into_bytes()) {
        IResult::Done(_, o) => Ok(o),
        IResult::Error(_) => Err("parse error"),
        IResult::Incomplete(_) => Err("incomplete query"),
    }
}

mod tests {
    use super::*;

    #[test]
    fn simple_select() {
        let qstring = "SELECT id, name FROM users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["id".into(), "name".into()],
                              table: String::from("users"),
                              cond: None } );
    }

    #[test]
    fn select_all() {
        let qstring = "SELECT * FROM users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["ALL".into()],
                              table: String::from("users"),
                              cond: None } );
    }

    #[test]
    fn spaces_optional() {
        let qstring = "SELECT id,name FROM users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["id".into(), "name".into()],
                              table: String::from("users"),
                              cond: None } );
    }

    #[test]
    fn case_sensitivity() {
        let qstring = "select id, name from users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["id".into(), "name".into()],
                              table: String::from("users"),
                              cond: None } );
    }

    #[test]
    fn termination() {
        let qstring_sem = "select id, name from users;";
        let qstring_linebreak = "select id, name from users\n";

        assert_eq!(parse_query(qstring_sem).unwrap(),
                   parse_query(qstring_linebreak).unwrap());
    }

    #[test]
    fn where_clause() {
        let qstring = "select * from ContactInfo where email=?";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["ALL".into(),],
                              table: String::from("ContactInfo").to_lowercase(),
                              cond: Some(ConditionTree {
                                  field: String::from("email"),
                                  expr: String::from("?") })
                            }
                   );
    }
}
<commit_msg>Note regarding selection targets.<commit_after>use nom::{IResult, alphanumeric, eof, line_ending, space};
use std::str;

#[derive(Debug, PartialEq)]
pub struct SqlQuery {
    fields: Vec<String>,
    table: String,
    cond: Option<ConditionTree>,
}

#[derive(Debug, PartialEq)]
pub struct ConditionTree {
    field: String,
    expr: String,
}

/// Parse rule for a comma-separated list.
named!(csvlist<&[u8], Vec<&str> >,
       many0!(
           map_res!(
               chain!(
                   fieldname: alphanumeric ~
                   opt!(
                       chain!(
                           tag!(",") ~
                           opt!(space),
                           ||{}
                       )
                   ),
                   ||{ fieldname }
               ),
               str::from_utf8
           )
       )
);

/// Parse list of columns/fields.
/// XXX(malte): add support for named table notation
named!(fieldlist<&[u8], Vec<&str> >,
       alt!(
           tag!("*") => { |_| vec!["ALL".into()] }
         | csvlist
       )
);

named!(where_clause<&[u8], ConditionTree>,
    chain!(
        tag!("where") ~
        space ~
        field: map_res!(alphanumeric, str::from_utf8) ~
        space? ~
        tag!("=") ~
        space? ~
        expr: map_res!(tag_s!(b"?"), str::from_utf8),
        || {
            ConditionTree {
                field: String::from(field),
                expr: String::from(expr),
            }
        }
    )
);

/// Parse rule for a SQL selection query.
/// TODO(malte): support nested queries as selection targets
named!(selection<&[u8], SqlQuery>,
    chain!(
        tag!("select") ~
        space ~
        fields: fieldlist ~
        space ~
        tag!("from") ~
        space ~
        table: map_res!(alphanumeric, str::from_utf8) ~
        space? ~
        cond: opt!(where_clause) ~
        space? ~
        alt!(eof | tag!(";") | line_ending),  // N.B.: eof must come FIRST
        || {
            SqlQuery {
                table: String::from(table),
                fields: fields.iter().map(|s| String::from(*s)).collect(),
                cond: cond
            }
        }
    )
);

pub fn parse_query(input: &str) -> Result<SqlQuery, &str> {
    // we process all queries in lowercase to avoid having to deal with capitalization in the
    // parser.
    let q_lower = input.to_lowercase();

    // TODO(malte): appropriately pass through errors from nom
    match selection(&q_lower.into_bytes()) {
        IResult::Done(_, o) => Ok(o),
        IResult::Error(_) => Err("parse error"),
        IResult::Incomplete(_) => Err("incomplete query"),
    }
}

mod tests {
    use super::*;

    #[test]
    fn simple_select() {
        let qstring = "SELECT id, name FROM users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["id".into(), "name".into()],
                              table: String::from("users"),
                              cond: None } );
    }

    #[test]
    fn select_all() {
        let qstring = "SELECT * FROM users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["ALL".into()],
                              table: String::from("users"),
                              cond: None } );
    }

    #[test]
    fn spaces_optional() {
        let qstring = "SELECT id,name FROM users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["id".into(), "name".into()],
                              table: String::from("users"),
                              cond: None } );
    }

    #[test]
    fn case_sensitivity() {
        let qstring = "select id, name from users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["id".into(), "name".into()],
                              table: String::from("users"),
                              cond: None } );
    }

    #[test]
    fn termination() {
        let qstring_sem = "select id, name from users;";
        let qstring_linebreak = "select id, name from users\n";

        assert_eq!(parse_query(qstring_sem).unwrap(),
                   parse_query(qstring_linebreak).unwrap());
    }

    #[test]
    fn where_clause() {
        let qstring = "select * from ContactInfo where email=?";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["ALL".into(),],
                              table: String::from("ContactInfo").to_lowercase(),
                              cond: Some(ConditionTree {
                                  field: String::from("email"),
                                  expr: String::from("?") })
                            }
                   );
    }
}
<|endoftext|>


<commit_before>// Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0.

use std::borrow::ToOwned;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};

use chrono::Local;
use clap::ArgMatches;
use collections::HashMap;
use tikv::config::{check_critical_config, persist_config, MetricConfig, TiKvConfig};
use tikv_util::{self, config, logger};

// A workaround for checking if log is initialized.
pub static LOG_INITIALIZED: AtomicBool = AtomicBool::new(false);

// The info log file names does not end with ".log" since it conflict with rocksdb WAL files.
pub const DEFAULT_ROCKSDB_LOG_FILE: &str = "rocksdb.info";
pub const DEFAULT_RAFTDB_LOG_FILE: &str = "raftdb.info";

#[macro_export]
macro_rules! fatal {
    ($lvl:expr $(, $arg:expr)*) => ({
        if $crate::setup::LOG_INITIALIZED.load(::std::sync::atomic::Ordering::SeqCst) {
            crit!($lvl $(, $arg)*);
        } else {
            eprintln!($lvl $(, $arg)*);
        }
        slog_global::clear_global();
        ::std::process::exit(1)
    })
}

// TODO: There is a very small chance that duplicate files will be generated if there are
// a lot of logs written in a very short time. Consider rename the rotated file with a version
// number while rotate by size.
fn rename_by_timestamp(path: &Path) -> io::Result<PathBuf> {
    let mut new_path = path.to_path_buf().into_os_string();
    new_path.push(format!(
        ".{}",
        Local::now().format(logger::DATETIME_ROTATE_SUFFIX)
    ));
    Ok(PathBuf::from(new_path))
}

fn make_engine_log_path(path: &str, sub_path: &str, filename: &str) -> String {
    let mut path = Path::new(path).to_path_buf();
    if !sub_path.is_empty() {
        path = path.join(Path::new(sub_path));
    }
    let path = path.to_str().unwrap_or_else(|| {
        fatal!(
            "failed to construct engine log dir {:?}, {:?}",
            path,
            sub_path
        );
    });
    config::ensure_dir_exist(path).unwrap_or_else(|e| {
        fatal!("failed to create engine log dir: {}", e);
    });
    config::canonicalize_log_dir(&path, filename).unwrap_or_else(|e| {
        fatal!("failed to canonicalize engine log dir {:?}: {}", path, e);
    })
}

#[allow(dead_code)]
pub fn initial_logger(config: &TiKvConfig) {
    fn build_logger<D>(drainer: D, config: &TiKvConfig)
    where
        D: slog::Drain + Send + 'static,
        <D as slog::Drain>::Err: std::fmt::Display,
    {
        // use async drainer and init std log.
        logger::init_log(
            drainer,
            config.log_level,
            true,
            true,
            vec![],
            config.slow_log_threshold.as_millis(),
        )
        .unwrap_or_else(|e| {
            fatal!("failed to initialize log: {}", e);
        });
    }

    if config.log_file.is_empty() {
        let writer = logger::term_writer();
        match config.log_format {
            config::LogFormat::Text => build_logger(logger::text_format(writer), config),
            config::LogFormat::Json => build_logger(logger::json_format(writer), config),
        };
    } else {
        let writer = logger::file_writer(
            &config.log_file,
            config.log_rotation_timespan,
            config.log_rotation_size,
            rename_by_timestamp,
        )
        .unwrap_or_else(|e| {
            fatal!(
                "failed to initialize log with file {}: {}",
                config.log_file,
                e
            );
        });

        let slow_log_writer = if config.slow_log_file.is_empty() {
            None
        } else {
            let slow_log_writer = logger::file_writer(
                &config.slow_log_file,
                config.log_rotation_timespan,
                config.log_rotation_size,
                rename_by_timestamp,
            )
            .unwrap_or_else(|e| {
                fatal!(
                    "failed to initialize slow-log with file {}: {}",
                    config.slow_log_file,
                    e
                );
            });
            Some(slow_log_writer)
        };

        let rocksdb_info_log_path = if !config.rocksdb.info_log_dir.is_empty() {
            make_engine_log_path(&config.rocksdb.info_log_dir, "", DEFAULT_ROCKSDB_LOG_FILE)
        } else {
            // Don't use `DEFAULT_ROCKSDB_SUB_DIR`, because of the logic of `RocksEngine::exists`.
            make_engine_log_path(&config.storage.data_dir, "", DEFAULT_ROCKSDB_LOG_FILE)
        };
        let raftdb_info_log_path = if !config.raftdb.info_log_dir.is_empty() {
            make_engine_log_path(&config.raftdb.info_log_dir, "", DEFAULT_RAFTDB_LOG_FILE)
        } else {
            make_engine_log_path(&config.storage.data_dir, "", DEFAULT_RAFTDB_LOG_FILE)
        };
        let rocksdb_log_writer = logger::file_writer(
            &rocksdb_info_log_path,
            config.log_rotation_timespan,
            config.log_rotation_size,
            rename_by_timestamp,
        )
        .unwrap_or_else(|e| {
            fatal!(
                "failed to initialize rocksdb log with file {}: {}",
                rocksdb_info_log_path,
                e
            );
        });

        let raftdb_log_writer = logger::file_writer(
            &raftdb_info_log_path,
            config.log_rotation_timespan,
            config.log_rotation_size,
            rename_by_timestamp,
        )
        .unwrap_or_else(|e| {
            fatal!(
                "failed to initialize raftdb log with file {}: {}",
                raftdb_info_log_path,
                e
            );
        });

        match config.log_format {
            config::LogFormat::Text => build_logger_with_slow_log(
                logger::text_format(writer),
                logger::rocks_text_format(rocksdb_log_writer),
                logger::text_format(raftdb_log_writer),
                slow_log_writer.map(logger::text_format),
                config,
            ),
            config::LogFormat::Json => build_logger_with_slow_log(
                logger::json_format(writer),
                logger::json_format(rocksdb_log_writer),
                logger::json_format(raftdb_log_writer),
                slow_log_writer.map(logger::json_format),
                config,
            ),
        };

        fn build_logger_with_slow_log<N, R, S, T>(
            normal: N,
            rocksdb: R,
            raftdb: T,
            slow: Option<S>,
            config: &TiKvConfig,
        ) where
            N: slog::Drain<Ok = (), Err = io::Error> + Send + 'static,
            R: slog::Drain<Ok = (), Err = io::Error> + Send + 'static,
            S: slog::Drain<Ok = (), Err = io::Error> + Send + 'static,
            T: slog::Drain<Ok = (), Err = io::Error> + Send + 'static,
        {
            let drainer = logger::LogDispatcher::new(normal, rocksdb, raftdb, slow);

            // use async drainer and init std log.
            logger::init_log(
                drainer,
                config.log_level,
                true,
                true,
                vec![],
                config.slow_log_threshold.as_millis(),
            )
            .unwrap_or_else(|e| {
                fatal!("failed to initialize log: {}", e);
            });
        }
    };

    // Set redact_info_log.
    let redact_info_log = config.security.redact_info_log.unwrap_or(false);
    log_wrappers::set_redact_info_log(redact_info_log);

    LOG_INITIALIZED.store(true, Ordering::SeqCst);
}

#[allow(dead_code)]
pub fn initial_metric(cfg: &MetricConfig) {
    tikv_util::metrics::monitor_process()
        .unwrap_or_else(|e| fatal!("failed to start process monitor: {}", e));
    tikv_util::metrics::monitor_threads("tikv")
        .unwrap_or_else(|e| fatal!("failed to start thread monitor: {}", e));
    tikv_util::metrics::monitor_allocator_stats("tikv")
        .unwrap_or_else(|e| fatal!("failed to monitor allocator stats: {}", e));

    if cfg.interval.as_secs() == 0 || cfg.address.is_empty() {
        return;
    }

    warn!("metrics push is not supported any more.");
}

#[allow(dead_code)]
pub fn overwrite_config_with_cmd_args(config: &mut TiKvConfig, matches: &ArgMatches<'_>) {
    if let Some(level) = matches.value_of("log-level") {
        config.log_level = logger::get_level_by_string(level).unwrap();
    }

    if let Some(file) = matches.value_of("log-file") {
        config.log_file = file.to_owned();
    }

    if let Some(addr) = matches.value_of("addr") {
        config.server.addr = addr.to_owned();
    }

    if let Some(advertise_addr) = matches.value_of("advertise-addr") {
        config.server.advertise_addr = advertise_addr.to_owned();
    }

    if let Some(status_addr) = matches.value_of("status-addr") {
        config.server.status_addr = status_addr.to_owned();
    }

    if let Some(advertise_status_addr) = matches.value_of("advertise-status-addr") {
        config.server.advertise_status_addr = advertise_status_addr.to_owned();
    }

    if let Some(data_dir) = matches.value_of("data-dir") {
        config.storage.data_dir = data_dir.to_owned();
    }

    if let Some(endpoints) = matches.values_of("pd-endpoints") {
        config.pd.endpoints = endpoints.map(ToOwned::to_owned).collect();
    }

    if let Some(labels_vec) = matches.values_of("labels") {
        let mut labels = HashMap::default();
        for label in labels_vec {
            let mut parts = label.split('=');
            let key = parts.next().unwrap().to_owned();
            let value = match parts.next() {
                None => fatal!("invalid label: {}", label),
                Some(v) => v.to_owned(),
            };
            if parts.next().is_some() {
                fatal!("invalid label: {}", label);
            }
            labels.insert(key, value);
        }
        config.server.labels = labels;
    }

    if let Some(capacity_str) = matches.value_of("capacity") {
        let capacity = capacity_str.parse().unwrap_or_else(|e| {
            fatal!("invalid capacity: {}", e);
        });
        config.raft_store.capacity = capacity;
    }

    if matches.value_of("metrics-addr").is_some() {
        warn!("metrics push is not supported any more.");
    }
}

#[allow(dead_code)]
pub fn validate_and_persist_config(config: &mut TiKvConfig, persist: bool) {
    config.compatible_adjust();
    if let Err(e) = config.validate() {
        fatal!("invalid configuration: {}", e);
    }

    if let Err(e) = check_critical_config(config) {
        fatal!("critical config check failed: {}", e);
    }

    if persist {
        if let Err(e) = persist_config(&config) {
            fatal!("persist critical config failed: {}", e);
        }
    }
}

pub fn ensure_no_unrecognized_config(unrecognized_keys: &[String]) {
    if !unrecognized_keys.is_empty() {
        fatal!(
            "unknown configuration options: {}",
            unrecognized_keys.join(", ")
        );
    }
}
<commit_msg>always put engine info log into data dir (#10218)<commit_after>// Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0.

use std::borrow::ToOwned;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};

use chrono::Local;
use clap::ArgMatches;
use collections::HashMap;
use tikv::config::{check_critical_config, persist_config, MetricConfig, TiKvConfig};
use tikv_util::{self, config, logger};

// A workaround for checking if log is initialized.
pub static LOG_INITIALIZED: AtomicBool = AtomicBool::new(false);

// The info log file names does not end with ".log" since it conflict with rocksdb WAL files.
pub const DEFAULT_ROCKSDB_LOG_FILE: &str = "rocksdb.info";
pub const DEFAULT_RAFTDB_LOG_FILE: &str = "raftdb.info";

#[macro_export]
macro_rules! fatal {
    ($lvl:expr $(, $arg:expr)*) => ({
        if $crate::setup::LOG_INITIALIZED.load(::std::sync::atomic::Ordering::SeqCst) {
            crit!($lvl $(, $arg)*);
        } else {
            eprintln!($lvl $(, $arg)*);
        }
        slog_global::clear_global();
        ::std::process::exit(1)
    })
}

// TODO: There is a very small chance that duplicate files will be generated if there are
// a lot of logs written in a very short time. Consider rename the rotated file with a version
// number while rotate by size.
fn rename_by_timestamp(path: &Path) -> io::Result<PathBuf> {
    let mut new_path = path.to_path_buf().into_os_string();
    new_path.push(format!(
        ".{}",
        Local::now().format(logger::DATETIME_ROTATE_SUFFIX)
    ));
    Ok(PathBuf::from(new_path))
}

fn make_engine_log_path(path: &str, sub_path: &str, filename: &str) -> String {
    let mut path = Path::new(path).to_path_buf();
    if !sub_path.is_empty() {
        path = path.join(Path::new(sub_path));
    }
    let path = path.to_str().unwrap_or_else(|| {
        fatal!(
            "failed to construct engine log dir {:?}, {:?}",
            path,
            sub_path
        );
    });
    config::ensure_dir_exist(path).unwrap_or_else(|e| {
        fatal!("failed to create engine log dir: {}", e);
    });
    config::canonicalize_log_dir(&path, filename).unwrap_or_else(|e| {
        fatal!("failed to canonicalize engine log dir {:?}: {}", path, e);
    })
}

#[allow(dead_code)]
pub fn initial_logger(config: &TiKvConfig) {
    let rocksdb_info_log_path = if !config.rocksdb.info_log_dir.is_empty() {
        make_engine_log_path(&config.rocksdb.info_log_dir, "", DEFAULT_ROCKSDB_LOG_FILE)
    } else {
        // Don't use `DEFAULT_ROCKSDB_SUB_DIR`, because of the logic of `RocksEngine::exists`.
        make_engine_log_path(&config.storage.data_dir, "", DEFAULT_ROCKSDB_LOG_FILE)
    };
    let raftdb_info_log_path = if !config.raftdb.info_log_dir.is_empty() {
        make_engine_log_path(&config.raftdb.info_log_dir, "", DEFAULT_RAFTDB_LOG_FILE)
    } else {
        make_engine_log_path(&config.storage.data_dir, "", DEFAULT_RAFTDB_LOG_FILE)
    };
    let rocksdb = logger::file_writer(
        &rocksdb_info_log_path,
        config.log_rotation_timespan,
        config.log_rotation_size,
        rename_by_timestamp,
    )
    .unwrap_or_else(|e| {
        fatal!(
            "failed to initialize rocksdb log with file {}: {}",
            rocksdb_info_log_path,
            e
        );
    });

    let raftdb = logger::file_writer(
        &raftdb_info_log_path,
        config.log_rotation_timespan,
        config.log_rotation_size,
        rename_by_timestamp,
    )
    .unwrap_or_else(|e| {
        fatal!(
            "failed to initialize raftdb log with file {}: {}",
            raftdb_info_log_path,
            e
        );
    });

    let slow_log_writer = if config.slow_log_file.is_empty() {
        None
    } else {
        let slow_log_writer = logger::file_writer(
            &config.slow_log_file,
            config.log_rotation_timespan,
            config.log_rotation_size,
            rename_by_timestamp,
        )
        .unwrap_or_else(|e| {
            fatal!(
                "failed to initialize slow-log with file {}: {}",
                config.slow_log_file,
                e
            );
        });
        Some(slow_log_writer)
    };

    fn build_logger_with_slow_log<N, R, S, T>(
        normal: N,
        rocksdb: R,
        raftdb: T,
        slow: Option<S>,
        config: &TiKvConfig,
    ) where
        N: slog::Drain<Ok = (), Err = io::Error> + Send + 'static,
        R: slog::Drain<Ok = (), Err = io::Error> + Send + 'static,
        S: slog::Drain<Ok = (), Err = io::Error> + Send + 'static,
        T: slog::Drain<Ok = (), Err = io::Error> + Send + 'static,
    {
        // Use async drainer and init std log.
        let drainer = logger::LogDispatcher::new(normal, rocksdb, raftdb, slow);
        let level = config.log_level;
        let slow_threshold = config.slow_log_threshold.as_millis();
        logger::init_log(drainer, level, true, true, vec![], slow_threshold).unwrap_or_else(|e| {
            fatal!("failed to initialize log: {}", e);
        });
    }

    macro_rules! do_build {
        ($log:expr, $rocksdb:expr, $raftdb:expr, $slow:expr) => {
            match config.log_format {
                config::LogFormat::Text => build_logger_with_slow_log(
                    logger::text_format($log),
                    logger::rocks_text_format($rocksdb),
                    logger::rocks_text_format($raftdb),
                    $slow.map(logger::text_format),
                    config,
                ),
                config::LogFormat::Json => build_logger_with_slow_log(
                    logger::json_format($log),
                    logger::json_format($rocksdb),
                    logger::json_format($raftdb),
                    $slow.map(logger::json_format),
                    config,
                ),
            }
        };
    }

    if config.log_file.is_empty() {
        let log = logger::term_writer();
        do_build!(log, rocksdb, raftdb, slow_log_writer);
    } else {
        let log = logger::file_writer(
            &config.log_file,
            config.log_rotation_timespan,
            config.log_rotation_size,
            rename_by_timestamp,
        )
        .unwrap_or_else(|e| {
            fatal!(
                "failed to initialize log with file {}: {}",
                config.log_file,
                e
            );
        });
        do_build!(log, rocksdb, raftdb, slow_log_writer);
    }

    // Set redact_info_log.
    let redact_info_log = config.security.redact_info_log.unwrap_or(false);
    log_wrappers::set_redact_info_log(redact_info_log);

    LOG_INITIALIZED.store(true, Ordering::SeqCst);
}

#[allow(dead_code)]
pub fn initial_metric(cfg: &MetricConfig) {
    tikv_util::metrics::monitor_process()
        .unwrap_or_else(|e| fatal!("failed to start process monitor: {}", e));
    tikv_util::metrics::monitor_threads("tikv")
        .unwrap_or_else(|e| fatal!("failed to start thread monitor: {}", e));
    tikv_util::metrics::monitor_allocator_stats("tikv")
        .unwrap_or_else(|e| fatal!("failed to monitor allocator stats: {}", e));

    if cfg.interval.as_secs() == 0 || cfg.address.is_empty() {
        return;
    }

    warn!("metrics push is not supported any more.");
}

#[allow(dead_code)]
pub fn overwrite_config_with_cmd_args(config: &mut TiKvConfig, matches: &ArgMatches<'_>) {
    if let Some(level) = matches.value_of("log-level") {
        config.log_level = logger::get_level_by_string(level).unwrap();
    }

    if let Some(file) = matches.value_of("log-file") {
        config.log_file = file.to_owned();
    }

    if let Some(addr) = matches.value_of("addr") {
        config.server.addr = addr.to_owned();
    }

    if let Some(advertise_addr) = matches.value_of("advertise-addr") {
        config.server.advertise_addr = advertise_addr.to_owned();
    }

    if let Some(status_addr) = matches.value_of("status-addr") {
        config.server.status_addr = status_addr.to_owned();
    }

    if let Some(advertise_status_addr) = matches.value_of("advertise-status-addr") {
        config.server.advertise_status_addr = advertise_status_addr.to_owned();
    }

    if let Some(data_dir) = matches.value_of("data-dir") {
        config.storage.data_dir = data_dir.to_owned();
    }

    if let Some(endpoints) = matches.values_of("pd-endpoints") {
        config.pd.endpoints = endpoints.map(ToOwned::to_owned).collect();
    }

    if let Some(labels_vec) = matches.values_of("labels") {
        let mut labels = HashMap::default();
        for label in labels_vec {
            let mut parts = label.split('=');
            let key = parts.next().unwrap().to_owned();
            let value = match parts.next() {
                None => fatal!("invalid label: {}", label),
                Some(v) => v.to_owned(),
            };
            if parts.next().is_some() {
                fatal!("invalid label: {}", label);
            }
            labels.insert(key, value);
        }
        config.server.labels = labels;
    }

    if let Some(capacity_str) = matches.value_of("capacity") {
        let capacity = capacity_str.parse().unwrap_or_else(|e| {
            fatal!("invalid capacity: {}", e);
        });
        config.raft_store.capacity = capacity;
    }

    if matches.value_of("metrics-addr").is_some() {
        warn!("metrics push is not supported any more.");
    }
}

#[allow(dead_code)]
pub fn validate_and_persist_config(config: &mut TiKvConfig, persist: bool) {
    config.compatible_adjust();
    if let Err(e) = config.validate() {
        fatal!("invalid configuration: {}", e);
    }

    if let Err(e) = check_critical_config(config) {
        fatal!("critical config check failed: {}", e);
    }

    if persist {
        if let Err(e) = persist_config(&config) {
            fatal!("persist critical config failed: {}", e);
        }
    }
}

pub fn ensure_no_unrecognized_config(unrecognized_keys: &[String]) {
    if !unrecognized_keys.is_empty() {
        fatal!(
            "unknown configuration options: {}",
            unrecognized_keys.join(", ")
        );
    }
}
<|endoftext|>