<commit_before>extern crate rand;
extern crate hyper;

use std::fs::{File, OpenOptions, remove_file};
use std::str::FromStr;
use std::path::Path;

mod pasteid;
use pasteid::PasteID;

use std::io;
use hyper::header::{ContentLength, ContentType};
use hyper::server::{Server, Request, Response};
use hyper::status::StatusCode;
use hyper::uri::RequestUri::*;

const ID_SIZE: usize = 8;
const USAGE: &'static [u8] = b"
pastabin 0.0.1 - Minimal pastebin clone in Rust. Manual post required. CLI recommended.

    USAGE

      POST /

        accepts raw data in the body of the request and responds with a URL of
        a page containing the body's content

      GET /<id>

        retrieves the content for the paste with id `<id>`

    EXAMPLE

        Upload a file:

            curl --data-binary @file.txt https://pasta.lol/

        Upload from stdin:

            echo \"Hellow, World\" | curl --data-binary @- https://pasta.lol/

        Delete an existing paste:

            curl -X DELETE https://pasta.lol/<id>
    ";

fn retrieve_paste(id: PasteID) -> Option<File> {
    File::open(&id.filename()).ok()
}

fn create_paste(id: &PasteID) -> io::Result<File> {
    OpenOptions::new()
        .write(true)
        .create_new(true)
        .open(&id.filename())
}

macro_rules! try_handle {
    ($res:ident, $e:expr) => {{
        match $e.ok() {
            Some(v) => v,
            None => {
                *$res.status_mut() = StatusCode::InternalServerError;
                return;
            },
        }
    }}
}

fn handle(mut req: Request, mut res: Response) {
    match req.method {
        hyper::Get => {
            match req.uri.clone() {
                AbsolutePath(path) => {
                    if path == "/" {
                        res.headers_mut().set(ContentType::plaintext());
                        res.send(USAGE).unwrap();
                    } else if path == "/favicon.ico" {
                        // todo favicon
                        *res.status_mut() = StatusCode::NotFound;
                    } else if let Ok(id) = PasteID::from_str(path.trim_left_matches("/")) {
                        match retrieve_paste(id) {
                            Some(mut file) => {
                                let metadata = try_handle!(res, file.metadata());
                                res.headers_mut().set(ContentLength(metadata.len()));
                                res.headers_mut().set(ContentType::plaintext());
                                io::copy(&mut file, &mut res.start().unwrap()).unwrap();
                            }
                            None => *res.status_mut() = StatusCode::NotFound,
                        }
                    } else {
                        *res.status_mut() = StatusCode::BadRequest;
                    }
                }
                _ => *res.status_mut() = StatusCode::BadRequest,
            }
        }
        hyper::Post => {
            match req.uri.clone() {
                AbsolutePath(path) => {
                    if path != "/" {
                        *res.status_mut() = StatusCode::BadRequest;
                    } else {
                        let mut tries = 0;
                        let mut id = PasteID::new(ID_SIZE);
                        let mut file = create_paste(&id);
                        while file.is_err() && tries < 3 {
                            id = PasteID::new(ID_SIZE);
                            file = create_paste(&id);
                            tries += 1;
                        }
                        if file.is_err() {
                            *res.status_mut() = StatusCode::InternalServerError;
                        } else {
                            // TODO: handle errors
                            try_handle!(res, io::copy(&mut req, &mut try_handle!(res, file)));
                            *res.status_mut() = StatusCode::Created;
                            res.send(format!("https://pasta.lol/{}\n", id).as_bytes())
                                .unwrap();
                        }
                    }
                }
                _ => *res.status_mut() = StatusCode::BadRequest,
            }
        }
        hyper::Delete => {
            match req.uri.clone() {
                AbsolutePath(path) => {
                    if let Ok(id) = PasteID::from_str(path.trim_left_matches("/")) {
                        let filename = id.filename();
                        let path = Path::new(&filename);
                        if path.exists() {
                            try_handle!(res, remove_file(path));
                        } else {
                            *res.status_mut() = StatusCode::NotFound;
                        }
                    } else {
                        *res.status_mut() = StatusCode::BadRequest;
                    }
                }
                _ => *res.status_mut() = StatusCode::BadRequest,
            }
        }
        _ => *res.status_mut() = StatusCode::MethodNotAllowed,
    }
}

fn main() {
    let mut ipv4 = Server::http("127.0.0.1:8080").unwrap();
    ipv4.keep_alive(None);
    let _guardipv4 = ipv4.handle(handle);
}
<commit_msg>add source code url to the index page<commit_after>extern crate rand;
extern crate hyper;

use std::fs::{File, OpenOptions, remove_file};
use std::str::FromStr;
use std::path::Path;

mod pasteid;
use pasteid::PasteID;

use std::io;
use hyper::header::{ContentLength, ContentType};
use hyper::server::{Server, Request, Response};
use hyper::status::StatusCode;
use hyper::uri::RequestUri::*;

const ID_SIZE: usize = 8;
const USAGE: &'static [u8] = b"
pastabin 0.0.1 - Minimal pastebin clone in Rust. Manual post required. CLI recommended.

    USAGE

      POST /

        accepts raw data in the body of the request and responds with a URL of
        a page containing the body's content

      GET /<id>

        retrieves the content for the paste with id `<id>`

    EXAMPLE

        Upload a file:

            curl --data-binary @file.txt https://pasta.lol/

        Upload from stdin:

            echo \"Hellow, World\" | curl --data-binary @- https://pasta.lol/

        Delete an existing paste:

            curl -X DELETE https://pasta.lol/<id>

    SOURCE CODE

        https://github.com/nopjmp/pastabin
    ";

fn retrieve_paste(id: PasteID) -> Option<File> {
    File::open(&id.filename()).ok()
}

fn create_paste(id: &PasteID) -> io::Result<File> {
    OpenOptions::new()
        .write(true)
        .create_new(true)
        .open(&id.filename())
}

macro_rules! try_handle {
    ($res:ident, $e:expr) => {{
        match $e.ok() {
            Some(v) => v,
            None => {
                *$res.status_mut() = StatusCode::InternalServerError;
                return;
            },
        }
    }}
}

fn handle(mut req: Request, mut res: Response) {
    match req.method {
        hyper::Get => {
            match req.uri.clone() {
                AbsolutePath(path) => {
                    if path == "/" {
                        res.headers_mut().set(ContentType::plaintext());
                        res.send(USAGE).unwrap();
                    } else if path == "/favicon.ico" {
                        // todo favicon
                        *res.status_mut() = StatusCode::NotFound;
                    } else if let Ok(id) = PasteID::from_str(path.trim_left_matches("/")) {
                        match retrieve_paste(id) {
                            Some(mut file) => {
                                let metadata = try_handle!(res, file.metadata());
                                res.headers_mut().set(ContentLength(metadata.len()));
                                res.headers_mut().set(ContentType::plaintext());
                                io::copy(&mut file, &mut res.start().unwrap()).unwrap();
                            }
                            None => *res.status_mut() = StatusCode::NotFound,
                        }
                    } else {
                        *res.status_mut() = StatusCode::BadRequest;
                    }
                }
                _ => *res.status_mut() = StatusCode::BadRequest,
            }
        }
        hyper::Post => {
            match req.uri.clone() {
                AbsolutePath(path) => {
                    if path != "/" {
                        *res.status_mut() = StatusCode::BadRequest;
                    } else {
                        let mut tries = 0;
                        let mut id = PasteID::new(ID_SIZE);
                        let mut file = create_paste(&id);
                        while file.is_err() && tries < 3 {
                            id = PasteID::new(ID_SIZE);
                            file = create_paste(&id);
                            tries += 1;
                        }
                        if file.is_err() {
                            *res.status_mut() = StatusCode::InternalServerError;
                        } else {
                            // TODO: handle errors
                            try_handle!(res, io::copy(&mut req, &mut try_handle!(res, file)));
                            *res.status_mut() = StatusCode::Created;
                            res.send(format!("https://pasta.lol/{}\n", id).as_bytes())
                                .unwrap();
                        }
                    }
                }
                _ => *res.status_mut() = StatusCode::BadRequest,
            }
        }
        hyper::Delete => {
            match req.uri.clone() {
                AbsolutePath(path) => {
                    if let Ok(id) = PasteID::from_str(path.trim_left_matches("/")) {
                        let filename = id.filename();
                        let path = Path::new(&filename);
                        if path.exists() {
                            try_handle!(res, remove_file(path));
                        } else {
                            *res.status_mut() = StatusCode::NotFound;
                        }
                    } else {
                        *res.status_mut() = StatusCode::BadRequest;
                    }
                }
                _ => *res.status_mut() = StatusCode::BadRequest,
            }
        }
        _ => *res.status_mut() = StatusCode::MethodNotAllowed,
    }
}

fn main() {
    let mut ipv4 = Server::http("127.0.0.1:8080").unwrap();
    ipv4.keep_alive(None);
    let _guardipv4 = ipv4.handle(handle);
}
<|endoftext|>


<commit_before>extern crate clap;
extern crate serde;

use clap::{App, Arg};
use std::collections::btree_map::BTreeMap;
use serde::json;

mod parse;

impl parse::Value {
    fn to_json(&self) -> json::Value {
        let mut map = BTreeMap::new();
        let (ty, value) = match *self {
            parse::Value::DataFile(ref s)      => ("data_file", s),
            parse::Value::DataString(ref s)    => ("data_string", s),
            parse::Value::FormFile(ref s)      => ("form_file", s),
            parse::Value::HttpHeader(ref s)    => ("http_header", s),
            parse::Value::RawJsonString(ref s) => ("raw_json_string", s),
            parse::Value::RawJsonFile(ref s)   => ("raw_json_file", s),
            parse::Value::UrlParam(ref s)      => ("url_param", s),
        };
        map.insert("type".to_string(), json::Value::String(ty.to_string()));
        map.insert("value".to_string(), json::Value::String(value.to_string()));
        json::Value::Object(map)
    }
}

fn dump(kvs: &[(String, Option<parse::Value>)]) -> String {
    let kv_arrays = kvs.iter().map(|kv| {
        let k = json::Value::String(kv.0.clone());
        let v = kv.1.clone().map(|v| v.to_json()).unwrap_or(json::Value::Null);
        json::Value::Array(vec![k, v])
    }).collect();
    let root = json::Value::Array(kv_arrays);
    json::ser::to_string(&root).unwrap_or("".to_string())
}

fn main() {
    let matches = App::new("skv")
        .version("0.1.0")
        .arg(Arg::with_name("key-value pairs")
            .index(1)
            .multiple(true))
        .get_matches();
    let pairs = matches.values_of("key-value pairs").unwrap_or(vec![]);
    let parsed = parse::parse(&pairs);
    println!("dump => {}", dump(&parsed));
}
<commit_msg>Print formatted JSON by default<commit_after>extern crate clap;
extern crate serde;

use clap::{App, Arg};
use std::collections::btree_map::BTreeMap;
use std::fs::File;
use std::io::Read;
use serde::json;

mod parse;

fn slurp(path: &str) -> Option<String> {
    File::open(path).ok().as_mut().and_then(|f| {
        let mut s = String::new();
        f.read_to_string(&mut s).ok().and_then(|_| Some(s))
    })
}

impl parse::Value {
    fn to_json(&self) -> json::Value {
        let maybe_s = match *self {
            parse::Value::DataFile(ref path)    => slurp(path),
            parse::Value::DataString(ref s)     => Some(s.to_string()),
            parse::Value::FormFile(ref path)    => slurp(path),
            parse::Value::HttpHeader(ref s)     => Some(s.to_string()),
            parse::Value::RawJsonString(ref s)  => Some(s.to_string()),
            parse::Value::RawJsonFile(ref path) => slurp(path).and_then(|s| json::from_str(&s).ok()),
            parse::Value::UrlParam(ref s)       => Some(s.to_string()),
        };
        match maybe_s {
            Some(s) => json::Value::String(s),
            None    => json::Value::Null,
        }
    }

    fn to_typed_json(&self) -> json::Value {
        let mut map = BTreeMap::new();
        let (ty, value) = match *self {
            parse::Value::DataFile(ref path)    => ("data_file", path),
            parse::Value::DataString(ref s)     => ("data_string", s),
            parse::Value::FormFile(ref path)    => ("form_file", path),
            parse::Value::HttpHeader(ref s)     => ("http_header", s),
            parse::Value::RawJsonString(ref s)  => ("raw_json_string", s),
            parse::Value::RawJsonFile(ref path) => ("raw_json_file", path),
            parse::Value::UrlParam(ref s)       => ("url_param", s),
        };
        map.insert("type".to_string(), json::Value::String(ty.to_string()));
        map.insert("value".to_string(), json::Value::String(value.to_string()));
        json::Value::Object(map)
    }
}

fn to_json(kvs: &[(String, Option<parse::Value>)]) -> String {
    let mut map = BTreeMap::new();
    for kv in kvs.iter() {
        let k = kv.0.clone();
        let v = kv.1.clone().map(|v| v.to_json()).unwrap_or(json::Value::Null);
        map.insert(k, v);
    }
    let root = json::Value::Object(map);
    json::ser::to_string(&root).unwrap_or("".to_string())
}

fn dump(kvs: &[(String, Option<parse::Value>)]) -> String {
    let kv_arrays = kvs.iter().map(|kv| {
        let k = json::Value::String(kv.0.clone());
        let v = kv.1.clone().map(|v| v.to_typed_json()).unwrap_or(json::Value::Null);
        json::Value::Array(vec![k, v])
    }).collect();
    let root = json::Value::Array(kv_arrays);
    json::ser::to_string(&root).unwrap_or("".to_string())
}

fn main() {
    let matches = App::new("skv")
        .version("0.1.0")
        .arg(Arg::with_name("dump")
            .long("dump")
            .takes_value(false)
            .required(false))
        .arg(Arg::with_name("key-value pairs")
            .index(1)
            .multiple(true))
        .get_matches();
    let items = matches.values_of("key-value pairs").unwrap_or(vec![]);
    let kvs = parse::parse(&items);
    if matches.is_present("dump") {
        println!("{}", dump(&kvs));
    } else {
        println!("{}", to_json(&kvs));
    }
}
<|endoftext|>


<commit_before>extern crate flexi_logger;
extern crate time;
#[macro_use]
extern crate log;
#[macro_use]
extern crate quick_error;
#[macro_use]
extern crate clap;

#[macro_use]
extern crate varnishslog;

use std::io::{self, stdin, Read, Write};
use std::error::Error;
use clap::{Arg, App};

use varnishslog::stream_buf::{StreamBuf, ReadStreamBuf, FillError, FillApplyError};
use varnishslog::vsl::record::VslRecord;
use varnishslog::vsl::record::parser::{binary_vsl_tag, vsl_record_v4};
use varnishslog::access_log::session_state::SessionState;
use varnishslog::access_log::record_state::RecordState;
use varnishslog::serialization::{log_session_record, Config, Format, OutputError, JsonError};

mod program;

quick_error! {
    #[derive(Debug)]
    pub enum ProcessingError {
        IO(err: io::Error) {
            display("IO Error while processing records: {}", err)
            description("I/O error")
            from()
        }
        InputBuffer(err: FillError) {
            display("Failed to fill parsing buffer: {}", err)
            description("Input buffer error")
        }
        Parsing(err: String) {
            display("Failed to parse VSL record: {}", err)
            description("Parser error")
        }
        Serialization(err: String) {
            display("Failed to serialize log record: {}", err)
            description("Serialization error")
        }
    }
}

impl<'b> From<FillApplyError<&'b[u8], u32>> for ProcessingError {
    fn from(err: FillApplyError<&'b[u8], u32>) -> ProcessingError {
        match err {
            FillApplyError::FillError(FillError::Io(err)) => ProcessingError::IO(err),
            FillApplyError::FillError(err) => ProcessingError::InputBuffer(err),
            // we need to convert it to string due to input reference
            FillApplyError::Parser(err) => ProcessingError::Parsing(format!("nom parser error: {}", err)),
        }
    }
}

impl From<OutputError> for ProcessingError {
    fn from(err: OutputError) -> ProcessingError {
        match err {
            OutputError::JsonSerialization(JsonError::Io(err)) |
            OutputError::Io(err) => ProcessingError::IO(err),
            err => ProcessingError::Serialization(format!("Serialization error: {}", err)),
        }
    }
}

impl ProcessingError {
    fn to_exit_code(&self) -> i32 {
        match *self {
            ProcessingError::IO(_) => 10,
            ProcessingError::InputBuffer(_) => 11,
            ProcessingError::Parsing(_) => 20,
            ProcessingError::Serialization(_) => 30,
        }
    }
}

impl ProcessingError {
    fn is_brokend_pipe(&self) -> bool {
        match *self {
            ProcessingError::IO(ref err) =>
                err.kind() == io::ErrorKind::UnexpectedEof || err.kind() == io::ErrorKind::BrokenPipe,
            _ => false
        }
    }
}

fn try_read_vsl_tag<R: Read>(stream: &mut ReadStreamBuf<R>) -> Result<(), ProcessingError> {
    loop {
        match try!(stream.fill_apply(binary_vsl_tag)) {
            None => continue,
            Some(Some(_)) => {
                info!("Found VSL tag");
                break
            }
            Some(_) => break,
        }
    }
    Ok(())
}

trait WriteRecord {
    fn write_record<W>(&mut self, record: VslRecord, output: &mut W) -> Result<(), ProcessingError> where W: Write;
    fn log_reports(&self) {}
}

fn process_vsl_records<R, W, P>(stream: &mut ReadStreamBuf<R>, mut writer: P, output: &mut W) -> Result<(), ProcessingError> where R: Read, W: Write, P: WriteRecord {
    loop {
        match stream.fill_apply(vsl_record_v4).map_err(ProcessingError::from) {
            Ok(None) => continue,
            Ok(Some(record)) => try!(writer.write_record(record, output)),
            Err(err) => {
                //TODO: need better tracking of orphan records and other stats
                if err.is_brokend_pipe() {
                    writer.log_reports();
                }
                return Err(err)
            }
        }
    }
}

fn process_vsl_stream<R, W>(input: R, mut output: W, stream_buf_size: usize, output_format: OutputFormat, config: Config) -> Result<(), ProcessingError> where R: Read, W: Write {
    //TODO: make buffer size configurable
    let mut stream = ReadStreamBuf::with_capacity(input, stream_buf_size);

    try!(try_read_vsl_tag(&mut stream));

    match output_format {
        OutputFormat::Log => process_vsl_records(&mut stream, LogWriter::default(), &mut output),
        OutputFormat::LogDebug => process_vsl_records(&mut stream, LogDebugWriter::default(), &mut output),
        OutputFormat::RecordDebug => process_vsl_records(&mut stream, RecordDebugWriter::default(), &mut output),
        OutputFormat::SessionDebug => process_vsl_records(&mut stream, SessionDebugWriter::default(), &mut output),
        OutputFormat::Json => process_vsl_records(&mut stream, SerdeWriter::new(Format::Json, config), &mut output),
        OutputFormat::JsonPretty => process_vsl_records(&mut stream, SerdeWriter::new(Format::JsonPretty, config), &mut output),
        OutputFormat::NcsaJson => process_vsl_records(&mut stream, SerdeWriter::new(Format::NcsaJson, config), &mut output),
    }
}

#[derive(Default)]
struct LogWriter;
impl WriteRecord for LogWriter {
    fn write_record<W>(&mut self, record: VslRecord, output: &mut W) -> Result<(), ProcessingError> where W: Write {
        writeln!(output, "{:#}", record).map_err(From::from)
    }
}

#[derive(Default)]
struct LogDebugWriter;
impl WriteRecord for LogDebugWriter {
    fn write_record<W>(&mut self, record: VslRecord, output: &mut W) -> Result<(), ProcessingError> where W: Write {
        writeln!(output, "{:#?}", record).map_err(From::from)
    }
}

#[derive(Default)]
struct RecordDebugWriter {
    state: RecordState,
}

impl WriteRecord for RecordDebugWriter {
    fn write_record<W>(&mut self, record: VslRecord, output: &mut W) -> Result<(), ProcessingError> where W: Write {
        if let Some(record) = self.state.apply(&record) {
            writeln!(output, "{:#?}", record).map_err(From::from)
        } else {
            Ok(())
        }
    }
}

#[derive(Default)]
struct SessionDebugWriter {
    state: SessionState,
}

impl WriteRecord for SessionDebugWriter {
    fn write_record<W>(&mut self, record: VslRecord, output: &mut W) -> Result<(), ProcessingError> where W: Write {
        if let Some(record) = self.state.apply(&record) {
            writeln!(output, "{:#?}", record).map_err(From::from)
        } else {
            Ok(())
        }
    }
}

struct SerdeWriter {
    state: SessionState,
    format: Format,
    config: Config,
}

impl SerdeWriter {
    fn new(format: Format, config: Config) -> SerdeWriter {
        SerdeWriter {
            state: SessionState::default(),
            format: format,
            config: config,
        }
    }
}

impl WriteRecord for SerdeWriter {
    fn write_record<W>(&mut self, record: VslRecord, output: &mut W) -> Result<(), ProcessingError> where W: Write {
        if let Some(session) = self.state.apply(&record) {
            log_session_record(&session, &self.format, output, &self.config).map_err(From::from)
        } else {
            Ok(())
        }
    }

    fn log_reports(&self) {
        for client in self.state.unmatched_client_access_records() {
            warn!("ClientAccessRecord without matching session left: {:?}", client)
        }

        for backend in self.state.unmatched_backend_access_records() {
            warn!("BackendAccessRecord without matching session left: {:?}", backend)
        }

        for session in self.state.unresolved_sessions() {
            warn!("SessionRecord with unresolved links to other objects left: {:?}", session)
        }
    }
}

arg_enum! {
    #[derive(Debug)]
    enum OutputFormat {
        Log,
        LogDebug,
        RecordDebug,
        SessionDebug,
        Json,
        JsonPretty,
        NcsaJson
    }
}

fn main() {
    let arguments = App::new("Varnish VSL log to syslog logger")
        .version(crate_version!())
        .author(crate_authors!())
        .about("Reads binary VSL log entreis, correlates them togeter and emits JSON log entry to syslog")
        .arg(Arg::with_name("log-spec")
             .short("d")
             .long("log-sepc")
             .value_name("LOG_LEVEL_SPEC")
             .help("Logging level specification, e.g: info")
             .takes_value(true)
             .default_value("warn"))
        .arg(Arg::with_name("output")
             .long("output-format")
             .short("o")
             .help("Format of the output")
             .takes_value(true)
             .possible_values(&OutputFormat::variants())
             .default_value(OutputFormat::variants().last().unwrap()))
        .arg(Arg::with_name("no-log-processing")
             .long("no-log-processing")
             .short("l")
             .help("Do not process VSL log into vars, messages and ACL matches"))
        .arg(Arg::with_name("keep-raw-log")
             .long("keep-raw-log")
             .short("L")
             .help("Include raw log messages"))
        .arg(Arg::with_name("no-header-indexing")
             .long("no-header-indexing")
             .short("i")
             .help("Do not make indices of request and response headers with normalized header names"))
        .arg(Arg::with_name("keep-raw-headers")
             .long("keep-raw-headers")
             .short("I")
             .help("Keep raw header name/value pairs; any indices are moved to top level"))
        .arg(Arg::with_name("stream-buffer-size")
             .long("stream-buffer-size")
             .short("s")
             .help("Size of stream buffer in bytes - must be bigger than biggest VSL record")
             .default_value("262144"))
        .get_matches();

    program::init(arguments.value_of("log-spec"));

    let output_format = value_t!(arguments, "output", OutputFormat).unwrap_or_else(|e| e.exit());
    let stream_buf_size = value_t!(arguments, "stream-buffer-size", usize).unwrap_or_else(|e| e.exit());

    let input = stdin();
    let input = input.lock();
    let output = std::io::stdout();

    let config = Config {
        no_log_processing: arguments.is_present("no-log-processing"),
        keep_raw_log: arguments.is_present("keep-raw-log"),
        no_header_indexing: arguments.is_present("no-header-indexing"),
        keep_raw_headers: arguments.is_present("keep-raw-headers"),
    };

    if let Err(err) = process_vsl_stream(input, output, stream_buf_size, output_format, config) {
        if err.is_brokend_pipe() {
            info!("Broken pipe")
        } else {
            error!("{}", err);
            program::exit_with_error(err.description(), err.to_exit_code())
        }
    }

    info!("Done");
}
<commit_msg>suport for running with log file<commit_after>extern crate flexi_logger;
extern crate time;
#[macro_use]
extern crate log;
#[macro_use]
extern crate quick_error;
#[macro_use]
extern crate clap;

#[macro_use]
extern crate varnishslog;

use std::io::{self, stdin, Read, Write};
use std::fs::File;
use std::error::Error;
use clap::{Arg, App};

use varnishslog::stream_buf::{StreamBuf, ReadStreamBuf, FillError, FillApplyError};
use varnishslog::vsl::record::VslRecord;
use varnishslog::vsl::record::parser::{binary_vsl_tag, vsl_record_v4};
use varnishslog::access_log::session_state::SessionState;
use varnishslog::access_log::record_state::RecordState;
use varnishslog::serialization::{log_session_record, Config, Format, OutputError, JsonError};

mod program;

quick_error! {
    #[derive(Debug)]
    pub enum ProcessingError {
        IO(err: io::Error) {
            display("IO Error while processing records: {}", err)
            description("I/O error")
            from()
        }
        InputBuffer(err: FillError) {
            display("Failed to fill parsing buffer: {}", err)
            description("Input buffer error")
        }
        Parsing(err: String) {
            display("Failed to parse VSL record: {}", err)
            description("Parser error")
        }
        Serialization(err: String) {
            display("Failed to serialize log record: {}", err)
            description("Serialization error")
        }
    }
}

impl<'b> From<FillApplyError<&'b[u8], u32>> for ProcessingError {
    fn from(err: FillApplyError<&'b[u8], u32>) -> ProcessingError {
        match err {
            FillApplyError::FillError(FillError::Io(err)) => ProcessingError::IO(err),
            FillApplyError::FillError(err) => ProcessingError::InputBuffer(err),
            // we need to convert it to string due to input reference
            FillApplyError::Parser(err) => ProcessingError::Parsing(format!("nom parser error: {}", err)),
        }
    }
}

impl From<OutputError> for ProcessingError {
    fn from(err: OutputError) -> ProcessingError {
        match err {
            OutputError::JsonSerialization(JsonError::Io(err)) |
            OutputError::Io(err) => ProcessingError::IO(err),
            err => ProcessingError::Serialization(format!("Serialization error: {}", err)),
        }
    }
}

impl ProcessingError {
    fn to_exit_code(&self) -> i32 {
        match *self {
            ProcessingError::IO(_) => 10,
            ProcessingError::InputBuffer(_) => 11,
            ProcessingError::Parsing(_) => 20,
            ProcessingError::Serialization(_) => 30,
        }
    }
}

impl ProcessingError {
    fn is_brokend_pipe(&self) -> bool {
        match *self {
            ProcessingError::IO(ref err) =>
                err.kind() == io::ErrorKind::UnexpectedEof || err.kind() == io::ErrorKind::BrokenPipe,
            _ => false
        }
    }
}

fn try_read_vsl_tag<R: Read>(stream: &mut ReadStreamBuf<R>) -> Result<(), ProcessingError> {
    loop {
        match try!(stream.fill_apply(binary_vsl_tag)) {
            None => continue,
            Some(Some(_)) => {
                info!("Found VSL tag");
                break
            }
            Some(_) => break,
        }
    }
    Ok(())
}

trait WriteRecord {
    fn write_record<W>(&mut self, record: VslRecord, output: &mut W) -> Result<(), ProcessingError> where W: Write;
    fn log_reports(&self) {}
}

fn process_vsl_records<R, W, P>(stream: &mut ReadStreamBuf<R>, mut writer: P, output: &mut W) -> Result<(), ProcessingError> where R: Read, W: Write, P: WriteRecord {
    loop {
        match stream.fill_apply(vsl_record_v4).map_err(ProcessingError::from) {
            Ok(None) => continue,
            Ok(Some(record)) => try!(writer.write_record(record, output)),
            Err(err) => {
                //TODO: need better tracking of orphan records and other stats
                if err.is_brokend_pipe() {
                    writer.log_reports();
                }
                return Err(err)
            }
        }
    }
}

fn process_vsl_stream<R, W>(input: R, mut output: W, stream_buf_size: usize, output_format: OutputFormat, config: Config) -> Result<(), ProcessingError> where R: Read, W: Write {
    //TODO: make buffer size configurable
    let mut stream = ReadStreamBuf::with_capacity(input, stream_buf_size);

    try!(try_read_vsl_tag(&mut stream));

    match output_format {
        OutputFormat::Log => process_vsl_records(&mut stream, LogWriter::default(), &mut output),
        OutputFormat::LogDebug => process_vsl_records(&mut stream, LogDebugWriter::default(), &mut output),
        OutputFormat::RecordDebug => process_vsl_records(&mut stream, RecordDebugWriter::default(), &mut output),
        OutputFormat::SessionDebug => process_vsl_records(&mut stream, SessionDebugWriter::default(), &mut output),
        OutputFormat::Json => process_vsl_records(&mut stream, SerdeWriter::new(Format::Json, config), &mut output),
        OutputFormat::JsonPretty => process_vsl_records(&mut stream, SerdeWriter::new(Format::JsonPretty, config), &mut output),
        OutputFormat::NcsaJson => process_vsl_records(&mut stream, SerdeWriter::new(Format::NcsaJson, config), &mut output),
    }
}

#[derive(Default)]
struct LogWriter;
impl WriteRecord for LogWriter {
    fn write_record<W>(&mut self, record: VslRecord, output: &mut W) -> Result<(), ProcessingError> where W: Write {
        writeln!(output, "{:#}", record).map_err(From::from)
    }
}

#[derive(Default)]
struct LogDebugWriter;
impl WriteRecord for LogDebugWriter {
    fn write_record<W>(&mut self, record: VslRecord, output: &mut W) -> Result<(), ProcessingError> where W: Write {
        writeln!(output, "{:#?}", record).map_err(From::from)
    }
}

#[derive(Default)]
struct RecordDebugWriter {
    state: RecordState,
}

impl WriteRecord for RecordDebugWriter {
    fn write_record<W>(&mut self, record: VslRecord, output: &mut W) -> Result<(), ProcessingError> where W: Write {
        if let Some(record) = self.state.apply(&record) {
            writeln!(output, "{:#?}", record).map_err(From::from)
        } else {
            Ok(())
        }
    }
}

#[derive(Default)]
struct SessionDebugWriter {
    state: SessionState,
}

impl WriteRecord for SessionDebugWriter {
    fn write_record<W>(&mut self, record: VslRecord, output: &mut W) -> Result<(), ProcessingError> where W: Write {
        if let Some(record) = self.state.apply(&record) {
            writeln!(output, "{:#?}", record).map_err(From::from)
        } else {
            Ok(())
        }
    }
}

struct SerdeWriter {
    state: SessionState,
    format: Format,
    config: Config,
}

impl SerdeWriter {
    fn new(format: Format, config: Config) -> SerdeWriter {
        SerdeWriter {
            state: SessionState::default(),
            format: format,
            config: config,
        }
    }
}

impl WriteRecord for SerdeWriter {
    fn write_record<W>(&mut self, record: VslRecord, output: &mut W) -> Result<(), ProcessingError> where W: Write {
        if let Some(session) = self.state.apply(&record) {
            log_session_record(&session, &self.format, output, &self.config).map_err(From::from)
        } else {
            Ok(())
        }
    }

    fn log_reports(&self) {
        for client in self.state.unmatched_client_access_records() {
            warn!("ClientAccessRecord without matching session left: {:?}", client)
        }

        for backend in self.state.unmatched_backend_access_records() {
            warn!("BackendAccessRecord without matching session left: {:?}", backend)
        }

        for session in self.state.unresolved_sessions() {
            warn!("SessionRecord with unresolved links to other objects left: {:?}", session)
        }
    }
}

arg_enum! {
    #[derive(Debug)]
    enum OutputFormat {
        Log,
        LogDebug,
        RecordDebug,
        SessionDebug,
        Json,
        JsonPretty,
        NcsaJson
    }
}

fn main() {
    let arguments = App::new("Varnish VSL log to syslog logger")
        .version(crate_version!())
        .author(crate_authors!())
        .about("Reads binary VSL log entreis, correlates them togeter and emits JSON log entry to syslog")
        .arg(Arg::with_name("log-spec")
             .short("d")
             .long("log-sepc")
             .value_name("LOG_LEVEL_SPEC")
             .help("Logging level specification, e.g: info")
             .takes_value(true)
             .default_value("warn"))
        .arg(Arg::with_name("output")
             .long("output-format")
             .short("o")
             .help("Format of the output")
             .takes_value(true)
             .possible_values(&OutputFormat::variants())
             .default_value(OutputFormat::variants().last().unwrap()))
        .arg(Arg::with_name("no-log-processing")
             .long("no-log-processing")
             .short("l")
             .help("Do not process VSL log into vars, messages and ACL matches"))
        .arg(Arg::with_name("keep-raw-log")
             .long("keep-raw-log")
             .short("L")
             .help("Include raw log messages"))
        .arg(Arg::with_name("no-header-indexing")
             .long("no-header-indexing")
             .short("i")
             .help("Do not make indices of request and response headers with normalized header names"))
        .arg(Arg::with_name("keep-raw-headers")
             .long("keep-raw-headers")
             .short("I")
             .help("Keep raw header name/value pairs; any indices are moved to top level"))
        .arg(Arg::with_name("stream-buffer-size")
             .long("stream-buffer-size")
             .short("s")
             .help("Size of stream buffer in bytes - must be bigger than biggest VSL record")
             .default_value("262144"))
        .arg(Arg::with_name("vsl-file")
             .value_name("VSL_FILE")
             .help("VSL file to process (read from standard input if not specified)"))
        .get_matches();

    program::init(arguments.value_of("log-spec"));

    let output_format = value_t!(arguments, "output", OutputFormat).unwrap_or_else(|e| e.exit());
    let stream_buf_size = value_t!(arguments, "stream-buffer-size", usize).unwrap_or_else(|e| e.exit());

    let output = std::io::stdout();

    let config = Config {
        no_log_processing: arguments.is_present("no-log-processing"),
        keep_raw_log: arguments.is_present("keep-raw-log"),
        no_header_indexing: arguments.is_present("no-header-indexing"),
        keep_raw_headers: arguments.is_present("keep-raw-headers"),
    };

    let result = if let Some(path) = arguments.value_of("vsl-file") {
        let file = File::open(path);
        match file {
            Ok(file) => process_vsl_stream(file, output, stream_buf_size, output_format, config),
            Err(err) => program::exit_with_error(&format!("Failed to open VSL file: {}: {}", path, err), 1),
        }
    } else {
        let stdin = stdin();
        let stdin = stdin.lock();
        process_vsl_stream(stdin, output, stream_buf_size, output_format, config)
    };

    if let Err(err) = result {
        if err.is_brokend_pipe() {
            info!("Broken pipe")
        } else {
            error!("{}", err);
            program::exit_with_error(err.description(), err.to_exit_code())
        }
    }

    info!("Done");
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate error_chain;

extern crate clap;
extern crate serde;
extern crate rmp_serde;
extern crate sodiumoxide;
extern crate xdg;

extern crate mine;

mod init;
mod insert;
mod show;
mod set_tag;

use clap::{Arg, App, AppSettings, SubCommand};

use mine::Mine;

mod errors {
    extern crate mine;
    // Create the Error, ErrorKind, ResultExt, and Result types
    error_chain! {
        links {
            MineLib(mine::errors::Error, mine::errors::ErrorKind);
        }
    }
}

use errors::*;

quick_main!(run);

fn run() -> Result<()> {
    sodiumoxide::init();

    let matches = cli().get_matches();

    let mine = Mine::new("mine")
        .chain_err(|| "failed to initialize mine")?;

    match matches.subcommand() {
        ("init", Some(_)) => init::init_run(mine)?,
        ("insert", Some(m)) => insert::insert_run(mine, m)?,
        ("show", Some(m)) => show::show_run(mine, m)?,
        ("set-tag", Some(m)) => set_tag::set_tag_run(mine, m)?,
        (_, _) => unreachable!(),
    }

    Ok(())
}

fn cli() -> App<'static, 'static> {
    App::new("mine")
        .version("0.0.0")
        .author("Johannes Löthberg <johannes@kyriasis.com>")
        .about("NaCL based password manager in Rust")
        .global_settings(&[AppSettings::ColoredHelp])
        .setting(AppSettings::SubcommandRequiredElseHelp)
        .subcommand(SubCommand::with_name("init")
                    .about("initialize password store"))
        .subcommand(SubCommand::with_name("insert")
                    .about("insert a password")
                    .arg(Arg::with_name("NAME")
                         .required(true)
                         .index(1))
                    .arg(Arg::with_name("PASSWORD")
                         .required(true)
                         .index(2)))
        .subcommand(SubCommand::with_name("show")
                    .about("show a password")
                    .arg(Arg::with_name("NAME")
                         .help("Password name")
                         .required(true)
                         .index(1)))
        .subcommand(SubCommand::with_name("set-tag")
                    .about("set a tag on a password")
                    .arg(Arg::with_name("PASSWORD")
                         .required(true)
                         .index(1))
                    .arg(Arg::with_name("TAG")
                         .required(true)
                         .index(2))
                    .arg(Arg::with_name("VALUE")
                         .required(true)
                         .index(3)))
}
<commit_msg>main: Add subcommand for generating shell completions<commit_after>#[macro_use]
extern crate error_chain;

extern crate clap;
extern crate serde;
extern crate rmp_serde;
extern crate sodiumoxide;
extern crate xdg;

extern crate mine;

mod init;
mod insert;
mod show;
mod set_tag;

use clap::{Arg, App, AppSettings, Shell, SubCommand};

use mine::Mine;

mod errors {
    extern crate mine;
    // Create the Error, ErrorKind, ResultExt, and Result types
    error_chain! {
        links {
            MineLib(mine::errors::Error, mine::errors::ErrorKind);
        }
    }
}

use errors::*;

quick_main!(run);

fn run() -> Result<()> {
    sodiumoxide::init();

    let matches = cli().get_matches();

    let mine = Mine::new("mine")
        .chain_err(|| "failed to initialize mine")?;

    match matches.subcommand() {
        ("init", Some(_)) => init::init_run(mine)?,
        ("insert", Some(m)) => insert::insert_run(mine, m)?,
        ("show", Some(m)) => show::show_run(mine, m)?,
        ("set-tag", Some(m)) => set_tag::set_tag_run(mine, m)?,
        ("completions", Some(m)) => {
            if let Some(shell) = m.value_of("shell") {
                cli().gen_completions_to("mine", shell.parse::<Shell>().unwrap(), &mut std::io::stdout());
            }
        }
        (_, _) => unreachable!(),
    }

    Ok(())
}

fn cli() -> App<'static, 'static> {
    App::new("mine")
        .version("0.0.0")
        .author("Johannes Löthberg <johannes@kyriasis.com>")
        .about("NaCL based password manager in Rust")
        .global_settings(&[AppSettings::ColoredHelp])
        .setting(AppSettings::SubcommandRequiredElseHelp)
        .subcommand(SubCommand::with_name("init")
                    .about("initialize password store"))
        .subcommand(SubCommand::with_name("insert")
                    .about("insert a password")
                    .arg(Arg::with_name("NAME")
                         .required(true)
                         .index(1))
                    .arg(Arg::with_name("PASSWORD")
                         .required(true)
                         .index(2)))
        .subcommand(SubCommand::with_name("show")
                    .about("show a password")
                    .arg(Arg::with_name("NAME")
                         .help("Password name")
                         .required(true)
                         .index(1)))
        .subcommand(SubCommand::with_name("set-tag")
                    .about("set a tag on a password")
                    .arg(Arg::with_name("PASSWORD")
                         .required(true)
                         .index(1))
                    .arg(Arg::with_name("TAG")
                         .required(true)
                         .index(2))
                    .arg(Arg::with_name("VALUE")
                         .required(true)
                         .index(3)))
        .subcommand(SubCommand::with_name("completions")
                    .about("generate shell completion scripts")
                    .setting(AppSettings::ArgRequiredElseHelp)
                    .arg(Arg::with_name("shell")
                         .possible_values(&Shell::variants())))
}
<|endoftext|>


<commit_before>#![deny(warnings)]

extern crate chan;
extern crate crypto;
extern crate walkdir;

use self::crypto::digest::Digest;
use self::crypto::sha1::Sha1;
use std::env;
use std::fs::File;
use std::io::Read;
use std::path::Path;
use std::io;
use walkdir::WalkDir;
use std::thread;
use std::sync::Arc;
use walkdir::DirEntry;

/// The purpose of this newtype is to hide the channel implementation.
struct GatherIter<T>(chan::Iter<T>);
impl<T> Iterator for GatherIter<T> {
    type Item = T;
    fn next(&mut self) -> Option<T> {
        self.0.next()
    }
}

/// producer_ctor and xform_ctor is needed to allow construction
/// in the correct producer / worker thread.
/// TODO: Figure out how to simplify this trait mess :)
fn scatter_gather<PC, XC, P, X, J, R>(producer_ctor: PC, xform_ctor: XC) -> GatherIter<R>
    where PC: 'static + std::marker::Send + FnOnce() -> P,
          XC: 'static + std::marker::Send + std::marker::Sync + Fn() -> X,
          X: FnMut(J) -> R,
          J: 'static + std::marker::Send,
          R: 'static + std::marker::Send,
          P: IntoIterator<Item = J>
{
    let jobs_rx = {
        let (tx, rx) = chan::sync(0);
        thread::spawn(move || {
            for e in producer_ctor().into_iter() {
                tx.send(e);
            }
        });
        rx
    };
    let results_rx = {
        let (tx, rx) = chan::sync(0);
        let xform_ctor = Arc::new(xform_ctor); // TODO: Investigate why this is needed.
        for _ in 0..8 {
            let tx = tx.clone();
            let jobs_rx = jobs_rx.clone();
            let xform_ctor = xform_ctor.clone();
            thread::spawn(move || {
                let mut xform = xform_ctor();
                for e in jobs_rx {
                    tx.send(xform(e));
                }
            });
        }
        rx
    };
    GatherIter(results_rx.iter())
}

fn calc_hash(p: &Path, hasher: &mut Sha1, buf: &mut [u8]) -> io::Result<String> {
    hasher.reset();
    let mut f = File::open(p)?;
    loop {
        let num_read = f.read(buf)?;
        if num_read == 0 {
            break;
        }
        hasher.input(&buf[..num_read]);
    }
    Ok(hasher.result_str())
}

type Result<T> = std::result::Result<T, Error>;

#[derive(Debug)]
enum Error {
    Io(io::Error),
    Ignored(DirEntry),
    WalkDir(walkdir::Error),
}

fn process_root(root: &Path) -> io::Result<()> {
    let pb = root.to_path_buf();
    let producer_ctor = || WalkDir::new(pb);
    let xform_ctor = || {
        let mut hasher = Sha1::new();
        let mut buf = [0; 1024 * 8];
        let f = move |e: walkdir::Result<DirEntry>| -> Result<String> {
            let e = e.map_err(Error::WalkDir)?;
            if !e.file_type().is_file() {
                return Err(Error::Ignored(e));
            }
            calc_hash(e.path(), &mut hasher, &mut buf).map_err(Error::Io)
        };
        f
    };
    let results = scatter_gather(producer_ctor, xform_ctor);
    for r in results {
        println!("{:?}", r);
    }
    Ok(())
}

fn main() {
    let root = env::args().nth(1).unwrap_or(".".to_string());
    let root = Path::new(root.as_str());
    process_root(root).expect("Hello error 2!");
}
<commit_msg>Comment update.<commit_after>#![deny(warnings)]

extern crate chan;
extern crate crypto;
extern crate walkdir;

use self::crypto::digest::Digest;
use self::crypto::sha1::Sha1;
use std::env;
use std::fs::File;
use std::io::Read;
use std::path::Path;
use std::io;
use walkdir::WalkDir;
use std::thread;
use std::sync::Arc;
use walkdir::DirEntry;

/// The purpose of this newtype is to hide the channel.
struct GatherIter<T>(chan::Iter<T>);
impl<T> Iterator for GatherIter<T> {
    type Item = T;
    fn next(&mut self) -> Option<T> {
        self.0.next()
    }
}

/// producer_ctor and xform_ctor is needed to allow construction
/// in the correct producer / worker thread.
/// TODO: Figure out how to simplify this trait mess :)
fn scatter_gather<PC, XC, P, X, J, R>(producer_ctor: PC, xform_ctor: XC) -> GatherIter<R>
    where PC: 'static + std::marker::Send + FnOnce() -> P,
          XC: 'static + std::marker::Send + std::marker::Sync + Fn() -> X,
          X: FnMut(J) -> R,
          J: 'static + std::marker::Send,
          R: 'static + std::marker::Send,
          P: IntoIterator<Item = J>
{
    let jobs_rx = {
        let (tx, rx) = chan::sync(0);
        thread::spawn(move || {
            for e in producer_ctor().into_iter() {
                tx.send(e);
            }
        });
        rx
    };
    let results_rx = {
        let (tx, rx) = chan::sync(0);
        let xform_ctor = Arc::new(xform_ctor); // TODO: Investigate why this is needed.
        for _ in 0..8 {
            let tx = tx.clone();
            let jobs_rx = jobs_rx.clone();
            let xform_ctor = xform_ctor.clone();
            thread::spawn(move || {
                let mut xform = xform_ctor();
                for e in jobs_rx {
                    tx.send(xform(e));
                }
            });
        }
        rx
    };
    GatherIter(results_rx.iter())
}

fn calc_hash(p: &Path, hasher: &mut Sha1, buf: &mut [u8]) -> io::Result<String> {
    hasher.reset();
    let mut f = File::open(p)?;
    loop {
        let num_read = f.read(buf)?;
        if num_read == 0 {
            break;
        }
        hasher.input(&buf[..num_read]);
    }
    Ok(hasher.result_str())
}

type Result<T> = std::result::Result<T, Error>;

#[derive(Debug)]
enum Error {
    Io(io::Error),
    Ignored(DirEntry),
    WalkDir(walkdir::Error),
}

fn process_root(root: &Path) -> io::Result<()> {
    let pb = root.to_path_buf();
    let producer_ctor = || WalkDir::new(pb);
    let xform_ctor = || {
        let mut hasher = Sha1::new();
        let mut buf = [0; 1024 * 8];
        let f = move |e: walkdir::Result<DirEntry>| -> Result<String> {
            let e = e.map_err(Error::WalkDir)?;
            if !e.file_type().is_file() {
                return Err(Error::Ignored(e));
            }
            calc_hash(e.path(), &mut hasher, &mut buf).map_err(Error::Io)
        };
        f
    };
    let results = scatter_gather(producer_ctor, xform_ctor);
    for r in results {
        println!("{:?}", r);
    }
    Ok(())
}

fn main() {
    let root = env::args().nth(1).unwrap_or(".".to_string());
    let root = Path::new(root.as_str());
    process_root(root).expect("Hello error 2!");
}
<|endoftext|>


<commit_before>//
// main.rs
// Copyright (C) 2017 Szymon Urbaś <szymon.urbas@aol.com>
// Distributed under terms of the BSD (2-clause) license.
//
// Created on: 13 Mar 2017 21:03:41 +0100 (CET)
//

extern crate rand;
extern crate cursive;

#[cfg(feature = "go-slow")]
use std::thread;
#[cfg(feature = "go-slow")]
use std::time;

mod process;
mod scheduler;
mod fcfs;
mod round_robin;
mod tui;

use scheduler::*;
use tui::*;

/// Length of a single system clock tick (frequency of actual, real time)
#[cfg(feature = "go-slow")]
const CLOCK_HZ: u64 = 1000;
/// Switch context every <value> clock ticks
const SYSTEM_HZ: usize = 8;

fn main() {
  let mut tui = tui::new();

  let scenarios = {
    (0..20).map(|_| (0..64).map(|_| rand::random::<usize>() % (SYSTEM_HZ * 4) + 1).collect::<Vec<usize>>()).collect::<Vec<Vec<usize>>>()
  };

  macro_rules! run_simulation_suite {
    ($scheduler:ident) => ({
      let mut average_waiting_time = 0f64;

      for (run, scenario) in scenarios.iter().enumerate() {
        let scheduler = $scheduler::new();
        let avg = run_simulation(&mut tui, scheduler, scenario.clone());
        tui.add_result(format!("{}: #{:02}: Average waiting time: {:02.2}", stringify!($scheduler), run, avg));

        average_waiting_time -= average_waiting_time / (run + 1) as f64;
        average_waiting_time += avg / (run + 1) as f64;
      }

      tui.add_result(format!("{}: Overall average: {:02.2}", stringify!($scheduler), average_waiting_time));
    })
  }

  run_simulation_suite!(fcfs);
  run_simulation_suite!(round_robin);

  tui.update();

  loop {}
}

fn run_simulation<S>(mut tui: &mut Tui, mut scheduler: S, mut process_list: Vec<usize>) -> f64
where S: Scheduler {
  let mut clock_tick = 0;
  let mut process_spawner = process::new_spawner();
  let mut average_waiting_time = 0f64;
  let mut num_of_spawned_procs = 1;

  tui.set_header(format!("Using algorithm: {}", scheduler.name()));
  tui.debug(format!("{:05}: Starting simulation using {}", clock_tick, scheduler.name()));

  // vectors only support the pop (from the back) operation so
  // reverse it (so we pop from the front)
  process_list.reverse();

  // Add the equivalent of the 'init' process
  scheduler.add_process(process_spawner.spawn(process_list.pop().unwrap()));

  loop {
    // update the process view
    scheduler.list_processes(&mut tui);

    if cfg!(feature = "go-slow") || clock_tick % (SYSTEM_HZ * 2) == 0 {
      // update all the views
      tui.update();
    }

    if !scheduler.has_processes() {
      // if there's no more processes - end the simulation
      // if a process can only be created by another process - there's
      // nothing else to do
      break;
    }

    // simulate executing the current process
    tui.debug(format!("{:05}: Executing {}", clock_tick, scheduler.current_proc().unwrap().name));
    scheduler.current_proc_mut().unwrap().record_execution();

    // Simulate other processes creating new processes
    // Assume that you can't create new processes, if there's none already
    if clock_tick % SYSTEM_HZ == 0 {
      if let Some(burst_time) = process_list.pop() {
        let new_proc = process_spawner.spawn(burst_time);
        tui.debug(format!("{:05}: Spawning {}", clock_tick, new_proc.name));
        scheduler.add_process(new_proc);
        num_of_spawned_procs += 1;
      }
    }

    if scheduler.current_proc().unwrap().done_executing() {
      tui.debug(format!("{:05}: Killing process {}", clock_tick, scheduler.current_proc().unwrap().name));
      scheduler.kill_current_proc();
    }

    if scheduler.has_processes() {
      if clock_tick % SYSTEM_HZ == SYSTEM_HZ - 1 {
        let prev_proc_name = scheduler.current_proc().unwrap().name.clone();
        scheduler.schedule();
        tui.debug(format!("{:05}: Switching context: {} -> {}", clock_tick, prev_proc_name, scheduler.current_proc().unwrap().name));
        tui.debug(format!("{:05}: {} was waiting {} clock ticks", clock_tick, scheduler.current_proc().unwrap().name, scheduler.current_proc().unwrap().waiting_time));

        average_waiting_time -= average_waiting_time / num_of_spawned_procs as f64;
        average_waiting_time += scheduler.current_proc().unwrap().waiting_time as f64 / num_of_spawned_procs as f64;
      }

      // increase the waiting time for every process
      scheduler.increase_waiting_times();
    }

    #[cfg(feature = "go-slow")]
    thread::sleep(time::Duration::from_millis(1000 / CLOCK_HZ));

    clock_tick += 1;
  }

  average_waiting_time
}

/*
 * vi: ts=2 sw=2 expandtab
 */

<commit_msg>Have a tendency to spawn more short-term processes<commit_after>//
// main.rs
// Copyright (C) 2017 Szymon Urbaś <szymon.urbas@aol.com>
// Distributed under terms of the BSD (2-clause) license.
//
// Created on: 13 Mar 2017 21:03:41 +0100 (CET)
//

extern crate rand;
extern crate cursive;

#[cfg(feature = "go-slow")]
use std::thread;
#[cfg(feature = "go-slow")]
use std::time;

mod process;
mod scheduler;
mod fcfs;
mod round_robin;
mod tui;

use scheduler::*;
use tui::*;

/// Length of a single system clock tick (frequency of actual, real time)
#[cfg(feature = "go-slow")]
const CLOCK_HZ: u64 = 1000;
/// Switch context every <value> clock ticks
const SYSTEM_HZ: usize = 8;

fn main() {
  let mut tui = tui::new();

  let scenarios = {
    (0..20).map(|_| (0..64).map(|_| {
      let upper_limit = rand::random::<usize>() % (SYSTEM_HZ * 4) + 1;
      let burst_time = rand::random::<usize>() % upper_limit;

      burst_time + 1 // + 1 so there's no chance of having a process with 0 burst time
    }).collect::<Vec<usize>>()).collect::<Vec<Vec<usize>>>()
  };

  macro_rules! run_simulation_suite {
    ($scheduler:ident) => ({
      let mut average_waiting_time = 0f64;

      for (run, scenario) in scenarios.iter().enumerate() {
        let scheduler = $scheduler::new();
        let avg = run_simulation(&mut tui, scheduler, scenario.clone());
        tui.add_result(format!("{}: #{:02}: Average waiting time: {:02.2}", stringify!($scheduler), run, avg));

        average_waiting_time -= average_waiting_time / (run + 1) as f64;
        average_waiting_time += avg / (run + 1) as f64;
      }

      tui.add_result(format!("{}: Overall average: {:02.2}", stringify!($scheduler), average_waiting_time));
    })
  }

  run_simulation_suite!(fcfs);
  run_simulation_suite!(round_robin);

  tui.update();

  loop {}
}

fn run_simulation<S>(mut tui: &mut Tui, mut scheduler: S, mut process_list: Vec<usize>) -> f64
where S: Scheduler {
  let mut clock_tick = 0;
  let mut process_spawner = process::new_spawner();
  let mut average_waiting_time = 0f64;
  let mut num_of_spawned_procs = 1;

  tui.set_header(format!("Using algorithm: {}", scheduler.name()));
  tui.debug(format!("{:05}: Starting simulation using {}", clock_tick, scheduler.name()));

  // vectors only support the pop (from the back) operation so
  // reverse it (so we pop from the front)
  process_list.reverse();

  // Add the equivalent of the 'init' process
  scheduler.add_process(process_spawner.spawn(process_list.pop().unwrap()));

  loop {
    // update the process view
    scheduler.list_processes(&mut tui);

    if cfg!(feature = "go-slow") || clock_tick % (SYSTEM_HZ * 2) == 0 {
      // update all the views
      tui.update();
    }

    if !scheduler.has_processes() {
      // if there's no more processes - end the simulation
      // if a process can only be created by another process - there's
      // nothing else to do
      break;
    }

    // simulate executing the current process
    tui.debug(format!("{:05}: Executing {}", clock_tick, scheduler.current_proc().unwrap().name));
    scheduler.current_proc_mut().unwrap().record_execution();

    // Simulate other processes creating new processes
    // Assume that you can't create new processes, if there's none already
    if clock_tick % SYSTEM_HZ == 0 {
      if let Some(burst_time) = process_list.pop() {
        let new_proc = process_spawner.spawn(burst_time);
        tui.debug(format!("{:05}: Spawning {}", clock_tick, new_proc.name));
        scheduler.add_process(new_proc);
        num_of_spawned_procs += 1;
      }
    }

    if scheduler.current_proc().unwrap().done_executing() {
      tui.debug(format!("{:05}: Killing process {}", clock_tick, scheduler.current_proc().unwrap().name));
      scheduler.kill_current_proc();
    }

    if scheduler.has_processes() {
      if clock_tick % SYSTEM_HZ == SYSTEM_HZ - 1 {
        let prev_proc_name = scheduler.current_proc().unwrap().name.clone();
        scheduler.schedule();
        tui.debug(format!("{:05}: Switching context: {} -> {}", clock_tick, prev_proc_name, scheduler.current_proc().unwrap().name));
        tui.debug(format!("{:05}: {} was waiting {} clock ticks", clock_tick, scheduler.current_proc().unwrap().name, scheduler.current_proc().unwrap().waiting_time));

        average_waiting_time -= average_waiting_time / num_of_spawned_procs as f64;
        average_waiting_time += scheduler.current_proc().unwrap().waiting_time as f64 / num_of_spawned_procs as f64;
      }

      // increase the waiting time for every process
      scheduler.increase_waiting_times();
    }

    #[cfg(feature = "go-slow")]
    thread::sleep(time::Duration::from_millis(1000 / CLOCK_HZ));

    clock_tick += 1;
  }

  average_waiting_time
}

/*
 * vi: ts=2 sw=2 expandtab
 */

<|endoftext|>


<commit_before>extern crate regex;
use regex::Regex;

use std::io;
use std::io::Write;

#[derive(Debug, PartialEq, Clone)]
pub enum Token {
    Unknown(String), // Invalid test (basically non-ascii)
    Literal(String), // Numeric literal number
    Func(Function), // Pre-defined function (like cos() )
    Const(Constant), // Constant like pi or e
    Var(String), // str arbitrary single char variable name
    Op(Operator), // Any of the 4 operators (+-*/)
    Open, // Open parens '('
    Close, // Closing parens ')'
    Comma,
}

#[derive(Debug, PartialEq, Clone)]
pub enum Function {
	Abs,
    Sqrt,
    Ln,
    Log,
    Sin,
    Csc,
    Cos,
    Sec,
    Tan,
    Cot,
    Asin,
    Acsc,
    Acos,
    Asec,
    Atan,
    Acot,
    Sinh,
    Csch,
    Cosh,
    Sech,
    Tanh,
    Coth,
    Asinh,
    Acsch,
    Acosh,
    Asech,
    Atanh,
    Acoth,
    Max,
}

#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum Constant {
	Pi,
	E,
}

#[derive(Debug, PartialEq, Eq, Clone, Copy)]
#[repr(u32)]
pub enum Operator {
    Add=0,
    Sub=1,
    Div=2,
    Mul=3,
    Pow=4, // Right
}

#[derive(Debug, PartialEq, Eq, Clone, Copy)]
#[repr(u32)]
pub enum Assoc {
    Left=0,
    Right=1,
}

#[derive(Debug, PartialEq, Eq, Clone, Copy)]
#[repr(u32)]
pub enum Order {
    AddSub = 1,
    MulDiv = 2,
    IMul = 3, // Implicit multiplication
    Pow = 4,
    Paren = 5,
}

#[derive(Debug, PartialEq, Clone)]
struct Expression {
    tokens: Vec<Token>,
}

struct ExpressionIter<'a> {
	expr: &'a Expression,
	count: usize,
}

impl<'a> Iterator for ExpressionIter<'a> {
	type Item = &'a Token;
    fn next(&mut self) -> Option<&'a Token> {
    	let token = self.expr.get_tokens().get(self.count);
    	self.count += 1;
    	token
    }
}

#[allow(dead_code)]
impl Expression {
	fn new(tokens: Vec<Token>) -> Self {
		Expression {tokens: tokens}
	}

	fn push(&mut self, token: Token) {
		self.tokens.push(token);
	}

	fn get_tokens(&self) -> &[Token] {
		self.tokens.as_slice()
	}

	fn get_mut_tokens(&mut self) -> &mut [Token] {
		self.tokens.as_mut_slice()
	}

	fn get_token(&self, index: usize) -> &Token {
		self.tokens.get(index).unwrap()
	}

	fn get_mut_token(&mut self, index: usize) -> &mut Token {
		self.tokens.get_mut(index).unwrap()
	}

	fn replace_token(&mut self, index: usize, new_token: Token) {
		self.tokens.remove(index);
		self.tokens.insert(index, new_token);
	}

	fn replace_all_tokens(&mut self, new_token: Vec<Token>) {
		self.tokens = new_token
	}

	fn find_first(&self, token: &Token) -> Option<usize> {
		for i in 0..self.tokens.len() {
			if self.tokens.get(i).unwrap() == token {
				return Some(i);
			}
		}
		None
	}

	fn find_last(&self, token: &Token) -> Option<usize> {
		for i in (0..self.tokens.len()).rev() {
			if self.tokens.get(i).unwrap() == token {
				return Some(i);
			}
		}
		None
	}

	fn split_at(&self, index: usize) -> (&[Token], &[Token]) {
		self.tokens.split_at(index)
	}

	fn len(&self) -> usize {
		self.tokens.len()
	}

	fn iter(&self) -> ExpressionIter {
		ExpressionIter {expr: &self, count: 0}
	}
}

fn main() {
	let numeric_regex: Regex = Regex::new(r"\d+\.\d+|\d+").unwrap();
	let function_regex: Regex = Regex::new(r"\w{2,}").unwrap();
    println!("Welcome to Rust-Calculus!");
    println!("To evaluate an expression, simply type one in and hit RETURN.");
    println!("To set a variable, simply type VAR_NAME=EXPRESSION and hit RETURN.");
    println!("Valid commands are: sym_int, int, sym_def, and def.");
    println!("Type 'quit' to exit.");
    let mut input = String::new();
    let stdin = io::stdin();
    let mut stdout = io::stdout();
    loop {
    	input.clear();
    	print!(">>>> ");
    	stdout.flush().ok();
        if let Err(x) = stdin.read_line(&mut input) {
        	println!("There was a problem reading stdin: {:?}", x);
        	print!("Exiting...");;
        	break;
        }
        input = strip_white_space(&input).to_lowercase();
        if input == "quit" {
        	print!("Exiting...");
        	break;
        }
        let expr = parse_input(&input, &numeric_regex, &function_regex);
        if expr.is_ok() {
        	println!("{:?}", &expr.unwrap());
        } else {
        	println!("Encountered an error while parsing: {:?}", expr.unwrap_err());
        	println!("Try Again...(type 'quit' to exit)");
        	continue;
        }
    }
}

fn strip_white_space(input: &String) -> String {
	input.split_whitespace().collect::<Vec<&str>>().join("")
}

fn parse_input(input: &String, numeric_regex: &Regex, function_regex: &Regex) -> Result<Expression, String> {
	// 1. Replace everthing except letters/numbers with their enums
	// 2. Then go through and replace things with Literals or functions
	let mut expr: Expression = Expression::new(Vec::new());
	let mut builder: String = String::new();
	for c in input.chars() {
		match c {
		    '(' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Open);
		    },
		    ')' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Close);
		    },
		    '+' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Op(Operator::Add));
		    },
		    '-' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Op(Operator::Sub));
		    },
		    '*' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Op(Operator::Mul));
		    },
		    '/' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Op(Operator::Div));
		    },
		    '^' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Op(Operator::Pow));
		    },
		    ',' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Comma);
		    },
		    _ => {
		    	builder.push(c);
		    }
		}
	}
	if builder.len() > 0 {
		if numeric_regex.is_match(&builder[..]) {
			expr.push(Token::Literal(builder.clone()));
		} else if function_regex.is_match(&builder[..]) {
			expr.push(map_string_to_func(&builder));
		}
		builder = String::new();
	}
	let mut op_stack: Vec<Token> = Vec::with_capacity(input.len());
	let mut out_queue: Vec<Token> = Vec::with_capacity(input.len());
	//println!("EXPR {:?}", &expr);
	for i in 0..expr.len() {
		let current_token = expr.get_token(i);
		//println!("BEFORE current_token: {:?}, OPSTACK {:?}, out_queue {:?}", &current_token, &op_stack, &out_queue);
		match current_token {
		    &Token::Literal(ref x) => out_queue.push(Token::Literal(x.clone())),
		    &Token::Func(ref x) => op_stack.push(Token::Func(x.clone())),
		    &Token::Comma => {
		    	let mut found_open = false;
		    	loop {
			    	let stack_token = try!{op_stack.pop().ok_or("Malformed Expression")};
			    	match stack_token {
			    	    Token::Open => {op_stack.push(stack_token); found_open=true; break;},
			    	    _ => out_queue.push(stack_token),
			    	}
		    	}
		    	if !found_open {return Err(String::from("Mismatch Parenthesis"))}
			},
		    &Token::Op(ref o1) =>  {
		    	loop {
			    	if op_stack.len() < 1 {
						break;
					}
			    	let o2 = try!{op_stack.pop().ok_or("Malformed Expression")}; // top of stack
			    	match o1 {
			    	    &Operator::Pow => match o2 {
			    	    	_ => { op_stack.push(o2); break; },
			    	    },
			    	    &Operator::Mul | &Operator::Div => match o2 {
			    	    	Token::Op(Operator::Pow) | Token::Op(Operator::Mul) | Token::Op(Operator::Div) => out_queue.push(o2),
			    	    	_ => { op_stack.push(o2); break; },
			    	    },
			    	    &Operator::Add | &Operator::Sub => match o2 {
			    	    	Token::Op(Operator::Add) | Token::Op(Operator::Sub) => out_queue.push(o2),
			    	    	_ => { op_stack.push(o2); break; },
			    	    }
			    	}
		    	};
		    	op_stack.push(Token::Op(o1.clone()));
			},
			&Token::Open => op_stack.push(Token::Open),
			&Token::Close => {
				let mut found_open = false;
				loop {
					let stack_token = try!{op_stack.pop().ok_or("Malformed Expression")};
					match stack_token {
						Token::Open => {op_stack.push(stack_token); found_open=true; break;},
						_ => out_queue.push(stack_token),
					}
				};
				if !found_open {return Err(String::from("Mismatch Parenthesis"))}
				let stack_token = try!{op_stack.pop().ok_or("Malformed Expression")};
				let next_stack_token = try!{op_stack.pop().ok_or("Malformed Expression")};
				match next_stack_token {
					Token::Func(ref x) => out_queue.push(Token::Func(x.clone())),
					_ => op_stack.push(next_stack_token),
				}
			},
			&Token::Unknown(ref x) => {
				let mut message: String = "You either misspelled a function, or it is not yet implemented. The unknown string was: ".to_owned();
				message.push_str(x);
				return Err(message);
			},
		    _ => break,
		}
		//println!("AFTER current_token: {:?}, OPSTACK {:?}, out_queue {:?}\n\n\n", &current_token, &op_stack, &out_queue);
	}
	//println!("OPSTACK {:?}", &op_stack);
	while op_stack.len() > 0 {
	    out_queue.push(try!{op_stack.pop().ok_or("Malformed Expression")});
	}
	Ok(Expression::new(out_queue))
}

fn map_string_to_func(input: &String) -> Token {
	match &(input.to_lowercase())[..] {
		"abs" => Token::Func(Function::Abs),
	    "sqrt" => Token::Func(Function::Sqrt),
	    "ln" => Token::Func(Function::Ln),
	    "log" => Token::Func(Function::Log),
	    "sin" => Token::Func(Function::Sin),
	    "csc" => Token::Func(Function::Csc),
	    "cos" => Token::Func(Function::Cos),
	    "sec" => Token::Func(Function::Sec),
	    "tan" => Token::Func(Function::Tan),
	    "cot" => Token::Func(Function::Cot),
	    "asin" => Token::Func(Function::Asin),
	    "acsc" => Token::Func(Function::Acsc),
	    "acos" => Token::Func(Function::Acos),
	    "asec" => Token::Func(Function::Asec),
	    "atan" => Token::Func(Function::Atan),
	    "acot" => Token::Func(Function::Acot),
	    "sinh" => Token::Func(Function::Sinh),
	    "csch" => Token::Func(Function::Csch),
	    "cosh" => Token::Func(Function::Cosh),
	    "sech" => Token::Func(Function::Sech),
	    "tanh" => Token::Func(Function::Tanh),
	    "coth" => Token::Func(Function::Coth),
	    "asinh" => Token::Func(Function::Asinh),
	    "acsch" => Token::Func(Function::Acsch),
	    "acosh" => Token::Func(Function::Acosh),
	    "asech" => Token::Func(Function::Asech),
	    "atanh" => Token::Func(Function::Atanh),
	    "acoth" => Token::Func(Function::Acoth),
	    "max" => Token::Func(Function::Max),
	    _ => Token::Unknown(input.clone()),
	}
}<commit_msg>Removed unused code<commit_after>extern crate regex;
use regex::Regex;

use std::io;
use std::io::Write;

#[derive(Debug, PartialEq, Clone)]
pub enum Token {
    Unknown(String), // Invalid test (basically non-ascii)
    Literal(String), // Numeric literal number
    Func(Function), // Pre-defined function (like cos() )
    Const(Constant), // Constant like pi or e
    Var(String), // str arbitrary single char variable name
    Op(Operator), // Any of the 4 operators (+-*/)
    Open, // Open parens '('
    Close, // Closing parens ')'
    Comma,
}

#[derive(Debug, PartialEq, Clone)]
pub enum Function {
	Abs,
    Sqrt,
    Ln,
    Log,
    Sin,
    Csc,
    Cos,
    Sec,
    Tan,
    Cot,
    Asin,
    Acsc,
    Acos,
    Asec,
    Atan,
    Acot,
    Sinh,
    Csch,
    Cosh,
    Sech,
    Tanh,
    Coth,
    Asinh,
    Acsch,
    Acosh,
    Asech,
    Atanh,
    Acoth,
    Max,
}

#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum Constant {
	Pi,
	E,
}

#[derive(Debug, PartialEq, Eq, Clone, Copy)]
#[repr(u32)]
pub enum Operator {
    Add=0,
    Sub=1,
    Div=2,
    Mul=3,
    Pow=4, // Right
}

#[derive(Debug, PartialEq, Clone)]
struct Expression {
    tokens: Vec<Token>,
}

struct ExpressionIter<'a> {
	expr: &'a Expression,
	count: usize,
}

impl<'a> Iterator for ExpressionIter<'a> {
	type Item = &'a Token;
    fn next(&mut self) -> Option<&'a Token> {
    	let token = self.expr.get_tokens().get(self.count);
    	self.count += 1;
    	token
    }
}

#[allow(dead_code)]
impl Expression {
	fn new(tokens: Vec<Token>) -> Self {
		Expression {tokens: tokens}
	}

	fn push(&mut self, token: Token) {
		self.tokens.push(token);
	}

	fn get_tokens(&self) -> &[Token] {
		self.tokens.as_slice()
	}

	fn get_mut_tokens(&mut self) -> &mut [Token] {
		self.tokens.as_mut_slice()
	}

	fn get_token(&self, index: usize) -> &Token {
		self.tokens.get(index).unwrap()
	}

	fn get_mut_token(&mut self, index: usize) -> &mut Token {
		self.tokens.get_mut(index).unwrap()
	}

	fn replace_token(&mut self, index: usize, new_token: Token) {
		self.tokens.remove(index);
		self.tokens.insert(index, new_token);
	}

	fn replace_all_tokens(&mut self, new_token: Vec<Token>) {
		self.tokens = new_token
	}

	fn find_first(&self, token: &Token) -> Option<usize> {
		for i in 0..self.tokens.len() {
			if self.tokens.get(i).unwrap() == token {
				return Some(i);
			}
		}
		None
	}

	fn find_last(&self, token: &Token) -> Option<usize> {
		for i in (0..self.tokens.len()).rev() {
			if self.tokens.get(i).unwrap() == token {
				return Some(i);
			}
		}
		None
	}

	fn split_at(&self, index: usize) -> (&[Token], &[Token]) {
		self.tokens.split_at(index)
	}

	fn len(&self) -> usize {
		self.tokens.len()
	}

	fn iter(&self) -> ExpressionIter {
		ExpressionIter {expr: &self, count: 0}
	}
}

fn main() {
	let numeric_regex: Regex = Regex::new(r"\d+\.\d+|\d+").unwrap();
	let function_regex: Regex = Regex::new(r"\w{2,}").unwrap();
    println!("Welcome to Rust-Calculus!");
    println!("To evaluate an expression, simply type one in and hit RETURN.");
    println!("To set a variable, simply type VAR_NAME=EXPRESSION and hit RETURN.");
    println!("Valid commands are: sym_int, int, sym_def, and def.");
    println!("Type 'quit' to exit.");
    let mut input = String::new();
    let stdin = io::stdin();
    let mut stdout = io::stdout();
    loop {
    	input.clear();
    	print!(">>>> ");
    	stdout.flush().ok();
        if let Err(x) = stdin.read_line(&mut input) {
        	println!("There was a problem reading stdin: {:?}", x);
        	print!("Exiting...");;
        	break;
        }
        input = strip_white_space(&input).to_lowercase();
        if input == "quit" {
        	print!("Exiting...");
        	break;
        }
        let expr = parse_input(&input, &numeric_regex, &function_regex);
        if expr.is_ok() {
        	println!("{:?}", &expr.unwrap());
        } else {
        	println!("Encountered an error while parsing: {:?}", expr.unwrap_err());
        	println!("Try Again...(type 'quit' to exit)");
        	continue;
        }
    }
}

fn strip_white_space(input: &String) -> String {
	input.split_whitespace().collect::<Vec<&str>>().join("")
}

fn parse_input(input: &String, numeric_regex: &Regex, function_regex: &Regex) -> Result<Expression, String> {
	// 1. Replace everthing except letters/numbers with their enums
	// 2. Then go through and replace things with Literals or functions
	let mut expr: Expression = Expression::new(Vec::new());
	let mut builder: String = String::new();
	for c in input.chars() {
		match c {
		    '(' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Open);
		    },
		    ')' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Close);
		    },
		    '+' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Op(Operator::Add));
		    },
		    '-' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Op(Operator::Sub));
		    },
		    '*' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Op(Operator::Mul));
		    },
		    '/' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Op(Operator::Div));
		    },
		    '^' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Op(Operator::Pow));
		    },
		    ',' => {
		    	if builder.len() > 0 {
		    		if numeric_regex.is_match(&builder[..]) {
		    			expr.push(Token::Literal(builder.clone()));
		    		} else if function_regex.is_match(&builder[..]) {
		    			expr.push(map_string_to_func(&builder));
		    		}
		    		builder = String::new();
		    	}
		    	expr.push(Token::Comma);
		    },
		    _ => {
		    	builder.push(c);
		    }
		}
	}
	if builder.len() > 0 {
		if numeric_regex.is_match(&builder[..]) {
			expr.push(Token::Literal(builder.clone()));
		} else if function_regex.is_match(&builder[..]) {
			expr.push(map_string_to_func(&builder));
		}
		builder = String::new();
	}
	let mut op_stack: Vec<Token> = Vec::with_capacity(input.len());
	let mut out_queue: Vec<Token> = Vec::with_capacity(input.len());
	//println!("EXPR {:?}", &expr);
	for i in 0..expr.len() {
		let current_token = expr.get_token(i);
		//println!("BEFORE current_token: {:?}, OPSTACK {:?}, out_queue {:?}", &current_token, &op_stack, &out_queue);
		match current_token {
		    &Token::Literal(ref x) => out_queue.push(Token::Literal(x.clone())),
		    &Token::Func(ref x) => op_stack.push(Token::Func(x.clone())),
		    &Token::Comma => {
		    	let mut found_open = false;
		    	loop {
			    	let stack_token = try!{op_stack.pop().ok_or("Malformed Expression")};
			    	match stack_token {
			    	    Token::Open => {op_stack.push(stack_token); found_open=true; break;},
			    	    _ => out_queue.push(stack_token),
			    	}
		    	}
		    	if !found_open {return Err(String::from("Mismatch Parenthesis"))}
			},
		    &Token::Op(ref o1) =>  {
		    	loop {
			    	if op_stack.len() < 1 {
						break;
					}
			    	let o2 = try!{op_stack.pop().ok_or("Malformed Expression")}; // top of stack
			    	match o1 {
			    	    &Operator::Pow => match o2 {
			    	    	_ => { op_stack.push(o2); break; },
			    	    },
			    	    &Operator::Mul | &Operator::Div => match o2 {
			    	    	Token::Op(Operator::Pow) | Token::Op(Operator::Mul) | Token::Op(Operator::Div) => out_queue.push(o2),
			    	    	_ => { op_stack.push(o2); break; },
			    	    },
			    	    &Operator::Add | &Operator::Sub => match o2 {
			    	    	Token::Op(Operator::Add) | Token::Op(Operator::Sub) => out_queue.push(o2),
			    	    	_ => { op_stack.push(o2); break; },
			    	    }
			    	}
		    	};
		    	op_stack.push(Token::Op(o1.clone()));
			},
			&Token::Open => op_stack.push(Token::Open),
			&Token::Close => {
				let mut found_open = false;
				loop {
					let stack_token = try!{op_stack.pop().ok_or("Malformed Expression")};
					match stack_token {
						Token::Open => {op_stack.push(stack_token); found_open=true; break;},
						_ => out_queue.push(stack_token),
					}
				};
				if !found_open {return Err(String::from("Mismatch Parenthesis"))}
				let stack_token = try!{op_stack.pop().ok_or("Malformed Expression")};
				let next_stack_token = try!{op_stack.pop().ok_or("Malformed Expression")};
				match next_stack_token {
					Token::Func(ref x) => out_queue.push(Token::Func(x.clone())),
					_ => op_stack.push(next_stack_token),
				}
			},
			&Token::Unknown(ref x) => {
				let mut message: String = "You either misspelled a function, or it is not yet implemented. The unknown string was: ".to_owned();
				message.push_str(x);
				return Err(message);
			},
		    _ => break,
		}
		//println!("AFTER current_token: {:?}, OPSTACK {:?}, out_queue {:?}\n\n\n", &current_token, &op_stack, &out_queue);
	}
	//println!("OPSTACK {:?}", &op_stack);
	while op_stack.len() > 0 {
	    out_queue.push(try!{op_stack.pop().ok_or("Malformed Expression")});
	}
	Ok(Expression::new(out_queue))
}

fn map_string_to_func(input: &String) -> Token {
	match &(input.to_lowercase())[..] {
		"abs" => Token::Func(Function::Abs),
	    "sqrt" => Token::Func(Function::Sqrt),
	    "ln" => Token::Func(Function::Ln),
	    "log" => Token::Func(Function::Log),
	    "sin" => Token::Func(Function::Sin),
	    "csc" => Token::Func(Function::Csc),
	    "cos" => Token::Func(Function::Cos),
	    "sec" => Token::Func(Function::Sec),
	    "tan" => Token::Func(Function::Tan),
	    "cot" => Token::Func(Function::Cot),
	    "asin" => Token::Func(Function::Asin),
	    "acsc" => Token::Func(Function::Acsc),
	    "acos" => Token::Func(Function::Acos),
	    "asec" => Token::Func(Function::Asec),
	    "atan" => Token::Func(Function::Atan),
	    "acot" => Token::Func(Function::Acot),
	    "sinh" => Token::Func(Function::Sinh),
	    "csch" => Token::Func(Function::Csch),
	    "cosh" => Token::Func(Function::Cosh),
	    "sech" => Token::Func(Function::Sech),
	    "tanh" => Token::Func(Function::Tanh),
	    "coth" => Token::Func(Function::Coth),
	    "asinh" => Token::Func(Function::Asinh),
	    "acsch" => Token::Func(Function::Acsch),
	    "acosh" => Token::Func(Function::Acosh),
	    "asech" => Token::Func(Function::Asech),
	    "atanh" => Token::Func(Function::Atanh),
	    "acoth" => Token::Func(Function::Acoth),
	    "max" => Token::Func(Function::Max),
	    _ => Token::Unknown(input.clone()),
	}
}<|endoftext|>


<commit_before>extern crate sdl2;
extern crate time;
extern crate cgmath;

mod mesh;
mod scene;
mod tests;
mod device;
mod memory;
mod shader;
mod material;
mod import3ds;
mod generator;
mod rasterization;

use cgmath::*;
use scene::Scene;
use shader::Shader;
use material::Material;
use generator::{generate_plane, generate_sphere};
use import3ds::Loader3ds;

pub fn main() {
    let eye;
    let center;
    let model;
    let ind = 0;
    let mut material = Material::gold();
    let mut init_matrix = Matrix4::<f32>::one();
    let mut add_angle = rad(2.0_f32 * std::f32::consts::PI / 180.0_f32);
    let light = Vector3::new(1.0_f32, -1.0_f32, 1.0_f32);
    match ind {
        0 => {
            eye = Point3::new(0.0_f32, 0.7_f32, -1.1_f32);
	        center = Point3::new(0.0_f32, 0.0_f32, 0.0_f32);
            material = Material::silver();
            add_angle = rad(0.0_f32);
            model = Loader3ds::load("ring.3ds").unwrap();
        },
        1 => {
            eye = Point3::new(0.0_f32, 0.0_f32, -10.1_f32);
	        center = Point3::new(0.0_f32, 0.0_f32, 0.0_f32);
            model = Loader3ds::load("tux.3ds").unwrap();
        },
        2 => {
            eye = Point3::new(0.0_f32, -2.0_f32, -4.1_f32);
	        center = Point3::new(0.0_f32, -2.0_f32, 0.0_f32);
            material = Material::monster_skin();
            init_matrix = Matrix4::from(Matrix3::from_angle_x(rad(-1.8_f32)));
            model = Loader3ds::load("monster.3ds").unwrap();
        },
        3 => {
            eye = Point3::new(0.0_f32, 0.0_f32, 0.5_f32);
	        center = Point3::new(0.0_f32, 0.0_f32, 0.0_f32);
            add_angle = rad(0.0_f32);
            model = generate_plane().unwrap();
        },
        4 => {
            eye = Point3::new(0.0_f32, 0.0_f32, -2.0_f32);
	        center = Point3::new(0.0_f32, 0.0_f32, 0.0_f32);
            model = generate_sphere(50).unwrap();
        },

        _ => return
    };
	let up = Vector3::new(0.0_f32, 1.0_f32, 0.0_f32);

    let mut angle = rad(0.0_f32);

    let mut scene = Scene::new(800, 600);
    scene.proj(deg(100.0_f32), 0.1_f32, 100.0_f32)
        .view(eye, center, up)
        .light(light);

    let mut shader = Shader::new(&material);
    shader.set_shaders(Shader::vertex_cook_torrance, Shader::pixel_cook_torrance);
    // shader.set_shaders(Shader::vertex_phong_blinn, Shader::pixel_phong_blinn);
    // shader.set_shaders(Shader::vertex_normals, Shader::pixel_normals);
    while scene.start(0xAAAAAA) {
        angle = angle + add_angle;
        scene.draw(&model, Matrix4::from(Matrix3::from_angle_y(angle)).mul_m(&init_matrix), &mut shader).present();
    }
}
<commit_msg>add auto rotate<commit_after>extern crate sdl2;
extern crate time;
extern crate cgmath;

mod mesh;
mod scene;
mod tests;
mod device;
mod memory;
mod shader;
mod material;
mod import3ds;
mod generator;
mod rasterization;

use cgmath::*;
use scene::Scene;
use shader::Shader;
use material::Material;
use generator::{generate_plane, generate_sphere};
use import3ds::Loader3ds;

pub fn main() {
    let eye;
    let center;
    let model;
    let ind = 0;
    let mut material = Material::gold();
    let mut init_matrix = Matrix4::<f32>::one();
    let add_angle = rad(2.0_f32 * std::f32::consts::PI / 180.0_f32);
    let light = Vector3::new(1.0_f32, -1.0_f32, 1.0_f32);
    match ind {
        0 => {
            eye = Point3::new(0.0_f32, 0.7_f32, -1.1_f32);
	          center = Point3::new(0.0_f32, 0.0_f32, 0.0_f32);
            material = Material::silver();
            model = Loader3ds::load("ring.3ds").unwrap();
        },
        1 => {
            eye = Point3::new(0.0_f32, 0.0_f32, -10.1_f32);
	          center = Point3::new(0.0_f32, 0.0_f32, 0.0_f32);
            model = Loader3ds::load("tux.3ds").unwrap();
        },
        2 => {
            eye = Point3::new(0.0_f32, -2.0_f32, -4.1_f32);
	          center = Point3::new(0.0_f32, -2.0_f32, 0.0_f32);
            material = Material::monster_skin();
            init_matrix = Matrix4::from(Matrix3::from_angle_x(rad(-1.8_f32)));
            model = Loader3ds::load("monster.3ds").unwrap();
        },
        3 => {
            eye = Point3::new(0.0_f32, 0.0_f32, 0.5_f32);
	          center = Point3::new(0.0_f32, 0.0_f32, 0.0_f32);
            model = generate_plane().unwrap();
        },
        4 => {
            eye = Point3::new(0.0_f32, 0.0_f32, -2.0_f32);
	          center = Point3::new(0.0_f32, 0.0_f32, 0.0_f32);
            model = generate_sphere(50).unwrap();
        },

        _ => return
    };
	  let up = Vector3::new(0.0_f32, 1.0_f32, 0.0_f32);

    let mut angle = rad(0.0_f32);

    let mut scene = Scene::new(800, 600);
    scene.proj(deg(100.0_f32), 0.1_f32, 100.0_f32)
        .view(eye, center, up)
        .light(light);

    let mut shader = Shader::new(&material);
    shader.set_shaders(Shader::vertex_cook_torrance, Shader::pixel_cook_torrance);
    // shader.set_shaders(Shader::vertex_phong_blinn, Shader::pixel_phong_blinn);
    // shader.set_shaders(Shader::vertex_normals, Shader::pixel_normals);
    while scene.start(0xAAAAAA) {
        angle = angle + add_angle;
        scene.draw(&model, Matrix4::from(Matrix3::from_angle_y(angle)).mul_m(&init_matrix), &mut shader).present();
    }
}
<|endoftext|>


<commit_before>use std::fs;
use std::env;
use std::fs::File;
use std::io::Read;
use std::hash::Hasher;
use std::path::PathBuf;
use std::collections::HashMap;
use std::os::unix::fs::MetadataExt;
use std::collections::hash_map::DefaultHasher;

#[macro_use]
extern crate log;
extern crate env_logger;

struct FileEntry {
    inode: u64,
    dev:  u64,
    path : String,
}

fn collect_files(dir: &String, h: &mut HashMap<u64, Vec<FileEntry>>) {

    let entries = match fs::read_dir(dir) {
        Ok(x) => x,
        _ => {
            warn!("{}: invalid directory", dir);
            return
        },
    };

    for path in  entries.filter( |x| x.is_ok() ).map( |x| x.unwrap().path() )
                .collect::<Vec<PathBuf>>() {
        if let (Some(path_str), Ok(metadata)) =
                (path.to_str(), path.symlink_metadata()) {
            let ft = metadata.file_type();
            if ft.is_symlink() {
                continue;
            } else if ft.is_file() {
                match metadata.len() {
                    0 => continue,
                    i => h.entry(i).or_insert_with(Vec::new)
                          .push(
                           FileEntry{
                               inode : metadata.ino(),
                               dev   : metadata.dev(),
                               path  : String::from(path_str),
                           }),
                };
            } else if ft.is_dir() {
                collect_files(&(String::from(path_str)), h);
            }
        }
    }
}

fn find_duplicates(duplicates: &mut HashMap<u64, Vec<FileEntry>>,
                   vec: &Vec<FileEntry>, thorough: bool ) {
    for file_entry in vec {
        if let Ok(mut file) = File::open(&file_entry.path) {
            let mut hash: u64 = 0;
            let mut hasher = DefaultHasher::new();
            if thorough {
                let mut contents: Vec<u8> = Vec::new();
                if let Ok(_) = file.read_to_end(&mut contents) {
                    hasher.write(&contents[..]);
                    hash = hasher.finish();
                }
            } else {
                let mut buf: [u8; 1024] = [0; 1024];
                if let Ok(nbytes) = file.read(&mut buf) {
                    hasher.write(&buf[..nbytes as usize]);
                    hash = hasher.finish();
                }
            }
            if hash != 0 {
                duplicates.entry(hash).or_insert_with(Vec::new)
                    .push(
                        FileEntry{
                            inode : file_entry.inode,
                            dev   : file_entry.dev,
                            path  : file_entry.path.to_string(),
                        });
            }
        }
    }
}

fn main() {
    env_logger::init().unwrap();

    let args: Vec<String> = env::args().collect();

    // Print out usage if no directories are given.
    if args.len() == 1 || ( args.len() == 2 && args[1] == "--thorough" ) {
        println!("Usage: {} [--thorough] <dir1> [dir2 [dir3 ...]]", args[0]);
        ::std::process::exit(0);
    }

    let mut hmap: HashMap<u64, Vec<FileEntry> > = HashMap::new();

    let mut thorough = false;
    for dir in &args[1..] {
        if dir == "--thorough" {
            thorough = true;
            continue;
        }
        collect_files(dir, &mut hmap);
    }

    // Get rid of all the single entries.
    hmap.retain(|_, v| v.len() >= 2);

    let mut cluster = 1;

    for (key, val) in hmap.iter() {
        let mut duplicates: HashMap<u64, Vec<FileEntry> > = HashMap::new();
        find_duplicates(&mut duplicates, &val, thorough);

        for (hash, vec) in duplicates {
            if vec.len() >= 2 {
                println!("{} files in cluster {} (size: {}, digest: {})",
                         vec.len(), cluster, key, hash);
                // for_each becomes stable v1.22.0 onwards. Should uncomment then.
                //vec.iter().for_each(|f| println!("{}", f.path));
                for file_entry in vec {
                    println!("{}", file_entry.path);
                }
                cluster += 1;
            }
        }
    }
}
<commit_msg>Add some log messages<commit_after>use std::fs;
use std::env;
use std::fs::File;
use std::io::Read;
use std::hash::Hasher;
use std::path::PathBuf;
use std::collections::HashMap;
use std::os::unix::fs::MetadataExt;
use std::collections::hash_map::DefaultHasher;

#[macro_use]
extern crate log;
extern crate env_logger;

struct FileEntry {
    inode: u64,
    dev:  u64,
    path : String,
}

fn collect_files(dir: &String, h: &mut HashMap<u64, Vec<FileEntry>>) {

    let entries = match fs::read_dir(dir) {
        Ok(x) => x,
        _ => {
            warn!("{}: invalid directory", dir);
            return
        },
    };

    for path in  entries.filter( |x| x.is_ok() ).map( |x| x.unwrap().path() )
                .collect::<Vec<PathBuf>>() {
        if let (Some(path_str), Ok(metadata)) =
                (path.to_str(), path.symlink_metadata()) {
            let ft = metadata.file_type();
            if ft.is_symlink() {
                continue;
            } else if ft.is_file() {
                match metadata.len() {
                    0 => continue,
                    i => h.entry(i).or_insert_with(Vec::new)
                          .push(
                           FileEntry{
                               inode : metadata.ino(),
                               dev   : metadata.dev(),
                               path  : String::from(path_str),
                           }),
                };
            } else if ft.is_dir() {
                collect_files(&(String::from(path_str)), h);
            }
        } else {
            warn!("Failed to retrieve metadata for {}", path.to_str().unwrap());
        }
    }
}

fn find_duplicates(duplicates: &mut HashMap<u64, Vec<FileEntry>>,
                   vec: &Vec<FileEntry>, thorough: bool ) {
    for file_entry in vec {
        if let Ok(mut file) = File::open(&file_entry.path) {
            let mut hash: u64 = 0;
            let mut hasher = DefaultHasher::new();
            if thorough {
                let mut contents: Vec<u8> = Vec::new();
                if let Ok(_) = file.read_to_end(&mut contents) {
                    hasher.write(&contents[..]);
                    hash = hasher.finish();
                } else {
                    warn!("Failed to read {}", file_entry.path);
                }
            } else {
                let mut buf: [u8; 1024] = [0; 1024];
                if let Ok(nbytes) = file.read(&mut buf) {
                    hasher.write(&buf[..nbytes as usize]);
                    hash = hasher.finish();
                } else {
                    warn!("Failed to read {} chunk", file_entry.path);
                }
            }
            if hash != 0 {
                duplicates.entry(hash).or_insert_with(Vec::new)
                    .push(
                        FileEntry{
                            inode : file_entry.inode,
                            dev   : file_entry.dev,
                            path  : file_entry.path.to_string(),
                        });
            }
        } else {
            warn!("Failed to open file {}", file_entry.path);
        }
    }
}

fn main() {
    env_logger::init().unwrap();

    let args: Vec<String> = env::args().collect();

    // Print out usage if no directories are given.
    if args.len() == 1 || ( args.len() == 2 && args[1] == "--thorough" ) {
        println!("Usage: {} [--thorough] <dir1> [dir2 [dir3 ...]]", args[0]);
        ::std::process::exit(0);
    }

    let mut hmap: HashMap<u64, Vec<FileEntry> > = HashMap::new();

    let mut thorough = false;
    for dir in &args[1..] {
        if dir == "--thorough" {
            thorough = true;
            continue;
        }
        collect_files(dir, &mut hmap);
    }

    // Get rid of all the single entries.
    hmap.retain(|_, v| v.len() >= 2);

    let mut cluster = 1;

    for (key, val) in hmap.iter() {
        let mut duplicates: HashMap<u64, Vec<FileEntry> > = HashMap::new();
        find_duplicates(&mut duplicates, &val, thorough);

        for (hash, vec) in duplicates {
            if vec.len() >= 2 {
                println!("{} files in cluster {} (size: {}, digest: {})",
                         vec.len(), cluster, key, hash);
                // for_each becomes stable v1.22.0 onwards. Should uncomment then.
                //vec.iter().for_each(|f| println!("{}", f.path));
                for file_entry in vec {
                    println!("{}", file_entry.path);
                }
                cluster += 1;
            }
        }
    }
}
<|endoftext|>