<commit_before>#![warn(rust_2018_idioms)]
#![cfg(feature = "full")]

use tokio::fs::File;
use tokio::prelude::*;

use std::io::prelude::*;
use tempfile::NamedTempFile;

const HELLO: &[u8] = b"hello world...";

#[tokio::test]
async fn basic_read() {
    let mut tempfile = tempfile();
    tempfile.write_all(HELLO).unwrap();

    let mut file = File::open(tempfile.path()).await.unwrap();

    let mut buf = [0; 1024];
    let n = file.read(&mut buf).await.unwrap();

    assert_eq!(n, HELLO.len());
    assert_eq!(&buf[..n], HELLO);
}

#[tokio::test]
async fn basic_write() {
    let tempfile = tempfile();

    let mut file = File::create(tempfile.path()).await.unwrap();

    file.write_all(HELLO).await.unwrap();
    file.flush().await.unwrap();

    let file = std::fs::read(tempfile.path()).unwrap();
    assert_eq!(file, HELLO);
}

fn tempfile() -> NamedTempFile {
    NamedTempFile::new().unwrap()
}

#[tokio::test]
#[cfg(unix)]
async fn unix_fd() {
    use std::os::unix::io::AsRawFd;
    let tempfile = tempfile();

    let file = File::create(tempfile.path()).await.unwrap();
    assert!(file.as_raw_fd() as u64 > 0);
}

#[tokio::test]
#[cfg(windows)]
async fn windows_handle() {
    use std::os::windows::io::AsRawHandle;
    let tempfile = tempfile();

    let file = File::create(tempfile.path()).await.unwrap();
    assert!(file.as_raw_handle() as u64 > 0);
}
<commit_msg>fs: add coop test (#2344)<commit_after>#![warn(rust_2018_idioms)]
#![cfg(feature = "full")]

use tokio::fs::File;
use tokio::prelude::*;
use tokio_test::task;

use std::io::prelude::*;
use tempfile::NamedTempFile;

const HELLO: &[u8] = b"hello world...";

#[tokio::test]
async fn basic_read() {
    let mut tempfile = tempfile();
    tempfile.write_all(HELLO).unwrap();

    let mut file = File::open(tempfile.path()).await.unwrap();

    let mut buf = [0; 1024];
    let n = file.read(&mut buf).await.unwrap();

    assert_eq!(n, HELLO.len());
    assert_eq!(&buf[..n], HELLO);
}

#[tokio::test]
async fn basic_write() {
    let tempfile = tempfile();

    let mut file = File::create(tempfile.path()).await.unwrap();

    file.write_all(HELLO).await.unwrap();
    file.flush().await.unwrap();

    let file = std::fs::read(tempfile.path()).unwrap();
    assert_eq!(file, HELLO);
}

#[tokio::test]
async fn coop() {
    let mut tempfile = tempfile();
    tempfile.write_all(HELLO).unwrap();

    let mut task = task::spawn(async {
        let mut file = File::open(tempfile.path()).await.unwrap();

        let mut buf = [0; 1024];

        loop {
            file.read(&mut buf).await.unwrap();
            file.seek(std::io::SeekFrom::Start(0)).await.unwrap();
        }
    });

    for _ in 0..1_000 {
        if task.poll().is_pending() {
            return;
        }
    }

    panic!("did not yield");
}

fn tempfile() -> NamedTempFile {
    NamedTempFile::new().unwrap()
}

#[tokio::test]
#[cfg(unix)]
async fn unix_fd() {
    use std::os::unix::io::AsRawFd;
    let tempfile = tempfile();

    let file = File::create(tempfile.path()).await.unwrap();
    assert!(file.as_raw_fd() as u64 > 0);
}

#[tokio::test]
#[cfg(windows)]
async fn windows_handle() {
    use std::os::windows::io::AsRawHandle;
    let tempfile = tempfile();

    let file = File::create(tempfile.path()).await.unwrap();
    assert!(file.as_raw_handle() as u64 > 0);
}
<|endoftext|>


<commit_before>//! Utility methods and constructors for Lua objects

use std::fmt::Display;
use std::convert::From;
use rlua::{self, Lua, Table, UserData, AnyUserData, Value,
           FromLua, ToLua, Function};
use super::signal;
use super::class::Class;
use super::property::Property;

/// All Lua objects can be cast to this.
#[derive(Clone, Debug)]
pub struct Object<'lua> {
    pub object: AnyUserData<'lua>
}

impl <'lua> From<AnyUserData<'lua>> for Object<'lua> {
    fn from(object: AnyUserData<'lua>) -> Self {
        Object { object }
    }
}

/// Construct a new object, used when using the default Objectable::new.
pub struct ObjectBuilder<'lua>{
    lua: &'lua Lua,
    object: Object<'lua>
}

impl <'lua> ObjectBuilder<'lua> {
    pub fn add_to_meta(self, new_meta: Table<'lua>) -> rlua::Result<Self> {
        let meta = self.object.table()?.get_metatable()
            .expect("Object had no meta table");
        for entry in new_meta.pairs::<rlua::Value, rlua::Value>() {
            let (key, value) = entry?;
            meta.set(key, value)?;
        }
        self.object.table()?.set_metatable(Some(meta));
        Ok(self)
    }

    #[allow(dead_code)]
    pub fn add_to_signals(self, name: String, func: rlua::Function)
                          -> rlua::Result<Self> {
        signal::connect_signal(self.lua, self.object.clone(), name, &[func])?;
        Ok(self)
    }

    pub fn handle_constructor_argument(self, args: Table) -> rlua::Result<Self> {
        let meta = self.object.table()?.get_metatable()
            .expect("Object had no meta table");
        let class = meta.get::<_, Table>("__class")?;
        let props = class.get::<_, Vec<Property>>("properties")?;

        // Handle all table entries that correspond to known properties,
        // silently ignore all other keys
        for pair in args.pairs() {
            let (key, value): (Value, Value) = pair?;
            if let rlua::Value::String(key) = key {
                if let Ok(key) = key.to_str() {
                    // Find the property
                    for prop in props.iter() {
                        if prop.name == key {
                            // Property exists and has a cb_new callback
                            if let Some(ref new) = prop.cb_new {
                                let _: () = new.bind(self.object.clone())?.call(value)?;
                                break;
                            }
                        }
                    }
                }
            }
        }
        Ok(self)
    }

    pub fn build(self) -> Object<'lua> {
        self.object
    }
}

/// Trait implemented by all objects that represent OO lua objects.
///
/// This trait allows casting an object gotten back from the Lua runtime
/// into a concrete object so that Rust can do things with it.
///
/// You can't do anything to the object until it has been converted into a
/// canonical form using this trait.
pub trait Objectable<'lua, T, S: UserData + Default + Display + Clone> {
    fn cast(obj: Object<'lua>) -> rlua::Result<T> {
        if obj.object.is::<S>()? {
            Ok(Self::_wrap(obj))
        } else {
            use rlua::Error::RuntimeError;
            Err(RuntimeError("Could not cast object to concrete type".into()))
        }
    }

    /// Given the internal object table, constructs a concrete object out
    /// of it.
    ///
    /// This is only used internally, which is why it's prefixed with a "_"
    /// Please do not use it outside of object.rs.
    fn _wrap(table: Object<'lua>) -> T;

    fn state(&self) -> rlua::Result<S> {
        Ok(self.get_object()?.clone())
    }

    fn set_state(&mut self, data: S) -> rlua::Result<()> {
        *self.get_object_mut()? = data;
        Ok(())
    }

    /// Gets the internal state for the concrete object.
    /// Used internally `state`, though there's nothing wrong with it being
    /// used outside of internal object use.
    fn get_object(&self) -> rlua::Result<S>;

    /// Gets a mutable reference to the internal state for the concrete object.
    /// Used internally `set_state`, though there's nothing wrong with it being
    /// used outside of internal object use.
    fn get_object_mut(&mut self) -> rlua::Result<::std::cell::RefMut<S>>;

    /// Lua objects in Way Cooler are just how they are in Awesome:
    /// * We expose the user data directly.
    /// * State for the object is stored using set_user_value in a "wrapper" table.
    /// * The wrapper table has a data field which hosts the data.
    /// * Class methods/attributes are on the meta table of the wrapper table.
    fn allocate(lua: &'lua Lua, class: Class) -> rlua::Result<ObjectBuilder<'lua>>
    {
        let object = lua.create_userdata(S::default())?;
        // TODO Increment the instance count
        let wrapper_table = lua.create_table()?;
        let data_table = lua.create_table()?;
        wrapper_table.set("data", data_table)?;
        let meta = lua.create_table()?;
        meta.set("__class", class)?;
        meta.set("properties", Vec::<Property>::new().to_lua(lua)?)?;
        meta.set("signals", lua.create_table()?)?;
        meta.set("connect_signal",
                 lua.create_function(connect_signal)?)?;
        meta.set("disconnect_signal",
                 lua.create_function(disconnect_signal)?)?;
        meta.set("emit_signal", lua.create_function(emit_signal)?)?;
        meta.set("__index", meta.clone())?;
        meta.set("__tostring", lua.create_function(|_, data: AnyUserData| {
            Ok(format!("{}", data.borrow::<S>()?.clone()))
        })?)?;
        wrapper_table.set_metatable(Some(meta));
        object.set_user_value(wrapper_table)?;
        // TODO Emit new signal event
        let object = Object { object };
        Ok(ObjectBuilder { object, lua })
    }
}

impl <'lua> ToLua<'lua> for Object<'lua> {
    fn to_lua(self, _: &'lua Lua) -> rlua::Result<Value<'lua>> {
        Ok(Value::UserData(self.object))
    }
}

impl <'lua> Object<'lua> {
    pub fn signals(&self) -> rlua::Result<rlua::Table<'lua>> {
        self.table()?.get::<_, Table>("signals")
    }

    pub fn table(&self) -> rlua::Result<Table<'lua>> {
        self.object.get_user_value::<Table<'lua>>()
    }
}

/// Default indexing of an Awesome object.
///
/// Automatically looks up contents in meta table, so instead of overriding this
/// it's easier to just add the required data in the meta table.
pub fn default_index<'lua>(lua: &'lua Lua,
                           (obj, index): (AnyUserData<'lua>, Value<'lua>))
                           -> rlua::Result<Value<'lua>> {
    // Look up in metatable first
    let obj: Object<'lua> = obj.into();
    let obj_table = obj.table()?;
    let meta = obj_table.get_metatable()
        .expect("Object had no metatable");
    warn!("Here");
    if meta.get::<_, Table>("__class").is_ok() {
        if let Ok(val) = meta.raw_get::<_, Value>(index.clone()) {
            match val {
                Value::Nil => {},
                val => return Ok(val)
            }
        }
    }
    let index = String::from_lua(index, lua)?;
    match index.as_str() {
        "valid" => {
            Ok(Value::Boolean(
                if let Ok(class) = meta.get::<_, Table>("__class") {
                    let class: Class = class.into();
                    class.checker()?
                        .map(|checker| checker(obj))
                        .unwrap_or(true)
                } else {
                    false
                }))
        },
        "data" => {
            obj_table.to_lua(lua)
        },
        index => {
            warn!("Here2");
            // Try see if there is a property of the class with the name
            if let Ok(class) = meta.get::<_, Table>("__class") {
                let props = class.get::<_, Vec<Property>>("properties")?;
                for prop in props {
                    if prop.name.as_str() == index {
                        // Property exists and has an index callback
                        if let Some(index) = prop.cb_index {
                            return index.call(obj)
                        }
                    }
                }
                match class.get::<_, Function>("__index_miss_handler") {
                    Ok(function) => {
                        error!("Calling {:?} w/ obj {:?} and arg {:?}",
                               function, obj, index
                        );
                        return function.bind(obj)?.call(index)
                    },
                    Err(_) => {}
                }
            }
            // TODO property miss handler if index doesn't exst
            Ok(rlua::Value::Nil)
        }
    }
}

/// Default new indexing (assignment) of an Awesome object.
///
/// Automatically looks up contents in meta table, so instead of overriding this
/// it's easier to just add the required data in the meta table.
pub fn default_newindex<'lua>(_: &'lua Lua,
                              (obj, index, val):
                              (AnyUserData<'lua>, String, Value<'lua>))
                              -> rlua::Result<Value<'lua>> {
    // Look up in metatable first
    let obj: Object<'lua> = obj.into();
    let obj_table = obj.table()?;
    if let Some(meta) = obj_table.get_metatable() {
        if let Ok(val) = meta.raw_get::<_, Value>(index.clone()) {
            match val {
                Value::Nil => {},
                val => return Ok(val)
            }
        }
        let class = meta.get::<_, Table>("__class")?;
        let props = class.get::<_, Vec<Property>>("properties")?;
        for prop in props {
            if prop.name.as_str() == index {
                // Property exists and has a newindex callback
                if let Some(newindex) = prop.cb_newindex {
                    return newindex.bind(obj.clone())?.call(val)
                }
            }
        }
        match class.get::<_, Function>("__newindex_miss_handler") {
            Ok(function) => {
                return function.bind(obj)?.call((index, val))
            },
            Err(_) => {}
        }
        // TODO property miss handler if index doesn't exst
    }
    Ok(Value::Nil)
}

fn connect_signal(lua: &Lua, (obj, signal, func): (AnyUserData, String, Function))
                  -> rlua::Result<()> {
    signal::connect_signal(lua, obj.into(), signal, &[func])
}

fn disconnect_signal(lua: &Lua, (obj, signal): (AnyUserData, String))
                     -> rlua::Result<()> {
    signal::disconnect_signal(lua, obj.into(), signal)
}

fn emit_signal(lua: &Lua, (obj, signal, args): (AnyUserData, String, Value))
               -> rlua::Result<()> {
    signal::emit_object_signal(lua, obj.into(), signal, args)
}
<commit_msg>Bind object table not object in miss handler<commit_after>//! Utility methods and constructors for Lua objects

use std::fmt::Display;
use std::convert::From;
use rlua::{self, Lua, Table, UserData, AnyUserData, Value,
           FromLua, ToLua, Function};
use super::signal;
use super::class::Class;
use super::property::Property;

/// All Lua objects can be cast to this.
#[derive(Clone, Debug)]
pub struct Object<'lua> {
    pub object: AnyUserData<'lua>
}

impl <'lua> From<AnyUserData<'lua>> for Object<'lua> {
    fn from(object: AnyUserData<'lua>) -> Self {
        Object { object }
    }
}

/// Construct a new object, used when using the default Objectable::new.
pub struct ObjectBuilder<'lua>{
    lua: &'lua Lua,
    object: Object<'lua>
}

impl <'lua> ObjectBuilder<'lua> {
    pub fn add_to_meta(self, new_meta: Table<'lua>) -> rlua::Result<Self> {
        let meta = self.object.table()?.get_metatable()
            .expect("Object had no meta table");
        for entry in new_meta.pairs::<rlua::Value, rlua::Value>() {
            let (key, value) = entry?;
            meta.set(key, value)?;
        }
        self.object.table()?.set_metatable(Some(meta));
        Ok(self)
    }

    #[allow(dead_code)]
    pub fn add_to_signals(self, name: String, func: rlua::Function)
                          -> rlua::Result<Self> {
        signal::connect_signal(self.lua, self.object.clone(), name, &[func])?;
        Ok(self)
    }

    pub fn handle_constructor_argument(self, args: Table) -> rlua::Result<Self> {
        let meta = self.object.table()?.get_metatable()
            .expect("Object had no meta table");
        let class = meta.get::<_, Table>("__class")?;
        let props = class.get::<_, Vec<Property>>("properties")?;

        // Handle all table entries that correspond to known properties,
        // silently ignore all other keys
        for pair in args.pairs() {
            let (key, value): (Value, Value) = pair?;
            if let rlua::Value::String(key) = key {
                if let Ok(key) = key.to_str() {
                    // Find the property
                    for prop in props.iter() {
                        if prop.name == key {
                            // Property exists and has a cb_new callback
                            if let Some(ref new) = prop.cb_new {
                                let _: () = new.bind(self.object.clone())?.call(value)?;
                                break;
                            }
                        }
                    }
                }
            }
        }
        Ok(self)
    }

    pub fn build(self) -> Object<'lua> {
        self.object
    }
}

/// Trait implemented by all objects that represent OO lua objects.
///
/// This trait allows casting an object gotten back from the Lua runtime
/// into a concrete object so that Rust can do things with it.
///
/// You can't do anything to the object until it has been converted into a
/// canonical form using this trait.
pub trait Objectable<'lua, T, S: UserData + Default + Display + Clone> {
    fn cast(obj: Object<'lua>) -> rlua::Result<T> {
        if obj.object.is::<S>()? {
            Ok(Self::_wrap(obj))
        } else {
            use rlua::Error::RuntimeError;
            Err(RuntimeError("Could not cast object to concrete type".into()))
        }
    }

    /// Given the internal object table, constructs a concrete object out
    /// of it.
    ///
    /// This is only used internally, which is why it's prefixed with a "_"
    /// Please do not use it outside of object.rs.
    fn _wrap(table: Object<'lua>) -> T;

    fn state(&self) -> rlua::Result<S> {
        Ok(self.get_object()?.clone())
    }

    fn set_state(&mut self, data: S) -> rlua::Result<()> {
        *self.get_object_mut()? = data;
        Ok(())
    }

    /// Gets the internal state for the concrete object.
    /// Used internally `state`, though there's nothing wrong with it being
    /// used outside of internal object use.
    fn get_object(&self) -> rlua::Result<S>;

    /// Gets a mutable reference to the internal state for the concrete object.
    /// Used internally `set_state`, though there's nothing wrong with it being
    /// used outside of internal object use.
    fn get_object_mut(&mut self) -> rlua::Result<::std::cell::RefMut<S>>;

    /// Lua objects in Way Cooler are just how they are in Awesome:
    /// * We expose the user data directly.
    /// * State for the object is stored using set_user_value in a "wrapper" table.
    /// * The wrapper table has a data field which hosts the data.
    /// * Class methods/attributes are on the meta table of the wrapper table.
    fn allocate(lua: &'lua Lua, class: Class) -> rlua::Result<ObjectBuilder<'lua>>
    {
        let object = lua.create_userdata(S::default())?;
        // TODO Increment the instance count
        let wrapper_table = lua.create_table()?;
        let data_table = lua.create_table()?;
        wrapper_table.set("data", data_table)?;
        let meta = lua.create_table()?;
        meta.set("__class", class)?;
        meta.set("properties", Vec::<Property>::new().to_lua(lua)?)?;
        meta.set("signals", lua.create_table()?)?;
        meta.set("connect_signal",
                 lua.create_function(connect_signal)?)?;
        meta.set("disconnect_signal",
                 lua.create_function(disconnect_signal)?)?;
        meta.set("emit_signal", lua.create_function(emit_signal)?)?;
        meta.set("__index", meta.clone())?;
        meta.set("__tostring", lua.create_function(|_, data: AnyUserData| {
            Ok(format!("{}", data.borrow::<S>()?.clone()))
        })?)?;
        wrapper_table.set_metatable(Some(meta));
        object.set_user_value(wrapper_table)?;
        // TODO Emit new signal event
        let object = Object { object };
        Ok(ObjectBuilder { object, lua })
    }
}

impl <'lua> ToLua<'lua> for Object<'lua> {
    fn to_lua(self, _: &'lua Lua) -> rlua::Result<Value<'lua>> {
        Ok(Value::UserData(self.object))
    }
}

impl <'lua> Object<'lua> {
    pub fn signals(&self) -> rlua::Result<rlua::Table<'lua>> {
        self.table()?.get::<_, Table>("signals")
    }

    pub fn table(&self) -> rlua::Result<Table<'lua>> {
        self.object.get_user_value::<Table<'lua>>()
    }
}

/// Default indexing of an Awesome object.
///
/// Automatically looks up contents in meta table, so instead of overriding this
/// it's easier to just add the required data in the meta table.
pub fn default_index<'lua>(lua: &'lua Lua,
                           (obj, index): (AnyUserData<'lua>, Value<'lua>))
                           -> rlua::Result<Value<'lua>> {
    // Look up in metatable first
    let obj: Object<'lua> = obj.into();
    let obj_table = obj.table()?;
    let meta = obj_table.get_metatable()
        .expect("Object had no metatable");
    if meta.get::<_, Table>("__class").is_ok() {
        if let Ok(val) = meta.raw_get::<_, Value>(index.clone()) {
            match val {
                Value::Nil => {},
                val => return Ok(val)
            }
        }
    }
    let index = String::from_lua(index, lua)?;
    match index.as_str() {
        "valid" => {
            Ok(Value::Boolean(
                if let Ok(class) = meta.get::<_, Table>("__class") {
                    let class: Class = class.into();
                    class.checker()?
                        .map(|checker| checker(obj))
                        .unwrap_or(true)
                } else {
                    false
                }))
        },
        "data" => {
            obj_table.to_lua(lua)
        },
        index => {
            // Try see if there is a property of the class with the name
            if let Ok(class) = meta.get::<_, Table>("__class") {
                let props = class.get::<_, Vec<Property>>("properties")?;
                for prop in props {
                    if prop.name.as_str() == index {
                        // Property exists and has an index callback
                        if let Some(index) = prop.cb_index {
                            return index.call(obj)
                        }
                    }
                }
                match class.get::<_, Function>("__index_miss_handler") {
                    Ok(function) => {
                        return function.bind(obj_table)?.call(index)
                    },
                    Err(_) => {}
                }
            }
            // TODO property miss handler if index doesn't exst
            Ok(rlua::Value::Nil)
        }
    }
}

/// Default new indexing (assignment) of an Awesome object.
///
/// Automatically looks up contents in meta table, so instead of overriding this
/// it's easier to just add the required data in the meta table.
pub fn default_newindex<'lua>(_: &'lua Lua,
                              (obj, index, val):
                              (AnyUserData<'lua>, String, Value<'lua>))
                              -> rlua::Result<Value<'lua>> {
    // Look up in metatable first
    let obj: Object<'lua> = obj.into();
    let obj_table = obj.table()?;
    if let Some(meta) = obj_table.get_metatable() {
        if let Ok(val) = meta.raw_get::<_, Value>(index.clone()) {
            match val {
                Value::Nil => {},
                val => return Ok(val)
            }
        }
        let class = meta.get::<_, Table>("__class")?;
        let props = class.get::<_, Vec<Property>>("properties")?;
        for prop in props {
            if prop.name.as_str() == index {
                // Property exists and has a newindex callback
                if let Some(newindex) = prop.cb_newindex {
                    return newindex.bind(obj.clone())?.call(val)
                }
            }
        }
        match class.get::<_, Function>("__newindex_miss_handler") {
            Ok(function) => {
                return function.bind(obj)?.call((index, val))
            },
            Err(_) => {}
        }
        // TODO property miss handler if index doesn't exst
    }
    Ok(Value::Nil)
}

fn connect_signal(lua: &Lua, (obj, signal, func): (AnyUserData, String, Function))
                  -> rlua::Result<()> {
    signal::connect_signal(lua, obj.into(), signal, &[func])
}

fn disconnect_signal(lua: &Lua, (obj, signal): (AnyUserData, String))
                     -> rlua::Result<()> {
    signal::disconnect_signal(lua, obj.into(), signal)
}

fn emit_signal(lua: &Lua, (obj, signal, args): (AnyUserData, String, Value))
               -> rlua::Result<()> {
    signal::emit_object_signal(lua, obj.into(), signal, args)
}
<|endoftext|>


<commit_before>// This program watches a configured directory for incoming files.
// When a file is detected the following procedure is used to handle it:
//
//  * First we digest the file, and add the digest to our manifest.
//    This way we can intelligently handle future events for this file
//
//  * Next we attempt to detect a mime type using various heuristics, incl:
//
//    - Quick detection w/ our magic bytes table
//    - Querying external tools (e.g: ffmpeg, imagemagick, et al.)
//    - etc.
//
//  * If we cannot handle the mime type: the file is moved to an exception
//    directory, and an error is logged somewhere the user will see it.
//
//  * If we can handle the mime type: the file is moved to the content store
//    and an entry is created in the database.
//
//    - This should ideally be done atomically so that other aqua utility
//      processes (e.g: sister agnes) don't mistakenly mark the file as an
//      orphan / missing / etc.
//

#[macro_use] extern crate log;
#[macro_use] extern crate serde_derive;

extern crate aqua;
extern crate clap;
extern crate diesel;
extern crate dotenv;
extern crate env_logger;
extern crate image;
extern crate notify;
extern crate serde;
extern crate serde_json;

use aqua::controllers::prelude::hash_file;
use aqua::models::{Entry, NewEntry};
use aqua::schema;
use clap::{Arg, App};
use diesel::prelude::*;
use diesel::pg::PgConnection;
use dotenv::dotenv;
use notify::{DebouncedEvent, Watcher, RecursiveMode, watcher};
use std::{env, fmt, process};
use std::collections::HashMap;
use std::convert::From;
use std::error::Error;
use std::fs::{self, File, OpenOptions};
use std::io::{self, Read, Write};
use std::path::{Path, PathBuf};
use std::sync::mpsc::channel;
use std::time::Duration;

#[derive(Debug)]
enum ProcessingError {
    DigestFailed,
    DetectionFailed,
    ThumbnailFailed,

    DbConnErr(diesel::ConnectionError),
    DbQueryErr(diesel::result::Error),
    IoErr(io::Error),
    ImageErr(image::ImageError),
    Misc(Box<Error>),
}

impl Error for ProcessingError {
    fn description(&self) -> &str {
        match *self {
            // internal errors
            ProcessingError::DigestFailed      => "Unhandled error while generating SHA256 digest",
            ProcessingError::DetectionFailed   => "The file's type could not be detected",
            ProcessingError::ThumbnailFailed   => "The thumbnail could not be generated",

            // external errors
            ProcessingError::DbConnErr(ref inner)  => inner.description(),
            ProcessingError::DbQueryErr(ref inner) => inner.description(),
            ProcessingError::IoErr(ref inner)      => inner.description(),
            ProcessingError::ImageErr(ref inner)   => inner.description(),
            ProcessingError::Misc(ref inner)       => inner.description(),
        }
    }

    fn cause(&self) -> Option<&Error> {
        match *self {
            ProcessingError::DbConnErr(ref err)  => Some(err),
            ProcessingError::DbQueryErr(ref err) => Some(err),
            ProcessingError::IoErr(ref err)      => Some(err),
            ProcessingError::ImageErr(ref err)   => Some(err),
            ProcessingError::Misc(ref err)       => Some(err.as_ref()),
            _ => None,
        }
    }
}

impl fmt::Display for ProcessingError {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        match *self {
            // TODO: better display impl.
            _ => write!(f, "{}", self.description()),
        }
    }
}

impl From<diesel::ConnectionError> for ProcessingError {
    fn from(err: diesel::ConnectionError) -> Self { ProcessingError::DbConnErr(err) }
}

impl From<diesel::result::Error> for ProcessingError {
    fn from(err: diesel::result::Error) -> Self { ProcessingError::DbQueryErr(err) }
}

impl From<image::ImageError> for ProcessingError {
    fn from(err: image::ImageError) -> Self { ProcessingError::ImageErr(err) }
}

impl From<io::Error> for ProcessingError {
    fn from(err: io::Error) -> Self { ProcessingError::IoErr(err) }
}

impl From<serde_json::Error> for ProcessingError {
    fn from(err: serde_json::Error) -> Self { ProcessingError::Misc(Box::new(err)) }
}

impl From<std::string::FromUtf8Error> for ProcessingError {
    fn from(err: std::string::FromUtf8Error) -> Self { ProcessingError::Misc(Box::new(err)) }
}

fn main() {
    dotenv().expect("must provide .env file, see README (TODO: haha jk)");
    env_logger::init().expect("could not initialize console logging");

    // read command line arguments
    let matches = App::new("aqua-watch")
        .version("0.1.0")
        .author("himechi <hime@localhost>")
        .about("Watches a directory for new files and moves them to the `aqua` content store.")
        .arg(Arg::with_name("INPUT")
             .help("Determines the input directory to be watched.")
             .required(true)
             .index(1))
        .arg(Arg::with_name("OUTPUT")
             .help("The root of the aqua content store where files will be moved.")
             .required(true)
             .index(2))
        .get_matches();

    let dropbox_path  = matches.value_of("INPUT").unwrap();
    let content_store = matches.value_of("OUTPUT").unwrap();

    // setup fs watcher
    let (fs_tx, fs_rx) = channel();
    let mut fs_watcher = watcher(fs_tx, Duration::from_millis(1000))
        .expect("could not create file system watcher!");

    fs_watcher.watch(dropbox_path, RecursiveMode::NonRecursive)
        .expect("could not enroll dropbox in fs events queue");

    // process filesystem events ...
    loop {
        match fs_rx.recv() {
            Ok(DebouncedEvent::Create(path)) => {
                if path.is_file() {
                    match handle_new_file(path, content_store) {
                        Ok(_res) => info!("file processed successfully ..."),
                        Err(msg) => warn!("could not process file: {:?} (inner: {:?}", msg, msg.cause()),
                    };
                }
                else { info!("directory created, ignoring ..."); }
            },
            Ok(event) => info!("unhandled evt: {:?}", event),
            Err(msg) => warn!("fs err: {}", msg),
        }
    }
}

fn handle_new_file(path: PathBuf, content_store: &str) -> Result<(), ProcessingError> {
    let digest = hash_file(path.as_path())
        .ok_or(ProcessingError::DigestFailed)?;

    let mut file = OpenOptions::new()
        .read(true)
        .write(false)
        .create_new(false)
        .open(path.as_path())?;

    // read file into memory
    let mut buf = vec![];
    file.read_to_end(&mut buf)?;

    if let Some(image_metadata) = aqua::util::mime_detect(&buf) {
        info!("got an image ...");
        process_image(content_store, &digest, &buf)?;
        move_file(path.as_path(), content_store, &digest, image_metadata.extension())?;

        let db_entry = create_db_entry(&digest, image_metadata.mime())?;
        info!("inserted: {:?} into database", db_entry);

        Ok(())
    } else if let Some(ffmpeg_metadata) = ffmpeg_detect(path.as_path())? {
        info!("got an video ...");
        process_video(content_store, &digest, &path)?;
        move_file(path.as_path(), content_store, &digest, ffmpeg_metadata.ext)?;

        let db_entry = create_db_entry(&digest, ffmpeg_metadata.mime)?;
        info!("inserted: {:?} into database", db_entry);

        Ok(())
    } else {
        Err(ProcessingError::DetectionFailed)
    }
}

#[derive(Debug, Serialize, Deserialize)]
struct FFProbeResult {
    format: Option<FFProbeFormat>,
}

#[derive(Debug, Serialize, Deserialize)]
struct FFProbeFormat {
    filename:    String,
    nb_streams:  i32,
    format_name: String,
    start_time:  String, // NOTE: these appear to be fixed decimal
    duration:    String, // NOTE: these appear to be fixed decimal
    size:        String, // NOTE: appears to be an integer
    bit_rate:    String, // NOTE: appears to be an integer
    probe_score: i32,    // NOTE: accuracy of the detection? (???)

    tags: HashMap<String, String>, // NOTE: not sure this is correct type
}

struct FFProbeMeta {
    pub mime: &'static str,
    pub ext:  &'static str,
}

fn ffmpeg_detect(path: &Path) -> Result<Option<FFProbeMeta>, ProcessingError> {
    let ffprobe_cmd = process::Command::new("ffprobe")
        .arg("-v").arg("quiet")            // silence debug output
        .arg("-hide_banner")               // don't print ffmpeg configuration
        .arg("-print_format").arg("json") // serialize to json
        .arg("-show_format")              // use the serializer
        .arg("-i").arg(path.as_os_str())  // set the input to current file
        .output()?;

    let json_str = String::from_utf8(ffprobe_cmd.stdout)?;
    let probe_result: FFProbeResult = serde_json::from_str(&json_str)?;
    info!("got result: {:?}", probe_result);

    // see if ffprobe was able to determine the file type ...
    let probe_result = match probe_result.format {
        Some(format_data) => format_data,
        None => return Ok(None),
    };

    // TODO: handle alternate streams
    if probe_result.nb_streams != 1 { return Err(ProcessingError::DetectionFailed) }

    let meta_data = match probe_result.format_name.as_ref() {
        "matroska,webm" => Some(FFProbeMeta { mime: "video/webm", ext: "webm" }),
        _ => None,
    };

    Ok(meta_data) // return the ffprobe results, if we know how to handle them ...
}

fn establish_connection() -> Result<PgConnection, ProcessingError> {
    let database_url = env::var("DATABASE_URL")
        .expect("DATABASE_URL not set in `.env` file !!!");

    Ok(PgConnection::establish(&database_url)?)
}

// create entry in database
fn create_db_entry(digest: &str, mime_ty: &str) -> Result<Entry, ProcessingError> {
    let pg_conn = establish_connection()?;
    let aqua_entry = NewEntry { hash: &digest, mime: Some(&mime_ty) };
    let entry: Result<Entry, diesel::result::Error> = diesel::insert(&aqua_entry)
        .into(schema::entries::table)
        .get_result(&pg_conn);

    Ok(entry?)
}

// moves the file from `src_path` to the `content_store` based on its digest
fn move_file(src_path: &Path, content_store: &str, digest: &str, file_ext: &str) -> Result<(), ProcessingError> {
    // carve out a bucket based on first byte of SHA256 digest
    // create the bucket if it does not exist
    let file_bucket    = format!("f{}", &digest[0..2]);
    let file_filename  = format!("{}.{}", &digest, file_ext);

    // create destination path
    let dest = PathBuf::from(content_store)
        .join(file_bucket)
        .join(file_filename);

    // TODO: bad error type ...
    let bucket_dir = dest.parent().ok_or(ProcessingError::ThumbnailFailed)?;
    fs::create_dir_all(bucket_dir)?;

    // move the file
    Ok(fs::rename(src_path, &dest)?)
}

// creates a thumbnail in the content store for the specified digest
// this expects an `ImageMeta` structure describing the input.
fn process_image(content_store: &str, digest: &str, buf: &[u8]) -> Result<(), ProcessingError> {
    // create in memory thumbnail
    let image = image::load_from_memory(&buf)?;

    let thumb = image.resize(200, 200, image::FilterType::Nearest);
    let thumb_bucket   = format!("t{}", &digest[0..2]);
    let thumb_filename = format!("{}.thumbnail", &digest);

    // store them in content store
    let dest = PathBuf::from(content_store)
        .join(thumb_bucket)
        .join(thumb_filename);

    // write thumbnail file to disk
    fs::create_dir_all(dest.parent().unwrap())?;
    let mut dest_file = File::create(dest)?;
    thumb.save(&mut dest_file, image::ImageFormat::JPEG)?;
    Ok(dest_file.flush()?)
}

fn process_video(content_store: &str, digest: &str, src: &Path) -> Result<(), ProcessingError> {
    let thumb_bucket   = format!("t{}", &digest[0..2]);
    let thumb_filename = format!("{}.thumbnail", &digest);

    // store them in content store
    let dest = PathBuf::from(content_store)
        .join(thumb_bucket)
        .join(thumb_filename);

    // write thumbnail file to disk
    fs::create_dir_all(dest.parent().unwrap())?;
    let ffmpeg_cmd = process::Command::new("ffmpeg")
        .arg("-i").arg(src.as_os_str())            // the input file
        .arg("-vf").arg("thumbnail,scale=200:200") // have ffmpeg seek for a "thumbnail"
        .arg("-frames:v").arg("1")                 // take a single frame
        .arg("-f").arg("mjpeg")                    // save it as jpeg
        .arg(dest.as_path().as_os_str())           // into the content store
        .output()?;

    debug!("ffmpeg stderr: {}", String::from_utf8_lossy(&ffmpeg_cmd.stderr));
    debug!("ffmpeg stdout: {}", String::from_utf8_lossy(&ffmpeg_cmd.stdout));

    info!("digest? {}", digest);
    info!("dest exists? {:?} => {}", dest, dest.is_file());

    match dest.is_file() {
        true  => Ok(()),
        false => Err(ProcessingError::ThumbnailFailed),
    }
}
<commit_msg>misc work on matching video types<commit_after>// This program watches a configured directory for incoming files.
// When a file is detected the following procedure is used to handle it:
//
//  * First we digest the file, and add the digest to our manifest.
//    This way we can intelligently handle future events for this file
//
//  * Next we attempt to detect a mime type using various heuristics, incl:
//
//    - Quick detection w/ our magic bytes table
//    - Querying external tools (e.g: ffmpeg, imagemagick, et al.)
//    - etc.
//
//  * If we cannot handle the mime type: the file is moved to an exception
//    directory, and an error is logged somewhere the user will see it.
//
//  * If we can handle the mime type: the file is moved to the content store
//    and an entry is created in the database.
//
//    - This should ideally be done atomically so that other aqua utility
//      processes (e.g: sister agnes) don't mistakenly mark the file as an
//      orphan / missing / etc.
//

#[macro_use] extern crate log;
#[macro_use] extern crate serde_derive;

extern crate aqua;
extern crate clap;
extern crate diesel;
extern crate dotenv;
extern crate env_logger;
extern crate image;
extern crate notify;
extern crate serde;
extern crate serde_json;

use aqua::controllers::prelude::hash_file;
use aqua::models::{Entry, NewEntry};
use aqua::schema;
use clap::{Arg, App};
use diesel::prelude::*;
use diesel::pg::PgConnection;
use dotenv::dotenv;
use notify::{DebouncedEvent, Watcher, RecursiveMode, watcher};
use std::{env, fmt, process};
use std::collections::HashMap;
use std::convert::From;
use std::error::Error;
use std::fs::{self, File, OpenOptions};
use std::io::{self, Read, Write};
use std::path::{Path, PathBuf};
use std::sync::mpsc::channel;
use std::time::Duration;

#[derive(Debug)]
enum ProcessingError {
    DigestFailed,
    DetectionFailed,
    ThumbnailFailed,

    DbConnErr(diesel::ConnectionError),
    DbQueryErr(diesel::result::Error),
    IoErr(io::Error),
    ImageErr(image::ImageError),
    Misc(Box<Error>),
}

impl Error for ProcessingError {
    fn description(&self) -> &str {
        match *self {
            // internal errors
            ProcessingError::DigestFailed      => "Unhandled error while generating SHA256 digest",
            ProcessingError::DetectionFailed   => "The file's type could not be detected",
            ProcessingError::ThumbnailFailed   => "The thumbnail could not be generated",

            // external errors
            ProcessingError::DbConnErr(ref inner)  => inner.description(),
            ProcessingError::DbQueryErr(ref inner) => inner.description(),
            ProcessingError::IoErr(ref inner)      => inner.description(),
            ProcessingError::ImageErr(ref inner)   => inner.description(),
            ProcessingError::Misc(ref inner)       => inner.description(),
        }
    }

    fn cause(&self) -> Option<&Error> {
        match *self {
            ProcessingError::DbConnErr(ref err)  => Some(err),
            ProcessingError::DbQueryErr(ref err) => Some(err),
            ProcessingError::IoErr(ref err)      => Some(err),
            ProcessingError::ImageErr(ref err)   => Some(err),
            ProcessingError::Misc(ref err)       => Some(err.as_ref()),
            _ => None,
        }
    }
}

impl fmt::Display for ProcessingError {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        match *self {
            // TODO: better display impl.
            _ => write!(f, "{}", self.description()),
        }
    }
}

impl From<diesel::ConnectionError> for ProcessingError {
    fn from(err: diesel::ConnectionError) -> Self { ProcessingError::DbConnErr(err) }
}

impl From<diesel::result::Error> for ProcessingError {
    fn from(err: diesel::result::Error) -> Self { ProcessingError::DbQueryErr(err) }
}

impl From<image::ImageError> for ProcessingError {
    fn from(err: image::ImageError) -> Self { ProcessingError::ImageErr(err) }
}

impl From<io::Error> for ProcessingError {
    fn from(err: io::Error) -> Self { ProcessingError::IoErr(err) }
}

impl From<serde_json::Error> for ProcessingError {
    fn from(err: serde_json::Error) -> Self { ProcessingError::Misc(Box::new(err)) }
}

impl From<std::string::FromUtf8Error> for ProcessingError {
    fn from(err: std::string::FromUtf8Error) -> Self { ProcessingError::Misc(Box::new(err)) }
}

fn main() {
    dotenv().expect("must provide .env file, see README (TODO: haha jk)");
    env_logger::init().expect("could not initialize console logging");

    // read command line arguments
    let matches = App::new("aqua-watch")
        .version("0.1.0")
        .author("himechi <hime@localhost>")
        .about("Watches a directory for new files and moves them to the `aqua` content store.")
        .arg(Arg::with_name("INPUT")
             .help("Determines the input directory to be watched.")
             .required(true)
             .index(1))
        .arg(Arg::with_name("OUTPUT")
             .help("The root of the aqua content store where files will be moved.")
             .required(true)
             .index(2))
        .get_matches();

    let dropbox_path  = matches.value_of("INPUT").unwrap();
    let content_store = matches.value_of("OUTPUT").unwrap();

    // setup fs watcher
    let (fs_tx, fs_rx) = channel();
    let mut fs_watcher = watcher(fs_tx, Duration::from_millis(1000))
        .expect("could not create file system watcher!");

    fs_watcher.watch(dropbox_path, RecursiveMode::NonRecursive)
        .expect("could not enroll dropbox in fs events queue");

    // process filesystem events ...
    loop {
        match fs_rx.recv() {
            Ok(DebouncedEvent::Create(path)) => {
                if path.is_file() {
                    match handle_new_file(path, content_store) {
                        Ok(_res) => info!("file processed successfully ..."),
                        Err(msg) => warn!("could not process file: {:?} (inner: {:?})", msg, msg.cause()),
                    };
                }
                else { info!("directory created, ignoring ..."); }
            },
            Ok(event) => info!("unhandled evt: {:?}", event),
            Err(msg) => warn!("fs err: {}", msg),
        }
    }
}

fn handle_new_file(path: PathBuf, content_store: &str) -> Result<(), ProcessingError> {
    let digest = hash_file(path.as_path())
        .ok_or(ProcessingError::DigestFailed)?;

    let mut file = OpenOptions::new()
        .read(true)
        .write(false)
        .create_new(false)
        .open(path.as_path())?;

    // read file into memory
    let mut buf = vec![];
    file.read_to_end(&mut buf)?;

    if let Some(image_metadata) = aqua::util::mime_detect(&buf) {
        info!("got an image ...");
        process_image(content_store, &digest, &buf)?;
        move_file(path.as_path(), content_store, &digest, image_metadata.extension())?;

        let db_entry = create_db_entry(&digest, image_metadata.mime())?;
        info!("inserted: {:?} into database", db_entry);

        Ok(())
    } else if let Some(ffmpeg_metadata) = ffmpeg_detect(path.as_path())? {
        info!("got an video ...");
        process_video(content_store, &digest, &path)?;
        move_file(path.as_path(), content_store, &digest, ffmpeg_metadata.ext)?;

        let db_entry = create_db_entry(&digest, ffmpeg_metadata.mime)?;
        info!("inserted: {:?} into database", db_entry);

        Ok(())
    } else {
        Err(ProcessingError::DetectionFailed)
    }
}

#[derive(Debug, Serialize, Deserialize)]
struct FFProbeResult {
    format: Option<FFProbeFormat>,
    streams: Option<Vec<FFProbeStream>>,
}

#[derive(Debug, Serialize, Deserialize)]
struct FFProbeFormat {
    filename:    String,
    nb_streams:  i32,
    format_name: String,
    start_time:  String, // NOTE: these appear to be fixed decimal
    duration:    String, // NOTE: these appear to be fixed decimal
    size:        String, // NOTE: appears to be an integer
    bit_rate:    String, // NOTE: appears to be an integer
    probe_score: i32,    // NOTE: accuracy of the detection? (???)

    tags: HashMap<String, String>, // NOTE: not sure this is correct type
}

#[derive(Debug, Serialize, Deserialize)]
struct FFProbeStream {
    codec_name: String,
    codec_type: String,
}

struct FFProbeMeta {
    pub mime: &'static str,
    pub ext:  &'static str,
}

fn ffmpeg_detect(path: &Path) -> Result<Option<FFProbeMeta>, ProcessingError> {
    let ffprobe_cmd = process::Command::new("ffprobe")
        .arg("-v").arg("quiet")            // silence debug output
        .arg("-hide_banner")               // don't print ffmpeg configuration
        .arg("-print_format").arg("json") // serialize to json
        .arg("-show_format")              // display format data
        .arg("-show_streams")             // display stream data
        .arg("-i").arg(path.as_os_str())  // set the input to current file
        .output()?;

    let json_str = String::from_utf8(ffprobe_cmd.stdout)?;
    let probe_result: FFProbeResult = serde_json::from_str(&json_str)?;
    info!("got result: {:?}", probe_result);

    // see if ffprobe was able to determine the file type ...
    let probe_format = match probe_result.format {
        Some(format_data) => format_data,
        None => return Ok(None),
    };

    let probe_streams = match probe_result.streams {
        Some(stream_data) => stream_data,
        None => return Ok(None),
    };

    // welp ... guess there's nothing to thumbnail (!!!)
    info!("got format: {:?}", probe_format);
    info!("got streams: {:?}", probe_streams);

    let number_of_videos = probe_streams
        .iter()
        .filter(|el| el.codec_type == "video")
        .count();

    if number_of_videos <= 0 { return Err(ProcessingError::DetectionFailed) }

    let meta_data = if probe_format.format_name.contains("matroska") {
        Some(FFProbeMeta { mime: "video/x-matroska", ext: "mkv" })
    } else if probe_format.format_name.contains("mp4") {
        Some(FFProbeMeta { mime: "video/mp4", ext: "mp4" })
    } else { None };

    Ok(meta_data)
}

fn establish_connection() -> Result<PgConnection, ProcessingError> {
    let database_url = env::var("DATABASE_URL")
        .expect("DATABASE_URL not set in `.env` file !!!");

    Ok(PgConnection::establish(&database_url)?)
}

// create entry in database
fn create_db_entry(digest: &str, mime_ty: &str) -> Result<Entry, ProcessingError> {
    let pg_conn = establish_connection()?;
    let aqua_entry = NewEntry { hash: &digest, mime: Some(&mime_ty) };
    let entry: Result<Entry, diesel::result::Error> = diesel::insert(&aqua_entry)
        .into(schema::entries::table)
        .get_result(&pg_conn);

    Ok(entry?)
}

// moves the file from `src_path` to the `content_store` based on its digest
fn move_file(src_path: &Path, content_store: &str, digest: &str, file_ext: &str) -> Result<(), ProcessingError> {
    // carve out a bucket based on first byte of SHA256 digest
    // create the bucket if it does not exist
    let file_bucket    = format!("f{}", &digest[0..2]);
    let file_filename  = format!("{}.{}", &digest, file_ext);

    // create destination path
    let dest = PathBuf::from(content_store)
        .join(file_bucket)
        .join(file_filename);

    // TODO: bad error type ...
    let bucket_dir = dest.parent().ok_or(ProcessingError::ThumbnailFailed)?;
    fs::create_dir_all(bucket_dir)?;

    // move the file
    Ok(fs::rename(src_path, &dest)?)
}

// creates a thumbnail in the content store for the specified digest
// this expects an `ImageMeta` structure describing the input.
fn process_image(content_store: &str, digest: &str, buf: &[u8]) -> Result<(), ProcessingError> {
    // create in memory thumbnail
    let image = image::load_from_memory(&buf)?;

    let thumb = image.resize(200, 200, image::FilterType::Nearest);
    let thumb_bucket   = format!("t{}", &digest[0..2]);
    let thumb_filename = format!("{}.thumbnail", &digest);

    // store them in content store
    let dest = PathBuf::from(content_store)
        .join(thumb_bucket)
        .join(thumb_filename);

    // write thumbnail file to disk
    fs::create_dir_all(dest.parent().unwrap())?;
    let mut dest_file = File::create(dest)?;
    thumb.save(&mut dest_file, image::ImageFormat::JPEG)?;
    Ok(dest_file.flush()?)
}

fn process_video(content_store: &str, digest: &str, src: &Path) -> Result<(), ProcessingError> {
    let thumb_bucket   = format!("t{}", &digest[0..2]);
    let thumb_filename = format!("{}.thumbnail", &digest);

    // store them in content store
    let dest = PathBuf::from(content_store)
        .join(thumb_bucket)
        .join(thumb_filename);

    // write thumbnail file to disk
    fs::create_dir_all(dest.parent().unwrap())?;
    let ffmpeg_cmd = process::Command::new("ffmpeg")
        .arg("-i").arg(src.as_os_str())            // the input file
        .arg("-vf").arg("thumbnail,scale=200:200") // have ffmpeg seek for a "thumbnail"
        .arg("-frames:v").arg("1")                 // take a single frame
        .arg("-f").arg("mjpeg")                    // save it as jpeg
        .arg(dest.as_path().as_os_str())           // into the content store
        .output()?;

    debug!("ffmpeg stderr: {}", String::from_utf8_lossy(&ffmpeg_cmd.stderr));
    debug!("ffmpeg stdout: {}", String::from_utf8_lossy(&ffmpeg_cmd.stdout));

    info!("digest? {}", digest);
    info!("dest exists? {:?} => {}", dest, dest.is_file());

    match dest.is_file() {
        true  => Ok(()),
        false => Err(ProcessingError::ThumbnailFailed),
    }
}
<|endoftext|>


<commit_before>extern crate bincode;
extern crate rustc_serialize;
extern crate rpsrtsrs;
extern crate docopt;

use std::net::TcpStream;
use std::ops::Deref;

use rpsrtsrs::state::{Game, ClientId};
use rpsrtsrs::network::{Message};

use docopt::Docopt;

use bincode::SizeLimit;
use bincode::rustc_serialize::{decode_from, encode_into};
use bincode::rustc_serialize::DecodingResult;

static USAGE: &'static str = "
Usage: cli_client [-p PORT] [-i IP] [-r ID]

Options:
    -p PORT  The port to connect to [default: 8080].
    -i IP    The ipv4 address to connect to [default: 127.0.0.1].
    -r ID    Reconnect with the given ID
";

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_p: u16,
    flag_i: String,
    flag_r: Option<ClientId>,
}

fn main() {
    let args: Args = Docopt::new(USAGE).and_then(|d| d.decode())
                                       .unwrap_or_else(|e| e.exit());
    let host = args.flag_i;
    let port = args.flag_p;
    let reconnect = args.flag_r;

    println!("connecting to host: {:?}:{:?} reconnect? {:?}", host, port, reconnect);

    let mut stream = TcpStream::connect((host.deref(), port)).unwrap();

    match reconnect {
        Some(id) => {
            encode_into(&Message::ClientReconnect(id), &mut stream, SizeLimit::Infinite).unwrap();
        },
        None => {
            encode_into(&Message::ClientHello, &mut stream, SizeLimit::Infinite).unwrap();
        }
    }
    let server_hello: DecodingResult<Message> = decode_from(&mut stream, SizeLimit::Infinite);
    println!("{:?}", server_hello);

    loop {
        let game_state: DecodingResult<Game> = decode_from(&mut stream, SizeLimit::Infinite);
        match game_state {
            Ok(game) => println!("{:?}", game),
            Err(e) => {
                println!("{:?}", e);
                return;
            }
        }
    }
}

<commit_msg>Fix panic when passing the reconnect flag (#28)<commit_after>extern crate bincode;
extern crate rustc_serialize;
extern crate rpsrtsrs;
extern crate docopt;

use std::net::TcpStream;
use std::ops::Deref;

use rpsrtsrs::state::{Game};
use rpsrtsrs::network::{Message};

use docopt::Docopt;

use bincode::SizeLimit;
use bincode::rustc_serialize::{decode_from, encode_into};
use bincode::rustc_serialize::DecodingResult;

static USAGE: &'static str = "
Usage: cli_client [-p PORT] [-i IP] [-r ID]

Options:
    -p PORT  The port to connect to [default: 8080].
    -i IP    The ipv4 address to connect to [default: 127.0.0.1].
    -r ID    Reconnect with the given ID
";

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_p: u16,
    flag_i: String,
    flag_r: Option<u32>,
}

fn main() {
    let args: Args = Docopt::new(USAGE).and_then(|d| d.decode())
                                       .unwrap_or_else(|e| e.exit());
    let host = args.flag_i;
    let port = args.flag_p;
    let reconnect = args.flag_r;

    println!("connecting to host: {:?}:{:?} reconnect? {:?}", host, port, reconnect);

    let mut stream = TcpStream::connect((host.deref(), port)).unwrap();

    match reconnect {
        Some(id) => {
            encode_into(&Message::ClientReconnect(id.into()), &mut stream, SizeLimit::Infinite).unwrap();
        },
        None => {
            encode_into(&Message::ClientHello, &mut stream, SizeLimit::Infinite).unwrap();
        }
    }
    let server_hello: DecodingResult<Message> = decode_from(&mut stream, SizeLimit::Infinite);
    println!("{:?}", server_hello);

    loop {
        let game_state: DecodingResult<Game> = decode_from(&mut stream, SizeLimit::Infinite);
        match game_state {
            Ok(game) => println!("{:?}", game),
            Err(e) => {
                println!("{:?}", e);
                return;
            }
        }
    }
}

<|endoftext|>


<commit_before>use std::collections::btree_map;
use std::collections::{BTreeMap, BTreeSet};

fn build_stopwords() -> BTreeSet<&'static str> {
    vec![
        "with",
        "sign",
        "small",
        "letter",
        "digit",
        "for",
        "symbol",
        "<control>",
    ]
    .into_iter()
    .collect()
}

#[test]
fn test_stopwords() {
    let sw = build_stopwords();
    assert!(sw.contains("with"));
}

fn add_component(result: &mut Vec<String>, component: &str) {
    lazy_static! {
        static ref STOPS: BTreeSet<&'static str> = build_stopwords();
    }
    let component = component.to_lowercase();
    let component = component.trim_end_matches(',');

    if STOPS.contains(component) || component.len() == 1 {
        return;
    }
    result.push(component.to_owned());

    if !component.contains('-') {
        return;
    }
    for parts in component.split('-') {
        add_component(result, parts);
    }
}

fn name_components(name: &str) -> Vec<String> {
    let mut result = vec![];
    result.push(name.to_lowercase());
    for component in name.split_whitespace() {
        add_component(&mut result, component);
    }
    result
}

#[test]
fn test_name_components() {
    assert_eq!(
        name_components("D WITH CURL, LATIN SMALL LETTER"),
        Vec::from_iter(
            vec!["d with curl, latin small letter", "curl", "latin"]
                .into_iter()
                .map(String::from)
        )
    );
    assert_eq!(
        name_components("Equals Sign"),
        Vec::from_iter(vec!["equals sign", "equals"].into_iter().map(String::from))
    );
    assert_eq!(
        name_components("UPSIDE-DOWN FACE"),
        Vec::from_iter(
            vec!["upside-down face", "upside-down", "upside", "down", "face"]
                .into_iter()
                .map(String::from)
        )
    );
}

#[derive(Clone, Debug, PartialEq)]
pub struct Names {
    map: BTreeMap<String, BTreeSet<char>>,
}

impl Names {
    pub fn new() -> Names {
        Names {
            map: BTreeMap::new(),
        }
    }

    ///Insert a character names' components into btree map, minus stopwords.
    pub fn insert(&mut self, names: Vec<String>, ch: char) {
        for name in names {
            for component in name_components(name.as_str()) {
                self.map
                    .entry(component)
                    .or_insert_with(BTreeSet::new)
                    .insert(ch);
            }
        }
    }

    pub fn iter(&self) -> btree_map::Iter<String, BTreeSet<char>> {
        self.map.iter()
    }
}
<commit_msg>Fix a problem I added in the last commit<commit_after>use std::collections::btree_map;
use std::collections::{BTreeMap, BTreeSet};

fn build_stopwords() -> BTreeSet<&'static str> {
    vec![
        "with",
        "sign",
        "small",
        "letter",
        "digit",
        "for",
        "symbol",
        "<control>",
    ]
    .into_iter()
    .collect()
}

#[test]
fn test_stopwords() {
    let sw = build_stopwords();
    assert!(sw.contains("with"));
}

fn add_component(result: &mut Vec<String>, component: &str) {
    lazy_static! {
        static ref STOPS: BTreeSet<&'static str> = build_stopwords();
    }
    let component = component.to_lowercase();
    let component = component.trim_end_matches(',');

    if STOPS.contains(component) || component.len() == 1 {
        return;
    }
    result.push(component.to_owned());

    if !component.contains('-') {
        return;
    }
    for parts in component.split('-') {
        add_component(result, parts);
    }
}

fn name_components(name: &str) -> Vec<String> {
    let mut result = vec![];
    result.push(name.to_lowercase());
    for component in name.split_whitespace() {
        add_component(&mut result, component);
    }
    result
}

#[test]
fn test_name_components() {
    assert_eq!(
        name_components("D WITH CURL, LATIN SMALL LETTER"),
        vec!["d with curl, latin small letter", "curl", "latin"]
            .into_iter()
            .map(String::from)
            .collect::<Vec<String>>()
    );
    assert_eq!(
        name_components("Equals Sign"),
        vec!["equals sign", "equals"]
            .into_iter()
            .map(String::from)
            .collect::<Vec<String>>()
    );
    assert_eq!(
        name_components("UPSIDE-DOWN FACE"),
        vec!["upside-down face", "upside-down", "upside", "down", "face"]
            .into_iter()
            .map(String::from)
            .collect::<Vec<String>>()
    );
}

#[derive(Clone, Debug, PartialEq)]
pub struct Names {
    map: BTreeMap<String, BTreeSet<char>>,
}

impl Names {
    pub fn new() -> Names {
        Names {
            map: BTreeMap::new(),
        }
    }

    ///Insert a character names' components into btree map, minus stopwords.
    pub fn insert(&mut self, names: Vec<String>, ch: char) {
        for name in names {
            for component in name_components(name.as_str()) {
                self.map
                    .entry(component)
                    .or_insert_with(BTreeSet::new)
                    .insert(ch);
            }
        }
    }

    pub fn iter(&self) -> btree_map::Iter<String, BTreeSet<char>> {
        self.map.iter()
    }
}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

extern crate docopt;
extern crate csv;
extern crate rustc_serialize;
extern crate curl;
extern crate mimirsbrunn;
#[macro_use]
extern crate log;
extern crate env_logger;

use std::path::Path;
use mimirsbrunn::rubber::Rubber;
use std::fs;

#[derive(RustcDecodable, RustcEncodable)]
pub struct Bano {
    pub id: String,
    pub nb: String,
    pub street: String,
    pub zip: String,
    pub city: String,
    pub src: String,
    pub lat: f64,
    pub lon: f64,
}

impl Bano {
    pub fn insee(&self) -> &str {
        assert!(self.id.len() >= 5);
        &self.id[..5]
    }
    pub fn fantoir(&self) -> &str {
        assert!(self.id.len() >= 10);
        &self.id[..10]
    }
    pub fn into_addr(self) -> mimirsbrunn::Addr {
        let street_name = format!("{}, {} {}", self.street, self.zip, self.city);
        let addr_name = format!("{} {}", self.nb, street_name);
        let street_id = format!("street:{}", self.fantoir().to_string());
        let admin = mimirsbrunn::Admin {
            id: format!("admin:{}", self.insee()),
            level: 8,
            name: self.city,
            zip_code: self.zip,
            weight: 1,
            coord: mimirsbrunn::Coord {
                lat: 0.0,
                lon: 0.0,
            },
        };
        let street = mimirsbrunn::Street {
            id: street_id,
            street_name: self.street,
            name: street_name,
            administrative_region: admin,
            weight: 1,
        };
        mimirsbrunn::Addr {
            id: format!("addr:{};{}", self.lat, self.lon),
            house_number: self.nb,
            street: street,
            name: addr_name,
            coord: mimirsbrunn::Coord {
                lat: self.lat,
                lon: self.lon,
            },
            weight: 1,
        }
    }
}

fn index_bano<I>(cnx_string: &str, files: I)
    where I: Iterator<Item = std::path::PathBuf>
{
    let mut rubber = Rubber::new(cnx_string);

    info!("purge and create Munin...");
    rubber.create_index().unwrap();

    for f in files {
        info!("importing {:?}...", &f);
        let mut rdr = csv::Reader::from_file(&f).unwrap().has_headers(false);

        let iter = rdr.decode().map(|r| {
            let b: Bano = r.unwrap();
            b.into_addr()
        });
        match rubber.index(iter) {
            Err(e) => panic!("failed to bulk insert file {:?} because: {}", &f, e),
            Ok(nb) => info!("importing {:?}: {} addresses added.", &f, nb),
        }
    }
}

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_connection_string: String,
}

static USAGE: &'static str = "
Usage:
    bano2mimir --input=<input> [--connection-string=<connection-string>]

    -i, --input=<input>           Bano files. Can be either a directory or a file.
    -c, --connection-string=<connection-string>
                                  Elasticsearch parameters, [default: http://localhost:9200/munin]
";

fn main() {
    env_logger::init().unwrap();
    info!("importing bano into Mimir");

    let args: Args = docopt::Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());

    let files_path = Path::new(&args.flag_input);
    if files_path.is_dir() {
        let paths: std::fs::ReadDir = fs::read_dir(&args.flag_input).unwrap();
        index_bano(&args.flag_connection_string,
                   paths.map(|p| p.unwrap().path()));
    } else {
        index_bano(&args.flag_connection_string,
                   [&args.flag_input].iter().map(|f| std::path::PathBuf::from(&f)));
    }

}
<commit_msg>corrections after review<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

extern crate docopt;
extern crate csv;
extern crate rustc_serialize;
extern crate curl;
extern crate mimirsbrunn;
#[macro_use]
extern crate log;
extern crate env_logger;

use std::path::Path;
use mimirsbrunn::rubber::Rubber;
use std::fs;

#[derive(RustcDecodable, RustcEncodable)]
pub struct Bano {
    pub id: String,
    pub nb: String,
    pub street: String,
    pub zip: String,
    pub city: String,
    pub src: String,
    pub lat: f64,
    pub lon: f64,
}

impl Bano {
    pub fn insee(&self) -> &str {
        assert!(self.id.len() >= 5);
        &self.id[..5]
    }
    pub fn fantoir(&self) -> &str {
        assert!(self.id.len() >= 10);
        &self.id[..10]
    }
    pub fn into_addr(self) -> mimirsbrunn::Addr {
        let street_name = format!("{}, {} {}", self.street, self.zip, self.city);
        let addr_name = format!("{} {}", self.nb, street_name);
        let street_id = format!("street:{}", self.fantoir().to_string());
        let admin = mimirsbrunn::Admin {
            id: format!("admin:{}", self.insee()),
            level: 8,
            name: self.city,
            zip_code: self.zip,
            weight: 1,
            coord: mimirsbrunn::Coord {
                lat: 0.0,
                lon: 0.0,
            },
        };
        let street = mimirsbrunn::Street {
            id: street_id,
            street_name: self.street,
            name: street_name,
            administrative_region: admin,
            weight: 1,
        };
        mimirsbrunn::Addr {
            id: format!("addr:{};{}", self.lat, self.lon),
            house_number: self.nb,
            street: street,
            name: addr_name,
            coord: mimirsbrunn::Coord {
                lat: self.lat,
                lon: self.lon,
            },
            weight: 1,
        }
    }
}

fn index_bano<I>(cnx_string: &str, files: I)
    where I: Iterator<Item = std::path::PathBuf>
{
    let mut rubber = Rubber::new(cnx_string);

    info!("purge and create Munin...");
    rubber.create_index().unwrap();

    for f in files {
        info!("importing {:?}...", &f);
        let mut rdr = csv::Reader::from_file(&f).unwrap().has_headers(false);

        let iter = rdr.decode().map(|r| {
            let b: Bano = r.unwrap();
            b.into_addr()
        });
        match rubber.index(iter) {
            Err(e) => panic!("failed to bulk insert file {:?} because: {}", &f, e),
            Ok(nb) => info!("importing {:?}: {} addresses added.", &f, nb),
        }
    }
}

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_connection_string: String,
}

static USAGE: &'static str = "
Usage:
    bano2mimir --input=<input> [--connection-string=<connection-string>]

    -i, --input=<input>           Bano files. Can be either a directory or a file.
    -c, --connection-string=<connection-string>
                                  Elasticsearch parameters, [default: http://localhost:9200/munin]
";

fn main() {
    env_logger::init().unwrap();
    info!("importing bano into Mimir");

    let args: Args = docopt::Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());

    let file_path = Path::new(&args.flag_input);
    if file_path.is_dir() {
        let paths: std::fs::ReadDir = fs::read_dir(&args.flag_input).unwrap();
        index_bano(&args.flag_connection_string,
                   paths.map(|p| p.unwrap().path()));
    } else {
        index_bano(&args.flag_connection_string,
                    std::iter::once(std::path::PathBuf::from(&args.flag_input)));
    }

}
<|endoftext|>


<commit_before>#![feature(inner_deref)]

extern crate cargo_metadata;

use std::path::{PathBuf, Path};
use std::io::{self, Write};
use std::process::Command;
use std::fs::{self, File};

const CARGO_MIRI_HELP: &str = r#"Interprets bin crates

Usage:
    cargo miri [subcommand] [options] [--] [<opts>...]

Subcommands:
    run                      Run binaries (default)
    test                     Run tests
    setup                    Only perform automatic setup, but without asking questions (for getting a proper libstd)

Common options:
    -h, --help               Print this message
    --features               Features to compile for the package
    -V, --version            Print version info and exit

Other options are the same as `cargo rustc`.

The feature `cargo-miri` is automatically defined for convenience. You can use
it to configure the resource limits

    #![cfg_attr(feature = "cargo-miri", memory_size = 42)]

available resource limits are `memory_size`, `step_limit`, `stack_limit`
"#;

#[derive(Copy, Clone, Debug, PartialEq, Eq)]
enum MiriCommand {
    Run,
    Test,
    Setup,
}

fn show_help() {
    println!("{}", CARGO_MIRI_HELP);
}

fn show_version() {
    println!("miri {} ({} {})",
        env!("CARGO_PKG_VERSION"), env!("VERGEN_SHA_SHORT"), env!("VERGEN_COMMIT_DATE"));
}

fn show_error(msg: String) -> ! {
    eprintln!("fatal error: {}", msg);
    std::process::exit(1)
}

fn list_targets(mut args: impl Iterator<Item=String>) -> impl Iterator<Item=cargo_metadata::Target> {
    // We need to get the manifest, and then the metadata, to enumerate targets.
    let manifest_path_arg = args.find(|val| {
        val.starts_with("--manifest-path=")
    });

    let mut metadata = if let Ok(metadata) = cargo_metadata::metadata(
        manifest_path_arg.as_ref().map(AsRef::as_ref),
    )
    {
        metadata
    } else {
        show_error(format!("error: Could not obtain cargo metadata."));
    };

    let manifest_path = manifest_path_arg.map(|arg| {
        PathBuf::from(Path::new(&arg["--manifest-path=".len()..]))
    });

    let current_dir = std::env::current_dir();

    let package_index = metadata
        .packages
        .iter()
        .position(|package| {
            let package_manifest_path = Path::new(&package.manifest_path);
            if let Some(ref manifest_path) = manifest_path {
                package_manifest_path == manifest_path
            } else {
                let current_dir = current_dir.as_ref().expect(
                    "could not read current directory",
                );
                let package_manifest_directory = package_manifest_path.parent().expect(
                    "could not find parent directory of package manifest",
                );
                package_manifest_directory == current_dir
            }
        })
        .expect("could not find matching package");
    let package = metadata.packages.remove(package_index);

    // Finally we got the list of targets to build
    package.targets.into_iter()
}

fn ask(question: &str) {
    let mut buf = String::new();
    print!("{} [Y/n] ", question);
    io::stdout().flush().unwrap();
    io::stdin().read_line(&mut buf).unwrap();
    match buf.trim().to_lowercase().as_ref() {
        "" | "y" | "yes" => {}, // proceed
        "n" | "no" => show_error(format!("Aborting as per your request")),
        a => show_error(format!("I do not understand `{}`", a))
    };
}

/// Perform the setup requires to make `cargo miri` work: Getting a custom-built libstd. Then sets MIRI_SYSROOT.
/// Skipped if MIRI_SYSROOT is already set, in that case we expect the user has done all this already.
fn setup(ask_user: bool) {
    if std::env::var("MIRI_SYSROOT").is_ok() {
        return;
    }

    // First, we need xargo
    if Command::new("xargo").arg("--version").output().is_err()
    {
        if ask_user {
            ask("It seems you do not have xargo installed. I will run `cargo install xargo`. Proceed?");
        } else {
            println!("Installing xargo: `cargo install xargo`");
        }
        if !Command::new("cargo").args(&["install", "xargo"]).status().unwrap().success() {
            show_error(format!("Failed to install xargo"));
        }
    }

    // Then, we also need rust-src.  Let's see if it is already installed.
    let sysroot = Command::new("rustc").args(&["--print", "sysroot"]).output().unwrap().stdout;
    let sysroot = std::str::from_utf8(&sysroot[..]).unwrap();
    let src = Path::new(sysroot.trim_end_matches('\n')).join("lib").join("rustlib").join("src");
    if !src.exists() {
        if ask_user {
            ask("It seems you do not have the rust-src component installed. I will run `rustup component add rust-src`. Proceed?");
        } else {
            println!("Installing rust-src component: `rustup component add rust-src`");
        }
        if !Command::new("rustup").args(&["component", "add", "rust-src"]).status().unwrap().success() {
            show_error(format!("Failed to install rust-src component"));
        }
    }

    // Next, we need our own libstd. We will do this work in whatever is a good cache dir for this platform.
    let dirs = directories::ProjectDirs::from("miri", "miri", "miri").unwrap();
    let dir = dirs.cache_dir();
    if !dir.exists() {
        fs::create_dir_all(&dir).unwrap();
    }
    // The interesting bit: Xargo.toml
    File::create(dir.join("Xargo.toml")).unwrap()
        .write_all(br#"
[dependencies.std]
features = ["panic_unwind"]

[dependencies.test]
stage = 1
        "#).unwrap();
    // The boring bits: A dummy project for xargo
    File::create(dir.join("Cargo.toml")).unwrap()
        .write_all(br#"
[package]
name = "miri-xargo"
description = "A dummy project for building libstd with xargo."
version = "0.0.0"

[lib]
path = "lib.rs"
        "#).unwrap();
    File::create(dir.join("lib.rs")).unwrap();
    // Run xargo
    if !Command::new("xargo").arg("build").arg("-q")
        .current_dir(&dir)
        .env("RUSTFLAGS", miri::miri_default_args().join(" "))
        .env("XARGO_HOME", dir.to_str().unwrap())
        .status().unwrap().success()
    {
        show_error(format!("Failed to run xargo"));
    }

    // That should be it!
    std::env::set_var("MIRI_SYSROOT", dir.join("HOST"));
}

fn main() {
    // Check for version and help flags even when invoked as 'cargo-miri'
    if std::env::args().any(|a| a == "--help" || a == "-h") {
        show_help();
        return;
    }
    if std::env::args().any(|a| a == "--version" || a == "-V") {
        show_version();
        return;
    }

    if let Some("miri") = std::env::args().nth(1).as_ref().map(AsRef::as_ref) {
        // this arm is when `cargo miri` is called.  We call `cargo rustc` for
        // each applicable target, but with the RUSTC env var set to the `cargo-miri`
        // binary so that we come back in the other branch, and dispatch
        // the invocations to rustc and miri, respectively.

        let (subcommand, skip) = match std::env::args().nth(2).deref() {
            Some("test") => (MiriCommand::Test, 3),
            Some("run") => (MiriCommand::Run, 3),
            Some("setup") => (MiriCommand::Setup, 3),
            // Default command, if there is an option or nothing
            Some(s) if s.starts_with("-") => (MiriCommand::Run, 2),
            None => (MiriCommand::Run, 2),
            // Unvalid command
            Some(s) => {
                show_error(format!("Unknown command `{}`", s))
            }
        };

        // We always setup
        let ask = subcommand != MiriCommand::Setup;
        setup(ask);

        // Now run the command.
        for target in list_targets(std::env::args().skip(skip)) {
            let args = std::env::args().skip(skip);
            let kind = target.kind.get(0).expect(
                "badly formatted cargo metadata: target::kind is an empty array",
            );
            match (subcommand, &kind[..]) {
                (MiriCommand::Test, "test") => {
                    // For test binaries we call `cargo rustc --test target -- <rustc args>`
                    if let Err(code) = process(
                        vec!["--test".to_string(), target.name].into_iter().chain(
                            args,
                        ),
                    )
                    {
                        std::process::exit(code);
                    }
                }
                (MiriCommand::Test, "lib") => {
                    // For libraries we call `cargo rustc -- --test <rustc args>`
                    // Notice now that `--test` is a rustc arg rather than a cargo arg. This tells
                    // rustc to build a test harness which calls all #[test] functions. We don't
                    // use the harness since we execute each #[test] function's MIR ourselves before
                    // compilation even completes, but this option is necessary to build the library.
                    if let Err(code) = process(
                        vec!["--".to_string(), "--test".to_string()].into_iter().chain(
                            args,
                        ),
                    )
                    {
                        std::process::exit(code);
                    }
                }
                (MiriCommand::Run, "bin") => {
                    // For ordinary binaries we call `cargo rustc --bin target -- <rustc args>`
                    if let Err(code) = process(
                        vec!["--bin".to_string(), target.name].into_iter().chain(
                            args,
                        ),
                    )
                    {
                        std::process::exit(code);
                    }
                }
                _ => {}
            }
        }
    } else {
        // This arm is executed when cargo-miri runs `cargo rustc` with the `RUSTC` env var set to itself:
        // Dependencies get dispatched to rustc, the final test/binary to miri.

        let home = option_env!("RUSTUP_HOME").or(option_env!("MULTIRUST_HOME"));
        let toolchain = option_env!("RUSTUP_TOOLCHAIN").or(option_env!("MULTIRUST_TOOLCHAIN"));
        let sys_root = if let Ok(sysroot) = ::std::env::var("MIRI_SYSROOT") {
            sysroot
        } else if let (Some(home), Some(toolchain)) = (home, toolchain) {
            format!("{}/toolchains/{}", home, toolchain)
        } else {
            option_env!("RUST_SYSROOT")
                .map(|s| s.to_owned())
                .or_else(|| {
                    Command::new("rustc")
                        .arg("--print")
                        .arg("sysroot")
                        .output()
                        .ok()
                        .and_then(|out| String::from_utf8(out.stdout).ok())
                        .map(|s| s.trim().to_owned())
                })
                .expect("need to specify RUST_SYSROOT env var during miri compilation, or use rustup or multirust")
        };

        // this conditional check for the --sysroot flag is there so users can call `cargo-miri` directly
        // without having to pass --sysroot or anything
        let mut args: Vec<String> = if std::env::args().any(|s| s == "--sysroot") {
            std::env::args().skip(1).collect()
        } else {
            std::env::args()
                .skip(1)
                .chain(Some("--sysroot".to_owned()))
                .chain(Some(sys_root))
                .collect()
        };
        args.splice(0..0, miri::miri_default_args().iter().map(ToString::to_string));

        // this check ensures that dependencies are built but not interpreted and the final crate is
        // interpreted but not built
        let miri_enabled = std::env::args().any(|s| s == "--emit=dep-info,metadata");

        let mut command = if miri_enabled {
            let mut path = std::env::current_exe().expect("current executable path invalid");
            path.set_file_name("miri");
            Command::new(path)
        } else {
            Command::new("rustc")
        };

        args.extend_from_slice(&["--cfg".to_owned(), r#"feature="cargo-miri""#.to_owned()]);

        match command.args(&args).status() {
            Ok(exit) => {
                if !exit.success() {
                    std::process::exit(exit.code().unwrap_or(42));
                }
            }
            Err(ref e) if miri_enabled => panic!("error during miri run: {:?}", e),
            Err(ref e) => panic!("error during rustc call: {:?}", e),
        }
    }
}

fn process<I>(old_args: I) -> Result<(), i32>
where
    I: Iterator<Item = String>,
{
    let mut args = vec!["rustc".to_owned()];

    let mut found_dashes = false;
    for arg in old_args {
        found_dashes |= arg == "--";
        args.push(arg);
    }
    if !found_dashes {
        args.push("--".to_owned());
    }
    args.push("--emit=dep-info,metadata".to_owned());
    args.push("--cfg".to_owned());
    args.push(r#"feature="cargo-miri""#.to_owned());

    let path = std::env::current_exe().expect("current executable path invalid");
    let exit_status = std::process::Command::new("cargo")
        .args(&args)
        .env("RUSTC", path)
        .spawn()
        .expect("could not run cargo")
        .wait()
        .expect("failed to wait for cargo?");

    if exit_status.success() {
        Ok(())
    } else {
        Err(exit_status.code().unwrap_or(-1))
    }
}
<commit_msg>inform the user when and where we are creating a cache dir<commit_after>#![feature(inner_deref)]

extern crate cargo_metadata;

use std::path::{PathBuf, Path};
use std::io::{self, Write};
use std::process::Command;
use std::fs::{self, File};

const CARGO_MIRI_HELP: &str = r#"Interprets bin crates

Usage:
    cargo miri [subcommand] [options] [--] [<opts>...]

Subcommands:
    run                      Run binaries (default)
    test                     Run tests
    setup                    Only perform automatic setup, but without asking questions (for getting a proper libstd)

Common options:
    -h, --help               Print this message
    --features               Features to compile for the package
    -V, --version            Print version info and exit

Other options are the same as `cargo rustc`.

The feature `cargo-miri` is automatically defined for convenience. You can use
it to configure the resource limits

    #![cfg_attr(feature = "cargo-miri", memory_size = 42)]

available resource limits are `memory_size`, `step_limit`, `stack_limit`
"#;

#[derive(Copy, Clone, Debug, PartialEq, Eq)]
enum MiriCommand {
    Run,
    Test,
    Setup,
}

fn show_help() {
    println!("{}", CARGO_MIRI_HELP);
}

fn show_version() {
    println!("miri {} ({} {})",
        env!("CARGO_PKG_VERSION"), env!("VERGEN_SHA_SHORT"), env!("VERGEN_COMMIT_DATE"));
}

fn show_error(msg: String) -> ! {
    eprintln!("fatal error: {}", msg);
    std::process::exit(1)
}

fn list_targets(mut args: impl Iterator<Item=String>) -> impl Iterator<Item=cargo_metadata::Target> {
    // We need to get the manifest, and then the metadata, to enumerate targets.
    let manifest_path_arg = args.find(|val| {
        val.starts_with("--manifest-path=")
    });

    let mut metadata = if let Ok(metadata) = cargo_metadata::metadata(
        manifest_path_arg.as_ref().map(AsRef::as_ref),
    )
    {
        metadata
    } else {
        show_error(format!("error: Could not obtain cargo metadata."));
    };

    let manifest_path = manifest_path_arg.map(|arg| {
        PathBuf::from(Path::new(&arg["--manifest-path=".len()..]))
    });

    let current_dir = std::env::current_dir();

    let package_index = metadata
        .packages
        .iter()
        .position(|package| {
            let package_manifest_path = Path::new(&package.manifest_path);
            if let Some(ref manifest_path) = manifest_path {
                package_manifest_path == manifest_path
            } else {
                let current_dir = current_dir.as_ref().expect(
                    "could not read current directory",
                );
                let package_manifest_directory = package_manifest_path.parent().expect(
                    "could not find parent directory of package manifest",
                );
                package_manifest_directory == current_dir
            }
        })
        .expect("could not find matching package");
    let package = metadata.packages.remove(package_index);

    // Finally we got the list of targets to build
    package.targets.into_iter()
}

fn ask(question: &str) {
    let mut buf = String::new();
    print!("{} [Y/n] ", question);
    io::stdout().flush().unwrap();
    io::stdin().read_line(&mut buf).unwrap();
    match buf.trim().to_lowercase().as_ref() {
        "" | "y" | "yes" => {}, // proceed
        "n" | "no" => show_error(format!("Aborting as per your request")),
        a => show_error(format!("I do not understand `{}`", a))
    };
}

/// Perform the setup requires to make `cargo miri` work: Getting a custom-built libstd. Then sets MIRI_SYSROOT.
/// Skipped if MIRI_SYSROOT is already set, in that case we expect the user has done all this already.
fn setup(ask_user: bool) {
    if std::env::var("MIRI_SYSROOT").is_ok() {
        return;
    }

    // First, we need xargo
    if Command::new("xargo").arg("--version").output().is_err()
    {
        if ask_user {
            ask("It seems you do not have xargo installed. I will run `cargo install xargo`. Proceed?");
        } else {
            println!("Installing xargo: `cargo install xargo`");
        }
        if !Command::new("cargo").args(&["install", "xargo"]).status().unwrap().success() {
            show_error(format!("Failed to install xargo"));
        }
    }

    // Then, we also need rust-src.  Let's see if it is already installed.
    let sysroot = Command::new("rustc").args(&["--print", "sysroot"]).output().unwrap().stdout;
    let sysroot = std::str::from_utf8(&sysroot[..]).unwrap();
    let src = Path::new(sysroot.trim_end_matches('\n')).join("lib").join("rustlib").join("src");
    if !src.exists() {
        if ask_user {
            ask("It seems you do not have the rust-src component installed. I will run `rustup component add rust-src`. Proceed?");
        } else {
            println!("Installing rust-src component: `rustup component add rust-src`");
        }
        if !Command::new("rustup").args(&["component", "add", "rust-src"]).status().unwrap().success() {
            show_error(format!("Failed to install rust-src component"));
        }
    }

    // Next, we need our own libstd. We will do this work in whatever is a good cache dir for this platform.
    let dirs = directories::ProjectDirs::from("miri", "miri", "miri").unwrap();
    let dir = dirs.cache_dir();
    if !dir.exists() {
        println!("Creating `{}` and using it for miri's build of libstd", dir.display());
        fs::create_dir_all(&dir).unwrap();
    }
    // The interesting bit: Xargo.toml
    File::create(dir.join("Xargo.toml")).unwrap()
        .write_all(br#"
[dependencies.std]
features = ["panic_unwind"]

[dependencies.test]
stage = 1
        "#).unwrap();
    // The boring bits: A dummy project for xargo
    File::create(dir.join("Cargo.toml")).unwrap()
        .write_all(br#"
[package]
name = "miri-xargo"
description = "A dummy project for building libstd with xargo."
version = "0.0.0"

[lib]
path = "lib.rs"
        "#).unwrap();
    File::create(dir.join("lib.rs")).unwrap();
    // Run xargo
    if !Command::new("xargo").arg("build").arg("-q")
        .current_dir(&dir)
        .env("RUSTFLAGS", miri::miri_default_args().join(" "))
        .env("XARGO_HOME", dir.to_str().unwrap())
        .status().unwrap().success()
    {
        show_error(format!("Failed to run xargo"));
    }

    // That should be it!
    std::env::set_var("MIRI_SYSROOT", dir.join("HOST"));
}

fn main() {
    // Check for version and help flags even when invoked as 'cargo-miri'
    if std::env::args().any(|a| a == "--help" || a == "-h") {
        show_help();
        return;
    }
    if std::env::args().any(|a| a == "--version" || a == "-V") {
        show_version();
        return;
    }

    if let Some("miri") = std::env::args().nth(1).as_ref().map(AsRef::as_ref) {
        // this arm is when `cargo miri` is called.  We call `cargo rustc` for
        // each applicable target, but with the RUSTC env var set to the `cargo-miri`
        // binary so that we come back in the other branch, and dispatch
        // the invocations to rustc and miri, respectively.

        let (subcommand, skip) = match std::env::args().nth(2).deref() {
            Some("test") => (MiriCommand::Test, 3),
            Some("run") => (MiriCommand::Run, 3),
            Some("setup") => (MiriCommand::Setup, 3),
            // Default command, if there is an option or nothing
            Some(s) if s.starts_with("-") => (MiriCommand::Run, 2),
            None => (MiriCommand::Run, 2),
            // Unvalid command
            Some(s) => {
                show_error(format!("Unknown command `{}`", s))
            }
        };

        // We always setup
        let ask = subcommand != MiriCommand::Setup;
        setup(ask);

        // Now run the command.
        for target in list_targets(std::env::args().skip(skip)) {
            let args = std::env::args().skip(skip);
            let kind = target.kind.get(0).expect(
                "badly formatted cargo metadata: target::kind is an empty array",
            );
            match (subcommand, &kind[..]) {
                (MiriCommand::Test, "test") => {
                    // For test binaries we call `cargo rustc --test target -- <rustc args>`
                    if let Err(code) = process(
                        vec!["--test".to_string(), target.name].into_iter().chain(
                            args,
                        ),
                    )
                    {
                        std::process::exit(code);
                    }
                }
                (MiriCommand::Test, "lib") => {
                    // For libraries we call `cargo rustc -- --test <rustc args>`
                    // Notice now that `--test` is a rustc arg rather than a cargo arg. This tells
                    // rustc to build a test harness which calls all #[test] functions. We don't
                    // use the harness since we execute each #[test] function's MIR ourselves before
                    // compilation even completes, but this option is necessary to build the library.
                    if let Err(code) = process(
                        vec!["--".to_string(), "--test".to_string()].into_iter().chain(
                            args,
                        ),
                    )
                    {
                        std::process::exit(code);
                    }
                }
                (MiriCommand::Run, "bin") => {
                    // For ordinary binaries we call `cargo rustc --bin target -- <rustc args>`
                    if let Err(code) = process(
                        vec!["--bin".to_string(), target.name].into_iter().chain(
                            args,
                        ),
                    )
                    {
                        std::process::exit(code);
                    }
                }
                _ => {}
            }
        }
    } else {
        // This arm is executed when cargo-miri runs `cargo rustc` with the `RUSTC` env var set to itself:
        // Dependencies get dispatched to rustc, the final test/binary to miri.

        let home = option_env!("RUSTUP_HOME").or(option_env!("MULTIRUST_HOME"));
        let toolchain = option_env!("RUSTUP_TOOLCHAIN").or(option_env!("MULTIRUST_TOOLCHAIN"));
        let sys_root = if let Ok(sysroot) = ::std::env::var("MIRI_SYSROOT") {
            sysroot
        } else if let (Some(home), Some(toolchain)) = (home, toolchain) {
            format!("{}/toolchains/{}", home, toolchain)
        } else {
            option_env!("RUST_SYSROOT")
                .map(|s| s.to_owned())
                .or_else(|| {
                    Command::new("rustc")
                        .arg("--print")
                        .arg("sysroot")
                        .output()
                        .ok()
                        .and_then(|out| String::from_utf8(out.stdout).ok())
                        .map(|s| s.trim().to_owned())
                })
                .expect("need to specify RUST_SYSROOT env var during miri compilation, or use rustup or multirust")
        };

        // this conditional check for the --sysroot flag is there so users can call `cargo-miri` directly
        // without having to pass --sysroot or anything
        let mut args: Vec<String> = if std::env::args().any(|s| s == "--sysroot") {
            std::env::args().skip(1).collect()
        } else {
            std::env::args()
                .skip(1)
                .chain(Some("--sysroot".to_owned()))
                .chain(Some(sys_root))
                .collect()
        };
        args.splice(0..0, miri::miri_default_args().iter().map(ToString::to_string));

        // this check ensures that dependencies are built but not interpreted and the final crate is
        // interpreted but not built
        let miri_enabled = std::env::args().any(|s| s == "--emit=dep-info,metadata");

        let mut command = if miri_enabled {
            let mut path = std::env::current_exe().expect("current executable path invalid");
            path.set_file_name("miri");
            Command::new(path)
        } else {
            Command::new("rustc")
        };

        args.extend_from_slice(&["--cfg".to_owned(), r#"feature="cargo-miri""#.to_owned()]);

        match command.args(&args).status() {
            Ok(exit) => {
                if !exit.success() {
                    std::process::exit(exit.code().unwrap_or(42));
                }
            }
            Err(ref e) if miri_enabled => panic!("error during miri run: {:?}", e),
            Err(ref e) => panic!("error during rustc call: {:?}", e),
        }
    }
}

fn process<I>(old_args: I) -> Result<(), i32>
where
    I: Iterator<Item = String>,
{
    let mut args = vec!["rustc".to_owned()];

    let mut found_dashes = false;
    for arg in old_args {
        found_dashes |= arg == "--";
        args.push(arg);
    }
    if !found_dashes {
        args.push("--".to_owned());
    }
    args.push("--emit=dep-info,metadata".to_owned());
    args.push("--cfg".to_owned());
    args.push(r#"feature="cargo-miri""#.to_owned());

    let path = std::env::current_exe().expect("current executable path invalid");
    let exit_status = std::process::Command::new("cargo")
        .args(&args)
        .env("RUSTC", path)
        .spawn()
        .expect("could not run cargo")
        .wait()
        .expect("failed to wait for cargo?");

    if exit_status.success() {
        Ok(())
    } else {
        Err(exit_status.code().unwrap_or(-1))
    }
}
<|endoftext|>


<commit_before>pub use self::config::Config;
pub use self::process_builder::{process, ProcessBuilder};
pub use self::result::{Wrap, Require};
pub use self::errors::{CargoResult, CargoError, BoxError, ChainError, CliResult};
pub use self::errors::{CliError, FromError, ProcessError};
pub use self::errors::{process_error, internal_error, internal, human, caused_human};
pub use self::paths::realpath;
pub use self::hex::{to_hex, short_hash};
pub use self::pool::TaskPool;
pub use self::dependency_queue::{DependencyQueue, Fresh, Dirty, Freshness};
pub use self::dependency_queue::Dependency;
pub use self::graph::Graph;
pub use self::to_url::ToUrl;
pub use self::to_semver::ToSemver;
pub use self::vcs::{GitRepo, HgRepo};
pub use self::sha256::Sha256;

pub mod graph;
pub mod process_builder;
pub mod config;
pub mod important_paths;
pub mod result;
pub mod toml;
pub mod paths;
pub mod errors;
pub mod hex;
pub mod profile;
pub mod to_semver;
mod pool;
mod dependency_queue;
mod to_url;
mod vcs;
mod sha256;
<commit_msg>Make to_url public for rustc<commit_after>pub use self::config::Config;
pub use self::process_builder::{process, ProcessBuilder};
pub use self::result::{Wrap, Require};
pub use self::errors::{CargoResult, CargoError, BoxError, ChainError, CliResult};
pub use self::errors::{CliError, FromError, ProcessError};
pub use self::errors::{process_error, internal_error, internal, human, caused_human};
pub use self::paths::realpath;
pub use self::hex::{to_hex, short_hash};
pub use self::pool::TaskPool;
pub use self::dependency_queue::{DependencyQueue, Fresh, Dirty, Freshness};
pub use self::dependency_queue::Dependency;
pub use self::graph::Graph;
pub use self::to_url::ToUrl;
pub use self::to_semver::ToSemver;
pub use self::vcs::{GitRepo, HgRepo};
pub use self::sha256::Sha256;

pub mod config;
pub mod errors;
pub mod graph;
pub mod hex;
pub mod important_paths;
pub mod paths;
pub mod process_builder;
pub mod profile;
pub mod result;
pub mod to_semver;
pub mod to_url;
pub mod toml;
mod dependency_queue;
mod pool;
mod sha256;
mod vcs;
<|endoftext|>