<commit_before>use std::collections::HashSet;
use std::io::Cursor;
use std::ops::Deref;
use std::rc::Rc;

use digest::digest_reader;
use serde_json;
use sha2::Sha256;
use url::Url;
use url_serde;

use resman::{AudioBuffer, ResMan};
use super::routegraph::RouteGraph;
use super::adjlist::AdjList;

#[derive(Debug)]
pub enum Error {
    /// No effect matches the metadata requested.
    NoMatchingEffect(EffectId),
}

/// Alias for a `Result` with our error type.
pub type ResultE<T> = Result<T, Error>;

/// Serializable info needed to look up an effect.
#[derive(Clone, Debug)]
#[derive(Serialize, Deserialize)]
pub struct EffectId {
    /// Canonical name of the effect
    name: String,
    /// Hash of the effect's definition file, or None if the effect is primitive
    sha256: Option<[u8; 32]>,
    // TODO: consider a smallset.
    // TODO: use #[serde(with = "url_serde")]
    /// List of URLs where the Effect can be obtained
    urls: HashSet<url_serde::Serde<Url>>,
}

/// All information that will be loaded from disk/network to describe how to
/// synthesize the Effect.
#[derive(Serialize, Deserialize)]
pub struct EffectDesc {
    meta: EffectMeta,
    adjlist: AdjList,
}

/// Validated version of EffectDesc. Guaranteed to be synthesizable.
pub struct Effect {
    id: EffectId,
    meta: EffectMeta,
    // TODO: Effects are immutable, so we can make this an AdjList and store
    // the connectivity information separately.
    // option, because effect MAY be primitive.
    data: EffectData,
}

#[derive(Clone, Debug)]
#[derive(Serialize, Deserialize)]
pub struct EffectMeta {
    /// Canonical name of the effect
    name: String,
    /// List of URLs where the Effect can be obtained
    urls: HashSet<url_serde::Serde<Url>>,
    inputs: Vec<EffectInput>,
    outputs: Vec<EffectOutput>,
}

#[derive(Clone, Debug)]
#[derive(Serialize, Deserialize)]
pub struct EffectIO {
    name: String,
    channel: u8,
}
pub type EffectInput = EffectIO;
pub type EffectOutput = EffectIO;

/// Implementation details of an Effect.
/// Either its route graph, a primitive effect, etc.
pub enum EffectData {
    RouteGraph(RouteGraph),
    Primitive(PrimitiveEffect),
    Buffer(AudioBuffer),
}

/// Effects that cannot be decomposed; they have no implementation details and
/// must be implemented directly by the renderer.
#[derive(Debug, Copy, Clone)]
pub enum PrimitiveEffect {
    /// Primitive Delay effect
    Delay,
    /// Primitive Constant effect.
    /// Also serves as a unit step;
    /// Returns the float value for t >= 0, else 0.
    F32Constant,
    /// Primitive effect to multiply TWO input streams sample-wise.
    Multiply,
    /// Primitive effect to calculate A/B.
    /// Note: because not all floating point numbers have inverses,
    /// A * (1/B) != A/B. Hence, we need division (not inversion) for proper
    /// precision.
    Divide,
    /// Primitive effect to calculate A%B (true modulo; not remainder.
    /// Result, y,  is always positive: y is bounded by [0, B).
    Modulo,
    /// Primitive effect to return the sample-wise minimum of two input streams.
    /// Max(A, B) can be implemented as -Min(-A, -B).
    /// The choice to define Min instead of Max was mostly arbitrary,
    /// and chosen because Min is more common in linear programming to avoid dealing
    /// with Inf.
    Minimum,
}

impl Effect {
    pub fn are_slots_connected(&self, from_slot: u32, to_slot: u32) -> bool {
        match self.data {
            EffectData::RouteGraph(ref g) => g.are_slots_connected(from_slot, to_slot),
            // For primitive effects, we assume ALL slots are connected.
            _ => true,
        }
    }
    pub fn id(&self) -> EffectId {
        EffectId {
            name: self.meta.name.clone(),
            sha256: None,
            urls: self.meta.urls.clone(),
        }
    }
    pub fn meta(&self) -> &EffectMeta {
        &self.meta
    }
    /// Given the effect's information, and an interface by which to load
    /// resources, return an actual Effect.
    pub fn from_id(id: EffectId, resman: &ResMan) -> ResultE<Rc<Self>> {
        // For primitive effects, don't attempt to locate their descriptions (they don't exist)
        let prim_effect = id.get_primitive_url().map(|url| match url.path() {
            "/F32Constant" => PrimitiveEffect::F32Constant,
            "/Delay" => PrimitiveEffect::Delay,
            "/Multiply" => PrimitiveEffect::Multiply,
            "/Divide" => PrimitiveEffect::Divide,
            "/Modulo" => PrimitiveEffect::Modulo,
            "/Minimum" => PrimitiveEffect::Minimum,
            _ => panic!("Unrecognized primitive effect: {} (full url: {})", url.path(), url),
        });
        if let Some(prim_effect) = prim_effect {
            let me = Self {
                meta: EffectMeta {
                    name: id.name.clone(),
                    urls: id.urls.clone(),
                    // Primitive effects have undocumented I/O;
                    inputs: Default::default(),
                    outputs: Default::default(),
                },
                id: id,
                data: EffectData::Primitive(prim_effect),
            };
            return Ok(Rc::new(me));
        }

        // Locate descriptions for non-primitive effects
        for (path, reader) in resman.find_effect(&id) {
            // Try to deserialize to an effect description
            let desc: Result<EffectDesc, serde_json::Error> = serde_json::from_reader(reader);
            match desc {
                Ok(desc) => {
                    // TODO: since we've matched the effect, we only need to recalculate
                    // the id if the original search had missing hashes.
                    let id = desc.id();
                    match RouteGraph::from_adjlist(desc.adjlist, resman) {
                        Ok(graph) => {
                            let me = Self {
                                id,
                                meta: desc.meta,
                                data: EffectData::RouteGraph(graph),
                            };
                            // TODO: implement some form of caching
                            return Ok(Rc::new(me));
                        },
                        Err(error) => println!("Warning: RouteGraph::from_adjlist failed: {:?}", error)
                    }
                },
                Err(error) => {
                    // Try parsing the file as an audio stream.
                    if let Ok(buffer) = AudioBuffer::from_path(path) {
                        let me = Self {
                            meta: EffectMeta {
                                name: id.name.clone(),
                                urls: id.urls.clone(),
                                // TODO: should be able to extract the number of outputs from the
                                // audio buffer
                                inputs: Default::default(),
                                outputs: Default::default(),
                            },
                            // TODO: refactor to avoid this clone.
                            id: id.clone(),
                            data: EffectData::Buffer(buffer),
                        };
                        // TODO: implement some form of caching
                        return Ok(Rc::new(me));
                    } else {
                        println!("Warning: unable to deserialize EffectDesc: {:?}", error)
                    }
                }
            }
        }
        // No matching effects
        Err(Error::NoMatchingEffect(id))
    }
    pub fn data(&self) -> &EffectData {
        &self.data
    }
}

impl EffectId {
    pub fn new<U>(name: String, sha256: Option<[u8; 32]>, urls: U) -> Self
        where U: IntoIterator<Item=Url>
    {
        Self {
            name,
            sha256,
            urls: urls.into_iter().map(|url| url_serde::Serde(url)).collect(),
        }
    }
    pub fn sha256(&self) -> &Option<[u8; 32]> {
        &self.sha256
    }
    /// Returns true if the effect cannot be decomposed.
    /// This is determined by the effect providing a SINGLE url, with the primitive:// Schema
    pub fn is_primitive(&self) -> bool {
        self.urls.len() == 1 && self.urls.iter().all(|url| {
            url.scheme() == "primitive"
        })
    }
    pub fn get_primitive_url(&self) -> Option<&Url> {
        if self.is_primitive() {
            self.urls.iter().next().map(|url| url.deref())
        } else {
            None
        }
    }
}

impl EffectDesc {
    pub fn new(meta: EffectMeta, adjlist: AdjList) -> Self {
        Self{ meta, adjlist }
    }
    pub fn id(&self) -> EffectId {
        // TODO: calculate sha using a smaller buffer
        let as_vec = serde_json::to_vec(self).unwrap();
        let result = digest_reader::<Sha256>(&mut Cursor::new(as_vec)).unwrap();
        let mut hash: [u8; 32] = Default::default();
        hash.copy_from_slice(result.as_slice());
        EffectId {
            name: self.meta.name.clone(),
            sha256: Some(hash),
            urls: self.meta.urls.clone(),
        }
    }
}

impl EffectMeta {
    pub fn new<U>(name: String, urls: U, inputs: Vec<EffectInput>, outputs: Vec<EffectOutput>) -> Self
        where U: IntoIterator<Item=Url>
    {
        Self {
            name,
            urls: urls.into_iter().map(|url| url_serde::Serde(url)).collect(),
            inputs,
            outputs,
        }
    }
    pub fn inputs(&self) -> &Vec<EffectInput> {
        &self.inputs
    }
    pub fn outputs(&self) -> &Vec<EffectOutput> {
        &self.outputs
    }
}

impl EffectIO {
    pub fn new(name: String, channel: u8) -> Self {
        Self{ name, channel }
    }
}

impl PartialEq for EffectMeta {
    // Equality implemented in a way where we can easily check things like
    //   "Is this the same primitive Delay effect this renderer knows how to implement?"
    fn eq(&self, other: &EffectMeta) -> bool {
        self.name == other.name &&
            self.urls == other.urls
    }
}
impl Eq for EffectMeta {}

impl PartialEq for Effect {
    fn eq(&self, other: &Effect) -> bool {
        self.meta == other.meta
    }
}
impl Eq for Effect {}

<commit_msg>Give way to filter I/Os<commit_after>use std::collections::HashSet;
use std::io::Cursor;
use std::ops::Deref;
use std::rc::Rc;

use digest::digest_reader;
use serde_json;
use sha2::Sha256;
use url::Url;
use url_serde;

use resman::{AudioBuffer, ResMan};
use super::routegraph::RouteGraph;
use super::adjlist::AdjList;

#[derive(Debug)]
pub enum Error {
    /// No effect matches the metadata requested.
    NoMatchingEffect(EffectId),
}

/// Alias for a `Result` with our error type.
pub type ResultE<T> = Result<T, Error>;

/// Serializable info needed to look up an effect.
#[derive(Clone, Debug)]
#[derive(Serialize, Deserialize)]
pub struct EffectId {
    /// Canonical name of the effect
    name: String,
    /// Hash of the effect's definition file, or None if the effect is primitive
    sha256: Option<[u8; 32]>,
    // TODO: consider a smallset.
    // TODO: use #[serde(with = "url_serde")]
    /// List of URLs where the Effect can be obtained
    urls: HashSet<url_serde::Serde<Url>>,
}

/// All information that will be loaded from disk/network to describe how to
/// synthesize the Effect.
#[derive(Serialize, Deserialize)]
pub struct EffectDesc {
    meta: EffectMeta,
    adjlist: AdjList,
}

/// Validated version of EffectDesc. Guaranteed to be synthesizable.
pub struct Effect {
    id: EffectId,
    meta: EffectMeta,
    // TODO: Effects are immutable, so we can make this an AdjList and store
    // the connectivity information separately.
    // option, because effect MAY be primitive.
    data: EffectData,
}

#[derive(Clone, Debug)]
#[derive(Serialize, Deserialize)]
pub struct EffectMeta {
    /// Canonical name of the effect
    name: String,
    /// List of URLs where the Effect can be obtained
    urls: HashSet<url_serde::Serde<Url>>,
    inputs: Vec<EffectInput>,
    outputs: Vec<EffectOutput>,
}

#[derive(Clone, Debug)]
#[derive(Serialize, Deserialize)]
pub struct EffectIO {
    name: String,
    channel: u8,
}
pub type EffectInput = EffectIO;
pub type EffectOutput = EffectIO;

/// Implementation details of an Effect.
/// Either its route graph, a primitive effect, etc.
pub enum EffectData {
    RouteGraph(RouteGraph),
    Primitive(PrimitiveEffect),
    Buffer(AudioBuffer),
}

/// Effects that cannot be decomposed; they have no implementation details and
/// must be implemented directly by the renderer.
#[derive(Debug, Copy, Clone)]
pub enum PrimitiveEffect {
    /// Primitive Delay effect
    Delay,
    /// Primitive Constant effect.
    /// Also serves as a unit step;
    /// Returns the float value for t >= 0, else 0.
    F32Constant,
    /// Primitive effect to multiply TWO input streams sample-wise.
    Multiply,
    /// Primitive effect to calculate A/B.
    /// Note: because not all floating point numbers have inverses,
    /// A * (1/B) != A/B. Hence, we need division (not inversion) for proper
    /// precision.
    Divide,
    /// Primitive effect to calculate A%B (true modulo; not remainder.
    /// Result, y,  is always positive: y is bounded by [0, B).
    Modulo,
    /// Primitive effect to return the sample-wise minimum of two input streams.
    /// Max(A, B) can be implemented as -Min(-A, -B).
    /// The choice to define Min instead of Max was mostly arbitrary,
    /// and chosen because Min is more common in linear programming to avoid dealing
    /// with Inf.
    Minimum,
}

impl Effect {
    pub fn are_slots_connected(&self, from_slot: u32, to_slot: u32) -> bool {
        match self.data {
            EffectData::RouteGraph(ref g) => g.are_slots_connected(from_slot, to_slot),
            // For primitive effects, we assume ALL slots are connected.
            _ => true,
        }
    }
    pub fn id(&self) -> EffectId {
        EffectId {
            name: self.meta.name.clone(),
            sha256: None,
            urls: self.meta.urls.clone(),
        }
    }
    pub fn meta(&self) -> &EffectMeta {
        &self.meta
    }
    /// Given the effect's information, and an interface by which to load
    /// resources, return an actual Effect.
    pub fn from_id(id: EffectId, resman: &ResMan) -> ResultE<Rc<Self>> {
        // For primitive effects, don't attempt to locate their descriptions (they don't exist)
        let prim_effect = id.get_primitive_url().map(|url| match url.path() {
            "/F32Constant" => PrimitiveEffect::F32Constant,
            "/Delay" => PrimitiveEffect::Delay,
            "/Multiply" => PrimitiveEffect::Multiply,
            "/Divide" => PrimitiveEffect::Divide,
            "/Modulo" => PrimitiveEffect::Modulo,
            "/Minimum" => PrimitiveEffect::Minimum,
            _ => panic!("Unrecognized primitive effect: {} (full url: {})", url.path(), url),
        });
        if let Some(prim_effect) = prim_effect {
            let me = Self {
                meta: EffectMeta {
                    name: id.name.clone(),
                    urls: id.urls.clone(),
                    // Primitive effects have undocumented I/O;
                    inputs: Default::default(),
                    outputs: Default::default(),
                },
                id: id,
                data: EffectData::Primitive(prim_effect),
            };
            return Ok(Rc::new(me));
        }

        // Locate descriptions for non-primitive effects
        for (path, reader) in resman.find_effect(&id) {
            // Try to deserialize to an effect description
            let desc: Result<EffectDesc, serde_json::Error> = serde_json::from_reader(reader);
            match desc {
                Ok(desc) => {
                    // TODO: since we've matched the effect, we only need to recalculate
                    // the id if the original search had missing hashes.
                    let id = desc.id();
                    match RouteGraph::from_adjlist(desc.adjlist, resman) {
                        Ok(graph) => {
                            let me = Self {
                                id,
                                meta: desc.meta,
                                data: EffectData::RouteGraph(graph),
                            };
                            // TODO: implement some form of caching
                            return Ok(Rc::new(me));
                        },
                        Err(error) => println!("Warning: RouteGraph::from_adjlist failed: {:?}", error)
                    }
                },
                Err(error) => {
                    // Try parsing the file as an audio stream.
                    if let Ok(buffer) = AudioBuffer::from_path(path) {
                        let me = Self {
                            meta: EffectMeta {
                                name: id.name.clone(),
                                urls: id.urls.clone(),
                                // TODO: should be able to extract the number of outputs from the
                                // audio buffer
                                inputs: Default::default(),
                                outputs: Default::default(),
                            },
                            // TODO: refactor to avoid this clone.
                            id: id.clone(),
                            data: EffectData::Buffer(buffer),
                        };
                        // TODO: implement some form of caching
                        return Ok(Rc::new(me));
                    } else {
                        println!("Warning: unable to deserialize EffectDesc: {:?}", error)
                    }
                }
            }
        }
        // No matching effects
        Err(Error::NoMatchingEffect(id))
    }
    pub fn data(&self) -> &EffectData {
        &self.data
    }
}

impl EffectId {
    pub fn new<U>(name: String, sha256: Option<[u8; 32]>, urls: U) -> Self
        where U: IntoIterator<Item=Url>
    {
        Self {
            name,
            sha256,
            urls: urls.into_iter().map(|url| url_serde::Serde(url)).collect(),
        }
    }
    pub fn sha256(&self) -> &Option<[u8; 32]> {
        &self.sha256
    }
    /// Returns true if the effect cannot be decomposed.
    /// This is determined by the effect providing a SINGLE url, with the primitive:// Schema
    pub fn is_primitive(&self) -> bool {
        self.urls.len() == 1 && self.urls.iter().all(|url| {
            url.scheme() == "primitive"
        })
    }
    pub fn get_primitive_url(&self) -> Option<&Url> {
        if self.is_primitive() {
            self.urls.iter().next().map(|url| url.deref())
        } else {
            None
        }
    }
}

impl EffectDesc {
    pub fn new(meta: EffectMeta, adjlist: AdjList) -> Self {
        Self{ meta, adjlist }
    }
    pub fn id(&self) -> EffectId {
        // TODO: calculate sha using a smaller buffer
        let as_vec = serde_json::to_vec(self).unwrap();
        let result = digest_reader::<Sha256>(&mut Cursor::new(as_vec)).unwrap();
        let mut hash: [u8; 32] = Default::default();
        hash.copy_from_slice(result.as_slice());
        EffectId {
            name: self.meta.name.clone(),
            sha256: Some(hash),
            urls: self.meta.urls.clone(),
        }
    }
}

impl EffectMeta {
    pub fn new<U>(name: String, urls: U, inputs: Vec<EffectInput>, outputs: Vec<EffectOutput>) -> Self
        where U: IntoIterator<Item=Url>
    {
        Self {
            name,
            urls: urls.into_iter().map(|url| url_serde::Serde(url)).collect(),
            inputs,
            outputs,
        }
    }
    pub fn inputs(&self) -> &Vec<EffectInput> {
        &self.inputs
    }
    pub fn outputs(&self) -> &Vec<EffectOutput> {
        &self.outputs
    }
    pub fn inputs_by_name<'a>(&'a self, name: &'a str) -> impl Iterator<Item=&EffectInput> + 'a {
        self.inputs.iter().filter(move |item| item.name() == name)
    }
    pub fn outputs_by_name<'a>(&'a self, name: &'a str) -> impl Iterator<Item=&EffectOutput> + 'a {
        self.outputs.iter().filter(move |item| item.name() == name)
    }
}

impl EffectIO {
    pub fn new(name: String, channel: u8) -> Self {
        Self{ name, channel }
    }
    pub fn name(&self) -> &str {
        &self.name
    }
    pub fn channel(&self) -> u8 {
        self.channel
    }
}

impl PartialEq for EffectMeta {
    // Equality implemented in a way where we can easily check things like
    //   "Is this the same primitive Delay effect this renderer knows how to implement?"
    fn eq(&self, other: &EffectMeta) -> bool {
        self.name == other.name &&
            self.urls == other.urls
    }
}
impl Eq for EffectMeta {}

impl PartialEq for Effect {
    fn eq(&self, other: &Effect) -> bool {
        self.meta == other.meta
    }
}
impl Eq for Effect {}

<|endoftext|>


<commit_before>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Implementation of the `build` subcommand, used to compile a book.

use std::env;
use std::fs::{self, File};
use std::io::prelude::*;
use std::io::{self, BufWriter};
use std::path::{Path, PathBuf};
use rustc_back::tempdir::TempDir;

use subcommand::Subcommand;
use term::Term;
use error::{err, CliResult, CommandResult};
use book;
use book::{Book, BookItem};
use css;
use javascript;

use rustdoc;

struct Build;

pub fn parse_cmd(name: &str) -> Option<Box<Subcommand>> {
    if name == "build" {
        Some(Box::new(Build))
    } else {
        None
    }
}

fn write_toc(book: &Book, path_to_root: &Path, out: &mut Write) -> io::Result<()> {
    fn walk_items(items: &[BookItem],
                  section: &str,
                  path_to_root: &Path,
                  out: &mut Write) -> io::Result<()> {
        for (i, item) in items.iter().enumerate() {
            try!(walk_item(item, &format!("{}{}.", section, i + 1)[..], path_to_root, out));
        }
        Ok(())
    }
    fn walk_item(item: &BookItem,
                 section: &str,
                 path_to_root: &Path,
                 out: &mut Write) -> io::Result<()> {
        try!(writeln!(out, "
{} {}",
                 path_to_root.join(&item.path.with_extension("html")).display(),
                 section,
                 item.title));
        if !item.children.is_empty() {
            try!(writeln!(out, "
"));
            let _ = walk_items(&item.children[..], section, path_to_root, out);
            try!(writeln!(out, "
"));
        }
        try!(writeln!(out, "
"));

        Ok(())
    }

    try!(writeln!(out, "
"));
    try!(writeln!(out, "
"));
    try!(walk_items(&book.chapters[..], "", path_to_root, out));
    try!(writeln!(out, "
"));
    try!(writeln!(out, "
"));

    Ok(())
}

fn render(book: &Book, tgt: &Path) -> CliResult<()> {
    let tmp = try!(TempDir::new("rust-book"));

    for (_section, item) in book.iter() {
        let out_path = match item.path.parent() {
            Some(p) => tgt.join(p),
            None => tgt.to_path_buf(),
        };

        let src;
        if env::args().len() < 3 {
            src = env::current_dir().unwrap().clone();
        } else {
            src = PathBuf::from(&env::args().nth(2).unwrap());
        }
        // preprocess the markdown, rerouting markdown references to html
        // references
        let mut markdown_data = String::new();
        try!(File::open(&src.join(&item.path)).and_then(|mut f| {
            f.read_to_string(&mut markdown_data)
        }));
        let preprocessed_path = tmp.path().join(item.path.file_name().unwrap());
        {
            let urls = markdown_data.replace(".md)", ".html)");
            try!(File::create(&preprocessed_path).and_then(|mut f| {
                f.write_all(urls.as_bytes())
            }));
        }

        // write the prelude to a temporary HTML file for rustdoc inclusion
        let prelude = tmp.path().join("prelude.html");
        {
            let mut toc = BufWriter::new(try!(File::create(&prelude)));
            try!(writeln!(&mut toc, r#"

                <button id="toggle-nav">
                  Toggle navigation

                </button>

"#));
            let _ = write_toc(book, &item.path_to_root, &mut toc);
            try!(writeln!(&mut toc, "
"));
            try!(writeln!(&mut toc, "
"));
        }

        // write the postlude to a temporary HTML file for rustdoc inclusion
        let postlude = tmp.path().join("postlude.html");
        {
            let mut toc = BufWriter::new(try!(File::create(&postlude)));
            try!(toc.write_all(javascript::JAVASCRIPT.as_bytes()));
            try!(writeln!(&mut toc, "

"));
        }

        try!(fs::create_dir_all(&out_path));

        let rustdoc_args: &[String] = &[
            "".to_string(),
            preprocessed_path.display().to_string(),
            format!("-o{}", out_path.display()),
            format!("--html-before-content={}", prelude.display()),
            format!("--html-after-content={}", postlude.display()),
            format!("--markdown-playground-url=http://play.rust-lang.org"),
            format!("--markdown-css={}", item.path_to_root.join("rust-book.css").display()),
            "--markdown-no-toc".to_string(),
        ];
        let output_result = rustdoc::main_args(rustdoc_args);
        if output_result != 0 {
            let message = format!("Could not execute `rustdoc` with {:?}: {}",
                                  rustdoc_args, output_result);
            return Err(err(&message));
        }
    }

    // create index.html from the root README
    try!(fs::copy(&tgt.join("README.html"), &tgt.join("index.html")));

    // Copy some js for playpen
    let mut jquery = try!(File::create(tgt.join("jquery.js")));
    let js = include_bytes!("../librustdoc/html/static/jquery-2.1.0.min.js");
    try!(jquery.write_all(js));
    let mut playpen = try!(File::create(tgt.join("playpen.js")));
    let js = include_bytes!("../librustdoc/html/static/playpen.js");
    try!(playpen.write_all(js));
    Ok(())
}

impl Subcommand for Build {
    fn parse_args(&mut self, _: &[String]) -> CliResult<()> {
        Ok(())
    }
    fn usage(&self) {}
    fn execute(&mut self, term: &mut Term) -> CommandResult<()> {
        let cwd = env::current_dir().unwrap();
        let src;
        let tgt;

        if env::args().len() < 3 {
            src = cwd.clone();
        } else {
            src = PathBuf::from(&env::args().nth(2).unwrap());
        }

        if env::args().len() < 4 {
            tgt = cwd.join("_book");
        } else {
            tgt = PathBuf::from(&env::args().nth(3).unwrap());
        }

        try!(fs::create_dir(&tgt));

        try!(File::create(&tgt.join("rust-book.css")).and_then(|mut f| {
            f.write_all(css::STYLE.as_bytes())
        }));

        let mut summary = try!(File::open(&src.join("SUMMARY.md")));
        match book::parse_summary(&mut summary, &src) {
            Ok(book) => {
                // execute rustdoc on the whole book
                render(&book, &tgt)
            }
            Err(errors) => {
                let n = errors.len();
                for err in errors {
                    term.err(&format!("error: {}", err)[..]);
                }

                Err(err(&format!("{} errors occurred", n)))
            }
        }
    }
}
<commit_msg>Delete `_book` folder if it exists from previous runs<commit_after>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Implementation of the `build` subcommand, used to compile a book.

use std::env;
use std::fs::{self, File};
use std::io::prelude::*;
use std::io::{self, BufWriter};
use std::path::{Path, PathBuf};
use rustc_back::tempdir::TempDir;

use subcommand::Subcommand;
use term::Term;
use error::{err, CliResult, CommandResult};
use book;
use book::{Book, BookItem};
use css;
use javascript;

use rustdoc;

struct Build;

pub fn parse_cmd(name: &str) -> Option<Box<Subcommand>> {
    if name == "build" {
        Some(Box::new(Build))
    } else {
        None
    }
}

fn write_toc(book: &Book, path_to_root: &Path, out: &mut Write) -> io::Result<()> {
    fn walk_items(items: &[BookItem],
                  section: &str,
                  path_to_root: &Path,
                  out: &mut Write) -> io::Result<()> {
        for (i, item) in items.iter().enumerate() {
            try!(walk_item(item, &format!("{}{}.", section, i + 1)[..], path_to_root, out));
        }
        Ok(())
    }
    fn walk_item(item: &BookItem,
                 section: &str,
                 path_to_root: &Path,
                 out: &mut Write) -> io::Result<()> {
        try!(writeln!(out, "
{} {}",
                 path_to_root.join(&item.path.with_extension("html")).display(),
                 section,
                 item.title));
        if !item.children.is_empty() {
            try!(writeln!(out, "
"));
            let _ = walk_items(&item.children[..], section, path_to_root, out);
            try!(writeln!(out, "
"));
        }
        try!(writeln!(out, "
"));

        Ok(())
    }

    try!(writeln!(out, "
"));
    try!(writeln!(out, "
"));
    try!(walk_items(&book.chapters[..], "", path_to_root, out));
    try!(writeln!(out, "
"));
    try!(writeln!(out, "
"));

    Ok(())
}

fn render(book: &Book, tgt: &Path) -> CliResult<()> {
    let tmp = try!(TempDir::new("rust-book"));

    for (_section, item) in book.iter() {
        let out_path = match item.path.parent() {
            Some(p) => tgt.join(p),
            None => tgt.to_path_buf(),
        };

        let src;
        if env::args().len() < 3 {
            src = env::current_dir().unwrap().clone();
        } else {
            src = PathBuf::from(&env::args().nth(2).unwrap());
        }
        // preprocess the markdown, rerouting markdown references to html
        // references
        let mut markdown_data = String::new();
        try!(File::open(&src.join(&item.path)).and_then(|mut f| {
            f.read_to_string(&mut markdown_data)
        }));
        let preprocessed_path = tmp.path().join(item.path.file_name().unwrap());
        {
            let urls = markdown_data.replace(".md)", ".html)");
            try!(File::create(&preprocessed_path).and_then(|mut f| {
                f.write_all(urls.as_bytes())
            }));
        }

        // write the prelude to a temporary HTML file for rustdoc inclusion
        let prelude = tmp.path().join("prelude.html");
        {
            let mut toc = BufWriter::new(try!(File::create(&prelude)));
            try!(writeln!(&mut toc, r#"

                <button id="toggle-nav">
                  Toggle navigation

                </button>

"#));
            let _ = write_toc(book, &item.path_to_root, &mut toc);
            try!(writeln!(&mut toc, "
"));
            try!(writeln!(&mut toc, "
"));
        }

        // write the postlude to a temporary HTML file for rustdoc inclusion
        let postlude = tmp.path().join("postlude.html");
        {
            let mut toc = BufWriter::new(try!(File::create(&postlude)));
            try!(toc.write_all(javascript::JAVASCRIPT.as_bytes()));
            try!(writeln!(&mut toc, "

"));
        }

        try!(fs::create_dir_all(&out_path));

        let rustdoc_args: &[String] = &[
            "".to_string(),
            preprocessed_path.display().to_string(),
            format!("-o{}", out_path.display()),
            format!("--html-before-content={}", prelude.display()),
            format!("--html-after-content={}", postlude.display()),
            format!("--markdown-playground-url=http://play.rust-lang.org"),
            format!("--markdown-css={}", item.path_to_root.join("rust-book.css").display()),
            "--markdown-no-toc".to_string(),
        ];
        let output_result = rustdoc::main_args(rustdoc_args);
        if output_result != 0 {
            let message = format!("Could not execute `rustdoc` with {:?}: {}",
                                  rustdoc_args, output_result);
            return Err(err(&message));
        }
    }

    // create index.html from the root README
    try!(fs::copy(&tgt.join("README.html"), &tgt.join("index.html")));

    // Copy some js for playpen
    let mut jquery = try!(File::create(tgt.join("jquery.js")));
    let js = include_bytes!("../librustdoc/html/static/jquery-2.1.0.min.js");
    try!(jquery.write_all(js));
    let mut playpen = try!(File::create(tgt.join("playpen.js")));
    let js = include_bytes!("../librustdoc/html/static/playpen.js");
    try!(playpen.write_all(js));
    Ok(())
}

impl Subcommand for Build {
    fn parse_args(&mut self, _: &[String]) -> CliResult<()> {
        Ok(())
    }
    fn usage(&self) {}
    fn execute(&mut self, term: &mut Term) -> CommandResult<()> {
        let cwd = env::current_dir().unwrap();
        let src;
        let tgt;

        if env::args().len() < 3 {
            src = cwd.clone();
        } else {
            src = PathBuf::from(&env::args().nth(2).unwrap());
        }

        if env::args().len() < 4 {
            tgt = cwd.join("_book");
        } else {
            tgt = PathBuf::from(&env::args().nth(3).unwrap());
        }

        // `_book` directory may already exist from previous runs. Check and
        // delete it if it exists.
        for entry in try!(fs::read_dir(&cwd)) {
            let path = try!(entry).path();
            if path == tgt { try!(fs::remove_dir_all(&tgt)) }
        }
        try!(fs::create_dir(&tgt));

        try!(File::create(&tgt.join("rust-book.css")).and_then(|mut f| {
            f.write_all(css::STYLE.as_bytes())
        }));

        let mut summary = try!(File::open(&src.join("SUMMARY.md")));
        match book::parse_summary(&mut summary, &src) {
            Ok(book) => {
                // execute rustdoc on the whole book
                render(&book, &tgt)
            }
            Err(errors) => {
                let n = errors.len();
                for err in errors {
                    term.err(&format!("error: {}", err)[..]);
                }

                Err(err(&format!("{} errors occurred", n)))
            }
        }
    }
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

//! Windows implementation of `tutil::screen`.
//!
//! Currently there are no tests written for this implementation.

use super::{Width, Height};

use winapi::{DWORD, HANDLE};
use winapi::{STD_INPUT_HANDLE, STD_OUTPUT_HANDLE};
use winapi::{CONSOLE_SCREEN_BUFFER_INFO, COORD, SMALL_RECT};
use kernel32::{GetStdHandle, GetConsoleScreenBufferInfo};

/// Returns the terminal screen size.
///
/// Returns `None` if the screen size is `(0, 0)` or is not able to be
/// determined.
pub fn size() -> Option<(Width, Height)> {
    // Retrieve a handle to STDOUT.
    let handle: HANDLE = unsafe { GetStdHandle(STD_OUTPUT_HANDLE) };

    // An empty COORD struct for use with CONSOLE_SCREEN_BUFFER_INFO.
    let coord = COORD { X: 0, Y: 0 };

    // An empty CONSOLE_SCREEN_BUFFER_INFO struct for use with
    // GetConsoleScreenBufferInfo.
    let mut csbi = CONSOLE_SCREEN_BUFFER_INFO {
        // A COORD structure that contains the size of the console screen
        // buffer, in character columns and rows.
        dwSize: coord.clone(),

        // A COORD structure that contains the column and row coordinates of
        // the cursor in the console screen buffer.
        dwCursorPosition: coord.clone(),

        // The attributes of the characters written to a screen buffer by the
        // WriteFile and WriteConsole functions, or echoed to a screen buffer
        // by the ReadFile and ReadConsole functions.
        wAttributes: 0,

        // A SMALL_RECT structure that contains the console screen buffer
        // coordinates of the upper-left and lower-right corners of the
        // display window.
        srWindow: SMALL_RECT{ Left: 0, Top: 0, Right: 0, Bottom: 0 },

        // A COORD structure that contains the maximum size of the console
        // window, in character columns and rows, given the current screen
        // buffer size and font and the screen size.
        dwMaximumWindowSize: coord,
    };

    let success: bool = unsafe {
        GetConsoleScreenBufferInfo(handle, &mut csbi) != 0
    };

    if success {
        let width = Width((csbi.srWindow.Right - csbi.srWindow.Left + 1) as u16);
        let height = Height((csbi.srWindow.Bottom - csbi.srWindow.Top + 1) as u16);
        Some((width, height))
    } else {
        None
    }
}

/// Returns the terminal screen width.
///
/// Returns `None` if the terminal width is detected as being <= 0 columns or is
/// not able to be determined at all.
pub fn width() -> Option<Width> {
    let size = size();

    if let Some((Width(width), Height(_))) = size {
        Some(Width(width))
    } else {
        None
    }
}

/// Returns the terminal height.
///
/// Returns `None` if the terminal height is detected as being <= 0 rows or is
/// not able to be determined at all.
pub fn height() -> Option<Height> {
    let size = size();

    if let Some((Width(_), Height(height))) = size {
        Some(Height(height))
    } else {
        None
    }
}

#[cfg(test)]
mod test {
    // TODO: Write tests.
}
<commit_msg>Remove unused imports<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

//! Windows implementation of `tutil::screen`.
//!
//! Currently there are no tests written for this implementation.

use super::{Width, Height};

use winapi::{HANDLE, STD_OUTPUT_HANDLE, COORD, SMALL_RECT, CONSOLE_SCREEN_BUFFER_INFO};
use kernel32::{GetStdHandle, GetConsoleScreenBufferInfo};

/// Returns the terminal screen size.
///
/// Returns `None` if the screen size is `(0, 0)` or is not able to be
/// determined.
pub fn size() -> Option<(Width, Height)> {
    // Retrieve a handle to STDOUT.
    let handle: HANDLE = unsafe { GetStdHandle(STD_OUTPUT_HANDLE) };

    // An empty COORD struct for use with CONSOLE_SCREEN_BUFFER_INFO.
    let coord = COORD { X: 0, Y: 0 };

    // An empty CONSOLE_SCREEN_BUFFER_INFO struct for use with
    // GetConsoleScreenBufferInfo.
    let mut csbi = CONSOLE_SCREEN_BUFFER_INFO {
        // A COORD structure that contains the size of the console screen
        // buffer, in character columns and rows.
        dwSize: coord.clone(),

        // A COORD structure that contains the column and row coordinates of
        // the cursor in the console screen buffer.
        dwCursorPosition: coord.clone(),

        // The attributes of the characters written to a screen buffer by the
        // WriteFile and WriteConsole functions, or echoed to a screen buffer
        // by the ReadFile and ReadConsole functions.
        wAttributes: 0,

        // A SMALL_RECT structure that contains the console screen buffer
        // coordinates of the upper-left and lower-right corners of the
        // display window.
        srWindow: SMALL_RECT{ Left: 0, Top: 0, Right: 0, Bottom: 0 },

        // A COORD structure that contains the maximum size of the console
        // window, in character columns and rows, given the current screen
        // buffer size and font and the screen size.
        dwMaximumWindowSize: coord,
    };

    let success: bool = unsafe {
        GetConsoleScreenBufferInfo(handle, &mut csbi) != 0
    };

    if success {
        let width = Width((csbi.srWindow.Right - csbi.srWindow.Left + 1) as u16);
        let height = Height((csbi.srWindow.Bottom - csbi.srWindow.Top + 1) as u16);
        Some((width, height))
    } else {
        None
    }
}

/// Returns the terminal screen width.
///
/// Returns `None` if the terminal width is detected as being <= 0 columns or is
/// not able to be determined at all.
pub fn width() -> Option<Width> {
    let size = size();

    if let Some((Width(width), Height(_))) = size {
        Some(Width(width))
    } else {
        None
    }
}

/// Returns the terminal height.
///
/// Returns `None` if the terminal height is detected as being <= 0 rows or is
/// not able to be determined at all.
pub fn height() -> Option<Height> {
    let size = size();

    if let Some((Width(_), Height(height))) = size {
        Some(Height(height))
    } else {
        None
    }
}

#[cfg(test)]
mod test {
    // TODO: Write tests.
}
<|endoftext|>


<commit_before>// This program watches a configured directory for incoming files.
// When a file is detected the following procedure is used to handle it:
//
//  * First we digest the file, and add the digest to our manifest.
//    This way we can intelligently handle future events for this file
//
//  * Next we attempt to detect a mime type using various heuristics, incl:
//
//    - Quick detection w/ our magic bytes table
//    - Querying external tools (e.g: ffmpeg, imagemagick, et al.)
//    - etc.
//
//  * If we cannot handle the mime type: the file is moved to an exception
//    directory, and an error is logged somewhere the user will see it.
//
//  * If we can handle the mime type: the file is moved to the content store
//    and an entry is created in the database.
//
//    - This should ideally be done atomically so that other aqua utility
//      processes (e.g: sister agnes) don't mistakenly mark the file as an
//      orphan / missing / etc.
//

#[macro_use] extern crate log;
#[macro_use] extern crate serde_derive;

extern crate aqua;
extern crate clap;
extern crate diesel;
extern crate dotenv;
extern crate env_logger;
extern crate image;
extern crate notify;
extern crate serde;
extern crate serde_json;

use aqua::controllers::prelude::hash_file;
use aqua::models::{Entry, NewEntry};
use aqua::schema;
use clap::{Arg, App};
use diesel::prelude::*;
use diesel::pg::PgConnection;
use dotenv::dotenv;
use notify::{DebouncedEvent, Watcher, RecursiveMode, watcher};
use std::{env, fmt, process};
use std::collections::HashMap;
use std::convert::From;
use std::error::Error;
use std::fs::{self, File, OpenOptions};
use std::io::{self, Read, Write};
use std::path::{Path, PathBuf};
use std::sync::mpsc::channel;
use std::time::Duration;

#[derive(Debug)]
enum ProcessingError {
    DigestFailed,
    DetectionFailed,
    ThumbnailFailed,

    DbConnErr(diesel::ConnectionError),
    DbQueryErr(diesel::result::Error),
    IoErr(io::Error),
    ImageErr(image::ImageError),
    Misc(Box<Error>),
}

impl Error for ProcessingError {
    fn description(&self) -> &str {
        match *self {
            // internal errors
            ProcessingError::DigestFailed      => "Unhandled error while generating SHA256 digest",
            ProcessingError::DetectionFailed   => "The file's type could not be detected",
            ProcessingError::ThumbnailFailed   => "The thumbnail could not be generated",

            // external errors
            ProcessingError::DbConnErr(ref inner)  => inner.description(),
            ProcessingError::DbQueryErr(ref inner) => inner.description(),
            ProcessingError::IoErr(ref inner)      => inner.description(),
            ProcessingError::ImageErr(ref inner)   => inner.description(),
            ProcessingError::Misc(ref inner)       => inner.description(),
        }
    }

    fn cause(&self) -> Option<&Error> {
        match *self {
            ProcessingError::DbConnErr(ref err)  => Some(err),
            ProcessingError::DbQueryErr(ref err) => Some(err),
            ProcessingError::IoErr(ref err)      => Some(err),
            ProcessingError::ImageErr(ref err)   => Some(err),
            ProcessingError::Misc(ref err)       => Some(err.as_ref()),
            _ => None,
        }
    }
}

impl fmt::Display for ProcessingError {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        match *self {
            // TODO: better display impl.
            _ => write!(f, "{}", self.description()),
        }
    }
}

impl From<diesel::ConnectionError> for ProcessingError {
    fn from(err: diesel::ConnectionError) -> Self { ProcessingError::DbConnErr(err) }
}

impl From<diesel::result::Error> for ProcessingError {
    fn from(err: diesel::result::Error) -> Self { ProcessingError::DbQueryErr(err) }
}

impl From<image::ImageError> for ProcessingError {
    fn from(err: image::ImageError) -> Self { ProcessingError::ImageErr(err) }
}

impl From<io::Error> for ProcessingError {
    fn from(err: io::Error) -> Self { ProcessingError::IoErr(err) }
}

impl From<serde_json::Error> for ProcessingError {
    fn from(err: serde_json::Error) -> Self { ProcessingError::Misc(Box::new(err)) }
}

impl From<std::string::FromUtf8Error> for ProcessingError {
    fn from(err: std::string::FromUtf8Error) -> Self { ProcessingError::Misc(Box::new(err)) }
}

fn main() {
    dotenv().expect("must provide .env file, see README (TODO: haha jk)");
    env_logger::init().expect("could not initialize console logging");

    // read command line arguments
    let matches = App::new("aqua-watch")
        .version("0.1.0")
        .author("himechi <hime@localhost>")
        .about("Watches a directory for new files and moves them to the `aqua` content store.")
        .arg(Arg::with_name("INPUT")
             .help("Determines the input directory to be watched.")
             .required(true)
             .index(1))
        .arg(Arg::with_name("OUTPUT")
             .help("The root of the aqua content store where files will be moved.")
             .required(true)
             .index(2))
        .get_matches();

    let dropbox_path  = matches.value_of("INPUT").unwrap();
    let content_store = matches.value_of("OUTPUT").unwrap();

    // setup fs watcher
    let (fs_tx, fs_rx) = channel();
    let mut fs_watcher = watcher(fs_tx, Duration::from_millis(1000))
        .expect("could not create file system watcher!");

    fs_watcher.watch(dropbox_path, RecursiveMode::NonRecursive)
        .expect("could not enroll dropbox in fs events queue");

    // process filesystem events ...
    loop {
        match fs_rx.recv() {
            Ok(DebouncedEvent::Create(path)) => {
                if path.is_file() {
                    match handle_new_file(path, content_store) {
                        Ok(_res) => info!("file processed successfully ..."),
                        Err(msg) => warn!("could not process file: {:?} (inner: {:?})", msg, msg.cause()),
                    };
                }
                else { info!("directory created, ignoring ..."); }
            },
            Ok(event) => info!("unhandled evt: {:?}", event),
            Err(msg) => warn!("fs err: {}", msg),
        }
    }
}

fn handle_new_file(path: PathBuf, content_store: &str) -> Result<(), ProcessingError> {
    let digest = hash_file(path.as_path())
        .ok_or(ProcessingError::DigestFailed)?;

    let mut file = OpenOptions::new()
        .read(true)
        .write(false)
        .create_new(false)
        .open(path.as_path())?;

    // read file into memory
    let mut buf = vec![];
    file.read_to_end(&mut buf)?;

    // TODO: move_file() & db() is probably going to be common to all handlers?
    if let Some(image_metadata) = aqua::util::mime_detect(&buf) {
        info!("got an image ...");
        process_image(content_store, &digest, &buf)?;
        move_file(path.as_path(), content_store, &digest, image_metadata.extension())?;

        let db_entry = create_db_entry(&digest, image_metadata.mime())?;
        info!("inserted: {:?} into database", db_entry);

        Ok(())
    } else if let Some(ffmpeg_metadata) = ffmpeg_detect(path.as_path())? {
        info!("got an video ...");
        process_video(content_store, &digest, &path)?;
        move_file(path.as_path(), content_store, &digest, ffmpeg_metadata.ext)?;

        let db_entry = create_db_entry(&digest, ffmpeg_metadata.mime)?;
        info!("inserted: {:?} into database", db_entry);

        Ok(())
    } else {
        Err(ProcessingError::DetectionFailed)
    }
}

#[derive(Debug, Serialize, Deserialize)]
struct FFProbeResult {
    format: Option<FFProbeFormat>,
    streams: Option<Vec<FFProbeStream>>,
}

#[derive(Debug, Serialize, Deserialize)]
struct FFProbeFormat {
    filename:    String,
    nb_streams:  i32,
    format_name: String,
    start_time:  String, // NOTE: these appear to be fixed decimal
    duration:    String, // NOTE: these appear to be fixed decimal
    size:        String, // NOTE: appears to be an integer
    bit_rate:    String, // NOTE: appears to be an integer
    probe_score: i32,    // NOTE: accuracy of the detection? (???)

    tags: HashMap<String, String>, // NOTE: not sure this is correct type
}

#[derive(Debug, Serialize, Deserialize)]
struct FFProbeStream {
    codec_name: String,
    codec_type: String,
}

struct FFProbeMeta {
    pub mime: &'static str,
    pub ext:  &'static str,
}

fn ffmpeg_detect(path: &Path) -> Result<Option<FFProbeMeta>, ProcessingError> {
    let ffprobe_cmd = process::Command::new("ffprobe")
        .arg("-v").arg("quiet")            // silence debug output
        .arg("-hide_banner")               // don't print ffmpeg configuration
        .arg("-print_format").arg("json") // serialize to json
        .arg("-show_format")              // display format data
        .arg("-show_streams")             // display stream data
        .arg("-i").arg(path.as_os_str())  // set the input to current file
        .output()?;

    let json_str = String::from_utf8(ffprobe_cmd.stdout)?;
    let probe_result: FFProbeResult = serde_json::from_str(&json_str)?;
    info!("got result: {:?}", probe_result);

    // see if ffprobe was able to determine the file type ...
    let probe_format = match probe_result.format {
        Some(format_data) => format_data,
        None => return Ok(None),
    };

    let probe_streams = match probe_result.streams {
        Some(stream_data) => stream_data,
        None => return Ok(None),
    };

    // welp ... guess there's nothing to thumbnail (!!!)
    info!("got format: {:?}", probe_format);
    info!("got streams: {:?}", probe_streams);

    let number_of_videos = probe_streams
        .iter()
        .filter(|el| el.codec_type == "video")
        .count();

    if number_of_videos <= 0 { return Err(ProcessingError::DetectionFailed) }

    // TODO: how do we correlate format_name w/ stream & stream position?
    // TODO: I believe this should be matching on containers (which is what will be moved
    //       to the content store; and therefore what will be played back ...)
    //
    let meta_data = if probe_format.format_name.contains("matroska") {
        Some(FFProbeMeta { mime: "video/x-matroska", ext: "mkv" })
    } else if probe_format.format_name.contains("mp4") {
        Some(FFProbeMeta { mime: "video/mp4", ext: "mp4" })
    } else { None };

    Ok(meta_data)
}

fn establish_connection() -> Result<PgConnection, ProcessingError> {
    let database_url = env::var("DATABASE_URL")
        .expect("DATABASE_URL not set in `.env` file !!!");

    Ok(PgConnection::establish(&database_url)?)
}

// create entry in database
fn create_db_entry(digest: &str, mime_ty: &str) -> Result<Entry, ProcessingError> {
    let pg_conn = establish_connection()?;
    let aqua_entry = NewEntry { hash: &digest, mime: Some(&mime_ty) };
    let entry: Result<Entry, diesel::result::Error> = diesel::insert(&aqua_entry)
        .into(schema::entries::table)
        .get_result(&pg_conn);

    Ok(entry?)
}

// moves the file from `src_path` to the `content_store` based on its digest
fn move_file(src_path: &Path, content_store: &str, digest: &str, file_ext: &str) -> Result<(), ProcessingError> {
    // carve out a bucket based on first byte of SHA256 digest
    // create the bucket if it does not exist
    let file_bucket    = format!("f{}", &digest[0..2]);
    let file_filename  = format!("{}.{}", &digest, file_ext);

    // create destination path
    let dest = PathBuf::from(content_store)
        .join(file_bucket)
        .join(file_filename);

    // TODO: bad error type ...
    let bucket_dir = dest.parent().ok_or(ProcessingError::ThumbnailFailed)?;
    fs::create_dir_all(bucket_dir)?;

    // move the file
    Ok(fs::rename(src_path, &dest)?)
}

// creates a thumbnail in the content store for the specified digest
// this expects an `ImageMeta` structure describing the input.
fn process_image(content_store: &str, digest: &str, buf: &[u8]) -> Result<(), ProcessingError> {
    // create in memory thumbnail
    let image = image::load_from_memory(&buf)?;

    let thumb = image.resize(200, 200, image::FilterType::Nearest);
    let thumb_bucket   = format!("t{}", &digest[0..2]);
    let thumb_filename = format!("{}.thumbnail", &digest);

    // store them in content store
    let dest = PathBuf::from(content_store)
        .join(thumb_bucket)
        .join(thumb_filename);

    // write thumbnail file to disk
    fs::create_dir_all(dest.parent().unwrap())?;
    let mut dest_file = File::create(dest)?;
    thumb.save(&mut dest_file, image::ImageFormat::JPEG)?;
    Ok(dest_file.flush()?)
}

fn process_video(content_store: &str, digest: &str, src: &Path) -> Result<(), ProcessingError> {
    let thumb_bucket   = format!("t{}", &digest[0..2]);
    let thumb_filename = format!("{}.thumbnail", &digest);

    // store them in content store
    let dest = PathBuf::from(content_store)
        .join(thumb_bucket)
        .join(thumb_filename);

    // TODO: seems weird to have "mjpeg" in here... but I couldn't find any other
    //       JPEG muxer/encoder in my ffmpeg install ...
    //
    // write thumbnail file to disk
    fs::create_dir_all(dest.parent().unwrap())?;
    let ffmpeg_cmd = process::Command::new("ffmpeg")
        .arg("-i").arg(src.as_os_str())            // the input file
        .arg("-vf").arg("thumbnail,scale=200:200") // have ffmpeg seek for a "thumbnail"
        .arg("-frames:v").arg("1")                 // take a single frame
        .arg("-f").arg("mjpeg")                    // save it as jpeg
        .arg(dest.as_path().as_os_str())           // into the content store
        .output()?;

    debug!("ffmpeg stderr: {}", String::from_utf8_lossy(&ffmpeg_cmd.stderr));
    debug!("ffmpeg stdout: {}", String::from_utf8_lossy(&ffmpeg_cmd.stdout));

    info!("digest? {}", digest);
    info!("dest exists? {:?} => {}", dest, dest.is_file());

    match dest.is_file() {
        true  => Ok(()),
        false => Err(ProcessingError::ThumbnailFailed),
    }
}
<commit_msg>check error on creating content buckets<commit_after>// This program watches a configured directory for incoming files.
// When a file is detected the following procedure is used to handle it:
//
//  * First we digest the file, and add the digest to our manifest.
//    This way we can intelligently handle future events for this file
//
//  * Next we attempt to detect a mime type using various heuristics, incl:
//
//    - Quick detection w/ our magic bytes table
//    - Querying external tools (e.g: ffmpeg, imagemagick, et al.)
//    - etc.
//
//  * If we cannot handle the mime type: the file is moved to an exception
//    directory, and an error is logged somewhere the user will see it.
//
//  * If we can handle the mime type: the file is moved to the content store
//    and an entry is created in the database.
//
//    - This should ideally be done atomically so that other aqua utility
//      processes (e.g: sister agnes) don't mistakenly mark the file as an
//      orphan / missing / etc.
//

#[macro_use] extern crate log;
#[macro_use] extern crate serde_derive;

extern crate aqua;
extern crate clap;
extern crate diesel;
extern crate dotenv;
extern crate env_logger;
extern crate image;
extern crate notify;
extern crate serde;
extern crate serde_json;

use aqua::controllers::prelude::hash_file;
use aqua::models::{Entry, NewEntry};
use aqua::schema;
use clap::{Arg, App};
use diesel::prelude::*;
use diesel::pg::PgConnection;
use dotenv::dotenv;
use notify::{DebouncedEvent, Watcher, RecursiveMode, watcher};
use std::{env, fmt, process};
use std::collections::HashMap;
use std::convert::From;
use std::error::Error;
use std::fs::{self, File, OpenOptions};
use std::io::{self, Read, Write};
use std::path::{Path, PathBuf};
use std::sync::mpsc::channel;
use std::time::Duration;

#[derive(Debug)]
enum ProcessingError {
    DigestFailed,
    DetectionFailed,
    ThumbnailFailed,

    DbConnErr(diesel::ConnectionError),
    DbQueryErr(diesel::result::Error),
    IoErr(io::Error),
    ImageErr(image::ImageError),
    Misc(Box<Error>),
}

impl Error for ProcessingError {
    fn description(&self) -> &str {
        match *self {
            // internal errors
            ProcessingError::DigestFailed      => "Unhandled error while generating SHA256 digest",
            ProcessingError::DetectionFailed   => "The file's type could not be detected",
            ProcessingError::ThumbnailFailed   => "The thumbnail could not be generated",

            // external errors
            ProcessingError::DbConnErr(ref inner)  => inner.description(),
            ProcessingError::DbQueryErr(ref inner) => inner.description(),
            ProcessingError::IoErr(ref inner)      => inner.description(),
            ProcessingError::ImageErr(ref inner)   => inner.description(),
            ProcessingError::Misc(ref inner)       => inner.description(),
        }
    }

    fn cause(&self) -> Option<&Error> {
        match *self {
            ProcessingError::DbConnErr(ref err)  => Some(err),
            ProcessingError::DbQueryErr(ref err) => Some(err),
            ProcessingError::IoErr(ref err)      => Some(err),
            ProcessingError::ImageErr(ref err)   => Some(err),
            ProcessingError::Misc(ref err)       => Some(err.as_ref()),
            _ => None,
        }
    }
}

impl fmt::Display for ProcessingError {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        match *self {
            // TODO: better display impl.
            _ => write!(f, "{}", self.description()),
        }
    }
}

impl From<diesel::ConnectionError> for ProcessingError {
    fn from(err: diesel::ConnectionError) -> Self { ProcessingError::DbConnErr(err) }
}

impl From<diesel::result::Error> for ProcessingError {
    fn from(err: diesel::result::Error) -> Self { ProcessingError::DbQueryErr(err) }
}

impl From<image::ImageError> for ProcessingError {
    fn from(err: image::ImageError) -> Self { ProcessingError::ImageErr(err) }
}

impl From<io::Error> for ProcessingError {
    fn from(err: io::Error) -> Self { ProcessingError::IoErr(err) }
}

impl From<serde_json::Error> for ProcessingError {
    fn from(err: serde_json::Error) -> Self { ProcessingError::Misc(Box::new(err)) }
}

impl From<std::string::FromUtf8Error> for ProcessingError {
    fn from(err: std::string::FromUtf8Error) -> Self { ProcessingError::Misc(Box::new(err)) }
}

fn main() {
    dotenv().expect("must provide .env file, see README (TODO: haha jk)");
    env_logger::init().expect("could not initialize console logging");

    // read command line arguments
    let matches = App::new("aqua-watch")
        .version("0.1.0")
        .author("himechi <hime@localhost>")
        .about("Watches a directory for new files and moves them to the `aqua` content store.")
        .arg(Arg::with_name("INPUT")
             .help("Determines the input directory to be watched.")
             .required(true)
             .index(1))
        .arg(Arg::with_name("OUTPUT")
             .help("The root of the aqua content store where files will be moved.")
             .required(true)
             .index(2))
        .get_matches();

    let dropbox_path  = matches.value_of("INPUT").unwrap();
    let content_store = matches.value_of("OUTPUT").unwrap();

    // setup fs watcher
    let (fs_tx, fs_rx) = channel();
    let mut fs_watcher = watcher(fs_tx, Duration::from_millis(1000))
        .expect("could not create file system watcher!");

    fs_watcher.watch(dropbox_path, RecursiveMode::NonRecursive)
        .expect("could not enroll dropbox in fs events queue");

    // process filesystem events ...
    loop {
        match fs_rx.recv() {
            Ok(DebouncedEvent::Create(path)) => {
                if path.is_file() {
                    match handle_new_file(path, content_store) {
                        Ok(_res) => info!("file processed successfully ..."),
                        Err(msg) => warn!("could not process file: {:?} (inner: {:?})", msg, msg.cause()),
                    };
                }
                else { info!("directory created, ignoring ..."); }
            },
            Ok(event) => info!("unhandled evt: {:?}", event),
            Err(msg) => warn!("fs err: {}", msg),
        }
    }
}

// TODO: check that file doesn't exist before moving it ...
fn handle_new_file(path: PathBuf, content_store: &str) -> Result<(), ProcessingError> {
    let digest = hash_file(path.as_path())
        .ok_or(ProcessingError::DigestFailed)?;

    let mut file = OpenOptions::new()
        .read(true)
        .write(false)
        .create_new(false)
        .open(path.as_path())?;

    // read file into memory
    let mut buf = vec![];
    file.read_to_end(&mut buf)?;

    // TODO: move_file() & db() is probably going to be common to all handlers?
    if let Some(image_metadata) = aqua::util::mime_detect(&buf) {
        info!("got an image ...");
        process_image(content_store, &digest, &buf)?;
        move_file(path.as_path(), content_store, &digest, image_metadata.extension())?;

        let db_entry = create_db_entry(&digest, image_metadata.mime())?;
        info!("inserted: {:?} into database", db_entry);

        Ok(())
    } else if let Some(ffmpeg_metadata) = ffmpeg_detect(path.as_path())? {
        info!("got an video ...");
        process_video(content_store, &digest, &path)?;
        move_file(path.as_path(), content_store, &digest, ffmpeg_metadata.ext)?;

        let db_entry = create_db_entry(&digest, ffmpeg_metadata.mime)?;
        info!("inserted: {:?} into database", db_entry);

        Ok(())
    } else {
        Err(ProcessingError::DetectionFailed)
    }
}

#[derive(Debug, Serialize, Deserialize)]
struct FFProbeResult {
    format: Option<FFProbeFormat>,
    streams: Option<Vec<FFProbeStream>>,
}

#[derive(Debug, Serialize, Deserialize)]
struct FFProbeFormat {
    filename:    String,
    nb_streams:  i32,
    format_name: String,
    start_time:  String, // NOTE: these appear to be fixed decimal
    duration:    String, // NOTE: these appear to be fixed decimal
    size:        String, // NOTE: appears to be an integer
    bit_rate:    String, // NOTE: appears to be an integer
    probe_score: i32,    // NOTE: accuracy of the detection? (???)

    tags: HashMap<String, String>, // NOTE: not sure this is correct type
}

#[derive(Debug, Serialize, Deserialize)]
struct FFProbeStream {
    codec_name: String,
    codec_type: String,
}

struct FFProbeMeta {
    pub mime: &'static str,
    pub ext:  &'static str,
}

fn ffmpeg_detect(path: &Path) -> Result<Option<FFProbeMeta>, ProcessingError> {
    let ffprobe_cmd = process::Command::new("ffprobe")
        .arg("-v").arg("quiet")            // silence debug output
        .arg("-hide_banner")               // don't print ffmpeg configuration
        .arg("-print_format").arg("json") // serialize to json
        .arg("-show_format")              // display format data
        .arg("-show_streams")             // display stream data
        .arg("-i").arg(path.as_os_str())  // set the input to current file
        .output()?;

    let json_str = String::from_utf8(ffprobe_cmd.stdout)?;
    let probe_result: FFProbeResult = serde_json::from_str(&json_str)?;
    info!("got result: {:?}", probe_result);

    // see if ffprobe was able to determine the file type ...
    let probe_format = match probe_result.format {
        Some(format_data) => format_data,
        None => return Ok(None),
    };

    let probe_streams = match probe_result.streams {
        Some(stream_data) => stream_data,
        None => return Ok(None),
    };

    // welp ... guess there's nothing to thumbnail (!!!)
    info!("got format: {:?}", probe_format);
    info!("got streams: {:?}", probe_streams);

    let number_of_videos = probe_streams
        .iter()
        .filter(|el| el.codec_type == "video")
        .count();

    if number_of_videos <= 0 { return Err(ProcessingError::DetectionFailed) }

    // TODO: how do we correlate format_name w/ stream & stream position?
    // TODO: I believe this should be matching on containers (which is what will be moved
    //       to the content store; and therefore what will be played back ...)
    //
    let meta_data = if probe_format.format_name.contains("matroska") {
        Some(FFProbeMeta { mime: "video/x-matroska", ext: "mkv" })
    } else if probe_format.format_name.contains("mp4") {
        Some(FFProbeMeta { mime: "video/mp4", ext: "mp4" })
    } else { None };

    Ok(meta_data)
}

fn establish_connection() -> Result<PgConnection, ProcessingError> {
    let database_url = env::var("DATABASE_URL")
        .expect("DATABASE_URL not set in `.env` file !!!");

    Ok(PgConnection::establish(&database_url)?)
}

// create entry in database
fn create_db_entry(digest: &str, mime_ty: &str) -> Result<Entry, ProcessingError> {
    let pg_conn = establish_connection()?;
    let aqua_entry = NewEntry { hash: &digest, mime: Some(&mime_ty) };
    let entry: Result<Entry, diesel::result::Error> = diesel::insert(&aqua_entry)
        .into(schema::entries::table)
        .get_result(&pg_conn);

    Ok(entry?)
}

// moves the file from `src_path` to the `content_store` based on its digest
fn move_file(src_path: &Path, content_store: &str, digest: &str, file_ext: &str) -> Result<(), ProcessingError> {
    // carve out a bucket based on first byte of SHA256 digest
    // create the bucket if it does not exist
    let file_bucket    = format!("f{}", &digest[0..2]);
    let file_filename  = format!("{}.{}", &digest, file_ext);

    // create destination path
    let dest = PathBuf::from(content_store)
        .join(file_bucket)
        .join(file_filename);

    // TODO: bad error type ...
    let bucket_dir = dest.parent().ok_or(ProcessingError::ThumbnailFailed)?;
    fs::create_dir_all(bucket_dir)?;

    // move the file
    Ok(fs::rename(src_path, &dest)?)
}

// creates a thumbnail in the content store for the specified digest
// this expects an `ImageMeta` structure describing the input.
fn process_image(content_store: &str, digest: &str, buf: &[u8]) -> Result<(), ProcessingError> {
    // create in memory thumbnail
    let image = image::load_from_memory(&buf)?;

    let thumb = image.resize(200, 200, image::FilterType::Nearest);
    let thumb_bucket   = format!("t{}", &digest[0..2]);
    let thumb_filename = format!("{}.thumbnail", &digest);

    // store them in content store
    let dest = PathBuf::from(content_store)
        .join(thumb_bucket)
        .join(thumb_filename);

    // write thumbnail file to disk
    let bucket_dir = dest.parent().ok_or(ProcessingError::ThumbnailFailed)?;
    fs::create_dir_all(bucket_dir)?;
    let mut dest_file = File::create(&dest)?;
    thumb.save(&mut dest_file, image::ImageFormat::JPEG)?;
    Ok(dest_file.flush()?)
}

fn process_video(content_store: &str, digest: &str, src: &Path) -> Result<(), ProcessingError> {
    let thumb_bucket   = format!("t{}", &digest[0..2]);
    let thumb_filename = format!("{}.thumbnail", &digest);

    // store them in content store
    let dest = PathBuf::from(content_store)
        .join(thumb_bucket)
        .join(thumb_filename);

    // TODO: seems weird to have "mjpeg" in here... but I couldn't find any other
    //       JPEG muxer/encoder in my ffmpeg install ...
    //
    // write thumbnail file to disk
    let bucket_dir = dest.parent().ok_or(ProcessingError::ThumbnailFailed)?;
    fs::create_dir_all(bucket_dir)?;
    let ffmpeg_cmd = process::Command::new("ffmpeg")
        .arg("-i").arg(src.as_os_str())            // the input file
        .arg("-vf").arg("thumbnail,scale=200:200") // have ffmpeg seek for a "thumbnail"
        .arg("-frames:v").arg("1")                 // take a single frame
        .arg("-f").arg("mjpeg")                    // save it as jpeg
        .arg(dest.as_path().as_os_str())           // into the content store
        .output()?;

    debug!("ffmpeg stderr: {}", String::from_utf8_lossy(&ffmpeg_cmd.stderr));
    debug!("ffmpeg stdout: {}", String::from_utf8_lossy(&ffmpeg_cmd.stdout));

    info!("digest? {}", digest);
    info!("dest exists? {:?} => {}", dest, dest.is_file());

    match dest.is_file() {
        true  => Ok(()),
        false => Err(ProcessingError::ThumbnailFailed),
    }
}
<|endoftext|>


<commit_before>use argparse::{ArgumentParser, Store, List, StoreTrue, StoreFalse, Print};
use config::types::Config;

pub struct Arguments {
  pub files: Vec<String>,
  pub message: String,
  pub service: String,
  pub private: bool,
  pub auth: bool
}

pub fn get_arguments(config: &Config) -> Arguments {
  let mut arguments = Arguments {
    files: Vec::new(),
    message: String::from(""),
    service: String::from(""),
    private: config.lookup_boolean_or("defaults.private", true),
    auth: config.lookup_boolean_or("default.auth", true)
  };
  {
    let mut ap = ArgumentParser::new();
    ap.set_description("paste a file, string, or pipe to a pastebin");
    ap.refer(&mut arguments.files)
      .add_argument("files", List, "files to paste")
      .required();
    ap.refer(&mut arguments.service)
      .add_option(&["-s", "--service"], Store, "pastebin service to use")
      .required();
    ap.refer(&mut arguments.message)
      .add_option(&["-m", "--message"], Store, "message to paste");
    ap.refer(&mut arguments.private)
      .add_option(&["-p", "--private"], StoreTrue, "if the paste should be private")
      .add_option(&["-P", "--public"], StoreFalse, "if the paste should be public");
    ap.refer(&mut arguments.auth)
      .add_option(&["-a", "--auth"], StoreTrue, "if authentication (like api keys and tokens) should be used")
      .add_option(&["-A", "--anon"], StoreFalse, "if pastes should be posted without authentication");
    ap.parse_args_or_exit();
  }
  arguments
}
<commit_msg>feat(arguments): add the --list-services (-l) option<commit_after>use argparse::{ArgumentParser, Store, List, StoreTrue, StoreFalse, Print};
use config::types::Config;

pub struct Arguments {
  pub files: Vec<String>,
  pub message: String,
  pub service: String,
  pub private: bool,
  pub auth: bool
}

pub fn get_arguments(config: &Config) -> Arguments {
  let mut arguments = Arguments {
    files: Vec::new(),
    message: String::from(""),
    service: String::from(""),
    private: config.lookup_boolean_or("defaults.private", true),
    auth: config.lookup_boolean_or("default.auth", true)
  };
  {
    let mut ap = ArgumentParser::new();
    ap.set_description("paste a file, string, or pipe to a pastebin");
    ap.refer(&mut arguments.files)
      .add_argument("files", List, "files to paste")
      .required();
    ap.refer(&mut arguments.service)
      .add_option(&["-s", "--service"], Store, "pastebin service to use")
      .required();
    ap.refer(&mut arguments.message)
      .add_option(&["-m", "--message"], Store, "message to paste");
    ap.refer(&mut arguments.private)
      .add_option(&["-p", "--private"], StoreTrue, "if the paste should be private")
      .add_option(&["-P", "--public"], StoreFalse, "if the paste should be public");
    ap.refer(&mut arguments.auth)
      .add_option(&["-a", "--auth"], StoreTrue, "if authentication (like api keys and tokens) should be used")
      .add_option(&["-A", "--anon"], StoreFalse, "if pastes should be posted without authentication");
    ap.add_option(
      &["-l", "--list-services"],
      Print(String::from("gist, hastebin, pastebin, pastie")),
      "lists pastebin services available"
    );
    ap.parse_args_or_exit();
  }
  arguments
}
<|endoftext|>


<commit_before>use bidir_map::BidirMap;
use crypto::sha2::Sha256;
use crypto::hmac::Hmac;
use crypto::mac::Mac;

use serde_json;
use serde_json::Value;
use serde_json::value::Map;

use std::thread;
use std::time::Duration;

use error::*;
use helpers;
use pair::Pair;
use pair::Pair::*;
use currency::Currency;

lazy_static! {
    static ref PAIRS_STRING: BidirMap<Pair, &'static str> = {
        let mut m = BidirMap::new();
        m.insert(BTC_USD, "btcusd");
        m.insert(BTC_EUR, "btceur");
        m.insert(EUR_USD, "eurusd");
        m.insert(XRP_USD, "xrpusd");
        m.insert(XRP_EUR, "xrpeur");
        m.insert(XRP_BTC, "xrpbtc");
        m
    };
}

/// Return the name associated to pair used by Bitstamp
/// If the Pair is not supported, None is returned.
pub fn get_pair_string(pair: &Pair) -> Option<&&str> {
    PAIRS_STRING.get_by_first(pair)
}

/// Return the Pair enum associated to the string used by Bitstamp
/// If the Pair is not supported, None is returned.
pub fn get_pair_enum(pair: &str) -> Option<&Pair> {
    PAIRS_STRING.get_by_second(&pair)
}

pub fn block_or_continue(last_request: i64) {
    let threshold = 1000; // 600 requests per 10 mins = 1 request per second
    let delay = helpers::get_unix_timestamp_ms() - last_request;
    if delay < threshold {
        let duration_ms = Duration::from_millis(delay as u64);
        thread::sleep(duration_ms);
    }
}

pub fn build_signature(nonce: &str,
                       customer_id: &str,
                       api_key: &str,
                       api_secret: &str)
                       -> Result<String> {
    const C: &'static [u8] = b"0123456789ABCDEF";

    let message = nonce.to_owned() + customer_id + api_key;
    let mut hmac = Hmac::new(Sha256::new(), api_secret.as_bytes());

    hmac.input(message.as_bytes());
    let result = hmac.result();

    let raw_signature = result.code();
    let mut signature = Vec::with_capacity(raw_signature.len() * 2);
    for &byte in raw_signature {
        signature.push(C[(byte >> 4) as usize]);
        signature.push(C[(byte & 0xf) as usize]);
    }
    // TODO: Handle correctly the from_utf8 errors with error_chain.
    Ok(String::from_utf8(signature)?)
}

pub fn build_url(method: &str, pair: &str) -> String {
    "https://www.bitstamp.net/api/v2/".to_string() + method + "/" + pair + "/"
}

pub fn deserialize_json(json_string: &str) -> Result<Map<String, Value>> {
    let data: Value = match serde_json::from_str(json_string) {
        Ok(data) => data,
        Err(_) => return Err(ErrorKind::BadParse.into()),
    };

    match data.as_object() {
        Some(value) => Ok(value.clone()),
        None => Err(ErrorKind::BadParse.into()),
    }
}

pub fn generate_nonce(fixed_nonce: Option<String>) -> String {
    match fixed_nonce {
        Some(v) => v,
        None => helpers::get_unix_timestamp_ms().to_string(),
    }
}

/// If error array is null, return the result (encoded in a json object)
/// else return the error string found in array
pub fn parse_result(response: &Map<String, Value>) -> Result<Map<String, Value>> {
    let error_msg = match response.get("error") {
        Some(error) => {
            error
                .as_str()
                .ok_or_else(|| ErrorKind::InvalidFieldFormat("error".to_string()))?
        }
        None => return Ok(response.clone()),
    };

    match error_msg.as_ref() {
        "Invalid command." => Err(ErrorKind::InvalidArguments.into()),
        "Invalid API key/secret pair." => Err(ErrorKind::BadCredentials.into()),
        "Total must be at least 0.0001." => Err(ErrorKind::InsufficientOrderSize.into()),
        other => Err(ErrorKind::ExchangeSpecificError(other.to_string()).into()),
    }
}

/// Return the currency enum associated with the
/// string used by Bitstamp. If no currency is found,
/// return None
pub fn get_currency_enum(currency: &str) -> Option<Currency> {
    match currency {
        "usd_balance" => Some(Currency::USD),
        "btc_balance" => Some(Currency::BTC),
        "eur_balance" => Some(Currency::EUR),
        "xrp_balance" => Some(Currency::XRP),
        _ => None,
    }
}
<commit_msg>Create a doc test for bitstamp get_currency_enum<commit_after>use bidir_map::BidirMap;
use crypto::sha2::Sha256;
use crypto::hmac::Hmac;
use crypto::mac::Mac;

use serde_json;
use serde_json::Value;
use serde_json::value::Map;

use std::thread;
use std::time::Duration;

use error::*;
use helpers;
use pair::Pair;
use pair::Pair::*;
use currency::Currency;

lazy_static! {
    static ref PAIRS_STRING: BidirMap<Pair, &'static str> = {
        let mut m = BidirMap::new();
        m.insert(BTC_USD, "btcusd");
        m.insert(BTC_EUR, "btceur");
        m.insert(EUR_USD, "eurusd");
        m.insert(XRP_USD, "xrpusd");
        m.insert(XRP_EUR, "xrpeur");
        m.insert(XRP_BTC, "xrpbtc");
        m
    };
}

/// Return the name associated to pair used by Bitstamp
/// If the Pair is not supported, None is returned.
pub fn get_pair_string(pair: &Pair) -> Option<&&str> {
    PAIRS_STRING.get_by_first(pair)
}

/// Return the Pair enum associated to the string used by Bitstamp
/// If the Pair is not supported, None is returned.
pub fn get_pair_enum(pair: &str) -> Option<&Pair> {
    PAIRS_STRING.get_by_second(&pair)
}

pub fn block_or_continue(last_request: i64) {
    let threshold = 1000; // 600 requests per 10 mins = 1 request per second
    let delay = helpers::get_unix_timestamp_ms() - last_request;
    if delay < threshold {
        let duration_ms = Duration::from_millis(delay as u64);
        thread::sleep(duration_ms);
    }
}

pub fn build_signature(nonce: &str,
                       customer_id: &str,
                       api_key: &str,
                       api_secret: &str)
                       -> Result<String> {
    const C: &'static [u8] = b"0123456789ABCDEF";

    let message = nonce.to_owned() + customer_id + api_key;
    let mut hmac = Hmac::new(Sha256::new(), api_secret.as_bytes());

    hmac.input(message.as_bytes());
    let result = hmac.result();

    let raw_signature = result.code();
    let mut signature = Vec::with_capacity(raw_signature.len() * 2);
    for &byte in raw_signature {
        signature.push(C[(byte >> 4) as usize]);
        signature.push(C[(byte & 0xf) as usize]);
    }
    // TODO: Handle correctly the from_utf8 errors with error_chain.
    Ok(String::from_utf8(signature)?)
}

pub fn build_url(method: &str, pair: &str) -> String {
    "https://www.bitstamp.net/api/v2/".to_string() + method + "/" + pair + "/"
}

pub fn deserialize_json(json_string: &str) -> Result<Map<String, Value>> {
    let data: Value = match serde_json::from_str(json_string) {
        Ok(data) => data,
        Err(_) => return Err(ErrorKind::BadParse.into()),
    };

    match data.as_object() {
        Some(value) => Ok(value.clone()),
        None => Err(ErrorKind::BadParse.into()),
    }
}

pub fn generate_nonce(fixed_nonce: Option<String>) -> String {
    match fixed_nonce {
        Some(v) => v,
        None => helpers::get_unix_timestamp_ms().to_string(),
    }
}

/// If error array is null, return the result (encoded in a json object)
/// else return the error string found in array
pub fn parse_result(response: &Map<String, Value>) -> Result<Map<String, Value>> {
    let error_msg = match response.get("error") {
        Some(error) => {
            error
                .as_str()
                .ok_or_else(|| ErrorKind::InvalidFieldFormat("error".to_string()))?
        }
        None => return Ok(response.clone()),
    };

    match error_msg.as_ref() {
        "Invalid command." => Err(ErrorKind::InvalidArguments.into()),
        "Invalid API key/secret pair." => Err(ErrorKind::BadCredentials.into()),
        "Total must be at least 0.0001." => Err(ErrorKind::InsufficientOrderSize.into()),
        other => Err(ErrorKind::ExchangeSpecificError(other.to_string()).into()),
    }
}

/// Return the currency enum associated with the
/// string used by Bitstamp. If no currency is found,
/// return None
/// # Examples
///
/// ```
/// use coinnect::bitstamp::utils::get_currency_enum;
/// use coinnect::currency::Currency;
///
/// let currency = get_currency_enum("usd_balance");
/// assert_eq!(Some(Currency::USD), currency);
/// ```
pub fn get_currency_enum(currency: &str) -> Option<Currency> {
    match currency {
        "usd_balance" => Some(Currency::USD),
        "btc_balance" => Some(Currency::BTC),
        "eur_balance" => Some(Currency::EUR),
        "xrp_balance" => Some(Currency::XRP),
        _ => None,
    }
}
<|endoftext|>


<commit_before>use nix::sys::ioctl::*;

// See C code for source of values for op calculations:
// https://gist.github.com/posborne/83ea6880770a1aef332e

#[test]
fn test_op_none() {
    assert_eq!(io!(b'q', 10), 0x0000710A);
    assert_eq!(io!(b'a', 255), 0x000061FF);
}

#[test]
fn test_op_write() {
    assert_eq!(iow!(b'z', 10, 1), 0x40017A0A);
    assert_eq!(iow!(b'z', 10, 512), 0x42007A0A);
}

#[cfg(target_pointer_width = "64")]
#[test]
fn test_op_write_64() {
    assert_eq!(iow!(b'z', 10, (1 as u64) << 32), 0x40007A0A);
}

#[test]
fn test_op_read() {
    assert_eq!(ior!(b'z', 10, 1), 0x80017A0A);
    assert_eq!(ior!(b'z', 10, 512), 0x82007A0A);
}

#[cfg(target_pointer_width = "64")]
#[test]
fn test_op_read_64() {
    assert_eq!(ior!(b'z', 10, (1 as u64) << 32), 0x80007A0A);
}

#[test]
fn test_op_read_write() {
    assert_eq!(iorw!(b'z', 10, 1), 0xC0017A0A);
    assert_eq!(iorw!(b'z', 10, 512), 0xC2007A0A);
}

#[cfg(target_pointer_width = "64")]
#[test]
fn test_op_read_write_64() {
    assert_eq!(iorw!(b'z', 10, (1 as u64) << 32), 0xC0007A0A);
}
<commit_msg>conditionally enable ioctl test for linux<commit_after>#![cfg(target_os = "linux")] // no ioctl support for osx yet
use nix::sys::ioctl::*;

// See C code for source of values for op calculations:
// https://gist.github.com/posborne/83ea6880770a1aef332e

#[test]
fn test_op_none() {
    assert_eq!(io!(b'q', 10), 0x0000710A);
    assert_eq!(io!(b'a', 255), 0x000061FF);
}

#[test]
fn test_op_write() {
    assert_eq!(iow!(b'z', 10, 1), 0x40017A0A);
    assert_eq!(iow!(b'z', 10, 512), 0x42007A0A);
}

#[cfg(target_pointer_width = "64")]
#[test]
fn test_op_write_64() {
    assert_eq!(iow!(b'z', 10, (1 as u64) << 32), 0x40007A0A);
}

#[test]
fn test_op_read() {
    assert_eq!(ior!(b'z', 10, 1), 0x80017A0A);
    assert_eq!(ior!(b'z', 10, 512), 0x82007A0A);
}

#[cfg(target_pointer_width = "64")]
#[test]
fn test_op_read_64() {
    assert_eq!(ior!(b'z', 10, (1 as u64) << 32), 0x80007A0A);
}

#[test]
fn test_op_read_write() {
    assert_eq!(iorw!(b'z', 10, 1), 0xC0017A0A);
    assert_eq!(iorw!(b'z', 10, 512), 0xC2007A0A);
}

#[cfg(target_pointer_width = "64")]
#[test]
fn test_op_read_write_64() {
    assert_eq!(iorw!(b'z', 10, (1 as u64) << 32), 0xC0007A0A);
}
<|endoftext|>


<commit_before>// Routines useful for garbage collection.

import lib::llvm::False;
import lib::llvm::True;
import lib::llvm::llvm::ValueRef;
import middle::trans::get_tydesc;
import middle::trans_common::*;
import middle::ty;
import std::option::none;
import std::option::some;
import std::ptr;
import std::str;
import std::unsafe;
import std::vec;

import lll = lib::llvm::llvm;

type ctxt = @{ mutable next_tydesc_num: uint };

fn mk_ctxt() -> ctxt {
    ret @{ mutable next_tydesc_num: 0u };
}

fn add_global(ccx: &@crate_ctxt, llval: ValueRef, name: str) -> ValueRef {
    let llglobal = lll::LLVMAddGlobal(ccx.llmod, val_ty(llval),
                                      str::buf(name));
    lll::LLVMSetInitializer(llglobal, llval);
    lll::LLVMSetGlobalConstant(llglobal, True);
    ret llglobal;
}

fn add_gc_root(cx: &@block_ctxt, llval: ValueRef, ty: ty::t) -> @block_ctxt {
    let bcx = cx;
    if !type_is_gc_relevant(bcx_tcx(cx), ty) ||
            ty::type_has_dynamic_size(bcx_tcx(cx), ty) {
        ret bcx;
    }

    let gc_cx = bcx_ccx(cx).gc_cx;

    let ti = none;
    let td_r = get_tydesc(bcx, ty, false, ti);
    bcx = td_r.result.bcx;
    let lltydesc = td_r.result.val;

    let gcroot = bcx_ccx(bcx).intrinsics.get("llvm.gcroot");
    let llvalptr = bcx.build.PointerCast(llval, T_ptr(T_ptr(T_i8())));

    alt td_r.kind {
        tk_derived. {
            // It's a derived type descriptor. First, spill it.
            let lltydescptr = trans::alloca(bcx, val_ty(lltydesc));
            bcx.build.Store(lltydesc, lltydescptr);

            let number = gc_cx.next_tydesc_num;
            gc_cx.next_tydesc_num += 1u;

            let lldestindex = add_global(bcx_ccx(bcx),
                                         C_struct(~[C_int(0),
                                                    C_uint(number)]),
                                         "rust_gc_tydesc_dest_index");
            let llsrcindex = add_global(bcx_ccx(bcx),
                                        C_struct(~[C_int(1), C_uint(number)]),
                                        "rust_gc_tydesc_src_index");

            lldestindex = lll::LLVMConstPointerCast(lldestindex,
                                                    T_ptr(T_i8()));
            llsrcindex = lll::LLVMConstPointerCast(llsrcindex,
                                                   T_ptr(T_i8()));

            lltydescptr = bcx.build.PointerCast(lltydescptr,
                                                T_ptr(T_ptr(T_i8())));

            bcx.build.Call(gcroot, ~[ lltydescptr, lldestindex ]);
            bcx.build.Call(gcroot, ~[ llvalptr, llsrcindex ]);
        }
        tk_param. {
            bcx_tcx(cx).sess.bug("we should never be trying to root values " +
                "of a type parameter");
        }
        tk_static. {
            // Static type descriptor.

            let llstaticgcmeta = add_global(bcx_ccx(bcx),
                                            C_struct(~[C_int(2), lltydesc]),
                                            "rust_gc_tydesc_static_gc_meta");
            let llstaticgcmetaptr = lll::LLVMConstPointerCast(llstaticgcmeta,
                                                              T_ptr(T_i8()));

            bcx.build.Call(gcroot, ~[ llvalptr, llstaticgcmetaptr ]);
        }
    }

    ret bcx;
}

fn type_is_gc_relevant(cx: &ty::ctxt, ty: &ty::t) -> bool {
    alt ty::struct(cx, ty) {
        ty::ty_nil. | ty::ty_bot. | ty::ty_bool. | ty::ty_int. |
        ty::ty_float. | ty::ty_uint. | ty::ty_machine(_) | ty::ty_char. |
        ty::ty_istr. | ty::ty_type. | ty::ty_native(_) | ty::ty_ptr(_) |
        ty::ty_type. | ty::ty_native(_) {
            ret false;
        }

        ty::ty_rec(fields) {
            for f in fields {
                if type_is_gc_relevant(cx, f.mt.ty) { ret true; }
            }
            ret false;
        }
        ty::ty_tup(elts) {
            for elt in elts {
                if type_is_gc_relevant(cx, elt) { ret true; }
            }
            ret false;
        }

        ty::ty_tag(did, tps) {
            let variants = ty::tag_variants(cx, did);
            for variant in variants {
                for aty in variant.args {
                    let arg_ty = ty::substitute_type_params(cx, tps, aty);
                    if type_is_gc_relevant(cx, arg_ty) { ret true; }
                }
            }
            ret false;
        }

        ty::ty_vec(tm) { ret type_is_gc_relevant(cx, tm.ty); }
        ty::ty_constr(sub, _) { ret type_is_gc_relevant(cx, sub); }

        ty::ty_str. | ty::ty_box(_) | ty::ty_uniq(_) |
        ty::ty_fn(_,_,_,_,_) | ty::ty_native_fn(_,_,_) | ty::ty_obj(_) |
        ty::ty_param(_,_) | ty::ty_res(_,_,_) { ret true; }

        ty::ty_var(_) { fail "ty_var in type_is_gc_relevant"; }
    }
}

<commit_msg>rustc: Unconditionally zero out all GC-relevant types for now<commit_after>// Routines useful for garbage collection.

import lib::llvm::False;
import lib::llvm::True;
import lib::llvm::llvm::ValueRef;
import middle::trans;
import middle::trans::get_tydesc;
import middle::trans_common::*;
import middle::ty;
import std::option::none;
import std::option::some;
import std::ptr;
import std::str;
import std::unsafe;
import std::vec;

import lll = lib::llvm::llvm;

type ctxt = @{ mutable next_tydesc_num: uint };

fn mk_ctxt() -> ctxt {
    ret @{ mutable next_tydesc_num: 0u };
}

fn add_global(ccx: &@crate_ctxt, llval: ValueRef, name: str) -> ValueRef {
    let llglobal = lll::LLVMAddGlobal(ccx.llmod, val_ty(llval),
                                      str::buf(name));
    lll::LLVMSetInitializer(llglobal, llval);
    lll::LLVMSetGlobalConstant(llglobal, True);
    ret llglobal;
}

fn add_gc_root(cx: &@block_ctxt, llval: ValueRef, ty: ty::t) -> @block_ctxt {
    let bcx = cx;
    if !type_is_gc_relevant(bcx_tcx(cx), ty) ||
            ty::type_has_dynamic_size(bcx_tcx(cx), ty) {
        ret bcx;
    }

    let gc_cx = bcx_ccx(cx).gc_cx;

    // FIXME (issue #839): For now, we are unconditionally zeroing out all
    // GC-relevant types. Eventually we should use typestate for this.
    bcx = trans::zero_alloca(bcx, llval, ty).bcx;

    let ti = none;
    let td_r = get_tydesc(bcx, ty, false, ti);
    bcx = td_r.result.bcx;
    let lltydesc = td_r.result.val;

    let gcroot = bcx_ccx(bcx).intrinsics.get("llvm.gcroot");
    let llvalptr = bcx.build.PointerCast(llval, T_ptr(T_ptr(T_i8())));

    alt td_r.kind {
        tk_derived. {
            // It's a derived type descriptor. First, spill it.
            let lltydescptr = trans::alloca(bcx, val_ty(lltydesc));
            bcx.build.Store(lltydesc, lltydescptr);

            let number = gc_cx.next_tydesc_num;
            gc_cx.next_tydesc_num += 1u;

            let lldestindex = add_global(bcx_ccx(bcx),
                                         C_struct(~[C_int(0),
                                                    C_uint(number)]),
                                         "rust_gc_tydesc_dest_index");
            let llsrcindex = add_global(bcx_ccx(bcx),
                                        C_struct(~[C_int(1), C_uint(number)]),
                                        "rust_gc_tydesc_src_index");

            lldestindex = lll::LLVMConstPointerCast(lldestindex,
                                                    T_ptr(T_i8()));
            llsrcindex = lll::LLVMConstPointerCast(llsrcindex,
                                                   T_ptr(T_i8()));

            lltydescptr = bcx.build.PointerCast(lltydescptr,
                                                T_ptr(T_ptr(T_i8())));

            bcx.build.Call(gcroot, ~[ lltydescptr, lldestindex ]);
            bcx.build.Call(gcroot, ~[ llvalptr, llsrcindex ]);
        }
        tk_param. {
            bcx_tcx(cx).sess.bug("we should never be trying to root values " +
                "of a type parameter");
        }
        tk_static. {
            // Static type descriptor.

            let llstaticgcmeta = add_global(bcx_ccx(bcx),
                                            C_struct(~[C_int(2), lltydesc]),
                                            "rust_gc_tydesc_static_gc_meta");
            let llstaticgcmetaptr = lll::LLVMConstPointerCast(llstaticgcmeta,
                                                              T_ptr(T_i8()));

            bcx.build.Call(gcroot, ~[ llvalptr, llstaticgcmetaptr ]);
        }
    }

    ret bcx;
}

fn type_is_gc_relevant(cx: &ty::ctxt, ty: &ty::t) -> bool {
    alt ty::struct(cx, ty) {
        ty::ty_nil. | ty::ty_bot. | ty::ty_bool. | ty::ty_int. |
        ty::ty_float. | ty::ty_uint. | ty::ty_machine(_) | ty::ty_char. |
        ty::ty_istr. | ty::ty_type. | ty::ty_native(_) | ty::ty_ptr(_) |
        ty::ty_type. | ty::ty_native(_) {
            ret false;
        }

        ty::ty_rec(fields) {
            for f in fields {
                if type_is_gc_relevant(cx, f.mt.ty) { ret true; }
            }
            ret false;
        }
        ty::ty_tup(elts) {
            for elt in elts {
                if type_is_gc_relevant(cx, elt) { ret true; }
            }
            ret false;
        }

        ty::ty_tag(did, tps) {
            let variants = ty::tag_variants(cx, did);
            for variant in variants {
                for aty in variant.args {
                    let arg_ty = ty::substitute_type_params(cx, tps, aty);
                    if type_is_gc_relevant(cx, arg_ty) { ret true; }
                }
            }
            ret false;
        }

        ty::ty_vec(tm) { ret type_is_gc_relevant(cx, tm.ty); }
        ty::ty_constr(sub, _) { ret type_is_gc_relevant(cx, sub); }

        ty::ty_str. | ty::ty_box(_) | ty::ty_uniq(_) |
        ty::ty_fn(_,_,_,_,_) | ty::ty_native_fn(_,_,_) | ty::ty_obj(_) |
        ty::ty_param(_,_) | ty::ty_res(_,_,_) { ret true; }

        ty::ty_var(_) { fail "ty_var in type_is_gc_relevant"; }
    }
}

<|endoftext|>