<commit_before>// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// CarbonReporter sends a message to a carbon end point at a regular basis.
use registry::{Registry, StdRegistry};
use std::time::Duration;
use std::thread;
use std::sync::Arc;
use reporter::Reporter;
use metrics::{CounterSnapshot, GaugeSnapshot, MeterSnapshot};
use histogram::Histogram;
use time;
use time::Timespec;
use std::net::TcpStream;
use std::io::Write;
use std::io::Error;

struct CarbonStream {
    graphite_stream: Option<TcpStream>,
    host_and_port: String,
}

// TODO perhaps we autodiscover the host and port
//
pub struct CarbonReporter {
    host_and_port: String,
    prefix: &'static str,
    registry: Arc<StdRegistry<'static>>,
    reporter_name: &'static str,
}

impl CarbonStream {
    pub fn new(host_and_port: String) -> Self {
        CarbonStream {
            host_and_port: host_and_port,
            graphite_stream: None,
        }
    }

    pub fn connect(&mut self) -> Result<String, Error> {
        let graphite_stream = try!(TcpStream::connect(&*self.host_and_port));
        self.graphite_stream = Some(graphite_stream);
        Ok(String::from(""))
    }

    pub fn write(&mut self,
                 metric_path: String,
                 value: String,
                 timespec: Timespec)
                 -> Result<String, Error> {
        let seconds_in_ms = (timespec.sec * 1000) as u32;
        let nseconds_in_ms = (timespec.nsec / 1000) as u32;
        let timestamp = seconds_in_ms + nseconds_in_ms;
        match self.graphite_stream {
            Some(ref mut stream) => {
                let carbon_command = format!("{} {} {}\n", metric_path, value, timestamp)
                    .into_bytes();
                try!(stream.write_all(&carbon_command));
            }
            None => {
                try!(self.reconnect_stream());
                try!(self.write(metric_path, value, timespec));
            }
        }
        Ok(String::from(""))
    }
    fn reconnect_stream(&mut self) -> Result<String, Error> {
        // TODO 123 is made up
        println!("Waiting 123ms and then reconnecting");
        thread::sleep(Duration::from_millis(123));
        self.connect()
    }
}

impl Reporter for CarbonReporter {
    fn get_unique_reporter_name(&self) -> &'static str {
        self.reporter_name
    }
}

fn prefix(metric_line: String, prefix_str: &'static str) -> String {
    format!("{}.{}", prefix_str, metric_line)
}

fn send_meter_metric(metric_name: &str,
                     meter: MeterSnapshot,
                     carbon: &mut CarbonStream,
                     prefix_str: &'static str,
                     ts: Timespec)
                     -> Result<String, Error> {

    let count = meter.count.to_string();
    let m1_rate = meter.rates[0].to_string();
    let m5_rate = meter.rates[1].to_string();
    let m15_rate = meter.rates[2].to_string();
    let mean_rate = meter.mean.to_string();
    try!(carbon.write(prefix(format!("{}.count", metric_name), prefix_str),
                      count,
                      ts));
    try!(carbon.write(prefix(format!("{}.m1", metric_name), prefix_str),
                      m1_rate,
                      ts));
    try!(carbon.write(prefix(format!("{}.m5", metric_name), prefix_str),
                      m5_rate,
                      ts));
    try!(carbon.write(prefix(format!("{}.m15", metric_name), prefix_str),
                      m15_rate,
                      ts));
    try!(carbon.write(prefix(format!("{}.mean", metric_name), prefix_str),
                      mean_rate,
                      ts));
    Ok(String::from(""))
}

fn send_gauge_metric(metric_name: &str,
                     gauge: GaugeSnapshot,
                     carbon: &mut CarbonStream,
                     prefix_str: &'static str,
                     ts: Timespec)
                     -> Result<String, Error> {
    try!(carbon.write(prefix(format!("{}", metric_name), prefix_str),
                      gauge.value.to_string(),
                      ts));
    Ok(String::from(""))
}

fn send_counter_metric(metric_name: &str,
                       counter: CounterSnapshot,
                       carbon: &mut CarbonStream,
                       prefix_str: &'static str,
                       ts: Timespec)
                       -> Result<String, Error> {
    try!(carbon.write(prefix(format!("{}", metric_name), prefix_str),
                      counter.value.to_string(),
                      ts));
    Ok(String::from(""))
}
fn send_histogram_metric(metric_name: &str,
                         histogram: &mut Histogram,
                         carbon: &mut CarbonStream,
                         prefix_str: &'static str,
                         ts: Timespec)
                         -> Result<String, Error> {
    let count = histogram.into_iter().count();
    // let sum = histogram.sum();
    // let mean = sum / count;
    let max = histogram.percentile(100.0).unwrap();
    let min = histogram.percentile(0.0).unwrap();

    let p50 = histogram.percentile(50.0).unwrap();
    let p75 = histogram.percentile(75.0).unwrap();
    let p95 = histogram.percentile(95.0).unwrap();
    let p98 = histogram.percentile(98.0).unwrap();
    let p99 = histogram.percentile(99.0).unwrap();
    let p999 = histogram.percentile(99.9).unwrap();
    let p9999 = histogram.percentile(99.99).unwrap();
    let p99999 = histogram.percentile(99.999).unwrap();

    try!(carbon.write(prefix(format!("{}.count", metric_name), prefix_str),
                      count.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.max", metric_name), prefix_str),
                      max.to_string(),
                      ts));

    // carbon
    // .write(prefix(format!("{}.mean", metric_name), prefix_str),
    // mean.into_string(),
    // ts);

    try!(carbon.write(prefix(format!("{}.min", metric_name), prefix_str),
                      min.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p50", metric_name), prefix_str),
                      p50.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p75", metric_name), prefix_str),
                      p75.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p95", metric_name), prefix_str),
                      p95.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p98", metric_name), prefix_str),
                      p98.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p99", metric_name), prefix_str),
                      p99.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p999", metric_name), prefix_str),
                      p999.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p9999", metric_name), prefix_str),
                      p9999.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p99999", metric_name), prefix_str),
                      p99999.to_string(),
                      ts));
    Ok(String::from(""))
}

impl CarbonReporter {
    pub fn new(registry: Arc<StdRegistry<'static>>,
               reporter_name: &'static str,
               host_and_port: String,
               prefix: &'static str)
               -> Self {
        CarbonReporter {
            host_and_port: host_and_port,
            prefix: prefix,
            registry: registry,
            reporter_name: reporter_name,
        }
    }

    fn report_to_carbon_continuously(self,
                                     delay_ms: u64)
                                     -> thread::JoinHandle<Result<String, Error>> {
        use metrics::MetricValue::{Counter, Gauge, Histogram, Meter};

        let prefix = self.prefix;
        let host_and_port = self.host_and_port.clone();
        let mut carbon = CarbonStream::new(host_and_port);
        let registry = self.registry.clone();
        thread::spawn(move || {
            loop {
                let ts = time::now().to_timespec();
                for metric_name in &registry.get_metrics_names() {
                    let metric = registry.get(metric_name);
                    try!(match metric.export_metric() {
                        Meter(x) => send_meter_metric(metric_name, x, &mut carbon, prefix, ts),
                        Gauge(x) => send_gauge_metric(metric_name, x, &mut carbon, prefix, ts),
                        Counter(x) => send_counter_metric(metric_name, x, &mut carbon, prefix, ts),
                        Histogram(mut x) => {
                            send_histogram_metric(metric_name, &mut x, &mut carbon, prefix, ts)
                        }
                    });
                }
                thread::sleep(Duration::from_millis(delay_ms));
            }
        })
    }

    pub fn start(self, delay_ms: u64) {
        self.report_to_carbon_continuously(delay_ms);
    }
}

#[cfg(test)]
mod test {
    use histogram::*;
    use metrics::{Counter, Gauge, Meter, StdCounter, StdGauge, StdMeter};
    use registry::{Registry, StdRegistry};
    use std::sync::Arc;
    use super::CarbonReporter;

    #[test]
    fn meter() {
        let m = StdMeter::new();
        m.mark(100);

        let mut c = StdCounter::new();
        c.inc();

        let mut g = StdGauge::default();
        g.set(1.2);

        let mut h = Histogram::configure()
            .max_value(100)
            .precision(1)
            .build()
            .unwrap();

        h.increment_by(1, 1).unwrap();

        let mut r = StdRegistry::new();
        r.insert("meter1", m);
        r.insert("counter1", c);
        r.insert("gauge1", g);
        r.insert("histogram", h);

        let arc_registry = Arc::new(r);
        CarbonReporter::new(arc_registry.clone(),
                            "test",
                            "localhost:0".to_string(),
                            "asd.asdf");
    }
}
<commit_msg>Use Result<(), Error>, not Result<String, Error>.<commit_after>// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// CarbonReporter sends a message to a carbon end point at a regular basis.
use registry::{Registry, StdRegistry};
use std::time::Duration;
use std::thread;
use std::sync::Arc;
use reporter::Reporter;
use metrics::{CounterSnapshot, GaugeSnapshot, MeterSnapshot};
use histogram::Histogram;
use time;
use time::Timespec;
use std::net::TcpStream;
use std::io::Write;
use std::io::Error;

struct CarbonStream {
    graphite_stream: Option<TcpStream>,
    host_and_port: String,
}

// TODO perhaps we autodiscover the host and port
//
pub struct CarbonReporter {
    host_and_port: String,
    prefix: &'static str,
    registry: Arc<StdRegistry<'static>>,
    reporter_name: &'static str,
}

impl CarbonStream {
    pub fn new(host_and_port: String) -> Self {
        CarbonStream {
            host_and_port: host_and_port,
            graphite_stream: None,
        }
    }

    pub fn connect(&mut self) -> Result<(), Error> {
        let graphite_stream = try!(TcpStream::connect(&*self.host_and_port));
        self.graphite_stream = Some(graphite_stream);
        Ok(())
    }

    pub fn write(&mut self,
                 metric_path: String,
                 value: String,
                 timespec: Timespec)
                 -> Result<(), Error> {
        let seconds_in_ms = (timespec.sec * 1000) as u32;
        let nseconds_in_ms = (timespec.nsec / 1000) as u32;
        let timestamp = seconds_in_ms + nseconds_in_ms;
        match self.graphite_stream {
            Some(ref mut stream) => {
                let carbon_command = format!("{} {} {}\n", metric_path, value, timestamp)
                    .into_bytes();
                try!(stream.write_all(&carbon_command));
            }
            None => {
                try!(self.reconnect_stream());
                try!(self.write(metric_path, value, timespec));
            }
        }
        Ok(())
    }
    fn reconnect_stream(&mut self) -> Result<(), Error> {
        // TODO 123 is made up
        println!("Waiting 123ms and then reconnecting");
        thread::sleep(Duration::from_millis(123));
        self.connect()
    }
}

impl Reporter for CarbonReporter {
    fn get_unique_reporter_name(&self) -> &'static str {
        self.reporter_name
    }
}

fn prefix(metric_line: String, prefix_str: &'static str) -> String {
    format!("{}.{}", prefix_str, metric_line)
}

fn send_meter_metric(metric_name: &str,
                     meter: MeterSnapshot,
                     carbon: &mut CarbonStream,
                     prefix_str: &'static str,
                     ts: Timespec)
                     -> Result<(), Error> {

    let count = meter.count.to_string();
    let m1_rate = meter.rates[0].to_string();
    let m5_rate = meter.rates[1].to_string();
    let m15_rate = meter.rates[2].to_string();
    let mean_rate = meter.mean.to_string();
    try!(carbon.write(prefix(format!("{}.count", metric_name), prefix_str),
                      count,
                      ts));
    try!(carbon.write(prefix(format!("{}.m1", metric_name), prefix_str),
                      m1_rate,
                      ts));
    try!(carbon.write(prefix(format!("{}.m5", metric_name), prefix_str),
                      m5_rate,
                      ts));
    try!(carbon.write(prefix(format!("{}.m15", metric_name), prefix_str),
                      m15_rate,
                      ts));
    try!(carbon.write(prefix(format!("{}.mean", metric_name), prefix_str),
                      mean_rate,
                      ts));
    Ok(())
}

fn send_gauge_metric(metric_name: &str,
                     gauge: GaugeSnapshot,
                     carbon: &mut CarbonStream,
                     prefix_str: &'static str,
                     ts: Timespec)
                     -> Result<(), Error> {
    try!(carbon.write(prefix(format!("{}", metric_name), prefix_str),
                      gauge.value.to_string(),
                      ts));
    Ok(())
}

fn send_counter_metric(metric_name: &str,
                       counter: CounterSnapshot,
                       carbon: &mut CarbonStream,
                       prefix_str: &'static str,
                       ts: Timespec)
                       -> Result<(), Error> {
    try!(carbon.write(prefix(format!("{}", metric_name), prefix_str),
                      counter.value.to_string(),
                      ts));
    Ok(())
}
fn send_histogram_metric(metric_name: &str,
                         histogram: &mut Histogram,
                         carbon: &mut CarbonStream,
                         prefix_str: &'static str,
                         ts: Timespec)
                         -> Result<(), Error> {
    let count = histogram.into_iter().count();
    // let sum = histogram.sum();
    // let mean = sum / count;
    let max = histogram.percentile(100.0).unwrap();
    let min = histogram.percentile(0.0).unwrap();

    let p50 = histogram.percentile(50.0).unwrap();
    let p75 = histogram.percentile(75.0).unwrap();
    let p95 = histogram.percentile(95.0).unwrap();
    let p98 = histogram.percentile(98.0).unwrap();
    let p99 = histogram.percentile(99.0).unwrap();
    let p999 = histogram.percentile(99.9).unwrap();
    let p9999 = histogram.percentile(99.99).unwrap();
    let p99999 = histogram.percentile(99.999).unwrap();

    try!(carbon.write(prefix(format!("{}.count", metric_name), prefix_str),
                      count.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.max", metric_name), prefix_str),
                      max.to_string(),
                      ts));

    // carbon
    // .write(prefix(format!("{}.mean", metric_name), prefix_str),
    // mean.into_string(),
    // ts);

    try!(carbon.write(prefix(format!("{}.min", metric_name), prefix_str),
                      min.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p50", metric_name), prefix_str),
                      p50.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p75", metric_name), prefix_str),
                      p75.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p95", metric_name), prefix_str),
                      p95.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p98", metric_name), prefix_str),
                      p98.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p99", metric_name), prefix_str),
                      p99.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p999", metric_name), prefix_str),
                      p999.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p9999", metric_name), prefix_str),
                      p9999.to_string(),
                      ts));

    try!(carbon.write(prefix(format!("{}.p99999", metric_name), prefix_str),
                      p99999.to_string(),
                      ts));
    Ok(())
}

impl CarbonReporter {
    pub fn new(registry: Arc<StdRegistry<'static>>,
               reporter_name: &'static str,
               host_and_port: String,
               prefix: &'static str)
               -> Self {
        CarbonReporter {
            host_and_port: host_and_port,
            prefix: prefix,
            registry: registry,
            reporter_name: reporter_name,
        }
    }

    fn report_to_carbon_continuously(self, delay_ms: u64) -> thread::JoinHandle<Result<(), Error>> {
        use metrics::MetricValue::{Counter, Gauge, Histogram, Meter};

        let prefix = self.prefix;
        let host_and_port = self.host_and_port.clone();
        let mut carbon = CarbonStream::new(host_and_port);
        let registry = self.registry.clone();
        thread::spawn(move || {
            loop {
                let ts = time::now().to_timespec();
                for metric_name in &registry.get_metrics_names() {
                    let metric = registry.get(metric_name);
                    try!(match metric.export_metric() {
                        Meter(x) => send_meter_metric(metric_name, x, &mut carbon, prefix, ts),
                        Gauge(x) => send_gauge_metric(metric_name, x, &mut carbon, prefix, ts),
                        Counter(x) => send_counter_metric(metric_name, x, &mut carbon, prefix, ts),
                        Histogram(mut x) => {
                            send_histogram_metric(metric_name, &mut x, &mut carbon, prefix, ts)
                        }
                    });
                }
                thread::sleep(Duration::from_millis(delay_ms));
            }
        })
    }

    pub fn start(self, delay_ms: u64) {
        self.report_to_carbon_continuously(delay_ms);
    }
}

#[cfg(test)]
mod test {
    use histogram::*;
    use metrics::{Counter, Gauge, Meter, StdCounter, StdGauge, StdMeter};
    use registry::{Registry, StdRegistry};
    use std::sync::Arc;
    use super::CarbonReporter;

    #[test]
    fn meter() {
        let m = StdMeter::new();
        m.mark(100);

        let mut c = StdCounter::new();
        c.inc();

        let mut g = StdGauge::default();
        g.set(1.2);

        let mut h = Histogram::configure()
            .max_value(100)
            .precision(1)
            .build()
            .unwrap();

        h.increment_by(1, 1).unwrap();

        let mut r = StdRegistry::new();
        r.insert("meter1", m);
        r.insert("counter1", c);
        r.insert("gauge1", g);
        r.insert("histogram", h);

        let arc_registry = Arc::new(r);
        CarbonReporter::new(arc_registry.clone(),
                            "test",
                            "localhost:0".to_string(),
                            "asd.asdf");
    }
}
<|endoftext|>


<commit_before><commit_msg>Review blackboard documentation<commit_after><|endoftext|>


<commit_before>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
    storepath: String,
}

impl StorageBackend {

    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
        let storepath = rt.get_rtp() + "/store/";
        debug!("Trying to create {}", storepath);
        create_dir_all(&storepath).and_then(|_| {
            debug!("Creating succeeded, constructing backend instance");
            Ok(StorageBackend {
                basepath: rt.get_rtp(),
                storepath: storepath.clone(),
            })
        }).or_else(|e| {
            debug!("Creating failed, constructing error instance");
            let mut serr = StorageBackendError::build(
                "create_dir_all()",
                "Could not create store directories",
                Some(storepath)
            );
            serr.caused_by = Some(Box::new(e));
            Err(serr)
        })
    }

    fn build<M: Module>(rt: &Runtime, m: &M) -> StorageBackend {
        let path = rt.get_rtp() + m.name() + "/store";
        // TODO: Don't use "/store" but value from configuration
        debug!("Building StorageBackend for {}", path);
        StorageBackend::new(path)
    }

    fn get_file_ids(&self, m: &Module) -> Option<Vec<FileID>> {
        let list = glob(&self.prefix_of_files_for_module(m)[..]);

        if let Ok(globlist) = list {
            let mut v = vec![];
            for entry in globlist {
                if let Ok(path) = entry {
                    debug!(" - File: {:?}", path);
                    v.push(from_pathbuf(&path));
                } else {
                    // Entry is not a path
                }
            }

            Some(v)
        } else {
            None
        }
    }

    pub fn iter_ids(&self, m: &Module) -> Option<IntoIter<FileID>>
    {
        glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| {
            let v = globlist.filter_map(Result::ok)
                            .map(|pbuf| from_pathbuf(&pbuf))
                            .collect::<Vec<FileID>>()
                            .into_iter();
            Ok(v)
        }).ok()
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Option<IntoIter<File<'a>>>
        where HP: FileHeaderParser
    {
        self.iter_ids(m).and_then(|ids| {
            Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                    .collect::<Vec<File>>()
                    .into_iter())
        })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: FileID, p: &Parser<HP>) -> Option<File<'a>>
        where HP: FileHeaderParser
    {
        debug!("Searching for file with id '{}'", id);
        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
            let mut s = String::new();
            fs.read_to_string(&mut s);
            debug!("Success reading file with id '{}'", id);
            debug!("Parsing to internal structure now");
            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
        } else {
            debug!("No file with id '{}'", id);
            None
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::build(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("  id      : '{}'", id);
        self.prefix_of_files_for_module(owner) + "-" + &id[..] + ".imag"
    }

    fn build_filepath_with_id(&self, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!("  id      : '{}'", id);
        self.basepath.clone() + &id[..]
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {
    fn new(action: String,
           desc  : String,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         action,
            desc:           desc,
            data_dump:      data,
            caused_by:      None,
        }
    }

    fn build(action: &'static str,
             desc:   &'static str,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         String::from(action),
            desc:           String::from(desc),
            dataDump:       data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::build(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<commit_msg>Pass owner to file path builder helper function<commit_after>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
    storepath: String,
}

impl StorageBackend {

    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
        let storepath = rt.get_rtp() + "/store/";
        debug!("Trying to create {}", storepath);
        create_dir_all(&storepath).and_then(|_| {
            debug!("Creating succeeded, constructing backend instance");
            Ok(StorageBackend {
                basepath: rt.get_rtp(),
                storepath: storepath.clone(),
            })
        }).or_else(|e| {
            debug!("Creating failed, constructing error instance");
            let mut serr = StorageBackendError::build(
                "create_dir_all()",
                "Could not create store directories",
                Some(storepath)
            );
            serr.caused_by = Some(Box::new(e));
            Err(serr)
        })
    }

    fn build<M: Module>(rt: &Runtime, m: &M) -> StorageBackend {
        let path = rt.get_rtp() + m.name() + "/store";
        // TODO: Don't use "/store" but value from configuration
        debug!("Building StorageBackend for {}", path);
        StorageBackend::new(path)
    }

    fn get_file_ids(&self, m: &Module) -> Option<Vec<FileID>> {
        let list = glob(&self.prefix_of_files_for_module(m)[..]);

        if let Ok(globlist) = list {
            let mut v = vec![];
            for entry in globlist {
                if let Ok(path) = entry {
                    debug!(" - File: {:?}", path);
                    v.push(from_pathbuf(&path));
                } else {
                    // Entry is not a path
                }
            }

            Some(v)
        } else {
            None
        }
    }

    pub fn iter_ids(&self, m: &Module) -> Option<IntoIter<FileID>>
    {
        glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| {
            let v = globlist.filter_map(Result::ok)
                            .map(|pbuf| from_pathbuf(&pbuf))
                            .collect::<Vec<FileID>>()
                            .into_iter();
            Ok(v)
        }).ok()
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Option<IntoIter<File<'a>>>
        where HP: FileHeaderParser
    {
        self.iter_ids(m).and_then(|ids| {
            Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                    .collect::<Vec<File>>()
                    .into_iter())
        })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: FileID, p: &Parser<HP>) -> Option<File<'a>>
        where HP: FileHeaderParser
    {
        debug!("Searching for file with id '{}'", id);
        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
            let mut s = String::new();
            fs.read_to_string(&mut s);
            debug!("Success reading file with id '{}'", id);
            debug!("Parsing to internal structure now");
            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
        } else {
            debug!("No file with id '{}'", id);
            None
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::build(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!("  id      : '{}'", id);
        self.basepath.clone() + owner.name() + "-" + &id[..] + ".imag"
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {
    fn new(action: String,
           desc  : String,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         action,
            desc:           desc,
            data_dump:      data,
            caused_by:      None,
        }
    }

    fn build(action: &'static str,
             desc:   &'static str,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         String::from(action),
            desc:           String::from(desc),
            dataDump:       data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::build(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<|endoftext|>


<commit_before>use std::error::Error;
use std::fmt::Display;

use super::file::FileID;
use super::file::File;
use module::Module;

type BackendOperationResult = Result<(), StorageBackendError>;

pub struct StorageBackend<'a> {
    basepath: String,
    module: &'a Module,
}

impl<'a> StorageBackend<'a> {

    fn new() -> StorageBackend<'a> {
    }

    fn getFileList() -> Vec<(String, FileID)> {
    }

    fn createEmpty() -> FileID {
    }

    fn createFile() -> File {
    }

    fn writeFile(f: File) -> BackendOperationResult {
    }

    fn createFileWithContent(content: String) -> BackendOperationResult {
    }

    fn readFile(id: FileID) -> String {
    }

    // TODO: Meta files are not covered yet

}

#[derive(Debug)]
pub struct StorageBackendError;

impl StorageBackendError {
}

impl Error for StorageBackendError {
}

impl Display for StorageBackendError {
}

<commit_msg>Implement StorageBackendError<commit_after>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;

use super::file::FileID;
use super::file::File;
use module::Module;

type BackendOperationResult = Result<(), StorageBackendError>;

pub struct StorageBackend<'a> {
    basepath: String,
    module: &'a Module,
}

impl<'a> StorageBackend<'a> {

    fn new() -> StorageBackend<'a> {
    }

    fn getFileList() -> Vec<(String, FileID)> {
    }

    fn createEmpty() -> FileID {
    }

    fn createFile() -> File {
    }

    fn writeFile(f: File) -> BackendOperationResult {
    }

    fn createFileWithContent(content: String) -> BackendOperationResult {
    }

    fn readFile(id: FileID) -> String {
    }

    // TODO: Meta files are not covered yet

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub explanation: String,        // A long, user friendly description
    pub dataDump: Option<String>    // Data dump, if any
}

impl StorageBackendError {
    fn new(action: &'static str,
           desc  : &'static str,
           explan: &'static str,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         String::from(action),
            desc:           String::from(desc),
            explanation:    String::from(explan),
            dataDump:       data,
        }
    }
}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        None
    }

}

impl Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}\n\n{}",
               self.action, self.desc, self.explanation)
    }
}

<|endoftext|>


<commit_before>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
    storepath: String,
}

impl StorageBackend {

    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
        let storepath = rt.get_rtp() + "/store/";
        debug!("Trying to create {}", storepath);
        create_dir_all(&storepath).and_then(|_| {
            debug!("Creating succeeded, constructing backend instance");
            Ok(StorageBackend {
                basepath: rt.get_rtp(),
                storepath: storepath.clone(),
            })
        }).or_else(|e| {
            debug!("Creating failed, constructing error instance");
            let mut serr = StorageBackendError::new(
                "create_dir_all()",
                "Could not create store directories",
                Some(storepath)
            );
            serr.caused_by = Some(Box::new(e));
            Err(serr)
        })
    }

    pub fn iter_ids(&self, m: &Module) -> Result<IntoIter<FileID>, StorageBackendError>
    {
        glob(&self.prefix_of_files_for_module(m)[..])
            .and_then(|globlist| {
                let v = globlist.filter_map(Result::ok)
                                .map(|pbuf| FileID::from(&pbuf))
                                .collect::<Vec<FileID>>()
                                .into_iter();
                Ok(v)
            })
            .map_err(|e| {
                let serr = StorageBackendError::new(
                        "iter_ids()",
                        "Cannot iter on file ids",
                        None);
                serr
            })
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Result<IntoIter<File<'a>>, StorageBackendError>
        where HP: FileHeaderParser
    {
        self.iter_ids(m)
            .and_then(|ids| {
                Ok(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                        .collect::<Vec<File>>()
                        .into_iter())
            })
            .map_err(|e| {
                let serr = StorageBackendError::new(
                        "iter_files()",
                        "Cannot iter on files",
                        None);
                serr
            })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::new(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::new(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::new(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::new(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser<HP>) -> Option<File<'a>>
        where HP: FileHeaderParser
    {
        debug!("Searching for file with id '{}'", id);
        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
            let mut s = String::new();
            fs.read_to_string(&mut s);
            debug!("Success opening file with id '{}'", id);
            debug!("Parsing to internal structure now");
            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
        } else {
            debug!("No file with id '{}'", id);
            None
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::new(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        let idstr   : String        = id.clone().into();
        let idtype  : FileIDType    = id.into();
        let typestr : String        = idtype.into();

        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("        id: '{}'", idstr);
        debug!("      type: '{}'", typestr);

        self.prefix_of_files_for_module(owner) +
            "-" + &typestr[..] +
            "-" + &idstr[..] +
            ".imag"
    }

    fn prefix_of_files_for_module(&self, m: &Module) -> String {
        self.storepath.clone() + m.name()
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {

    fn new<S>(action: S, desc: S, data: Option<String>) -> StorageBackendError
        where S: Into<String>
    {
        StorageBackendError {
            action:         action.into(),
            desc:           desc.into(),
            data_dump:      data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl<'a> Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::new(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<commit_msg>StorageBackend::iter_ids(): Add caused_by() usage<commit_after>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
    storepath: String,
}

impl StorageBackend {

    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
        let storepath = rt.get_rtp() + "/store/";
        debug!("Trying to create {}", storepath);
        create_dir_all(&storepath).and_then(|_| {
            debug!("Creating succeeded, constructing backend instance");
            Ok(StorageBackend {
                basepath: rt.get_rtp(),
                storepath: storepath.clone(),
            })
        }).or_else(|e| {
            debug!("Creating failed, constructing error instance");
            let mut serr = StorageBackendError::new(
                "create_dir_all()",
                "Could not create store directories",
                Some(storepath)
            );
            serr.caused_by = Some(Box::new(e));
            Err(serr)
        })
    }

    pub fn iter_ids(&self, m: &Module) -> Result<IntoIter<FileID>, StorageBackendError>
    {
        glob(&self.prefix_of_files_for_module(m)[..])
            .and_then(|globlist| {
                Ok(globlist.filter_map(Result::ok)
                           .map(|pbuf| FileID::from(&pbuf))
                           .collect::<Vec<FileID>>()
                           .into_iter())
            })
            .map_err(|e| {
                let serr = StorageBackendError::new(
                        "iter_ids()",
                        "Cannot iter on file ids",
                        None);
                // Why the hack is Error not implemented for glob::PatternError
                // serr.caused_by = Some(Box::new(e));
                serr
            })
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Result<IntoIter<File<'a>>, StorageBackendError>
        where HP: FileHeaderParser
    {
        self.iter_ids(m)
            .and_then(|ids| {
                Ok(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                        .collect::<Vec<File>>()
                        .into_iter())
            })
            .map_err(|e| {
                let serr = StorageBackendError::new(
                        "iter_files()",
                        "Cannot iter on files",
                        None);
                serr
            })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::new(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::new(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::new(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::new(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser<HP>) -> Option<File<'a>>
        where HP: FileHeaderParser
    {
        debug!("Searching for file with id '{}'", id);
        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
            let mut s = String::new();
            fs.read_to_string(&mut s);
            debug!("Success opening file with id '{}'", id);
            debug!("Parsing to internal structure now");
            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
        } else {
            debug!("No file with id '{}'", id);
            None
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::new(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        let idstr   : String        = id.clone().into();
        let idtype  : FileIDType    = id.into();
        let typestr : String        = idtype.into();

        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("        id: '{}'", idstr);
        debug!("      type: '{}'", typestr);

        self.prefix_of_files_for_module(owner) +
            "-" + &typestr[..] +
            "-" + &idstr[..] +
            ".imag"
    }

    fn prefix_of_files_for_module(&self, m: &Module) -> String {
        self.storepath.clone() + m.name()
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {

    fn new<S>(action: S, desc: S, data: Option<String>) -> StorageBackendError
        where S: Into<String>
    {
        StorageBackendError {
            action:         action.into(),
            desc:           desc.into(),
            data_dump:      data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl<'a> Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::new(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<|endoftext|>


<commit_before>use std::path::{Path, PathBuf};

pub type FileID = String;

pub fn from_path_string(s: &String) -> FileID {
    String::from("")
}

pub fn from_path(p: &Path) -> FileID {
    String::from("")
}

pub fn from_pathbuf(p: &PathBuf) -> FileID {
    from_path(p.as_path())
}

<commit_msg>Fix: use unimplemented instead of false implementation<commit_after>use std::path::{Path, PathBuf};

pub type FileID = String;

pub fn from_path_string(s: &String) -> FileID {
    unimplemented!()
}

pub fn from_path(p: &Path) -> FileID {
    unimplemented!()
}

pub fn from_pathbuf(p: &PathBuf) -> FileID {
    from_path(p.as_path())
}

<|endoftext|>


<commit_before><commit_msg>Added an example for test rewrite<commit_after>extern crate sourcemap;

use std::env;
use std::fs;
use std::io::Read;
use std::path::Path;

use sourcemap::{decode, DecodedMap, RewriteOptions, SourceMap};

fn test(sm: &SourceMap) {
    for (src_id, source) in sm.sources().enumerate() {
        let path = Path::new(source);
        if path.is_file() {
            let mut f = fs::File::open(&path).unwrap();
            let mut contents = String::new();
            if f.read_to_string(&mut contents).ok().is_none() {
                continue;
            }
            if Some(contents.as_str()) != sm.get_source_contents(src_id as u32) {
                println!("  !!! {}", source);
            }
        }
    }
}

fn load_from_reader<R: Read>(mut rdr: R) -> SourceMap {
    match decode(&mut rdr).unwrap() {
        DecodedMap::Regular(sm) => sm,
        DecodedMap::Index(idx) => idx.flatten_and_rewrite(&RewriteOptions {
            load_local_source_contents: true,
            ..Default::default()
        }).unwrap(),
    }
}

fn main() {
    let args : Vec<_> = env::args().collect();
    let mut f = fs::File::open(&args[1]).unwrap();
    let sm = load_from_reader(&mut f);
    println!("before dump");
    test(&sm);

    println!("after dump");
    let mut json : Vec<u8> = vec![];
    sm.to_writer(&mut json).unwrap();
    let sm = load_from_reader(json.as_slice());
    test(&sm);
}
<|endoftext|>


<commit_before><commit_msg>udata is a thing<commit_after><|endoftext|>


<commit_before><commit_msg>Added experimental parallel IBBA. Experiments show speedups ranging from 4x-10x depending on the problem (on fractus with 11 worker threads). Short running problems experience a slight slow down relative to the serial implementation, but usually returns tighter bounds.<commit_after>// Cooperative optimization solver
use std::collections::BinaryHeap;
use std::io::Write;
extern crate rand;

#[macro_use(max)]
extern crate gelpia_utils;
extern crate ga;
extern crate gr;

use ga::{ea, Individual};

use gelpia_utils::{Quple, INF, NINF, Flt, Parameters, eps_tol};

use gr::{GI, width_box, split_box, midpoint_box};

use std::sync::{Barrier, RwLock, Arc, RwLockWriteGuard};

use std::sync::atomic::{AtomicBool, Ordering, AtomicUsize};

use std::thread;

use std::time::Duration;

extern crate function;
use function::FuncObj;

extern crate args;
use args::{process_args};

extern crate threadpool;
use threadpool::ThreadPool;
use std::sync::mpsc::channel;
extern crate time;

/// Returns the guaranteed upperbound for the algorithm
/// from the queue.
fn get_upper_bound(q: &RwLockWriteGuard<Vec<Quple>>,
                   f_best_high: f64) -> f64{
    let mut max = f_best_high;
    for qi in q.iter() {
        max = max!{max, qi.fdata.upper()};
    }
    max
}

fn log_max(q: &RwLockWriteGuard<Vec<Quple>>,
           f_best_low: f64,
           f_best_high: f64) {
    let max = get_upper_bound(q, f_best_high);
    let _ = writeln!(&mut std::io::stderr(),
                     "lb: {}, possible ub: {}, guaranteed ub: {}",
                     f_best_low,
                     f_best_high,
                     max);
}

fn print_q(q: &RwLockWriteGuard<BinaryHeap<Quple>>) {
    let mut lq: BinaryHeap<Quple> = (*q).clone();
    while lq.len() != 0 {
        let qi = lq.pop().unwrap();
        let (gen, v, fx) = (qi.pf, qi.p, qi.fdata);
        print!("[{}, {}, {}], ", v, gen, qi.fdata.to_string());
    }
    println!("\n");
}

/// Returns a tuple (function_estimate, eval_interval)
/// # Arguments
/// * `f` - The function to evaluate with
/// * `input` - The input domain
fn est_func(f: &FuncObj, input: &Vec<GI>) -> (Flt, GI) {
    let mid = midpoint_box(input);
    let est_m = f.call(&mid);
    let fsx = f.call(&input);
    let fsx_u = f.call(&input.iter()
                       .map(|&si| GI::new_p(si.upper()))
                       .collect::<Vec<_>>());
    let fsx_l = f.call(&input.iter()
                       .map(|&si| GI::new_p(si.lower()))
                       .collect::<Vec<_>>());
    let est_max = est_m.lower().max(fsx_u.lower()).max(fsx_l.lower());
    (est_max, fsx)
}

// Returns the upper bound, the domain where this bound occurs and a status
// flag indicating whether the answer is complete for the problem.
fn ibba(x_0: Vec<GI>, e_x: Flt, e_f: Flt, e_f_r: Flt,
        f_bestag: Arc<RwLock<Flt>>,
        f_best_shared: Arc<RwLock<Flt>>,
        x_bestbb: Arc<RwLock<Vec<GI>>>,
        b1: Arc<Barrier>, b2: Arc<Barrier>,
        q: Arc<RwLock<Vec<Quple>>>,
        sync: Arc<AtomicBool>, stop: Arc<AtomicBool>,
        f: FuncObj,
        logging: bool, max_iters: u32)
        -> (Flt, Flt, Vec<GI>) {
    let mut best_x = x_0.clone();

    let iters = Arc::new(AtomicUsize::new(0));
    let (est_max, first_val) = est_func(&f, &x_0);
    {
        q.write().unwrap().push(Quple{p: est_max, pf: 0, data: x_0.clone(),
                                      fdata: first_val});
    }
    let mut f_best_low = est_max;
    let mut f_best_high = est_max;

    let n_workers = 11;
    let n_jobs = n_workers;
    let pool = ThreadPool::new(n_workers);

    while q.read().unwrap().len() != 0 && !stop.load(Ordering::Acquire) {
        if max_iters != 0 && iters.load(Ordering::Acquire) as u32 >= max_iters {
            break;
        }
        if sync.load(Ordering::Acquire) {
            // Ugly: Update the update thread's view of the best branch bound.
            *f_best_shared.write().unwrap() = f_best_low;
            b1.wait();
            b2.wait();
        }
        {
            // Take q as writable during an iteration
            let q = q.write().unwrap();

            let fbl_orig = f_best_low;
            f_best_low = max!(f_best_low, *f_bestag.read().unwrap());

            if iters.load(Ordering::Acquire) % 2048 == 0 {
                let guaranteed_bound =
                    get_upper_bound(&q, f_best_high);
                if (guaranteed_bound - f_best_high).abs() < e_f {
                    f_best_high = guaranteed_bound;
                    break;
                }
            }

            if logging && fbl_orig != f_best_low {
                log_max(&q, f_best_low, f_best_high);
            }
        }

        let p_q_len = {
            let mut q = q.write().unwrap();
            q.sort();
            q.len()/n_workers + 1
        };

/*        let mut p_q = vec![];
        {
            let mut total_len = 0;
            let q = q.write().unwrap();
            let q_len = q.len();
            for i in 0..n_workers {
                let mut elems = vec![];
                for j in 0..p_q_len {
                    if i + j*n_workers >= q_len {
                        break;
                    }
                    elems.push(q[i+j*n_workers].clone());
                }
                total_len += elems.len();
                p_q.push(elems);
            }
        } */

        let outer_barr = Arc::new(Barrier::new(n_workers + 1));

        let (qtx, qrx) = channel();
        let (htx, hrx) = channel();
        let (ltx, lrx) = channel();

        for i in 0..n_workers {
            let inner_barr = outer_barr.clone();
//            let elems = p_q[i].clone();
            let _f = f.clone();
            let qtx = qtx.clone();
            let htx = htx.clone();
            let ltx = ltx.clone();
            let f_bestag = f_bestag.clone();
            let iters = iters.clone();
            let lqi = q.clone();
            pool.execute(move || {
                let mut l_f_best_high = f_best_high;
                let mut l_best_x = vec![];

                let mut l_f_best_low = f_best_low;
                let mut l_best_low_x = vec![];

                let mut lqo = vec![];
                let mut used = false;
                let lqi = lqi.read().unwrap();
//                let elems_len = elems.len();

                for j in 0..p_q_len {
                    if i + j*n_workers >= lqi.len() { break };
                    used = true;
                    let ref elem = lqi[i + j*n_workers];
                    let ref x = elem.data;
                    let ref iter_est = elem.p;
                    let ref fx = elem.fdata;
                    let ref gen = elem.pf;
                    //let (ref x, iter_est, fx, gen) = ;

                    if fx.upper() < l_f_best_low ||
                        width_box(&x, e_x) ||
                        eps_tol(*fx, *iter_est, e_f, e_f_r) {
                            if l_f_best_high < fx.upper() {
                                l_f_best_high = fx.upper();
                                l_best_x = x.clone();
                                // htx.send((fx.upper(), x.clone())).unwrap();
                            }
                        }
                    else {
                        let (x_s, is_split) = split_box(&x);
                        for sx in x_s {
                            let (est_max, fsx) = est_func(&_f, &sx);
                            if l_f_best_low < est_max {
                                l_f_best_low = est_max;
                                l_best_low_x = sx.clone();
                                // ltx.send((est_max, sx.clone())).unwrap();
                            }
                            iters.fetch_add(1, Ordering::Release);
                            if is_split && fsx.upper() > f_best_low &&
                                fsx.upper() > *f_bestag.read().unwrap() {
                                    lqo.push(Quple{p: est_max,
                                                   pf: gen+1,
                                                   data: sx,
                                                   fdata: fsx});
                                }
                        }
                    }
                }
                ltx.send((l_f_best_low, l_best_low_x, used)).unwrap();
                htx.send((l_f_best_high, l_best_x, used)).unwrap();
                lqo.sort();
                qtx.send(lqo).unwrap();
                inner_barr.wait();
            });
        }
        outer_barr.wait();
        drop(qtx);
        drop(htx);
        drop(ltx);

        for li in lrx.iter() {
            let (lb, lx, non_empty) = li;
            if non_empty && f_best_low < lb {
                f_best_low = lb;
                *x_bestbb.write().unwrap() = lx.clone();
            }
        }

        for hi in hrx.iter() {
            let (ub, ux, non_empty) = hi;
            if non_empty && f_best_high < ub {
                f_best_high = ub;
                best_x = ux.clone();
            }
        }
        {
            let mut lq = q.write().unwrap();
            *lq = vec![];
            for qis in qrx.iter() {
                for qi in &qis {
                    if qi.fdata.upper() > f_best_low {
                        lq.push(qi.clone());
                    }
                }
            }
        }
    }
    println!("{}", iters.load(Ordering::Acquire));
    stop.store(true, Ordering::Release);
    (f_best_low, f_best_high, best_x)
}

fn update(stop: Arc<AtomicBool>, _sync: Arc<AtomicBool>,
          _b1: Arc<Barrier>, _b2: Arc<Barrier>,
          _f: FuncObj,
          timeout: u32) {
    let start = time::get_time();
    let one_sec = Duration::new(1, 0);
    'out: while !stop.load(Ordering::Acquire) {
        // Timer code...
        thread::sleep(one_sec);
        if timeout > 0 &&
            (time::get_time() - start).num_seconds() >= timeout as i64 {
                let _ = writeln!(&mut std::io::stderr(), "Stopping early...");
                stop.store(true, Ordering::Release);
                break 'out;
            }
    }
}

fn main() {
    let args = process_args();

    let ref x_0 = args.domain;
    let ref fo = args.function;
    let x_err = args.x_error;
    let y_err = args.y_error;
    let y_rel = args.y_error_rel;
    let seed = args.seed;

    // Early out if there are no input variables...
    if x_0.len() == 0 {
        let result = fo.call(&x_0);
        println!("[[{},{}], {{}}]", result.lower(), result.upper());
        return
    }

    let q_inner: Vec<Quple> = Vec::new();
    let q = Arc::new(RwLock::new(q_inner));

    let population_inner: Vec<Individual> = Vec::new();
    let population = Arc::new(RwLock::new(population_inner));

    let b1 = Arc::new(Barrier::new(3));
    let b2 = Arc::new(Barrier::new(3));

    let sync = Arc::new(AtomicBool::new(false));
    let stop = Arc::new(AtomicBool::new(false));

    let f_bestag: Arc<RwLock<Flt>> = Arc::new(RwLock::new(NINF));
    let f_best_shared: Arc<RwLock<Flt>> = Arc::new(RwLock::new(NINF));

    let x_e = x_0.clone();
    let x_i = x_0.clone();

    let x_bestbb = Arc::new(RwLock::new(x_0.clone()));

    let ibba_thread =
    {
        let q = q.clone();
        let b1 = b1.clone();
        let b2 = b2.clone();
        let f_bestag = f_bestag.clone();
        let f_best_shared = f_best_shared.clone();
        let x_bestbb = x_bestbb.clone();
        let sync = sync.clone();
        let stop = stop.clone();
        let fo_c = fo.clone();
        let logging = args.logging;
        let iters= args.iters;
        thread::Builder::new().name("IBBA".to_string()).spawn(move || {
            ibba(x_i, x_err, y_err, y_rel,
                 f_bestag, f_best_shared,
                 x_bestbb,
                 b1, b2, q, sync, stop, fo_c, logging, iters)
        })};

    let ea_thread =
    {
        let population = population.clone();
        let f_bestag = f_bestag.clone();
        let x_bestbb = x_bestbb.clone();
        let sync = sync.clone();
        let stop = stop.clone();
        let b1 = b1.clone();
        let b2 = b2.clone();
        let fo_c = fo.clone();
        let factor = x_e.len();
        thread::Builder::new().name("EA".to_string()).spawn(move || {
            ea(x_e, Parameters{population: 50*factor, //1000,
                               selection: 8, //4,
                               elitism: 5, //2,
                               mutation: 0.4_f64,//0.3_f64,
                               crossover: 0.0_f64, // 0.5_f64
                               seed:  seed,
            },
               population,
               f_bestag,
               x_bestbb,
               b1, b2,
               stop, sync, fo_c)
        })};

    // pending finding out how to kill threads
    //let update_thread =
    {
        let sync = sync.clone();
        let stop = stop.clone();
        let b1 = b1.clone();
        let b2 = b2.clone();
        let fo_c = fo.clone();
        let to = args.timeout.clone();
        let _ = thread::Builder::new().name("Update".to_string()).spawn(move || {
            update(stop, sync, b1, b2, fo_c, to)
        });};

    let result = ibba_thread.unwrap().join();
    let ea_result = ea_thread.unwrap().join();

    // Join EA and Update here pending stop signaling.
    if result.is_ok() {
        let (min, mut max, mut interval) = result.unwrap();
        // Go through all remaining intervals from IBBA to find the true
        // max
        let mut lq = q.write().unwrap();
        while lq.len() != 0 {
            let ref top = lq.pop().unwrap();
            let (ub, dom) = (top.fdata.upper(), &top.data);
            if ub > max {
            max = ub;
            interval = dom.clone();
        }
    }
    println!("[[{},{}], {{", min, max);
    for i in 0..args.names.len() {
        println!("'{}' : {},", args.names[i], interval[i].to_string());
    }
    println!("}}]");

}
else {println!("error")}
}
<|endoftext|>


<commit_before><commit_msg>blank message<commit_after><|endoftext|>


<commit_before><commit_msg>an empy message<commit_after><|endoftext|>


<commit_before><commit_msg>Fix: Do not print data if output is pipe, except if that behaviour is overridden<commit_after><|endoftext|>