<commit_before>use std::iter;

/// LSD radix sort
pub fn lsd_string_sort(a: &mut [&str]) {
    let w = a[0].as_bytes().len();
    let n = a.len();
    let r = 256;

    let mut aux = iter::repeat("").take(n).collect::<Vec<&str>>();

    for d in (0 .. w).rev() {
        // sort by key-indexed counting on dth character

        // frequency counts
        let mut count = iter::repeat(0).take(r+1).collect::<Vec<usize>>();
        for i in 0 .. n {
            count[a[i].as_bytes()[d] as usize + 1] += 1;
        }

        // cumulates
        for r in 0 .. r {
            count[r+1] += count[r];
        }

        // move data
        for i in 0 .. n {
            aux[count[a[i].as_bytes()[d] as usize]] = a[i];
            count[a[i].as_bytes()[d] as usize] += 1;
        }

        // copy back
        for i in 0 .. n {
            a[i] = aux[i];
        }
    }
}

pub trait LSD: Clone + Default {
    fn r() -> usize;
    fn width(&self) -> usize;
    fn item_at(&self, n: usize) -> usize;

    fn sort(a: &mut [Self]) {
        let w = a[0].width();
        // assume all width equals
        let n = a.len();
        let r = Self::r();

        let mut aux = iter::repeat(Default::default()).take(n).collect::<Vec<Self>>();

        // NOTE: no .rev() here
        for d in 0 .. w {
            // sort by ey-indexed counting on dth character

            // frequency counts
            let mut count = iter::repeat(0).take(r+1).collect::<Vec<usize>>();
            for i in 0 .. n {
                count[a[i].item_at(d) + 1] += 1;
            }

            // cumulates
            for r in 0 .. r {
                count[r+1] += count[r];
            }

            // move data
            for i in 0 .. n {
                aux[count[a[i].item_at(d)]] = a[i].clone();
                count[a[i].item_at(d)] += 1;
            }

            // copy back
            for i in 0 .. n {
                a[i] = aux[i].clone();
            }
        }
    }
}

impl LSD for u32 {
    fn r() -> usize { 256 }
    fn width(&self) -> usize { 4 }
    fn item_at(&self, n: usize) -> usize {
        // 0:   lowest byte
        // n-1: highest byte
        ((self >> (8 * n)) & 0xFF) as usize
    }
}

impl LSD for i16 {
    fn r() -> usize { 256 }
    fn width(&self) -> usize { 2 }
    fn item_at(&self, n: usize) -> usize {
        // 0:   lowest byte
        // n-1: highest byte
        let mut ret = (self >> (8 * n)) & 0xFF;
        // handle highest byte negative
        if n == 1 {
            if *self < 0 {
                ret -= 128;
            } else {
                ret += 128;
            }
        }
        ret as usize
    }
}

#[cfg(test)]
fn is_sorted<T: PartialOrd>(a: &[T]) -> bool {
    for i in 1 .. a.len() {
        if a[i] < a[i-1] {
            return false;
        }
    }
    true
}

#[test]
fn test_lsd_string_sort() {
    let mut to_be_sorted = vec![
        "dab", "add", "cab", "fad", "fee", "bad",
        "bee", "fed", "bed", "ebb", "ace", "dad"];
    assert!(!is_sorted(&to_be_sorted));
    lsd_string_sort(&mut to_be_sorted);
    assert!(is_sorted(&to_be_sorted));
}

#[test]
fn test_lsd_ints_sort() {
    let mut to_be_sorted: Vec<u32> = vec![
        2830665856, 3165123597, 780293973, 3412877992, 3781910346,
        2837710927, 560466804, 2778567425, 265098057, 18593204];

    assert!(!is_sorted(&to_be_sorted));
    LSD::sort(&mut to_be_sorted);
    assert!(is_sorted(&to_be_sorted));

    let mut to_be_sorted: Vec<i16> = vec![
        19195, -5970, 2060, 2553, 26078, 15152,
        2318, 9647, -7906, -8452, -127, 128, 127];

    assert!(!is_sorted(&to_be_sorted));
    LSD::sort(&mut to_be_sorted);
    assert!(is_sorted(&to_be_sorted));

}

const CUTOFF: usize = 15;

pub trait MSD: Clone + Default + PartialOrd {
    fn r() -> usize;
    fn width(&self) -> usize;
    fn item_at(&self, n: usize) -> usize;

    fn insertion(a: &mut [Self], lo: usize, hi: usize) {
        for i in lo .. hi+1 {
            for j in (lo .. i).rev() {
                if !(j > lo && a[j] < a[j-1]) {
                    break;
                }
                a.swap(j, j-1);
            }
        }
    }

    fn sort_helper(a: &mut [Self], lo: usize, hi: usize, d: usize, aux: &mut [Self]) {
        if hi <= lo + CUTOFF {
            Self::insertion(a, lo, hi);
            return;
        }

        let n = a.len();
        let r = Self::r();

        // compute frequency counts (need R = 256)
        let mut count = iter::repeat(0).take(r+1).collect::<Vec<usize>>();
        for i in 0 .. n {
            count[a[i].item_at(d) + 1] += 1;
        }

        // transform counts to indicies
        for r in 0 .. r {
            count[r+1] += count[r];
        }

        // distribute
        for i in lo .. hi+1 {
            aux[count[a[i].item_at(d)]] = a[i].clone();
            count[a[i].item_at(d)] += 1;
        }

        // copy back
        for i in lo .. hi+1 {
            a[i] = aux[i-lo].clone();
        }

        // no more bits
        if d == a[0].width() {
            return ;
        }

        if count[0] > 0 {
            Self::sort_helper(a, lo, lo + count[0] - 1, d+1, aux);
        }
        for r in 0 .. r {
            if count[r+1] > count[r] {
                Self::sort_helper(a, lo + count[r], lo + count[r+1], d+1, aux);
            }
        }
    }

    fn sort(a: &mut [Self]) {
        let n = a.len();
        let mut aux = iter::repeat(Default::default()).take(n).collect::<Vec<Self>>();

        Self::sort_helper(a, 0, n-1, 0, &mut aux);
    }
}

impl MSD for u32 {
    fn r() -> usize { 256 }
    fn width(&self) -> usize { 4 }
    fn item_at(&self, n: usize) -> usize {
        // 0:   lowest byte
        // n-1: highest byte
        ((self >> (8 * n)) & 0xFF) as usize
    }
}

impl MSD for i16 {
    fn r() -> usize { 256 }
    fn width(&self) -> usize { 2 }
    fn item_at(&self, n: usize) -> usize {
        // 0:   lowest byte
        // n-1: highest byte
        let mut ret = (self >> (8 * n)) & 0xFF;
        // handle highest byte negative
        if n == 1 {
            if *self < 0 {
                ret -= 128;
            } else {
                ret += 128;
            }
        }
        ret as usize
    }
}

#[test]
fn test_msd_ints_sort() {
    extern crate rand;
    use rand::{thread_rng, Rng};

    let mut rng = thread_rng();
    for sz in vec![20, 1000] {
        let mut array = rng.gen_iter().take(sz).collect::<Vec<u32>>();
        assert!(!is_sorted(&array));
        LSD::sort(&mut array);
        assert!(is_sorted(&array));
    }

}
<commit_msg>radix_sorts: fix bug in radix msd sort; refactor LSD/MSD into RadixSort trait<commit_after>use std::iter;

/// LSD radix sort
pub fn lsd_string_sort(a: &mut [&str]) {
    let w = a[0].as_bytes().len();
    let n = a.len();
    let r = 256;

    let mut aux = iter::repeat("").take(n).collect::<Vec<&str>>();

    for d in (0 .. w).rev() {
        // sort by key-indexed counting on dth character

        // frequency counts
        let mut count = iter::repeat(0).take(r+1).collect::<Vec<usize>>();
        for i in 0 .. n {
            count[a[i].as_bytes()[d] as usize + 1] += 1;
        }

        // cumulates
        for r in 0 .. r {
            count[r+1] += count[r];
        }

        // move data
        for i in 0 .. n {
            aux[count[a[i].as_bytes()[d] as usize]] = a[i];
            count[a[i].as_bytes()[d] as usize] += 1;
        }

        // copy back
        for i in 0 .. n {
            a[i] = aux[i];
        }
    }
}

const CUTOFF: usize = 15;

pub trait RadixSort: Clone + Default + PartialOrd + ::std::fmt::Debug {
    fn r() -> usize;
    fn width(&self) -> usize;
    fn item_at(&self, n: usize) -> usize;

    // sort algorithms
    fn lsd_sort(a: &mut [Self]) {
        let w = a[0].width();
        // assume all width equals
        let n = a.len();
        let r = Self::r();

        let mut aux = iter::repeat(Default::default()).take(n).collect::<Vec<Self>>();

        // NOTE: no .rev() here
        for d in 0 .. w {
            // sort by ey-indexed counting on dth character

            // frequency counts
            let mut count = iter::repeat(0).take(r+1).collect::<Vec<usize>>();
            for i in 0 .. n {
                count[a[i].item_at(d) + 1] += 1;
            }

            // cumulates
            for r in 0 .. r {
                count[r+1] += count[r];
            }

            // move data
            for i in 0 .. n {
                aux[count[a[i].item_at(d)]] = a[i].clone();
                count[a[i].item_at(d)] += 1;
            }

            // copy back
            for i in 0 .. n {
                a[i] = aux[i].clone();
            }
        }
    }

    fn msd_insertion(a: &mut [Self], lo: usize, hi: usize, _d: usize) {
        for i in lo .. hi+1 {
            for j in (0 .. i+1).rev() {
                // if !(j > lo && a[j].item_at(d) < a[j-1].item_at(d)) {
                if !(j > lo && a[j] < a[j-1]) {
                    break;
                }
                a.swap(j, j-1);
            }
        }
    }

    fn msd_sort_helper(a: &mut [Self], lo: usize, hi: usize, d: usize, aux: &mut [Self]) {
        if hi <= lo + CUTOFF {
            Self::msd_insertion(a, lo, hi, d);
            return;
        }

        let r = Self::r();

        // compute frequency counts (need R = 256)
        let mut count = iter::repeat(0).take(r+1).collect::<Vec<usize>>();
        for i in lo .. hi+1 {
            count[a[i].item_at(d) + 1] += 1;
        }

        // transform counts to indicies
        for r in 0 .. r {
            count[r+1] += count[r];
        }

        // distribute
        for i in lo .. hi+1 {
            aux[count[a[i].item_at(d)]] = a[i].clone();
            count[a[i].item_at(d)] += 1;
        }

        // copy back
        for i in lo .. hi+1 {
            a[i] = aux[i-lo].clone();
        }

        // no more bits
        if d == 0 {
            return ;
        }

        if count[0] > 1 {
            Self::msd_sort_helper(a, lo, lo + count[0] - 1, d-1, aux);
        }
        for r in 0 .. r {
            if count[r+1] > count[r] {
                Self::msd_sort_helper(a, lo + count[r], lo + count[r+1] - 1, d-1, aux);
            }
        }
    }

    fn msd_sort(a: &mut [Self]) {
        let n = a.len();
        let w = a[0].width();
        let mut aux = iter::repeat(Default::default()).take(n).collect::<Vec<Self>>();
        Self::msd_sort_helper(a, 0, n-1, w-1, &mut aux);
    }

}

impl RadixSort for u32 {
    fn r() -> usize { 256 }
    fn width(&self) -> usize { 4 }
    fn item_at(&self, n: usize) -> usize {
        // 0:   lowest byte
        // n-1: highest byte
        // ((self >> (8 * (4-n))) & 0xFF) as usize
        ((self >> (8 * n)) & 0xFF) as usize
    }
}

impl RadixSort for u16 {
    fn r() -> usize { 256 }
    fn width(&self) -> usize { 2 }
    fn item_at(&self, n: usize) -> usize {
        // 0:   lowest byte
        // n-1: highest byte
        ((self >> (8 * n)) & 0xFF) as usize
    }
}

impl RadixSort for i16 {
    fn r() -> usize { 256 }
    fn width(&self) -> usize { 2 }
    fn item_at(&self, n: usize) -> usize {
        // 0:   lowest byte
        // n-1: highest byte
        let mut ret = (self >> (8 * n)) & 0xFF;
        // handle highest byte negative
        if n == 1 {
            if *self < 0 {
                ret -= 128;
            } else {
                ret += 128;
            }
        }
        ret as usize
    }
}

#[cfg(test)]
fn is_sorted<T: PartialOrd>(a: &[T]) -> bool {
    for i in 1 .. a.len() {
        if a[i] < a[i-1] {
            return false;
        }
    }
    true
}

#[test]
fn test_lsd_string_sort() {
    let mut to_be_sorted = vec![
        "dab", "add", "cab", "fad", "fee", "bad",
        "bee", "fed", "bed", "ebb", "ace", "dad"];
    assert!(!is_sorted(&to_be_sorted));
    lsd_string_sort(&mut to_be_sorted);
    assert!(is_sorted(&to_be_sorted));
}

#[test]
fn test_lsd_ints_sort() {
    let mut to_be_sorted: Vec<u32> = vec![
        2830665856, 3165123597, 780293973, 3412877992, 3781910346,
        2837710927, 560466804, 2778567425, 265098057, 18593204];

    assert!(!is_sorted(&to_be_sorted));
    RadixSort::lsd_sort(&mut to_be_sorted);
    assert!(is_sorted(&to_be_sorted));

    let mut to_be_sorted: Vec<i16> = vec![
        19195, -5970, 2060, 2553, 26078, 15152,
        2318, 9647, -7906, -8452, -127, 128, 127];

    assert!(!is_sorted(&to_be_sorted));
    RadixSort::lsd_sort(&mut to_be_sorted);
    assert!(is_sorted(&to_be_sorted));
}

#[test]
fn test_msd_ints_sort() {
    extern crate rand;
    use rand::{thread_rng, Rng};

    let mut rng = thread_rng();
    for sz in vec![6, 20, 1000] {
        let mut array = rng.gen_iter().take(sz).collect::<Vec<i16>>();
        assert!(!is_sorted(&array));
        RadixSort::msd_sort(&mut array);
        assert!(is_sorted(&array));

        let mut array = rng.gen_iter().take(sz).collect::<Vec<u32>>();
        assert!(!is_sorted(&array));
        RadixSort::msd_sort(&mut array);
        assert!(is_sorted(&array));
    }
}
<|endoftext|>


<commit_before>// Copyright (C) 2018 O.S. Systems Sofware LTDA
//
// SPDX-License-Identifier: MPL-2.0
//

use client::Api;
use failure::Error;
use states::idle::Idle;
use states::install::Install;
use states::{State, StateChangeImpl, StateMachine};
use std::fs::remove_file;
use update_package::{ObjectStatus, UpdatePackage};
use walkdir::WalkDir;

#[derive(Debug, PartialEq)]
pub struct Download {
    pub update_package: UpdatePackage,
}

create_state_step!(Download => Idle);
create_state_step!(Download => Install(update_package));

impl StateChangeImpl for State<Download> {
    fn to_next_state(self) -> Result<StateMachine, Error> {
        // Prune left over from previous installations
        WalkDir::new(&self.settings.update.download_dir)
            .follow_links(true)
            .min_depth(1)
            .into_iter()
            .filter_entry(|e| e.file_type().is_file())
            .filter_map(|e| e.ok())
            .filter(|e| {
                !self.state
                    .update_package
                    .objects()
                    .iter()
                    .map(|o| o.sha256sum())
                    .collect::<Vec<_>>()
                    .contains(&e.file_name().to_str().unwrap_or(""))
            })
            .for_each(|e| {
                remove_file(e.path()).unwrap_or_else(|err| {
                    error!("Fail to remove file: {} (err: {})", e.path().display(), err)
                })
            });

        // Prune corrupted files
        self.state
            .update_package
            .filter_objects(&self.settings, ObjectStatus::Corrupted)
            .into_iter()
            .for_each(|o| {
                remove_file(&self.settings.update.download_dir.join(o.sha256sum())).unwrap_or_else(
                    |err| error!("Fail to remove file: {} (err: {})", o.sha256sum(), err),
                )
            });

        // Download the missing or incomplete objects
        self.state
            .update_package
            .filter_objects(&self.settings, ObjectStatus::Missing)
            .into_iter()
            .chain(
                self.state
                    .update_package
                    .filter_objects(&self.settings, ObjectStatus::Incomplete),
            )
            .for_each(|o| {
                Api::new(&self.settings, &self.runtime_settings, &self.firmware)
                    .download_object(
                        &self.state.update_package.package_uid().unwrap(),
                        o.sha256sum(),
                    )
                    .unwrap_or_else(|err| {
                        error!("Fail to download object: {} (err: {})", o.sha256sum(), err);
                    });
            });

        // FIXME: Must return error when failing to download
        Ok(StateMachine::Install(self.into()))
    }
}

#[test]
fn skip_download_if_ready() {
    use super::*;
    use firmware::tests::{create_fake_metadata, FakeDevice};
    use std::fs::create_dir_all;
    use update_package::tests::{create_fake_object, create_fake_settings, get_update_package};

    let settings = create_fake_settings();
    let tmpdir = settings.update.download_dir.clone();
    let _ = create_dir_all(&tmpdir);
    let _ = create_fake_object(&settings);

    let machine = StateMachine::Download(State {
        settings: settings,
        runtime_settings: RuntimeSettings::default(),
        firmware: Metadata::new(&create_fake_metadata(FakeDevice::NoUpdate)).unwrap(),
        applied_package_uid: None,
        state: Download {
            update_package: get_update_package(),
        },
    }).step();

    assert_eq!(
        WalkDir::new(&tmpdir)
            .follow_links(true)
            .min_depth(1)
            .into_iter()
            .filter_entry(|e| e.file_type().is_file())
            .filter_map(|e| e.ok())
            .count(),
        1,
        "Number of objects is wrong"
    );

    assert_state!(machine, Install);
}

#[test]
fn download_objects() {
    use super::*;
    use crypto_hash::{hex_digest, Algorithm};
    use firmware::tests::{create_fake_metadata, FakeDevice};
    use mockito::mock;
    use std::fs::create_dir_all;
    use std::fs::File;
    use std::io::Read;
    use update_package::tests::{create_fake_settings, get_update_package};

    let settings = create_fake_settings();
    let update_package = get_update_package();
    let sha256sum = "c775e7b757ede630cd0aa1113bd102661ab38829ca52a6422ab782862f268646";
    let tmpdir = settings.update.download_dir.clone();
    let _ = create_dir_all(&tmpdir);

    // leftover file to ensure it is removed
    let _ = File::create(&tmpdir.join("leftover-file"));

    let mock = mock(
        "GET",
        format!(
            "/products/{}/packages/{}/objects/{}",
            "229ffd7e08721d716163fc81a2dbaf6c90d449f0a3b009b6a2defe8a0b0d7381",
            &update_package.package_uid().unwrap(),
            &sha256sum
        ).as_str(),
    ).match_header("Content-Type", "application/json")
        .match_header("Api-Content-Type", "application/vnd.updatehub-v1+json")
        .with_status(200)
        .with_body("1234567890")
        .create();

    let machine = StateMachine::Download(State {
        settings: settings,
        runtime_settings: RuntimeSettings::default(),
        firmware: Metadata::new(&create_fake_metadata(FakeDevice::NoUpdate)).unwrap(),
        applied_package_uid: None,
        state: Download {
            update_package: update_package,
        },
    }).step();

    mock.assert();

    assert_eq!(
        WalkDir::new(&tmpdir)
            .follow_links(true)
            .min_depth(1)
            .into_iter()
            .filter_entry(|e| e.file_type().is_file())
            .filter_map(|e| e.ok())
            .count(),
        1,
        "Failed to remove the corrupted object"
    );

    let mut object_content = String::new();
    let _ = File::open(&tmpdir.join(&sha256sum))
        .unwrap()
        .read_to_string(&mut object_content);

    assert_eq!(
        &hex_digest(Algorithm::SHA256, object_content.as_bytes()),
        &sha256sum,
        "Checksum mismatch"
    );

    assert_state!(machine, Install);
}
<commit_msg>states: download: Improve error handling<commit_after>// Copyright (C) 2018 O.S. Systems Sofware LTDA
//
// SPDX-License-Identifier: MPL-2.0
//

use client::Api;
use failure::Error;
use states::idle::Idle;
use states::install::Install;
use states::{State, StateChangeImpl, StateMachine};
use std::fs::remove_file;
use update_package::{ObjectStatus, UpdatePackage};
use walkdir::WalkDir;

#[derive(Debug, PartialEq)]
pub struct Download {
    pub update_package: UpdatePackage,
}

create_state_step!(Download => Idle);
create_state_step!(Download => Install(update_package));

impl StateChangeImpl for State<Download> {
    fn to_next_state(self) -> Result<StateMachine, Error> {
        // Prune left over from previous installations
        for entry in WalkDir::new(&self.settings.update.download_dir)
            .follow_links(true)
            .min_depth(1)
            .into_iter()
            .filter_entry(|e| e.file_type().is_file())
            .filter_map(|e| e.ok())
            .filter(|e| {
                !self.state
                    .update_package
                    .objects()
                    .iter()
                    .map(|o| o.sha256sum())
                    .collect::<Vec<_>>()
                    .contains(&e.file_name().to_str().unwrap_or(""))
            }) {
            remove_file(entry.path())?;
        }

        // Prune corrupted files
        for object in self.state
            .update_package
            .filter_objects(&self.settings, ObjectStatus::Corrupted)
            .into_iter()
        {
            remove_file(&self.settings.update.download_dir.join(object.sha256sum()))?;
        }

        // Download the missing or incomplete objects
        for object in self.state
            .update_package
            .filter_objects(&self.settings, ObjectStatus::Missing)
            .into_iter()
            .chain(
                self.state
                    .update_package
                    .filter_objects(&self.settings, ObjectStatus::Incomplete),
            ) {
            Api::new(&self.settings, &self.runtime_settings, &self.firmware).download_object(
                &self.state.update_package.package_uid().unwrap(),
                object.sha256sum(),
            )?;
        }

        if self.state
            .update_package
            .objects()
            .iter()
            .all(|o| o.status(&self.settings.update.download_dir).ok() == Some(ObjectStatus::Ready))
        {
            Ok(StateMachine::Install(self.into()))
        } else {
            bail!("Not all objects are ready for use")
        }
    }
}

#[test]
fn skip_download_if_ready() {
    use super::*;
    use firmware::tests::{create_fake_metadata, FakeDevice};
    use std::fs::create_dir_all;
    use update_package::tests::{create_fake_object, create_fake_settings, get_update_package};

    let settings = create_fake_settings();
    let tmpdir = settings.update.download_dir.clone();
    let _ = create_dir_all(&tmpdir);
    let _ = create_fake_object(&settings);

    let machine = StateMachine::Download(State {
        settings: settings,
        runtime_settings: RuntimeSettings::default(),
        firmware: Metadata::new(&create_fake_metadata(FakeDevice::NoUpdate)).unwrap(),
        applied_package_uid: None,
        state: Download {
            update_package: get_update_package(),
        },
    }).step();

    assert_state!(machine, Install);

    assert_eq!(
        WalkDir::new(&tmpdir)
            .follow_links(true)
            .min_depth(1)
            .into_iter()
            .filter_entry(|e| e.file_type().is_file())
            .filter_map(|e| e.ok())
            .count(),
        1,
        "Number of objects is wrong"
    );
}

#[test]
fn download_objects() {
    use super::*;
    use crypto_hash::{hex_digest, Algorithm};
    use firmware::tests::{create_fake_metadata, FakeDevice};
    use mockito::mock;
    use std::fs::create_dir_all;
    use std::fs::File;
    use std::io::Read;
    use update_package::tests::{create_fake_settings, get_update_package};

    let settings = create_fake_settings();
    let update_package = get_update_package();
    let sha256sum = "c775e7b757ede630cd0aa1113bd102661ab38829ca52a6422ab782862f268646";
    let tmpdir = settings.update.download_dir.clone();
    let _ = create_dir_all(&tmpdir);

    // leftover file to ensure it is removed
    let _ = File::create(&tmpdir.join("leftover-file"));

    let mock = mock(
        "GET",
        format!(
            "/products/{}/packages/{}/objects/{}",
            "229ffd7e08721d716163fc81a2dbaf6c90d449f0a3b009b6a2defe8a0b0d7381",
            &update_package.package_uid().unwrap(),
            &sha256sum
        ).as_str(),
    ).match_header("Content-Type", "application/json")
        .match_header("Api-Content-Type", "application/vnd.updatehub-v1+json")
        .with_status(200)
        .with_body("1234567890")
        .create();

    let machine = StateMachine::Download(State {
        settings: settings,
        runtime_settings: RuntimeSettings::default(),
        firmware: Metadata::new(&create_fake_metadata(FakeDevice::NoUpdate)).unwrap(),
        applied_package_uid: None,
        state: Download {
            update_package: update_package,
        },
    }).step();

    mock.assert();

    assert_state!(machine, Install);

    assert_eq!(
        WalkDir::new(&tmpdir)
            .follow_links(true)
            .min_depth(1)
            .into_iter()
            .filter_entry(|e| e.file_type().is_file())
            .filter_map(|e| e.ok())
            .count(),
        1,
        "Failed to remove the corrupted object"
    );

    let mut object_content = String::new();
    let _ = File::open(&tmpdir.join(&sha256sum))
        .unwrap()
        .read_to_string(&mut object_content);

    assert_eq!(
        &hex_digest(Algorithm::SHA256, object_content.as_bytes()),
        &sha256sum,
        "Checksum mismatch"
    );
}
<|endoftext|>


<commit_before>// Copyright 2014-2016 Johannes Köster.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Handling log-probabilities.

pub mod cdf;

use std::mem;
use std::f64;
use std::iter;
use std::ops::{Add, Sub, Mul, Div};

use itertools::linspace;
use itertools::Itertools;
use itertools::misc::ToFloat;

/// A factor to convert log-probabilities to PHRED-scale (phred = p * `LOG_TO_PHRED_FACTOR`).
const LOG_TO_PHRED_FACTOR: f64 = -4.3429448190325175; // -10 * 1 / ln(10)

/// A factor to convert PHRED-scale to log-probabilities (p = phred * `PHRED_TO_LOG_FACTOR`).
const PHRED_TO_LOG_FACTOR: f64 = -0.23025850929940456; // 1 / (-10 * log10(e))

/// Calculate log(1 - p) with p given in log space without loss of precision as described in
/// http://cran.r-project.org/web/packages/Rmpfr/vignettes/log1mexp-note.pdf.
fn ln_1m_exp(p: f64) -> f64 {
    assert!(p <= 0.0);
    if p < -0.693 {
        (-p.exp()).ln_1p()
    } else {
        (-p.exp_m1()).ln()
    }
}

custom_derive! {
    /// A newtype for probabilities.
    ///
    /// # Example
    ///
    /// ```
    /// #[macro_use]
    /// extern crate approx;
    /// # extern crate bio;
    /// # fn main() {
    /// use bio::stats::Prob;
    ///
    /// let p = Prob(0.5);
    /// let q = Prob(0.2);
    ///
    /// assert_relative_eq!(*(p + q), *Prob(0.7));
    /// # }
    /// ```
    #[derive(
        NewtypeFrom,
        NewtypeDeref,
        NewtypeAdd(*),
        NewtypeSub(*),
        NewtypeMul(*),
        NewtypeDiv(*),
        PartialEq,
        PartialOrd,
        Copy,
        Clone,
        Debug,
        Default
    )]
    pub struct Prob(pub f64);
}

custom_derive! {
    /// A newtype for log-scale probabilities.
    ///
    /// # Example
    ///
    /// ```
    /// #[macro_use]
    /// extern crate approx;
    /// # extern crate bio;
    /// # fn main() {
    /// use bio::stats::{LogProb, Prob};
    ///
    /// // convert from probability
    /// let p = LogProb::from(Prob(0.5));
    /// // convert manually
    /// let q = LogProb(0.2f64.ln());
    /// // obtain zero probability in log-space
    /// let o = LogProb::ln_one();
    ///
    /// assert_relative_eq!(*Prob::from(p.ln_add_exp(q) + o), *Prob(0.7));
    /// # }
    /// ```
    #[derive(
        NewtypeFrom,
        NewtypeDeref,
        NewtypeAdd(*),
        NewtypeSub(*),
        PartialEq,
        PartialOrd,
        Copy,
        Clone,
        Debug
    )]
    pub struct LogProb(pub f64);
}

custom_derive! {
    /// A newtype for PHRED-scale probabilities.
    ///
    /// # Example
    ///
    /// ```
    /// #[macro_use]
    /// extern crate approx;
    /// # extern crate bio;
    /// # fn main() {
    /// use bio::stats::{PHREDProb, Prob};
    ///
    /// let p = PHREDProb::from(Prob(0.5));
    ///
    /// assert_relative_eq!(*Prob::from(p), *Prob(0.5));
    /// # }
    /// ```
    #[derive(
        NewtypeFrom,
        NewtypeDeref,
        NewtypeAdd(*),
        NewtypeSub(*),
        PartialEq,
        PartialOrd,
        Copy,
        Clone,
        Debug
    )]
    pub struct PHREDProb(pub f64);
}

/// Iterator returned by scans over logprobs.
pub type ScanIter<I> = iter::Scan<<I as IntoIterator>::IntoIter, LogProb, fn(&mut LogProb, LogProb) -> Option<LogProb>>;

static LOGPROB_LN_ZERO: LogProb = LogProb(f64::NEG_INFINITY);
static LOGPROB_LN_ONE: LogProb = LogProb(0.0);

impl LogProb {
    /// Log-space representation of Pr=0
    pub fn ln_zero() -> LogProb {
        LOGPROB_LN_ZERO
    }

    /// Log-space representation of Pr=1
    pub fn ln_one() -> LogProb {
        LOGPROB_LN_ONE
    }

    /// Numerically stable calculation of 1 - p in log-space.
    pub fn ln_one_minus_exp(&self) -> LogProb {
        LogProb(ln_1m_exp(**self))
    }

    /// Numerically stable sum of probabilities in log-space.
    pub fn ln_sum_exp(probs: &[LogProb]) -> LogProb {
        if probs.is_empty() {
            Self::ln_zero()
        } else {
            let mut pmax = probs[0];
            let mut imax = 0;
            for (i, &p) in probs.iter().enumerate().skip(1) {
                if p > pmax {
                    pmax = p;
                    imax = i;
                }
            }
            if pmax == Self::ln_zero() {
                Self::ln_zero()
            } else if *pmax == f64::INFINITY {
                LogProb(f64::INFINITY)
            } else {
                // TODO use sum() once it has been stabilized: .sum::<usize>()
                pmax + LogProb(
                    (probs.iter()
                          .enumerate()
                          .filter_map(|(i, p)| {
                              if i == imax {
                                  None
                              } else {
                                  Some((p - pmax).exp())
                              }
                          })
                          .fold(0.0, |s, e| s + e)
                     ).ln_1p()
                 )
            }
        }
    }

    /// Numerically stable addition probabilities in log-space.
    pub fn ln_add_exp(self, other: LogProb) -> LogProb {
        let (mut p0, mut p1) = (self, other);
        if p1 > p0 {
            mem::swap(&mut p0, &mut p1);
        }
        if p0 == Self::ln_zero() {
            Self::ln_zero()
        } else if *p0 == f64::INFINITY {
            LogProb(f64::INFINITY)
        } else {
            p0 + LogProb((p1 - p0).exp().ln_1p())
        }
    }

    /// Numerically stable subtraction of probabilities in log-space.
    pub fn ln_sub_exp(self, other: LogProb) -> LogProb {
        let (p0, p1) = (self, other);
        assert!(p0 >= p1,
                "Subtraction would lead to negative probability, which is undefined in log space.");
        if relative_eq!(*p0, *p1) || p0 == Self::ln_zero() {
            // the first case leads to zero,
            // in the second case p0 and p1 are -inf, which is fine
            Self::ln_zero()
        } else if *p0 == f64::INFINITY {
            LogProb(f64::INFINITY)
        } else {
            p0 + (p1 - p0).ln_one_minus_exp()
        }
    }

    /// Calculate the cumulative sum of the given probabilities in a numerically stable way (Durbin 1998).
    pub fn ln_cumsum_exp<I: IntoIterator<Item = LogProb>>(probs: I) -> ScanIter<I> {
        probs.into_iter().scan(Self::ln_zero(), Self::scan_ln_add_exp)
    }

    /// Integrate numerically stable over given log-space density in the interval [a, b]. Uses the trapezoidal rule with n grid points.
    pub fn ln_integrate_exp<T, D>(density: &D, a: T, b: T, n: usize) -> LogProb where
        T: Copy + Add<Output=T> + Sub<Output=T> + Div<Output=T> + Mul<Output=T>,
        D: Fn(T) -> LogProb,
        usize: ToFloat<T>,
        f64: From<T>
    {
        let mut probs = linspace(a, b, n).dropping(1).dropping_back(1).map(|v| LogProb(*density(v) + 2.0f64.ln())).collect_vec();
        probs.push(density(a));
        probs.push(density(b));
        let width = f64::from(b - a);

        LogProb(*Self::ln_sum_exp(&probs) + width.ln() - (2.0 * (n - 1) as f64).ln())
    }

    fn scan_ln_add_exp(s: &mut LogProb, p: LogProb) -> Option<LogProb> {
        *s = s.ln_add_exp(p);
        Some(*s)
    }
}

impl From<LogProb> for Prob {
    fn from(p: LogProb) -> Prob {
        Prob(p.exp())
    }
}

impl From<PHREDProb> for Prob {
    fn from(p: PHREDProb) -> Prob {
        Prob(10.0f64.powf(-*p / 10.0))
    }
}

impl From<Prob> for LogProb {
    fn from(p: Prob) -> LogProb {
        LogProb(p.ln())
    }
}

impl From<PHREDProb> for LogProb {
    fn from(p: PHREDProb) -> LogProb {
        LogProb(*p * PHRED_TO_LOG_FACTOR)
    }
}

impl From<Prob> for PHREDProb {
    fn from(p: Prob) -> PHREDProb {
        PHREDProb(-10.0 * p.log10())
    }
}

impl From<LogProb> for PHREDProb {
    fn from(p: LogProb) -> PHREDProb {
        PHREDProb(*p * LOG_TO_PHRED_FACTOR)
    }
}

impl Default for LogProb {
    fn default() -> LogProb {
        LogProb::ln_zero()
    }
}

impl Default for PHREDProb {
    fn default() -> PHREDProb {
        PHREDProb::from(Prob(0.0))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use itertools::Itertools;

    #[test]
    fn test_sum() {
        let probs = [LogProb::ln_zero(), LogProb::ln_one(), LogProb::ln_zero()];
        assert_eq!(LogProb::ln_sum_exp(&probs), LogProb::ln_one());
    }

    #[test]
    fn test_empty_sum() {
        assert_eq!(LogProb::ln_sum_exp(&[]), LogProb::ln_zero());
    }

    #[test]
    fn test_cumsum() {
        let probs = vec![LogProb::ln_zero(), LogProb(0.01f64.ln()), LogProb(0.001f64.ln())];
        assert_eq!(LogProb::ln_cumsum_exp(probs).collect_vec(),
                   [LogProb::ln_zero(), LogProb(0.01f64.ln()), LogProb(0.011f64.ln())]);
    }

    #[test]
    fn test_sub() {
        assert_eq!(LogProb::ln_one().ln_sub_exp(LogProb::ln_one()), LogProb::ln_zero());
        assert_relative_eq!(*LogProb::ln_one().ln_sub_exp(LogProb(0.5f64.ln())), *LogProb(0.5f64.ln()));
    }

    #[test]
    fn test_one_minus() {
        assert_eq!(LogProb::ln_zero().ln_one_minus_exp(), LogProb::ln_one());
        assert_eq!(LogProb::ln_one().ln_one_minus_exp(), LogProb::ln_zero());
    }

    #[test]
    fn test_integrate() {
        let density = |_| LogProb(0.1f64.ln());
        let prob = LogProb::ln_integrate_exp(&density, 0.0, 10.0, 5);
        assert_relative_eq!(*prob, *LogProb::ln_one(), epsilon=0.0000001);
    }
}
<commit_msg>Make Prob, LogProb and PHREDProb de- and encodable.<commit_after>// Copyright 2014-2016 Johannes Köster.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Handling log-probabilities.

pub mod cdf;

use std::mem;
use std::f64;
use std::iter;
use std::ops::{Add, Sub, Mul, Div};

use itertools::linspace;
use itertools::Itertools;
use itertools::misc::ToFloat;

/// A factor to convert log-probabilities to PHRED-scale (phred = p * `LOG_TO_PHRED_FACTOR`).
const LOG_TO_PHRED_FACTOR: f64 = -4.3429448190325175; // -10 * 1 / ln(10)

/// A factor to convert PHRED-scale to log-probabilities (p = phred * `PHRED_TO_LOG_FACTOR`).
const PHRED_TO_LOG_FACTOR: f64 = -0.23025850929940456; // 1 / (-10 * log10(e))

/// Calculate log(1 - p) with p given in log space without loss of precision as described in
/// http://cran.r-project.org/web/packages/Rmpfr/vignettes/log1mexp-note.pdf.
fn ln_1m_exp(p: f64) -> f64 {
    assert!(p <= 0.0);
    if p < -0.693 {
        (-p.exp()).ln_1p()
    } else {
        (-p.exp_m1()).ln()
    }
}

custom_derive! {
    /// A newtype for probabilities.
    ///
    /// # Example
    ///
    /// ```
    /// #[macro_use]
    /// extern crate approx;
    /// # extern crate bio;
    /// # fn main() {
    /// use bio::stats::Prob;
    ///
    /// let p = Prob(0.5);
    /// let q = Prob(0.2);
    ///
    /// assert_relative_eq!(*(p + q), *Prob(0.7));
    /// # }
    /// ```
    #[derive(
        NewtypeFrom,
        NewtypeDeref,
        NewtypeAdd(*),
        NewtypeSub(*),
        NewtypeMul(*),
        NewtypeDiv(*),
        PartialEq,
        PartialOrd,
        Copy,
        Clone,
        Debug,
        Default,
        RustcDecodable,
        RustcEncodable
    )]
    #[cfg_attr(feature = "serde_macros", derive(Serialize, Deserialize))]
    pub struct Prob(pub f64);
}

custom_derive! {
    /// A newtype for log-scale probabilities.
    ///
    /// # Example
    ///
    /// ```
    /// #[macro_use]
    /// extern crate approx;
    /// # extern crate bio;
    /// # fn main() {
    /// use bio::stats::{LogProb, Prob};
    ///
    /// // convert from probability
    /// let p = LogProb::from(Prob(0.5));
    /// // convert manually
    /// let q = LogProb(0.2f64.ln());
    /// // obtain zero probability in log-space
    /// let o = LogProb::ln_one();
    ///
    /// assert_relative_eq!(*Prob::from(p.ln_add_exp(q) + o), *Prob(0.7));
    /// # }
    /// ```
    #[derive(
        NewtypeFrom,
        NewtypeDeref,
        NewtypeAdd(*),
        NewtypeSub(*),
        PartialEq,
        PartialOrd,
        Copy,
        Clone,
        Debug,
        RustcDecodable,
        RustcEncodable
    )]
    #[cfg_attr(feature = "serde_macros", derive(Serialize, Deserialize))]
    pub struct LogProb(pub f64);
}

custom_derive! {
    /// A newtype for PHRED-scale probabilities.
    ///
    /// # Example
    ///
    /// ```
    /// #[macro_use]
    /// extern crate approx;
    /// # extern crate bio;
    /// # fn main() {
    /// use bio::stats::{PHREDProb, Prob};
    ///
    /// let p = PHREDProb::from(Prob(0.5));
    ///
    /// assert_relative_eq!(*Prob::from(p), *Prob(0.5));
    /// # }
    /// ```
    #[derive(
        NewtypeFrom,
        NewtypeDeref,
        NewtypeAdd(*),
        NewtypeSub(*),
        PartialEq,
        PartialOrd,
        Copy,
        Clone,
        Debug,
        RustcDecodable,
        RustcEncodable
    )]
    #[cfg_attr(feature = "serde_macros", derive(Serialize, Deserialize))]
    pub struct PHREDProb(pub f64);
}

/// Iterator returned by scans over logprobs.
pub type ScanIter<I> = iter::Scan<<I as IntoIterator>::IntoIter, LogProb, fn(&mut LogProb, LogProb) -> Option<LogProb>>;

static LOGPROB_LN_ZERO: LogProb = LogProb(f64::NEG_INFINITY);
static LOGPROB_LN_ONE: LogProb = LogProb(0.0);

impl LogProb {
    /// Log-space representation of Pr=0
    pub fn ln_zero() -> LogProb {
        LOGPROB_LN_ZERO
    }

    /// Log-space representation of Pr=1
    pub fn ln_one() -> LogProb {
        LOGPROB_LN_ONE
    }

    /// Numerically stable calculation of 1 - p in log-space.
    pub fn ln_one_minus_exp(&self) -> LogProb {
        LogProb(ln_1m_exp(**self))
    }

    /// Numerically stable sum of probabilities in log-space.
    pub fn ln_sum_exp(probs: &[LogProb]) -> LogProb {
        if probs.is_empty() {
            Self::ln_zero()
        } else {
            let mut pmax = probs[0];
            let mut imax = 0;
            for (i, &p) in probs.iter().enumerate().skip(1) {
                if p > pmax {
                    pmax = p;
                    imax = i;
                }
            }
            if pmax == Self::ln_zero() {
                Self::ln_zero()
            } else if *pmax == f64::INFINITY {
                LogProb(f64::INFINITY)
            } else {
                // TODO use sum() once it has been stabilized: .sum::<usize>()
                pmax + LogProb(
                    (probs.iter()
                          .enumerate()
                          .filter_map(|(i, p)| {
                              if i == imax {
                                  None
                              } else {
                                  Some((p - pmax).exp())
                              }
                          })
                          .fold(0.0, |s, e| s + e)
                     ).ln_1p()
                 )
            }
        }
    }

    /// Numerically stable addition probabilities in log-space.
    pub fn ln_add_exp(self, other: LogProb) -> LogProb {
        let (mut p0, mut p1) = (self, other);
        if p1 > p0 {
            mem::swap(&mut p0, &mut p1);
        }
        if p0 == Self::ln_zero() {
            Self::ln_zero()
        } else if *p0 == f64::INFINITY {
            LogProb(f64::INFINITY)
        } else {
            p0 + LogProb((p1 - p0).exp().ln_1p())
        }
    }

    /// Numerically stable subtraction of probabilities in log-space.
    pub fn ln_sub_exp(self, other: LogProb) -> LogProb {
        let (p0, p1) = (self, other);
        assert!(p0 >= p1,
                "Subtraction would lead to negative probability, which is undefined in log space.");
        if relative_eq!(*p0, *p1) || p0 == Self::ln_zero() {
            // the first case leads to zero,
            // in the second case p0 and p1 are -inf, which is fine
            Self::ln_zero()
        } else if *p0 == f64::INFINITY {
            LogProb(f64::INFINITY)
        } else {
            p0 + (p1 - p0).ln_one_minus_exp()
        }
    }

    /// Calculate the cumulative sum of the given probabilities in a numerically stable way (Durbin 1998).
    pub fn ln_cumsum_exp<I: IntoIterator<Item = LogProb>>(probs: I) -> ScanIter<I> {
        probs.into_iter().scan(Self::ln_zero(), Self::scan_ln_add_exp)
    }

    /// Integrate numerically stable over given log-space density in the interval [a, b]. Uses the trapezoidal rule with n grid points.
    pub fn ln_integrate_exp<T, D>(density: &D, a: T, b: T, n: usize) -> LogProb where
        T: Copy + Add<Output=T> + Sub<Output=T> + Div<Output=T> + Mul<Output=T>,
        D: Fn(T) -> LogProb,
        usize: ToFloat<T>,
        f64: From<T>
    {
        let mut probs = linspace(a, b, n).dropping(1).dropping_back(1).map(|v| LogProb(*density(v) + 2.0f64.ln())).collect_vec();
        probs.push(density(a));
        probs.push(density(b));
        let width = f64::from(b - a);

        LogProb(*Self::ln_sum_exp(&probs) + width.ln() - (2.0 * (n - 1) as f64).ln())
    }

    fn scan_ln_add_exp(s: &mut LogProb, p: LogProb) -> Option<LogProb> {
        *s = s.ln_add_exp(p);
        Some(*s)
    }
}

impl From<LogProb> for Prob {
    fn from(p: LogProb) -> Prob {
        Prob(p.exp())
    }
}

impl From<PHREDProb> for Prob {
    fn from(p: PHREDProb) -> Prob {
        Prob(10.0f64.powf(-*p / 10.0))
    }
}

impl From<Prob> for LogProb {
    fn from(p: Prob) -> LogProb {
        LogProb(p.ln())
    }
}

impl From<PHREDProb> for LogProb {
    fn from(p: PHREDProb) -> LogProb {
        LogProb(*p * PHRED_TO_LOG_FACTOR)
    }
}

impl From<Prob> for PHREDProb {
    fn from(p: Prob) -> PHREDProb {
        PHREDProb(-10.0 * p.log10())
    }
}

impl From<LogProb> for PHREDProb {
    fn from(p: LogProb) -> PHREDProb {
        PHREDProb(*p * LOG_TO_PHRED_FACTOR)
    }
}

impl Default for LogProb {
    fn default() -> LogProb {
        LogProb::ln_zero()
    }
}

impl Default for PHREDProb {
    fn default() -> PHREDProb {
        PHREDProb::from(Prob(0.0))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use itertools::Itertools;

    #[test]
    fn test_sum() {
        let probs = [LogProb::ln_zero(), LogProb::ln_one(), LogProb::ln_zero()];
        assert_eq!(LogProb::ln_sum_exp(&probs), LogProb::ln_one());
    }

    #[test]
    fn test_empty_sum() {
        assert_eq!(LogProb::ln_sum_exp(&[]), LogProb::ln_zero());
    }

    #[test]
    fn test_cumsum() {
        let probs = vec![LogProb::ln_zero(), LogProb(0.01f64.ln()), LogProb(0.001f64.ln())];
        assert_eq!(LogProb::ln_cumsum_exp(probs).collect_vec(),
                   [LogProb::ln_zero(), LogProb(0.01f64.ln()), LogProb(0.011f64.ln())]);
    }

    #[test]
    fn test_sub() {
        assert_eq!(LogProb::ln_one().ln_sub_exp(LogProb::ln_one()), LogProb::ln_zero());
        assert_relative_eq!(*LogProb::ln_one().ln_sub_exp(LogProb(0.5f64.ln())), *LogProb(0.5f64.ln()));
    }

    #[test]
    fn test_one_minus() {
        assert_eq!(LogProb::ln_zero().ln_one_minus_exp(), LogProb::ln_one());
        assert_eq!(LogProb::ln_one().ln_one_minus_exp(), LogProb::ln_zero());
    }

    #[test]
    fn test_integrate() {
        let density = |_| LogProb(0.1f64.ln());
        let prob = LogProb::ln_integrate_exp(&density, 0.0, 10.0, 5);
        assert_relative_eq!(*prob, *LogProb::ln_one(), epsilon=0.0000001);
    }
}
<|endoftext|>


<commit_before>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
    storepath: String,
}

impl StorageBackend {

    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
        let storepath = rt.get_rtp() + "/store/";
        debug!("Trying to create {}", storepath);
        create_dir_all(&storepath).and_then(|_| {
            debug!("Creating succeeded, constructing backend instance");
            Ok(StorageBackend {
                basepath: rt.get_rtp(),
                storepath: storepath.clone(),
            })
        }).or_else(|e| {
            debug!("Creating failed, constructing error instance");
            let mut serr = StorageBackendError::build(
                "create_dir_all()",
                "Could not create store directories",
                Some(storepath)
            );
            serr.caused_by = Some(Box::new(e));
            Err(serr)
        })
    }

    fn get_file_ids(&self, m: &Module) -> Option<Vec<FileID>> {
        let list = glob(&self.prefix_of_files_for_module(m)[..]);

        if let Ok(globlist) = list {
            let mut v = vec![];
            for entry in globlist {
                if let Ok(path) = entry {
                    debug!(" - File: {:?}", path);
                    v.push(from_pathbuf(&path));
                } else {
                    // Entry is not a path
                }
            }

            Some(v)
        } else {
            None
        }
    }

    pub fn iter_ids(&self, m: &Module) -> Option<IntoIter<FileID>>
    {
        glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| {
            let v = globlist.filter_map(Result::ok)
                            .map(|pbuf| from_pathbuf(&pbuf))
                            .collect::<Vec<FileID>>()
                            .into_iter();
            Ok(v)
        }).ok()
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Option<IntoIter<File<'a>>>
        where HP: FileHeaderParser
    {
        self.iter_ids(m).and_then(|ids| {
            Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                    .collect::<Vec<File>>()
                    .into_iter())
        })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser<HP>) -> Option<File<'a>>
        where HP: FileHeaderParser
    {
        debug!("Searching for file with id '{}'", id);
        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
            let mut s = String::new();
            fs.read_to_string(&mut s);
            debug!("Success reading file with id '{}'", id);
            debug!("Parsing to internal structure now");
            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
        } else {
            debug!("No file with id '{}'", id);
            None
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::build(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("  id      : '{}'", id);
        self.prefix_of_files_for_module(owner) + "-" + &id[..] + ".imag"
    }

    fn prefix_of_files_for_module(&self, m: &Module) -> String {
        self.storepath.clone() + m.name()
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {
    fn new(action: String,
           desc  : String,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         action,
            desc:           desc,
            data_dump:      data,
            caused_by:      None,
        }
    }

    fn build(action: &'static str,
             desc:   &'static str,
             data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         String::from(action),
            desc:           String::from(desc),
            dataDump:       data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl<'a> Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::build(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<commit_msg>Fixup: Member name<commit_after>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
    storepath: String,
}

impl StorageBackend {

    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
        let storepath = rt.get_rtp() + "/store/";
        debug!("Trying to create {}", storepath);
        create_dir_all(&storepath).and_then(|_| {
            debug!("Creating succeeded, constructing backend instance");
            Ok(StorageBackend {
                basepath: rt.get_rtp(),
                storepath: storepath.clone(),
            })
        }).or_else(|e| {
            debug!("Creating failed, constructing error instance");
            let mut serr = StorageBackendError::build(
                "create_dir_all()",
                "Could not create store directories",
                Some(storepath)
            );
            serr.caused_by = Some(Box::new(e));
            Err(serr)
        })
    }

    fn get_file_ids(&self, m: &Module) -> Option<Vec<FileID>> {
        let list = glob(&self.prefix_of_files_for_module(m)[..]);

        if let Ok(globlist) = list {
            let mut v = vec![];
            for entry in globlist {
                if let Ok(path) = entry {
                    debug!(" - File: {:?}", path);
                    v.push(from_pathbuf(&path));
                } else {
                    // Entry is not a path
                }
            }

            Some(v)
        } else {
            None
        }
    }

    pub fn iter_ids(&self, m: &Module) -> Option<IntoIter<FileID>>
    {
        glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| {
            let v = globlist.filter_map(Result::ok)
                            .map(|pbuf| from_pathbuf(&pbuf))
                            .collect::<Vec<FileID>>()
                            .into_iter();
            Ok(v)
        }).ok()
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Option<IntoIter<File<'a>>>
        where HP: FileHeaderParser
    {
        self.iter_ids(m).and_then(|ids| {
            Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                    .collect::<Vec<File>>()
                    .into_iter())
        })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser<HP>) -> Option<File<'a>>
        where HP: FileHeaderParser
    {
        debug!("Searching for file with id '{}'", id);
        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
            let mut s = String::new();
            fs.read_to_string(&mut s);
            debug!("Success reading file with id '{}'", id);
            debug!("Parsing to internal structure now");
            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
        } else {
            debug!("No file with id '{}'", id);
            None
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::build(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("  id      : '{}'", id);
        self.prefix_of_files_for_module(owner) + "-" + &id[..] + ".imag"
    }

    fn prefix_of_files_for_module(&self, m: &Module) -> String {
        self.storepath.clone() + m.name()
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {
    fn new(action: String,
           desc  : String,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         action,
            desc:           desc,
            data_dump:      data,
            caused_by:      None,
        }
    }

    fn build(action: &'static str,
             desc:   &'static str,
             data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         String::from(action),
            desc:           String::from(desc),
            data_dump:      data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl<'a> Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::build(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate askama;
#[macro_use]
extern crate serde_json;

use askama::Template;
use serde_json::Value;

#[derive(Template)]
#[template(path = "simple.html")]
struct VariablesTemplate<'a> {
    strvar: &'a str,
    num: i64,
    i18n: String,
}

#[test]
fn test_variables() {
    let s = VariablesTemplate {
        strvar: "foo",
        num: 42,
        i18n: "Iñtërnâtiônàlizætiøn".to_string(),
    };
    assert_eq!(s.render().unwrap(), "\nhello world, foo\n\
                                     with number: 42\n\
                                     Iñtërnâtiônàlizætiøn is important\n\
                                     in vars too: Iñtërnâtiônàlizætiøn");
}

#[derive(Template)]
#[template(path = "if.html")]
struct IfTemplate {
    cond: bool,
}

#[test]
fn test_if() {
    let s = IfTemplate { cond: true };
    assert_eq!(s.render().unwrap(), "true");
}

#[derive(Template)]
#[template(path = "else.html")]
struct ElseTemplate {
    cond: bool,
}

#[test]
fn test_else() {
    let s = ElseTemplate { cond: false };
    assert_eq!(s.render().unwrap(), "false");
}

#[derive(Template)]
#[template(path = "else-if.html")]
struct ElseIfTemplate {
    cond: bool,
    check: bool,
}

#[test]
fn test_else_if() {
    let s = ElseIfTemplate { cond: false, check: true };
    assert_eq!(s.render().unwrap(), "checked");
}

#[derive(Template)]
#[template(path = "literals.html")]
struct LiteralsTemplate {}

#[test]
fn test_literals() {
    let s = LiteralsTemplate {};
    assert_eq!(s.render().unwrap(), "a");
}

struct Holder {
    a: usize,
}

#[derive(Template)]
#[template(path = "attr.html")]
struct AttrTemplate {
    inner: Holder,
}

#[test]
fn test_attr() {
    let t = AttrTemplate { inner: Holder { a: 5 } };
    assert_eq!(t.render().unwrap(), "5");
}

struct NestedHolder {
    holder: Holder,
}

#[derive(Template)]
#[template(path = "nested-attr.html")]
struct NestedAttrTemplate {
    inner: NestedHolder,
}

#[test]
fn test_nested_attr() {
    let t = NestedAttrTemplate {
        inner: NestedHolder { holder: Holder { a: 5 } }
    };
    assert_eq!(t.render().unwrap(), "5");
}

#[derive(Template)]
#[template(path = "option.html")]
struct OptionTemplate<'a> {
    var: Option<&'a str>,
}

#[test]
fn test_option() {
    let some = OptionTemplate { var: Some("foo") };
    assert_eq!(some.render().unwrap(), "some: foo");
    let none = OptionTemplate { var: None };
    assert_eq!(none.render().unwrap(), "none");
}

#[derive(Template)]
#[template(path = "generics.html")]
struct GenericsTemplate<T: std::fmt::Display, U = u8>
    where U: std::fmt::Display {
    t: T,
    u: U,
}

#[test]
fn test_generics() {
    let t = GenericsTemplate { t: "a", u: 42 };
    assert_eq!(t.render().unwrap(), "a42");
}

#[derive(Template)]
#[template(path = "json.html")]
struct JsonTemplate<'a> {
    foo: &'a str,
    bar: &'a Value,
}

#[test]
fn test_json() {
    let val =  json!({"arr": [ "one", 2, true, null ]});
    let t = JsonTemplate { foo: "a", bar: &val };
    // Note: the json filter lacks a way to specify initial indentation
    assert_eq!(t.render().unwrap(), r#"{
  "foo": "a",
  "bar": {
  "arr": [
    "one",
    2,
    true,
    null
  ]
}
}"#);
}

#[derive(Template)]
#[template(path = "composition.html")]
struct CompositionTemplate {
    foo: IfTemplate,
}

#[test]
fn test_composition() {
    let t = CompositionTemplate { foo: IfTemplate { cond: true } };
    assert_eq!(t.render().unwrap(), "composed: true");
}

#[derive(Template)]
#[template(source = "{{ foo }}", ext = "html")]
struct ImplicitEscapedTemplate<'a> {
    foo: &'a str,
}

#[test]
fn test_implicit_escaped() {
    let t = ImplicitEscapedTemplate { foo: "foo & bar" };
    assert_eq!(t.render().unwrap(), "foo & bar");
}

#[derive(Template)]
#[template(source = "{{ foo }}", ext = "html", escape = "html")]
struct ExplicitEscapedTemplate<'a> {
    foo: &'a str,
}

#[test]
fn test_explicit_escaped() {
    let t = ExplicitEscapedTemplate { foo: "foo & bar" };
    assert_eq!(t.render().unwrap(), "foo & bar");
}

#[derive(Template)]
#[template(source = "{{ foo }}", ext = "txt")]
struct UnescapedTemplate<'a> {
    foo: &'a str,
}

#[test]
fn test_unescaped() {
    let t = UnescapedTemplate { foo: "foo & bar" };
    assert_eq!(t.render().unwrap(), "foo & bar");
}
<commit_msg>Add test case for inferred escape mode<commit_after>#[macro_use]
extern crate askama;
#[macro_use]
extern crate serde_json;

use askama::Template;
use serde_json::Value;

#[derive(Template)]
#[template(path = "simple.html")]
struct VariablesTemplate<'a> {
    strvar: &'a str,
    num: i64,
    i18n: String,
}

#[test]
fn test_variables() {
    let s = VariablesTemplate {
        strvar: "foo",
        num: 42,
        i18n: "Iñtërnâtiônàlizætiøn".to_string(),
    };
    assert_eq!(s.render().unwrap(), "\nhello world, foo\n\
                                     with number: 42\n\
                                     Iñtërnâtiônàlizætiøn is important\n\
                                     in vars too: Iñtërnâtiônàlizætiøn");
}

#[derive(Template)]
#[template(path = "if.html")]
struct IfTemplate {
    cond: bool,
}

#[test]
fn test_if() {
    let s = IfTemplate { cond: true };
    assert_eq!(s.render().unwrap(), "true");
}

#[derive(Template)]
#[template(path = "else.html")]
struct ElseTemplate {
    cond: bool,
}

#[test]
fn test_else() {
    let s = ElseTemplate { cond: false };
    assert_eq!(s.render().unwrap(), "false");
}

#[derive(Template)]
#[template(path = "else-if.html")]
struct ElseIfTemplate {
    cond: bool,
    check: bool,
}

#[test]
fn test_else_if() {
    let s = ElseIfTemplate { cond: false, check: true };
    assert_eq!(s.render().unwrap(), "checked");
}

#[derive(Template)]
#[template(path = "literals.html")]
struct LiteralsTemplate {}

#[test]
fn test_literals() {
    let s = LiteralsTemplate {};
    assert_eq!(s.render().unwrap(), "a");
}

struct Holder {
    a: usize,
}

#[derive(Template)]
#[template(path = "attr.html")]
struct AttrTemplate {
    inner: Holder,
}

#[test]
fn test_attr() {
    let t = AttrTemplate { inner: Holder { a: 5 } };
    assert_eq!(t.render().unwrap(), "5");
}

struct NestedHolder {
    holder: Holder,
}

#[derive(Template)]
#[template(path = "nested-attr.html")]
struct NestedAttrTemplate {
    inner: NestedHolder,
}

#[test]
fn test_nested_attr() {
    let t = NestedAttrTemplate {
        inner: NestedHolder { holder: Holder { a: 5 } }
    };
    assert_eq!(t.render().unwrap(), "5");
}

#[derive(Template)]
#[template(path = "option.html")]
struct OptionTemplate<'a> {
    var: Option<&'a str>,
}

#[test]
fn test_option() {
    let some = OptionTemplate { var: Some("foo") };
    assert_eq!(some.render().unwrap(), "some: foo");
    let none = OptionTemplate { var: None };
    assert_eq!(none.render().unwrap(), "none");
}

#[derive(Template)]
#[template(path = "generics.html")]
struct GenericsTemplate<T: std::fmt::Display, U = u8>
    where U: std::fmt::Display {
    t: T,
    u: U,
}

#[test]
fn test_generics() {
    let t = GenericsTemplate { t: "a", u: 42 };
    assert_eq!(t.render().unwrap(), "a42");
}

#[derive(Template)]
#[template(path = "json.html")]
struct JsonTemplate<'a> {
    foo: &'a str,
    bar: &'a Value,
}

#[test]
fn test_json() {
    let val =  json!({"arr": [ "one", 2, true, null ]});
    let t = JsonTemplate { foo: "a", bar: &val };
    // Note: the json filter lacks a way to specify initial indentation
    assert_eq!(t.render().unwrap(), r#"{
  "foo": "a",
  "bar": {
  "arr": [
    "one",
    2,
    true,
    null
  ]
}
}"#);
}

#[derive(Template)]
#[template(path = "composition.html")]
struct CompositionTemplate {
    foo: IfTemplate,
}

#[test]
fn test_composition() {
    let t = CompositionTemplate { foo: IfTemplate { cond: true } };
    assert_eq!(t.render().unwrap(), "composed: true");
}

#[derive(Template)]
#[template(source = "{{ foo }}", ext = "html")]
struct ImplicitEscapedTemplate<'a> {
    foo: &'a str,
}

#[test]
fn test_implicit_escaped() {
    let t = ImplicitEscapedTemplate { foo: "foo & bar" };
    assert_eq!(t.render().unwrap(), "foo & bar");
}

#[derive(Template)]
#[template(source = "{{ foo }}", ext = "html", escape = "html")]
struct ExplicitEscapedTemplate<'a> {
    foo: &'a str,
}

#[test]
fn test_explicit_escaped() {
    let t = ExplicitEscapedTemplate { foo: "foo & bar" };
    assert_eq!(t.render().unwrap(), "foo & bar");
}

#[derive(Template)]
#[template(source = "{{ foo }}", ext = "txt")]
struct UnescapedTemplate<'a> {
    foo: &'a str,
}

#[test]
fn test_unescaped() {
    let t = UnescapedTemplate { foo: "foo & bar" };
    assert_eq!(t.render().unwrap(), "foo & bar");
}

#[derive(Template)]
#[template(path = "hello.html")]
struct PathBasedEscapedTemplate<'a> {
    name: &'a str,
}

#[test]
fn test_path_based_escaped() {
    let t = PathBasedEscapedTemplate { name: "you & me" };
    assert_eq!(t.render().unwrap(), "Hello, you & me!");
}
<|endoftext|>


<commit_before>
/// A Doc comment
#[AnAttribute]
pub struct Foo {
    #[rustfmt_skip]
    f :   SomeType, // Comment beside a field
    f: SomeType, // Comment beside a field
    // Comment on a field
    #[AnAttribute]
    g: SomeOtherType,
    /// A doc comment on a field
    h: AThirdType,
}

struct Bar;

// With a where clause and generics.
pub struct Foo<'a, Y: Baz>
    where X: Whatever
{
    f: SomeType, // Comment beside a field
}

struct Baz {
    a: A, // Comment A
    b: B, // Comment B
    c: C, // Comment C
}

struct Baz {
    // Comment A
    a: A,
    // Comment B
    b: B,
    // Comment C
    c: C,
}
<commit_msg>Add idempotent regression test for pub struct fields<commit_after>
/// A Doc comment
#[AnAttribute]
pub struct Foo {
    #[rustfmt_skip]
    f :   SomeType, // Comment beside a field
    f: SomeType, // Comment beside a field
    // Comment on a field
    #[AnAttribute]
    g: SomeOtherType,
    /// A doc comment on a field
    h: AThirdType,
    pub i: TypeForPublicField,
}

struct Bar;

// With a where clause and generics.
pub struct Foo<'a, Y: Baz>
    where X: Whatever
{
    f: SomeType, // Comment beside a field
}

struct Baz {
    a: A, // Comment A
    b: B, // Comment B
    c: C, // Comment C
}

struct Baz {
    // Comment A
    a: A,
    // Comment B
    b: B,
    // Comment C
    c: C,
}
<|endoftext|>


<commit_before>use std::net::{SocketAddr, UdpSocket};
use std::io;
use self::io::{Read, Write};
use num::bigint::BigUint;
use {amy, tracker, CONFIG};
use std::{thread, time};
use std::fs::OpenOptions;
use std::path::Path;

mod rt;
mod proto;

type ID = BigUint;

const BUCKET_MAX: usize = 8;
const VERSION: &'static str = "SY";
const SESSION_FILE: &'static str = "dht_data";
const MIN_BOOTSTRAP_BKTS: usize = 3;
const TX_TIMEOUT_SECS: i64 = 20;

pub struct Manager {
    id: usize,
    table: rt::RoutingTable,
    dht_flush: time::Instant,
    sock: UdpSocket,
    buf: Vec<u8>,
}

impl Manager {
    pub fn new(reg: &amy::Registrar) -> io::Result<Manager> {
        let sock = UdpSocket::bind(("0.0.0.0", CONFIG.dht.port))?;
        sock.set_nonblocking(true)?;
        let id = reg.register(&sock, amy::Event::Read)?;

        let p = Path::new(&CONFIG.disk.session[..]).join(SESSION_FILE);
        let mut data = Vec::new();
        if let Ok(mut f) = OpenOptions::new().read(true).open(&p) {
            f.read_to_end(&mut data)?;
        }
        let table = if let Some(t) = rt::RoutingTable::deserialize(&data[..]) {
            info!("DHT table loaded from disk!");
            t
        } else {
            info!("DHT table could not be read from disk, creating new table!");
            let mut t = rt::RoutingTable::new();
            if let Some(addr) = CONFIG.dht.bootstrap_node {
                info!("Using bootstrap node!");
                let (msg, _) = t.add_addr(addr);
                sock.send_to(&msg.encode(), addr)?;
            }
            t
        };

        Ok(Manager {
            table,
            sock,
            id,
            buf: vec![0u8; 500],
            dht_flush: time::Instant::now(),
        })
    }

    pub fn init(&mut self) {
        debug!("Initializing DHT nodes!");
        for (q, a) in self.table.init() {
            self.send_msg(&q.encode(), a);
        }
    }

    pub fn id(&self) -> usize {
        self.id
    }

    pub fn readable(&mut self) -> Vec<tracker::Response> {
        let mut resps = Vec::new();
        loop {
            match self.sock.recv_from(&mut self.buf[..]) {
                Ok((v, addr)) => {
                    trace!("Processing msg from {}", addr);
                    if let Ok(req) = proto::Request::decode(&self.buf[..v]) {
                        let resp = self.table.handle_req(req, addr).encode();
                        self.send_msg(&resp, addr);
                    } else if let Ok(resp) = proto::Response::decode(&self.buf[..v]) {
                        match self.table.handle_resp(resp, addr) {
                            Ok(r) => resps.push(r),
                            Err(q) => {
                                for (req, a) in q {
                                    self.send_msg(&req.encode(), a);
                                }
                            }
                        }
                    } else {
                        debug!("Received invalid message from {:?}!", addr);
                    }
                }
                Err(e) => {
                    if e.kind() == io::ErrorKind::WouldBlock {
                        break;
                    } else {
                        error!(
                            "Encountered unexpected error reading from UDP socket: {:?}!",
                            e
                        );
                        break;
                    }
                }
            }
        }
        resps
    }

    pub fn get_peers(&mut self, tid: usize, hash: [u8; 20]) {
        for (req, a) in self.table.get_peers(tid, hash) {
            self.send_msg(&req.encode(), a);
        }
    }

    pub fn add_addr(&mut self, addr: SocketAddr) {
        self.table.add_addr(addr);
    }

    pub fn announce(&mut self, hash: [u8; 20]) {
        for (req, a) in self.table.announce(hash) {
            self.send_msg(&req.encode(), a);
        }
    }

    pub fn tick(&mut self) {
        if self.dht_flush.elapsed() > time::Duration::from_secs(60) {
            let data = self.table.serialize();
            // TODO: Transmit this to the disk thread rather than spawning a thread.
            thread::spawn(move || {
                let p = Path::new(&CONFIG.disk.session[..]).join(SESSION_FILE);
                if let Err(e) = OpenOptions::new()
                    .write(true)
                    .create(true)
                    .open(&p)
                    .and_then(|mut f| f.write(&data[..]))
                {
                    error!("DHT serialization failed: {:?}!", e);
                }
            });
            self.dht_flush = time::Instant::now();
        }
        for (req, a) in self.table.tick() {
            self.send_msg(&req.encode(), a);
        }
    }

    fn send_msg(&mut self, msg: &[u8], addr: SocketAddr) {
        loop {
            if let Err(e) = self.sock.send_to(msg, addr) {
                if e.raw_os_error().map(|c| c != 11).unwrap_or(true) {
                    error!("Failed to send message on UDP socket: {:?}", e);
                    break;
                }
            }
        }
    }
}
<commit_msg>Ensure dht socket is shutdown<commit_after>use std::net::{SocketAddr, UdpSocket};
use std::io;
use self::io::{Read, Write};
use num::bigint::BigUint;
use {amy, tracker, CONFIG};
use std::{thread, time};
use std::fs::OpenOptions;
use std::path::Path;

mod rt;
mod proto;

type ID = BigUint;

const BUCKET_MAX: usize = 8;
const VERSION: &'static str = "SY";
const SESSION_FILE: &'static str = "dht_data";
const MIN_BOOTSTRAP_BKTS: usize = 3;
const TX_TIMEOUT_SECS: i64 = 20;

pub struct Manager {
    id: usize,
    table: rt::RoutingTable,
    dht_flush: time::Instant,
    sock: UdpSocket,
    buf: Vec<u8>,
}

impl Manager {
    pub fn new(reg: &amy::Registrar) -> io::Result<Manager> {
        let sock = UdpSocket::bind(("0.0.0.0", CONFIG.dht.port))?;
        sock.set_nonblocking(true)?;
        let id = reg.register(&sock, amy::Event::Read)?;

        let p = Path::new(&CONFIG.disk.session[..]).join(SESSION_FILE);
        let mut data = Vec::new();
        if let Ok(mut f) = OpenOptions::new().read(true).open(&p) {
            f.read_to_end(&mut data)?;
        }
        let table = if let Some(t) = rt::RoutingTable::deserialize(&data[..]) {
            info!("DHT table loaded from disk!");
            t
        } else {
            info!("DHT table could not be read from disk, creating new table!");
            let mut t = rt::RoutingTable::new();
            if let Some(addr) = CONFIG.dht.bootstrap_node {
                info!("Using bootstrap node!");
                let (msg, _) = t.add_addr(addr);
                sock.send_to(&msg.encode(), addr)?;
            }
            t
        };

        Ok(Manager {
            table,
            sock,
            id,
            buf: vec![0u8; 500],
            dht_flush: time::Instant::now(),
        })
    }

    pub fn init(&mut self) {
        debug!("Initializing DHT nodes!");
        for (q, a) in self.table.init() {
            self.send_msg(&q.encode(), a);
        }
    }

    pub fn id(&self) -> usize {
        self.id
    }

    pub fn readable(&mut self) -> Vec<tracker::Response> {
        let mut resps = Vec::new();
        loop {
            match self.sock.recv_from(&mut self.buf[..]) {
                Ok((v, addr)) => {
                    trace!("Processing msg from {}", addr);
                    if let Ok(req) = proto::Request::decode(&self.buf[..v]) {
                        let resp = self.table.handle_req(req, addr).encode();
                        self.send_msg(&resp, addr);
                    } else if let Ok(resp) = proto::Response::decode(&self.buf[..v]) {
                        match self.table.handle_resp(resp, addr) {
                            Ok(r) => resps.push(r),
                            Err(q) => {
                                for (req, a) in q {
                                    self.send_msg(&req.encode(), a);
                                }
                            }
                        }
                    } else {
                        debug!("Received invalid message from {:?}!", addr);
                    }
                }
                Err(e) => {
                    if e.kind() == io::ErrorKind::WouldBlock {
                        break;
                    } else {
                        error!(
                            "Encountered unexpected error reading from UDP socket: {:?}!",
                            e
                        );
                        break;
                    }
                }
            }
        }
        resps
    }

    pub fn get_peers(&mut self, tid: usize, hash: [u8; 20]) {
        for (req, a) in self.table.get_peers(tid, hash) {
            self.send_msg(&req.encode(), a);
        }
    }

    pub fn add_addr(&mut self, addr: SocketAddr) {
        self.table.add_addr(addr);
    }

    pub fn announce(&mut self, hash: [u8; 20]) {
        for (req, a) in self.table.announce(hash) {
            self.send_msg(&req.encode(), a);
        }
    }

    pub fn tick(&mut self) {
        if self.dht_flush.elapsed() > time::Duration::from_secs(60) {
            let data = self.table.serialize();
            // TODO: Transmit this to the disk thread rather than spawning a thread.
            thread::spawn(move || {
                let p = Path::new(&CONFIG.disk.session[..]).join(SESSION_FILE);
                if let Err(e) = OpenOptions::new()
                    .write(true)
                    .create(true)
                    .open(&p)
                    .and_then(|mut f| f.write(&data[..]))
                {
                    error!("DHT serialization failed: {:?}!", e);
                }
            });
            self.dht_flush = time::Instant::now();
        }
        for (req, a) in self.table.tick() {
            self.send_msg(&req.encode(), a);
        }
    }

    fn send_msg(&mut self, msg: &[u8], addr: SocketAddr) {
        // Cap tries to avoid burning CPU
        for _ in 0..25 {
            if let Err(e) = self.sock.send_to(msg, addr) {
                if e.raw_os_error().map(|c| c != 11).unwrap_or(true) {
                    error!("Failed to send message on UDP socket: {:?}", e);
                    break;
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>extern crate erlang_nif_sys;
use ::{ NifEnv, NifTerm, NifEncoder, NifDecoder, NifResult, NifError };
use ::types::atom;

macro_rules! impl_number_transcoder {
    ($dec_type:ty, $nif_type:ty, $encode_fun:ident, $decode_fun:ident) => {
        impl NifEncoder for $dec_type {
            fn encode<'a>(&self, env: NifEnv<'a>) -> NifTerm<'a> {
                unsafe { NifTerm::new(env, erlang_nif_sys::$encode_fun(env.as_c_arg(), *self as $nif_type)) }
            }
        }
        impl<'a> NifDecoder<'a> for $dec_type {
            fn decode(term: NifTerm) -> NifResult<$dec_type> {
                #![allow(unused_unsafe)]
                let mut res: $nif_type = Default::default();
                if unsafe { erlang_nif_sys::$decode_fun(term.get_env().as_c_arg(), term.as_c_arg(), &mut res) } == 0 {
                    return Err(NifError::BadArg);
                }
                Ok(res as $dec_type)
            }
        }
    }
}

// Base number types
impl_number_transcoder!(i32, i32, enif_make_int, enif_get_int);
impl_number_transcoder!(u32, u32, enif_make_uint, enif_get_uint);
impl_number_transcoder!(i64, i64, enif_make_int64, enif_get_int64);
impl_number_transcoder!(u64, u64, enif_make_uint64, enif_get_uint64);
impl_number_transcoder!(f64, f64, enif_make_double, enif_get_double);

// Casted number types
impl_number_transcoder!(i8, i32, enif_make_int, enif_get_int);
impl_number_transcoder!(u8, u32, enif_make_uint, enif_get_uint);
impl_number_transcoder!(i16, i32, enif_make_int, enif_get_int);
impl_number_transcoder!(u16, u32, enif_make_uint, enif_get_uint);
impl_number_transcoder!(f32, f64, enif_make_double, enif_get_double);

impl NifEncoder for bool {
    fn encode<'a>(&self, env: NifEnv<'a>) -> NifTerm<'a> {
        if *self {
            atom::true_().to_term(env)
        } else {
            atom::false_().to_term(env)
        }
    }
}
impl<'a> NifDecoder<'a> for bool {
    fn decode(term: NifTerm<'a>) -> NifResult<bool> {
        Ok(atom::is_truthy(term))
    }
}
<commit_msg>implement encoder/decoder for usize and isize<commit_after>extern crate erlang_nif_sys;
use ::{ NifEnv, NifTerm, NifEncoder, NifDecoder, NifResult, NifError };
use ::types::atom;

macro_rules! impl_number_transcoder {
    ($dec_type:ty, $nif_type:ty, $encode_fun:ident, $decode_fun:ident) => {
        impl NifEncoder for $dec_type {
            fn encode<'a>(&self, env: NifEnv<'a>) -> NifTerm<'a> {
                unsafe { NifTerm::new(env, erlang_nif_sys::$encode_fun(env.as_c_arg(), *self as $nif_type)) }
            }
        }
        impl<'a> NifDecoder<'a> for $dec_type {
            fn decode(term: NifTerm) -> NifResult<$dec_type> {
                #![allow(unused_unsafe)]
                let mut res: $nif_type = Default::default();
                if unsafe { erlang_nif_sys::$decode_fun(term.get_env().as_c_arg(), term.as_c_arg(), &mut res) } == 0 {
                    return Err(NifError::BadArg);
                }
                Ok(res as $dec_type)
            }
        }
    }
}

// Base number types
impl_number_transcoder!(i32, i32, enif_make_int, enif_get_int);
impl_number_transcoder!(u32, u32, enif_make_uint, enif_get_uint);
impl_number_transcoder!(i64, i64, enif_make_int64, enif_get_int64);
impl_number_transcoder!(u64, u64, enif_make_uint64, enif_get_uint64);
impl_number_transcoder!(f64, f64, enif_make_double, enif_get_double);

// Casted number types
impl_number_transcoder!(i8, i32, enif_make_int, enif_get_int);
impl_number_transcoder!(u8, u32, enif_make_uint, enif_get_uint);
impl_number_transcoder!(i16, i32, enif_make_int, enif_get_int);
impl_number_transcoder!(u16, u32, enif_make_uint, enif_get_uint);
impl_number_transcoder!(f32, f64, enif_make_double, enif_get_double);
impl_number_transcoder!(usize, u64, enif_make_uint64, enif_get_uint64);
impl_number_transcoder!(isize, i64, enif_make_int64, enif_get_int64);

impl NifEncoder for bool {
    fn encode<'a>(&self, env: NifEnv<'a>) -> NifTerm<'a> {
        if *self {
            atom::true_().to_term(env)
        } else {
            atom::false_().to_term(env)
        }
    }
}
impl<'a> NifDecoder<'a> for bool {
    fn decode(term: NifTerm<'a>) -> NifResult<bool> {
        Ok(atom::is_truthy(term))
    }
}
<|endoftext|>