<commit_before>use std;
use super::ParallelIterator;
use super::len::*;
use super::internal::*;

/// Specifies a "reduce operator". This is the combination of a start
/// value and a reduce function. The reduce function takes two items
/// and computes a reduced version. The start value `S` is a kind of
/// "zero" or "identity" value that may be intermingled as needed;
/// idealy, `reduce(S, X)` for any item `X` yields `X`.
///
/// Example: to sum up the values, use a `start_value` of `0` and a
/// reduce function of `reduce(a, b) = a + b`.
///
/// The order in which the reduce function will be applied is not
/// specified. For example, the input `[ 0 1 2 ]` might be reduced in a
/// sequential fashion:
///
/// ```ignore
/// reduce(reduce(reduce(S, 0), 1), 2)
/// ```
///
/// or it might be reduced in a tree-like way:
///
/// ```ignore
/// reduce(reduce(0, 1), reduce(S, 2))
/// ```
///
/// etc.
pub trait ReduceOp<T>: Sync {
    fn start_value(&self) -> T;
    fn reduce(&self, value1: T, value2: T) -> T;
}

pub fn reduce<PAR_ITER,REDUCE_OP,T>(pi: PAR_ITER, reduce_op: &REDUCE_OP) -> T
    where PAR_ITER: ParallelIterator<Item=T>,
          REDUCE_OP: ReduceOp<T>,
          T: Send,
{
    let consumer = ReduceConsumer { reduce_op: reduce_op };
    pi.drive_unindexed(consumer)
}

struct ReduceConsumer<'r, REDUCE_OP: 'r> {
    reduce_op: &'r REDUCE_OP,
}

impl<'r, REDUCE_OP> Copy for ReduceConsumer<'r, REDUCE_OP> {
}

impl<'r, REDUCE_OP> Clone for ReduceConsumer<'r, REDUCE_OP> {
    fn clone(&self) -> Self { *self }
}

impl<'r, REDUCE_OP, ITEM> Consumer<ITEM> for ReduceConsumer<'r, REDUCE_OP>
    where REDUCE_OP: ReduceOp<ITEM>,
          ITEM: Send,
{
    type Folder = ReduceFolder<'r, REDUCE_OP, ITEM>;
    type Reducer = Self;
    type Result = ITEM;

    fn cost(&mut self, cost: f64) -> f64 {
        // This isn't quite right, as we will do more than O(n) reductions, but whatever.
        cost * FUNC_ADJUSTMENT
    }

    fn split_at(self, _index: usize) -> (Self, Self, Self) {
        (self, self, self)
    }

    fn into_folder(self) -> ReduceFolder<'r, REDUCE_OP, ITEM> {
        ReduceFolder { reduce_op: self.reduce_op,
                       item: self.reduce_op.start_value() }
    }
}

impl<'r, REDUCE_OP, ITEM> UnindexedConsumer<ITEM> for ReduceConsumer<'r, REDUCE_OP>
    where REDUCE_OP: ReduceOp<ITEM>,
          ITEM: Send,
{
    fn split_off(&self) -> Self {
        ReduceConsumer { reduce_op: self.reduce_op }
    }

    fn to_reducer(&self) -> Self::Reducer {
        *self
    }
}

impl<'r, REDUCE_OP, ITEM> Reducer<ITEM> for ReduceConsumer<'r, REDUCE_OP>
    where REDUCE_OP: ReduceOp<ITEM>,
{
    fn reduce(self, left: ITEM, right: ITEM) -> ITEM {
        self.reduce_op.reduce(left, right)
    }
}

pub struct ReduceFolder<'r, REDUCE_OP: 'r, ITEM> {
    reduce_op: &'r REDUCE_OP,
    item: ITEM,
}

impl<'r, REDUCE_OP, ITEM> Folder<ITEM> for ReduceFolder<'r, REDUCE_OP, ITEM>
    where REDUCE_OP: ReduceOp<ITEM>,
{
    type Result = ITEM;

    fn consume(self, item: ITEM) -> Self {
        let item = self.reduce_op.reduce(self.item, item);
        ReduceFolder { reduce_op: self.reduce_op, item: item }
    }

    fn complete(self) -> ITEM {
        self.item
    }
}

///////////////////////////////////////////////////////////////////////////
// Specific operations

pub struct SumOp;

pub const SUM: &'static SumOp = &SumOp;

macro_rules! sum_rule {
    ($i:ty, $z:expr) => {
        impl ReduceOp<$i> for SumOp {
            fn start_value(&self) -> $i {
                $z
            }
            fn reduce(&self, value1: $i, value2: $i) -> $i {
                value1 + value2
            }
        }
    }
}

sum_rule!(i8, 0);
sum_rule!(i16, 0);
sum_rule!(i32, 0);
sum_rule!(i64, 0);
sum_rule!(isize, 0);
sum_rule!(u8, 0);
sum_rule!(u16, 0);
sum_rule!(u32, 0);
sum_rule!(u64, 0);
sum_rule!(usize, 0);
sum_rule!(f32, 0.0);
sum_rule!(f64, 0.0);

pub struct MulOp;

pub const MUL: &'static MulOp = &MulOp;

macro_rules! mul_rule {
    ($i:ty, $z:expr) => {
        impl ReduceOp<$i> for MulOp {
            fn start_value(&self) -> $i {
                $z
            }
            fn reduce(&self, value1: $i, value2: $i) -> $i {
                value1 * value2
            }
        }
    }
}

mul_rule!(i8, 1);
mul_rule!(i16, 1);
mul_rule!(i32, 1);
mul_rule!(i64, 1);
mul_rule!(isize, 1);
mul_rule!(u8, 1);
mul_rule!(u16, 1);
mul_rule!(u32, 1);
mul_rule!(u64, 1);
mul_rule!(usize, 1);
mul_rule!(f32, 1.0);
mul_rule!(f64, 1.0);

pub struct MinOp;

pub const MIN: &'static MinOp = &MinOp;

macro_rules! min_rule {
    ($i:ty, $z:expr, $f:expr) => {
        impl ReduceOp<$i> for MinOp {
            fn start_value(&self) -> $i {
                $z
            }
            fn reduce(&self, value1: $i, value2: $i) -> $i {
                $f(value1, value2)
            }
        }
    }
}

min_rule!(i8, std::i8::MAX, std::cmp::min);
min_rule!(i16, std::i16::MAX, std::cmp::min);
min_rule!(i32, std::i32::MAX, std::cmp::min);
min_rule!(i64, std::i64::MAX, std::cmp::min);
min_rule!(isize, std::isize::MAX, std::cmp::min);
min_rule!(u8, std::u8::MAX, std::cmp::min);
min_rule!(u16, std::u16::MAX, std::cmp::min);
min_rule!(u32, std::u32::MAX, std::cmp::min);
min_rule!(u64, std::u64::MAX, std::cmp::min);
min_rule!(usize, std::usize::MAX, std::cmp::min);
min_rule!(f32, std::f32::INFINITY, f32::min);
min_rule!(f64, std::f64::INFINITY, f64::min);

pub struct MaxOp;

pub const MAX: &'static MaxOp = &MaxOp;

macro_rules! max_rule {
    ($i:ty, $z:expr, $f:expr) => {
        impl ReduceOp<$i> for MaxOp {
            fn start_value(&self) -> $i {
                $z
            }
            fn reduce(&self, value1: $i, value2: $i) -> $i {
                $f(value1, value2)
            }
        }
    }
}

max_rule!(i8, std::i8::MIN, std::cmp::max);
max_rule!(i16, std::i16::MIN, std::cmp::max);
max_rule!(i32, std::i32::MIN, std::cmp::max);
max_rule!(i64, std::i64::MIN, std::cmp::max);
max_rule!(isize, std::isize::MIN, std::cmp::max);
max_rule!(u8, std::u8::MIN, std::cmp::max);
max_rule!(u16, std::u16::MIN, std::cmp::max);
max_rule!(u32, std::u32::MIN, std::cmp::max);
max_rule!(u64, std::u64::MIN, std::cmp::max);
max_rule!(usize, std::usize::MIN, std::cmp::max);
max_rule!(f32, std::f32::NEG_INFINITY, f32::max);
max_rule!(f64, std::f64::NEG_INFINITY, f64::max);

pub struct ReduceWithOp<'r, OP: 'r> {
    op: &'r OP
}

impl<'r, OP> ReduceWithOp<'r, OP> {
    pub fn new(op: &'r OP) -> ReduceWithOp<'r, OP> {
        ReduceWithOp {
            op: op
        }
    }
}

impl<'r, ITEM,OP> ReduceOp<Option<ITEM>> for ReduceWithOp<'r, OP>
    where OP: Fn(ITEM, ITEM) -> ITEM + Sync + 'r
{
    fn start_value(&self) -> Option<ITEM> {
        None
    }

    fn reduce(&self, value1: Option<ITEM>, value2: Option<ITEM>) -> Option<ITEM> {
        if let Some(value1) = value1 {
            if let Some(value2) = value2 {
                Some((self.op)(value1, value2))
            } else {
                Some(value1)
            }
        } else {
            value2
        }
    }
}

pub struct ReduceWithIdentityOp<'r, ITEM: 'r, OP: 'r> {
    identity: &'r ITEM,
    op: &'r OP,
}

impl<'r, ITEM, OP> ReduceWithIdentityOp<'r, ITEM, OP> {
    pub fn new(identity: &'r ITEM, op: &'r OP) -> ReduceWithIdentityOp<'r, ITEM, OP> {
        ReduceWithIdentityOp {
            identity: identity,
            op: op,
        }
    }
}

impl<'r, ITEM, OP> ReduceOp<ITEM> for ReduceWithIdentityOp<'r, ITEM, OP>
    where OP: Fn(ITEM, ITEM) -> ITEM + Sync,
          ITEM: 'r + Clone + Sync,
{
    fn start_value(&self) -> ITEM {
        self.identity.clone()
    }

    fn reduce(&self, value1: ITEM, value2: ITEM) -> ITEM {
        (self.op)(value1, value2)
    }
}

<commit_msg>Change 'idealy' to 'ideally' in src/par_iter/reduce.rs<commit_after>use std;
use super::ParallelIterator;
use super::len::*;
use super::internal::*;

/// Specifies a "reduce operator". This is the combination of a start
/// value and a reduce function. The reduce function takes two items
/// and computes a reduced version. The start value `S` is a kind of
/// "zero" or "identity" value that may be intermingled as needed;
/// ideally, `reduce(S, X)` for any item `X` yields `X`.
///
/// Example: to sum up the values, use a `start_value` of `0` and a
/// reduce function of `reduce(a, b) = a + b`.
///
/// The order in which the reduce function will be applied is not
/// specified. For example, the input `[ 0 1 2 ]` might be reduced in a
/// sequential fashion:
///
/// ```ignore
/// reduce(reduce(reduce(S, 0), 1), 2)
/// ```
///
/// or it might be reduced in a tree-like way:
///
/// ```ignore
/// reduce(reduce(0, 1), reduce(S, 2))
/// ```
///
/// etc.
pub trait ReduceOp<T>: Sync {
    fn start_value(&self) -> T;
    fn reduce(&self, value1: T, value2: T) -> T;
}

pub fn reduce<PAR_ITER,REDUCE_OP,T>(pi: PAR_ITER, reduce_op: &REDUCE_OP) -> T
    where PAR_ITER: ParallelIterator<Item=T>,
          REDUCE_OP: ReduceOp<T>,
          T: Send,
{
    let consumer = ReduceConsumer { reduce_op: reduce_op };
    pi.drive_unindexed(consumer)
}

struct ReduceConsumer<'r, REDUCE_OP: 'r> {
    reduce_op: &'r REDUCE_OP,
}

impl<'r, REDUCE_OP> Copy for ReduceConsumer<'r, REDUCE_OP> {
}

impl<'r, REDUCE_OP> Clone for ReduceConsumer<'r, REDUCE_OP> {
    fn clone(&self) -> Self { *self }
}

impl<'r, REDUCE_OP, ITEM> Consumer<ITEM> for ReduceConsumer<'r, REDUCE_OP>
    where REDUCE_OP: ReduceOp<ITEM>,
          ITEM: Send,
{
    type Folder = ReduceFolder<'r, REDUCE_OP, ITEM>;
    type Reducer = Self;
    type Result = ITEM;

    fn cost(&mut self, cost: f64) -> f64 {
        // This isn't quite right, as we will do more than O(n) reductions, but whatever.
        cost * FUNC_ADJUSTMENT
    }

    fn split_at(self, _index: usize) -> (Self, Self, Self) {
        (self, self, self)
    }

    fn into_folder(self) -> ReduceFolder<'r, REDUCE_OP, ITEM> {
        ReduceFolder { reduce_op: self.reduce_op,
                       item: self.reduce_op.start_value() }
    }
}

impl<'r, REDUCE_OP, ITEM> UnindexedConsumer<ITEM> for ReduceConsumer<'r, REDUCE_OP>
    where REDUCE_OP: ReduceOp<ITEM>,
          ITEM: Send,
{
    fn split_off(&self) -> Self {
        ReduceConsumer { reduce_op: self.reduce_op }
    }

    fn to_reducer(&self) -> Self::Reducer {
        *self
    }
}

impl<'r, REDUCE_OP, ITEM> Reducer<ITEM> for ReduceConsumer<'r, REDUCE_OP>
    where REDUCE_OP: ReduceOp<ITEM>,
{
    fn reduce(self, left: ITEM, right: ITEM) -> ITEM {
        self.reduce_op.reduce(left, right)
    }
}

pub struct ReduceFolder<'r, REDUCE_OP: 'r, ITEM> {
    reduce_op: &'r REDUCE_OP,
    item: ITEM,
}

impl<'r, REDUCE_OP, ITEM> Folder<ITEM> for ReduceFolder<'r, REDUCE_OP, ITEM>
    where REDUCE_OP: ReduceOp<ITEM>,
{
    type Result = ITEM;

    fn consume(self, item: ITEM) -> Self {
        let item = self.reduce_op.reduce(self.item, item);
        ReduceFolder { reduce_op: self.reduce_op, item: item }
    }

    fn complete(self) -> ITEM {
        self.item
    }
}

///////////////////////////////////////////////////////////////////////////
// Specific operations

pub struct SumOp;

pub const SUM: &'static SumOp = &SumOp;

macro_rules! sum_rule {
    ($i:ty, $z:expr) => {
        impl ReduceOp<$i> for SumOp {
            fn start_value(&self) -> $i {
                $z
            }
            fn reduce(&self, value1: $i, value2: $i) -> $i {
                value1 + value2
            }
        }
    }
}

sum_rule!(i8, 0);
sum_rule!(i16, 0);
sum_rule!(i32, 0);
sum_rule!(i64, 0);
sum_rule!(isize, 0);
sum_rule!(u8, 0);
sum_rule!(u16, 0);
sum_rule!(u32, 0);
sum_rule!(u64, 0);
sum_rule!(usize, 0);
sum_rule!(f32, 0.0);
sum_rule!(f64, 0.0);

pub struct MulOp;

pub const MUL: &'static MulOp = &MulOp;

macro_rules! mul_rule {
    ($i:ty, $z:expr) => {
        impl ReduceOp<$i> for MulOp {
            fn start_value(&self) -> $i {
                $z
            }
            fn reduce(&self, value1: $i, value2: $i) -> $i {
                value1 * value2
            }
        }
    }
}

mul_rule!(i8, 1);
mul_rule!(i16, 1);
mul_rule!(i32, 1);
mul_rule!(i64, 1);
mul_rule!(isize, 1);
mul_rule!(u8, 1);
mul_rule!(u16, 1);
mul_rule!(u32, 1);
mul_rule!(u64, 1);
mul_rule!(usize, 1);
mul_rule!(f32, 1.0);
mul_rule!(f64, 1.0);

pub struct MinOp;

pub const MIN: &'static MinOp = &MinOp;

macro_rules! min_rule {
    ($i:ty, $z:expr, $f:expr) => {
        impl ReduceOp<$i> for MinOp {
            fn start_value(&self) -> $i {
                $z
            }
            fn reduce(&self, value1: $i, value2: $i) -> $i {
                $f(value1, value2)
            }
        }
    }
}

min_rule!(i8, std::i8::MAX, std::cmp::min);
min_rule!(i16, std::i16::MAX, std::cmp::min);
min_rule!(i32, std::i32::MAX, std::cmp::min);
min_rule!(i64, std::i64::MAX, std::cmp::min);
min_rule!(isize, std::isize::MAX, std::cmp::min);
min_rule!(u8, std::u8::MAX, std::cmp::min);
min_rule!(u16, std::u16::MAX, std::cmp::min);
min_rule!(u32, std::u32::MAX, std::cmp::min);
min_rule!(u64, std::u64::MAX, std::cmp::min);
min_rule!(usize, std::usize::MAX, std::cmp::min);
min_rule!(f32, std::f32::INFINITY, f32::min);
min_rule!(f64, std::f64::INFINITY, f64::min);

pub struct MaxOp;

pub const MAX: &'static MaxOp = &MaxOp;

macro_rules! max_rule {
    ($i:ty, $z:expr, $f:expr) => {
        impl ReduceOp<$i> for MaxOp {
            fn start_value(&self) -> $i {
                $z
            }
            fn reduce(&self, value1: $i, value2: $i) -> $i {
                $f(value1, value2)
            }
        }
    }
}

max_rule!(i8, std::i8::MIN, std::cmp::max);
max_rule!(i16, std::i16::MIN, std::cmp::max);
max_rule!(i32, std::i32::MIN, std::cmp::max);
max_rule!(i64, std::i64::MIN, std::cmp::max);
max_rule!(isize, std::isize::MIN, std::cmp::max);
max_rule!(u8, std::u8::MIN, std::cmp::max);
max_rule!(u16, std::u16::MIN, std::cmp::max);
max_rule!(u32, std::u32::MIN, std::cmp::max);
max_rule!(u64, std::u64::MIN, std::cmp::max);
max_rule!(usize, std::usize::MIN, std::cmp::max);
max_rule!(f32, std::f32::NEG_INFINITY, f32::max);
max_rule!(f64, std::f64::NEG_INFINITY, f64::max);

pub struct ReduceWithOp<'r, OP: 'r> {
    op: &'r OP
}

impl<'r, OP> ReduceWithOp<'r, OP> {
    pub fn new(op: &'r OP) -> ReduceWithOp<'r, OP> {
        ReduceWithOp {
            op: op
        }
    }
}

impl<'r, ITEM,OP> ReduceOp<Option<ITEM>> for ReduceWithOp<'r, OP>
    where OP: Fn(ITEM, ITEM) -> ITEM + Sync + 'r
{
    fn start_value(&self) -> Option<ITEM> {
        None
    }

    fn reduce(&self, value1: Option<ITEM>, value2: Option<ITEM>) -> Option<ITEM> {
        if let Some(value1) = value1 {
            if let Some(value2) = value2 {
                Some((self.op)(value1, value2))
            } else {
                Some(value1)
            }
        } else {
            value2
        }
    }
}

pub struct ReduceWithIdentityOp<'r, ITEM: 'r, OP: 'r> {
    identity: &'r ITEM,
    op: &'r OP,
}

impl<'r, ITEM, OP> ReduceWithIdentityOp<'r, ITEM, OP> {
    pub fn new(identity: &'r ITEM, op: &'r OP) -> ReduceWithIdentityOp<'r, ITEM, OP> {
        ReduceWithIdentityOp {
            identity: identity,
            op: op,
        }
    }
}

impl<'r, ITEM, OP> ReduceOp<ITEM> for ReduceWithIdentityOp<'r, ITEM, OP>
    where OP: Fn(ITEM, ITEM) -> ITEM + Sync,
          ITEM: 'r + Clone + Sync,
{
    fn start_value(&self) -> ITEM {
        self.identity.clone()
    }

    fn reduce(&self, value1: ITEM, value2: ITEM) -> ITEM {
        (self.op)(value1, value2)
    }
}
<|endoftext|>


<commit_before>extern crate centralgithook;
extern crate tempdir;

mod helpers;

use centralgithook::migrate::RepoHost;
use centralgithook::migrate;
use tempdir::TempDir;

#[test]
fn test_initial_import()
{
    let td = TempDir::new("cgh_test").expect("folder cgh_test should be created");
    let central = helpers::TestRepo::new(&td.path().join("central"));
    let host = helpers::TestHost::new();
    let scratch = migrate::Scratch::new(&td.path().join("scratch"), &host);
    let shell = migrate::Shell { cwd: td.path().to_path_buf() };

    host.create_project("central").expect("create central failed");
    central.shell.command(&format!("git remote add origin {}", &host.remote_url("central")));

    central.add_file("modules/module_a/initial_a");
    central.add_file("modules/module_b/initial_b");
    let head = central.commit("initial");

    central.shell.command("git push origin master");
    migrate::central_submit(&scratch,
                            scratch.transfer(&head, &host.repo_dir("central")))
        .expect("call central_submit");

    shell.command(&format!("git clone {}", &host.remote_url("modules/module_a")));
    shell.command(&format!("git clone {}", &host.remote_url("modules/module_b")));
    let module_a = helpers::TestRepo::new(&td.path().join("module_a"));
    let module_b = helpers::TestRepo::new(&td.path().join("module_b"));

    assert!(module_a.has_file("initial_a"));
    assert!(module_b.has_file("initial_b"));

    central.add_file("modules/module_a/added_a");
    central.add_file("modules/module_c/added_c");
    let head = central.commit("add");

    central.shell.command("git push origin master");
    migrate::central_submit(&scratch, scratch.transfer(&head, &host.repo_dir("central")))
        .expect("call central_submit");

    module_a.shell.command("git pull");
    assert!(module_a.has_file("added_a"));

    shell.command(&format!("git clone {}", &host.remote_url("modules/module_c")));
    let module_c = helpers::TestRepo::new(&td.path().join("module_c"));

    assert!(module_c.has_file("added_c"));

    module_a.add_file("added/in_mod_a");
    let head = module_a.commit("module_a_commit");

    migrate::module_review_upload(&scratch,
                                  scratch.transfer(&head, &module_a.path),
                                  "modules/module_a",
                                  "central")
        .expect("module_review_upload failed");

    central.shell.command("git fetch origin for/master:for/master");
    let for_master = central.rev("for/master");

    migrate::central_submit(&scratch, scratch.transfer(&for_master, &host.repo_dir("central")))
        .expect("call central_submit");

    module_a.shell.command("git pull");
    assert_eq!(module_a.rev("origin/master"), head);

}
<commit_msg>add more asserts in tests<commit_after>extern crate centralgithook;
extern crate tempdir;

mod helpers;

use centralgithook::migrate::RepoHost;
use centralgithook::migrate;
use tempdir::TempDir;

#[test]
fn test_initial_import()
{
    let td = TempDir::new("cgh_test").expect("folder cgh_test should be created");
    let central = helpers::TestRepo::new(&td.path().join("central"));
    let host = helpers::TestHost::new();
    let scratch = migrate::Scratch::new(&td.path().join("scratch"), &host);
    let shell = migrate::Shell { cwd: td.path().to_path_buf() };

    host.create_project("central").expect("create central failed");
    central.shell.command(&format!("git remote add origin {}", &host.remote_url("central")));

    central.add_file("modules/module_a/initial_a");
    central.add_file("modules/module_b/initial_b");
    let head = central.commit("initial");

    central.shell.command("git push origin master");
    migrate::central_submit(&scratch,
                            scratch.transfer(&head, &host.repo_dir("central")))
        .expect("call central_submit");

    shell.command(&format!("git clone {}", &host.remote_url("modules/module_a")));
    shell.command(&format!("git clone {}", &host.remote_url("modules/module_b")));
    let module_a = helpers::TestRepo::new(&td.path().join("module_a"));
    let module_b = helpers::TestRepo::new(&td.path().join("module_b"));

    assert!(module_a.has_file("initial_a"));
    assert!(module_b.has_file("initial_b"));

    central.add_file("modules/module_a/added_a");
    central.add_file("modules/module_c/added_c");
    let head = central.commit("add");

    central.shell.command("git push origin master");
    migrate::central_submit(&scratch, scratch.transfer(&head, &host.repo_dir("central")))
        .expect("call central_submit");

    module_a.shell.command("git pull");
    assert!(module_a.has_file("added_a"));

    shell.command(&format!("git clone {}", &host.remote_url("modules/module_c")));
    let module_c = helpers::TestRepo::new(&td.path().join("module_c"));

    assert!(module_c.has_file("added_c"));

    module_a.add_file("added/in_mod_a");
    let head = module_a.commit("module_a_commit");

    migrate::module_review_upload(&scratch,
                                  scratch.transfer(&head, &module_a.path),
                                  "modules/module_a",
                                  "central")
        .expect("module_review_upload failed");

    central.shell.command("git fetch origin for/master:for/master");
    let for_master = central.rev("for/master");

    migrate::central_submit(&scratch, scratch.transfer(&for_master, &host.repo_dir("central")))
        .expect("call central_submit");

    central.shell.command("git rebase for/master");
    assert_eq!(central.rev("master"), central.rev("for/master"));
    assert_eq!(central.rev("master"), central.rev("HEAD"));
    assert!(central.has_file("modules/module_a/added/in_mod_a"));

    module_a.shell.command("git pull");
    assert_eq!(module_a.rev("origin/master"), head);
}
<|endoftext|>


<commit_before>use std::cmp;
use std::collections::HashMap;
use std::sync::{
    atomic::{AtomicUsize, Ordering},
    Arc, Mutex,
};

use futures::{future::FutureExt, select_biased};
use tbot::{
    connectors::Connector,
    types::parameters::{self, WebPagePreviewState},
    Bot,
};
use tokio::{
    self,
    stream::StreamExt,
    sync::Notify,
    time::{self, delay_for, delay_queue::DelayQueue, Duration, Instant},
};

use crate::client::pull_feed;
use crate::data::{Database, Feed, FeedUpdate};
use crate::messages::{format_large_msg, Escape};

pub fn start(
    bot: Bot<impl Connector>,
    db: Arc<Mutex<Database>>,
    min_interval: u32,
    max_interval: u32,
) {
    let mut queue = FetchQueue::new();
    // TODO: Don't use interval, it can accumulate ticks
    // replace it with delay_until
    let mut interval = time::interval_at(Instant::now(), Duration::from_secs(min_interval as u64));
    let throttle = Throttle::new(min_interval as usize);
    tokio::spawn(async move {
        loop {
            select_biased! {
                feed = queue.next().fuse() => {
                    let feed = feed.expect("unreachable");
                    let bot = bot.clone();
                    let db = db.clone();
                    let opportunity = throttle.acquire();
                    tokio::spawn(async move {
                        opportunity.wait().await;
                        if let Err(e) = fetch_and_push_updates(bot, db, feed).await {
                            dbg!(e);
                        }
                    });
                }
                _ = interval.tick().fuse() => {
                    let feeds = db.lock().unwrap().all_feeds();
                    for feed in feeds {
                        let feed_interval = cmp::min(
                            feed.ttl.map(|ttl| ttl * 60).unwrap_or(min_interval),
                            max_interval,
                        ) as u64 - 1; // after -1, we can stagger with `interval`
                        queue.enqueue(feed, Duration::from_secs(feed_interval));
                    }
                }
            }
        }
    });
}

async fn fetch_and_push_updates(
    bot: Bot<impl Connector>,
    db: Arc<Mutex<Database>>,
    feed: Feed,
) -> Result<(), tbot::errors::MethodCall> {
    let new_feed = match pull_feed(&feed.link).await {
        Ok(feed) => feed,
        Err(e) => {
            let down_time = db.lock().unwrap().get_or_update_down_time(&feed.link);
            // 5 days
            if down_time.as_secs() > 5 * 24 * 60 * 60 {
                db.lock().unwrap().reset_down_time(&feed.link);
                let msg = format!(
                    "《{}》\
                     已经连续 5 天拉取出错 ({}),\
                     可能已经关闭, 请取消订阅",
                    Escape(&feed.link),
                    Escape(&feed.title),
                    Escape(&e.to_string())
                );
                push_updates(&bot, &db, feed.subscribers, parameters::Text::html(&msg)).await?;
            }
            return Ok(());
        }
    };

    let updates = db.lock().unwrap().update(&feed.link, new_feed);
    for update in updates {
        match update {
            FeedUpdate::Items(items) => {
                let msgs =
                    format_large_msg(format!("{}", Escape(&feed.title)), &items, |item| {
                        let title = item
                            .title
                            .as_ref()
                            .map(|s| s.as_str())
                            .unwrap_or_else(|| &feed.title);
                        let link = item
                            .link
                            .as_ref()
                            .map(|s| s.as_str())
                            .unwrap_or_else(|| &feed.link);
                        format!("{}", Escape(link), Escape(title))
                    });
                for msg in msgs {
                    push_updates(
                        &bot,
                        &db,
                        feed.subscribers.iter().copied(),
                        parameters::Text::html(&msg),
                    )
                    .await?;
                }
            }
            FeedUpdate::Title(new_title) => {
                let msg = format!(
                    "{} 已更名为 {}",
                    Escape(&feed.link),
                    Escape(&feed.title),
                    Escape(&new_title)
                );
                push_updates(
                    &bot,
                    &db,
                    feed.subscribers.iter().copied(),
                    parameters::Text::html(&msg),
                )
                .await?;
            }
        }
    }
    Ok(())
}

async fn push_updates<I: IntoIterator<Item = i64>>(
    bot: &Bot<impl Connector>,
    db: &Arc<Mutex<Database>>,
    subscribers: I,
    msg: parameters::Text<'_>,
) -> Result<(), tbot::errors::MethodCall> {
    use tbot::errors::MethodCall;
    for mut subscriber in subscribers {
        'retry: for _ in 0..3 {
            match bot
                .send_message(tbot::types::chat::Id(subscriber), msg)
                .web_page_preview(WebPagePreviewState::Disabled)
                .call()
                .await
            {
                Err(MethodCall::RequestError { description, .. })
                    if chat_is_unavailable(&description) =>
                {
                    db.lock().unwrap().delete_subscriber(subscriber);
                }
                Err(MethodCall::RequestError {
                    migrate_to_chat_id: Some(new_chat_id),
                    ..
                }) => {
                    db.lock()
                        .unwrap()
                        .update_subscriber(subscriber, new_chat_id.0);
                    subscriber = new_chat_id.0;
                    continue 'retry;
                }
                Err(MethodCall::RequestError {
                    retry_after: Some(delay),
                    ..
                }) => {
                    time::delay_for(Duration::from_secs(delay)).await;
                    continue 'retry;
                }
                other => {
                    other?;
                }
            }
            break 'retry;
        }
    }
    Ok(())
}

pub fn chat_is_unavailable(s: &str) -> bool {
    s.contains("Forbidden") || s.contains("chat not found")
}

#[derive(Default)]
struct FetchQueue {
    feeds: HashMap<String, Feed>,
    notifies: DelayQueue<String>,
    wakeup: Notify,
}

impl FetchQueue {
    fn new() -> Self {
        Self::default()
    }

    fn enqueue(&mut self, feed: Feed, delay: Duration) -> bool {
        let exists = self.feeds.contains_key(&feed.link);
        if !exists {
            self.notifies.insert(feed.link.clone(), delay);
            self.feeds.insert(feed.link.clone(), feed);
            self.wakeup.notify();
        }
        !exists
    }

    async fn next(&mut self) -> Result<Feed, time::Error> {
        loop {
            if let Some(feed_id) = self.notifies.next().await {
                let feed = self.feeds.remove(feed_id?.get_ref()).unwrap();
                break Ok(feed);
            } else {
                self.wakeup.notified().await;
            }
        }
    }
}

struct Throttle {
    pieces: usize,
    counter: Arc<AtomicUsize>,
}

impl Throttle {
    fn new(pieces: usize) -> Self {
        Throttle {
            pieces,
            counter: Arc::new(AtomicUsize::new(0)),
        }
    }

    fn acquire(&self) -> Opportunity {
        Opportunity {
            n: self.counter.fetch_add(1, Ordering::AcqRel) % self.pieces,
            counter: self.counter.clone(),
        }
    }
}

#[must_use = "Don't lose your opportunity"]
struct Opportunity {
    n: usize,
    counter: Arc<AtomicUsize>,
}

impl Opportunity {
    async fn wait(&self) {
        delay_for(Duration::from_secs(self.n as u64)).await
    }
}

impl Drop for Opportunity {
    fn drop(&mut self) {
        self.counter.fetch_sub(1, Ordering::SeqCst);
    }
}
<commit_msg>Update `chat_is_unavailable`, cover the new errors<commit_after>use std::cmp;
use std::collections::HashMap;
use std::sync::{
    atomic::{AtomicUsize, Ordering},
    Arc, Mutex,
};

use futures::{future::FutureExt, select_biased};
use tbot::{
    connectors::Connector,
    types::parameters::{self, WebPagePreviewState},
    Bot,
};
use tokio::{
    self,
    stream::StreamExt,
    sync::Notify,
    time::{self, delay_for, delay_queue::DelayQueue, Duration, Instant},
};

use crate::client::pull_feed;
use crate::data::{Database, Feed, FeedUpdate};
use crate::messages::{format_large_msg, Escape};

pub fn start(
    bot: Bot<impl Connector>,
    db: Arc<Mutex<Database>>,
    min_interval: u32,
    max_interval: u32,
) {
    let mut queue = FetchQueue::new();
    // TODO: Don't use interval, it can accumulate ticks
    // replace it with delay_until
    let mut interval = time::interval_at(Instant::now(), Duration::from_secs(min_interval as u64));
    let throttle = Throttle::new(min_interval as usize);
    tokio::spawn(async move {
        loop {
            select_biased! {
                feed = queue.next().fuse() => {
                    let feed = feed.expect("unreachable");
                    let bot = bot.clone();
                    let db = db.clone();
                    let opportunity = throttle.acquire();
                    tokio::spawn(async move {
                        opportunity.wait().await;
                        if let Err(e) = fetch_and_push_updates(bot, db, feed).await {
                            dbg!(e);
                        }
                    });
                }
                _ = interval.tick().fuse() => {
                    let feeds = db.lock().unwrap().all_feeds();
                    for feed in feeds {
                        let feed_interval = cmp::min(
                            feed.ttl.map(|ttl| ttl * 60).unwrap_or(min_interval),
                            max_interval,
                        ) as u64 - 1; // after -1, we can stagger with `interval`
                        queue.enqueue(feed, Duration::from_secs(feed_interval));
                    }
                }
            }
        }
    });
}

async fn fetch_and_push_updates(
    bot: Bot<impl Connector>,
    db: Arc<Mutex<Database>>,
    feed: Feed,
) -> Result<(), tbot::errors::MethodCall> {
    let new_feed = match pull_feed(&feed.link).await {
        Ok(feed) => feed,
        Err(e) => {
            let down_time = db.lock().unwrap().get_or_update_down_time(&feed.link);
            // 5 days
            if down_time.as_secs() > 5 * 24 * 60 * 60 {
                db.lock().unwrap().reset_down_time(&feed.link);
                let msg = format!(
                    "《{}》\
                     已经连续 5 天拉取出错 ({}),\
                     可能已经关闭, 请取消订阅",
                    Escape(&feed.link),
                    Escape(&feed.title),
                    Escape(&e.to_string())
                );
                push_updates(&bot, &db, feed.subscribers, parameters::Text::html(&msg)).await?;
            }
            return Ok(());
        }
    };

    let updates = db.lock().unwrap().update(&feed.link, new_feed);
    for update in updates {
        match update {
            FeedUpdate::Items(items) => {
                let msgs =
                    format_large_msg(format!("{}", Escape(&feed.title)), &items, |item| {
                        let title = item
                            .title
                            .as_ref()
                            .map(|s| s.as_str())
                            .unwrap_or_else(|| &feed.title);
                        let link = item
                            .link
                            .as_ref()
                            .map(|s| s.as_str())
                            .unwrap_or_else(|| &feed.link);
                        format!("{}", Escape(link), Escape(title))
                    });
                for msg in msgs {
                    push_updates(
                        &bot,
                        &db,
                        feed.subscribers.iter().copied(),
                        parameters::Text::html(&msg),
                    )
                    .await?;
                }
            }
            FeedUpdate::Title(new_title) => {
                let msg = format!(
                    "{} 已更名为 {}",
                    Escape(&feed.link),
                    Escape(&feed.title),
                    Escape(&new_title)
                );
                push_updates(
                    &bot,
                    &db,
                    feed.subscribers.iter().copied(),
                    parameters::Text::html(&msg),
                )
                .await?;
            }
        }
    }
    Ok(())
}

async fn push_updates<I: IntoIterator<Item = i64>>(
    bot: &Bot<impl Connector>,
    db: &Arc<Mutex<Database>>,
    subscribers: I,
    msg: parameters::Text<'_>,
) -> Result<(), tbot::errors::MethodCall> {
    use tbot::errors::MethodCall;
    for mut subscriber in subscribers {
        'retry: for _ in 0..3 {
            match bot
                .send_message(tbot::types::chat::Id(subscriber), msg)
                .web_page_preview(WebPagePreviewState::Disabled)
                .call()
                .await
            {
                Err(MethodCall::RequestError { description, .. })
                    if chat_is_unavailable(&description) =>
                {
                    db.lock().unwrap().delete_subscriber(subscriber);
                }
                Err(MethodCall::RequestError {
                    migrate_to_chat_id: Some(new_chat_id),
                    ..
                }) => {
                    db.lock()
                        .unwrap()
                        .update_subscriber(subscriber, new_chat_id.0);
                    subscriber = new_chat_id.0;
                    continue 'retry;
                }
                Err(MethodCall::RequestError {
                    retry_after: Some(delay),
                    ..
                }) => {
                    time::delay_for(Duration::from_secs(delay)).await;
                    continue 'retry;
                }
                other => {
                    other?;
                }
            }
            break 'retry;
        }
    }
    Ok(())
}

pub fn chat_is_unavailable(s: &str) -> bool {
    s.contains("Forbidden")
        || s.contains("chat not found")
        || s.contains("have no rights")
        || s.contains("need administrator rights")
}

#[derive(Default)]
struct FetchQueue {
    feeds: HashMap<String, Feed>,
    notifies: DelayQueue<String>,
    wakeup: Notify,
}

impl FetchQueue {
    fn new() -> Self {
        Self::default()
    }

    fn enqueue(&mut self, feed: Feed, delay: Duration) -> bool {
        let exists = self.feeds.contains_key(&feed.link);
        if !exists {
            self.notifies.insert(feed.link.clone(), delay);
            self.feeds.insert(feed.link.clone(), feed);
            self.wakeup.notify();
        }
        !exists
    }

    async fn next(&mut self) -> Result<Feed, time::Error> {
        loop {
            if let Some(feed_id) = self.notifies.next().await {
                let feed = self.feeds.remove(feed_id?.get_ref()).unwrap();
                break Ok(feed);
            } else {
                self.wakeup.notified().await;
            }
        }
    }
}

struct Throttle {
    pieces: usize,
    counter: Arc<AtomicUsize>,
}

impl Throttle {
    fn new(pieces: usize) -> Self {
        Throttle {
            pieces,
            counter: Arc::new(AtomicUsize::new(0)),
        }
    }

    fn acquire(&self) -> Opportunity {
        Opportunity {
            n: self.counter.fetch_add(1, Ordering::AcqRel) % self.pieces,
            counter: self.counter.clone(),
        }
    }
}

#[must_use = "Don't lose your opportunity"]
struct Opportunity {
    n: usize,
    counter: Arc<AtomicUsize>,
}

impl Opportunity {
    async fn wait(&self) {
        delay_for(Duration::from_secs(self.n as u64)).await
    }
}

impl Drop for Opportunity {
    fn drop(&mut self) {
        self.counter.fetch_sub(1, Ordering::SeqCst);
    }
}
<|endoftext|>


<commit_before>//! Futures integration.

use crate::stream::Stream;
use crate::sync::Mutex;
use std::future::Future;
use std::mem;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{LocalWaker, Poll, Waker};

/// The state a stream future.
#[derive(Debug)]
enum FutureValue<T> {
    Pending,
    Ready(T),
    Finished,
}

/// The storage of a stream future.
#[derive(Debug)]
struct StreamFutureStorage<T> {
    value: FutureValue<T>,
    waker: Option<Waker>,
}

impl<T> StreamFutureStorage<T> {
    fn new() -> Arc<Mutex<Self>> {
        Arc::new(Mutex::new(StreamFutureStorage {
            value: FutureValue::Pending,
            waker: None,
        }))
    }
}

/// A future that waits for a stream value.
///
/// This is created by `Stream::next`.
#[derive(Debug)]
pub struct StreamFuture<T> {
    storage: Arc<Mutex<StreamFutureStorage<T>>>,
    stream: Stream<T>,
}

impl<T> StreamFuture<T> {
    /// Creates a future that returns the next value sent to this stream.
    pub(crate) fn new(stream: Stream<T>) -> Self
    where
        T: Clone + Send + 'static,
    {
        let storage = StreamFutureStorage::new();
        let weak = Arc::downgrade(&storage);
        stream.observe(move |val| {
            if let Some(st) = weak.upgrade() {
                let mut storage = st.lock();
                storage.value = FutureValue::Ready(val.into_owned());
                if let Some(waker) = &storage.waker {
                    waker.wake();
                }
            }
            false
        });
        StreamFuture { storage, stream }
    }

    /// Obtains the source stream.
    pub fn get_source(&self) -> &Stream<T> {
        &self.stream
    }
}

impl<T> Future for StreamFuture<T> {
    type Output = T;

    fn poll(self: Pin<&mut Self>, lw: &LocalWaker) -> Poll<Self::Output> {
        let mut storage = self.storage.lock();
        match mem::replace(&mut storage.value, FutureValue::Pending) {
            FutureValue::Ready(value) => {
                storage.value = FutureValue::Finished;
                Poll::Ready(value)
            }
            FutureValue::Pending => {
                storage.waker = Some(lw.clone().into_waker());
                Poll::Pending
            }
            FutureValue::Finished => {
                storage.value = FutureValue::Finished;
                panic!("future polled again after completion");
            }
        }
    }
}

impl<T> Unpin for StreamFuture<T> {}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::stream::Sink;
    use futures::executor::block_on;

    #[test]
    fn basic() {
        let sink = Sink::new();
        let future = StreamFuture::new(sink.stream());

        sink.send(42);
        sink.send(13);
        assert_eq!(block_on(future), 42);
    }

    #[test]
    #[should_panic]
    fn invalid_poll() {
        let sink = Sink::new();
        let mut future = StreamFuture::new(sink.stream());

        sink.send(42);
        let _a = block_on(&mut future);
        let _b = block_on(&mut future);
    }
}
<commit_msg>added new method StreamFuture::reload<commit_after>//! Futures integration.

use crate::stream::Stream;
use crate::sync::Mutex;
use std::future::Future;
use std::mem;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{LocalWaker, Poll, Waker};

/// The state a stream future.
#[derive(Debug)]
enum FutureValue<T> {
    Pending,
    Ready(T),
    Finished,
}

/// The storage of a stream future.
#[derive(Debug)]
struct StreamFutureStorage<T> {
    value: FutureValue<T>,
    waker: Option<Waker>,
}

impl<T> Default for StreamFutureStorage<T> {
    fn default() -> Self {
        StreamFutureStorage {
            value: FutureValue::Pending,
            waker: None,
        }
    }
}

/// A future that waits for a stream value.
///
/// This is created by `Stream::next`.
#[derive(Debug)]
pub struct StreamFuture<T> {
    storage: Arc<Mutex<StreamFutureStorage<T>>>,
    stream: Stream<T>,
}

impl<T: Clone + Send + 'static> StreamFuture<T> {
    /// Creates a future that returns the next value sent to this stream.
    pub(crate) fn new(stream: Stream<T>) -> Self {
        let this = StreamFuture {
            storage: Default::default(),
            stream,
        };
        this.register_callback();
        this
    }

    /// Registers the stream observer that will update this future.
    fn register_callback(&self) {
        let weak = Arc::downgrade(&self.storage);
        self.stream.observe(move |val| {
            if let Some(st) = weak.upgrade() {
                let mut storage = st.lock();
                storage.value = FutureValue::Ready(val.into_owned());
                if let Some(waker) = &storage.waker {
                    waker.wake();
                }
            }
            false
        });
    }

    /// Obtains the source stream.
    pub fn get_source(&self) -> &Stream<T> {
        &self.stream
    }

    /// Reuses a finished future so it can wait for another value.
    ///
    /// Normally calling `poll` on a future after it returned `Poll::Ready` will panic.
    /// This method will restart this future so it can be polled again for another value.
    /// This allows awaiting for multiple values without having to create and allocate multiple
    /// future objects.
    ///
    /// Calling this on a pending (or ready but unread) future will have no effect.
    pub fn reload(&self) {
        let mut storage = self.storage.lock();
        if let FutureValue::Finished = storage.value {
            *storage = Default::default();
            self.register_callback();
        }
    }
}

impl<T> Future for StreamFuture<T> {
    type Output = T;

    fn poll(self: Pin<&mut Self>, lw: &LocalWaker) -> Poll<Self::Output> {
        let mut storage = self.storage.lock();
        match mem::replace(&mut storage.value, FutureValue::Pending) {
            FutureValue::Ready(value) => {
                storage.value = FutureValue::Finished;
                Poll::Ready(value)
            }
            FutureValue::Pending => {
                storage.waker = Some(lw.clone().into_waker());
                Poll::Pending
            }
            FutureValue::Finished => {
                storage.value = FutureValue::Finished;
                panic!("future polled again after completion");
            }
        }
    }
}

impl<T> Unpin for StreamFuture<T> {}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::stream::Sink;
    use futures::executor::block_on;

    #[test]
    fn basic() {
        let sink = Sink::new();
        let future = StreamFuture::new(sink.stream());

        sink.send(42);
        sink.send(13);
        assert_eq!(block_on(future), 42);
    }

    #[test]
    #[should_panic]
    fn invalid_poll() {
        let sink = Sink::new();
        let mut future = StreamFuture::new(sink.stream());

        sink.send(42);
        let _a = block_on(&mut future);
        let _b = block_on(&mut future);
    }

    #[test]
    fn reload() {
        let sink = Sink::new();
        let mut future = StreamFuture::new(sink.stream());

        sink.send(42);
        assert_eq!(block_on(&mut future), 42);

        future.reload();
        sink.send(13);
        assert_eq!(block_on(&mut future), 13);
    }
}
<|endoftext|>


<commit_before>//! ### Note: not stable APIS
//! The items exported in this module are not considered part of this crate's public API
//! and may receive breaking changes in semver-compatible versions.
use futures::{stream, Stream, StreamExt};
use futures::Poll::*;
use futures_test::stream::StreamTestExt;
use futures_test::task::noop_context;

use std::cmp;

pub use crate::server::fuzzing::*;

pub use crate::StringError;
use crate::helpers::show_bytes;

/// Deterministically chunk test data so the fuzzer can discover new code paths
pub fn chunk_test_data<'d>(mut data: &'d [u8]) -> impl Stream<Item = Result<&'d [u8], StringError>> + 'd {
    // this ensures the test boundary will always be split between chunks
    stream::iter(data.chunks(BOUNDARY.len() - 1))
        .map(Ok)
        .interleave_pending()
}

pub fn fuzz_boundary_finder(test_data: &[u8]) {
    let finder = BoundaryFinder::new(chunk_test_data(test_data), BOUNDARY);
    pin_mut!(finder);

    let ref mut cx = noop_context();

    loop {
        match finder.as_mut().consume_boundary(cx) {
            Ready(Ok(false)) | Ready(Err(_)) => return,
            Ready(Ok(true)) => (),
            Pending => continue,
        }

        loop {
            match finder.as_mut().body_chunk(cx) {
                Ready(Some(Ok(chunk))) => {
                    assert_ne!(chunk, &[]);
                    assert_eq!(twoway::find_bytes(chunk, BOUNDARY.as_bytes()), None)
                },
                Pending => (),
                Ready(None) | Ready(Some(Err(_))) => return,
            }
        }
    }
}

/// Fuzz BoundaryFinder taking the input as the data of a field
pub fn fuzz_boundary_finder_field(test_data: &[u8]) {
    // ensure the boundary doesn't appear in the input data
    if twoway::find_bytes(test_data, BOUNDARY.as_bytes()).is_some() { return; }

    let start = format!("{}\r\n", BOUNDARY);
    let end = format!("\r\n{}--", BOUNDARY);
    let stream = chunk_test_data(start.as_bytes())
        .chain(chunk_test_data(test_data))
        .chain(chunk_test_data(end.as_bytes()));

    let finder = BoundaryFinder::new(stream, BOUNDARY);
    pin_mut!(finder);

    let ref mut cx = noop_context();

    loop {
        match finder.as_mut().consume_boundary(cx) {
            Ready(res) => {
                assert!(
                    res.expect("failed to read starting boundary"),
                    "didn't find starting boundary"
                );
                break
            },
            Pending => (),
        }
    }

    let mut remaining = test_data;

    loop {
        match finder.as_mut().body_chunk(cx) {
            Ready(Some(res)) => {
                let chunk = res.expect("failed to read body chunk");
                assert_ne!(chunk, &[]);
                assert!(
                    remaining.starts_with(chunk),
                    "expected chunk \"{}\" to be a prefix of remaining data \"{}\"",
                    show_bytes(chunk), show_bytes(remaining)
                );
                remaining = &remaining[chunk.len()..];
            },
            Ready(None) => assert_eq!(remaining, &[]),
            Pending => (),
        }
    }

    loop {
        match finder.as_mut().consume_boundary(cx) {
            Ready(res) => {
                assert!(
                    !res.expect("failed to read ending boundary"),
                    "didn't find ending boundary"
                );
                break
            },
            Pending => (),
        }
    }
}

pub const BOUNDARY: &str = "--boundary";

#[test]
fn test_fuzz_boundary_finder() {
    let _ = env_logger::init();
    fuzz_boundary_finder(b"--boundary\r\n");
}

#[test]
fn test_fuzz_boundary_finder_field() {
    let _ = env_logger::init();
    fuzz_boundary_finder_field(b"asdf1234ghjk5678zxcvnm90-=`023458nsdzfdl-");
}
<commit_msg>fix infinite loop in `fuzz_boundary_finder_field`<commit_after>//! ### Note: not stable APIS
//! The items exported in this module are not considered part of this crate's public API
//! and may receive breaking changes in semver-compatible versions.
use futures::{stream, Stream, StreamExt};
use futures::Poll::*;
use futures_test::stream::StreamTestExt;
use futures_test::task::noop_context;

use std::cmp;

pub use crate::server::fuzzing::*;

pub use crate::StringError;
use crate::helpers::show_bytes;

/// Deterministically chunk test data so the fuzzer can discover new code paths
pub fn chunk_test_data<'d>(mut data: &'d [u8]) -> impl Stream<Item = Result<&'d [u8], StringError>> + 'd {
    // this ensures the test boundary will always be split between chunks
    stream::iter(data.chunks(BOUNDARY.len() - 1))
        .map(Ok)
        .interleave_pending()
}

pub fn fuzz_boundary_finder(test_data: &[u8]) {
    let finder = BoundaryFinder::new(chunk_test_data(test_data), BOUNDARY);
    pin_mut!(finder);

    let ref mut cx = noop_context();

    loop {
        match finder.as_mut().consume_boundary(cx) {
            Ready(Ok(false)) | Ready(Err(_)) => return,
            Ready(Ok(true)) => (),
            Pending => continue,
        }

        loop {
            match finder.as_mut().body_chunk(cx) {
                Ready(Some(Ok(chunk))) => {
                    assert_ne!(chunk, &[]);
                    assert_eq!(twoway::find_bytes(chunk, BOUNDARY.as_bytes()), None)
                },
                Pending => (),
                Ready(None) | Ready(Some(Err(_))) => return,
            }
        }
    }
}

/// Fuzz BoundaryFinder taking the input as the data of a field
pub fn fuzz_boundary_finder_field(test_data: &[u8]) {
    // ensure the boundary doesn't appear in the input data
    if twoway::find_bytes(test_data, BOUNDARY.as_bytes()).is_some() { return; }

    let start = format!("{}\r\n", BOUNDARY);
    let end = format!("\r\n{}--", BOUNDARY);
    let stream = chunk_test_data(start.as_bytes())
        .chain(chunk_test_data(test_data))
        .chain(chunk_test_data(end.as_bytes()));

    let finder = BoundaryFinder::new(stream, BOUNDARY);
    pin_mut!(finder);

    let ref mut cx = noop_context();

    loop {
        match finder.as_mut().consume_boundary(cx) {
            Ready(res) => {
                assert!(
                    res.expect("failed to read starting boundary"),
                    "didn't find starting boundary"
                );
                break
            },
            Pending => (),
        }
    }

    let mut remaining = test_data;

    loop {
        match finder.as_mut().body_chunk(cx) {
            Ready(Some(res)) => {
                let chunk = res.expect("failed to read body chunk");
                assert_ne!(chunk, &[]);
                assert!(
                    remaining.starts_with(chunk),
                    "expected chunk \"{}\" to be a prefix of remaining data \"{}\"",
                    show_bytes(chunk), show_bytes(remaining)
                );
                remaining = &remaining[chunk.len()..];
            },
            Ready(None) => {
                assert_eq!(remaining, &[]);
                break;
            },
            Pending => (),
        }
    }

    loop {
        match finder.as_mut().consume_boundary(cx) {
            Ready(res) => {
                assert!(
                    !res.expect("failed to read ending boundary"),
                    "didn't find ending boundary"
                );
                break
            },
            Pending => (),
        }
    }
}

pub const BOUNDARY: &str = "--boundary";

#[test]
fn test_fuzz_boundary_finder() {
    let _ = env_logger::init();
    fuzz_boundary_finder(b"--boundary\r\n");
}

#[test]
fn test_fuzz_boundary_finder_field() {
    let _ = env_logger::init();
    fuzz_boundary_finder_field(b"asdf1234ghjk5678zxcvnm90-=`023458nsdzfdl-");
}
<|endoftext|>


<commit_before>extern crate ws;

use message::PeerMessage;
use std::collections::hash_map::HashMap;
use std::sync::{Arc, Mutex};

use bincode::rustc_serialize::decode;

/// Connection handler for incoming messages.
pub struct MessageHandler {
    /// Sender that is used to communicate for the handler.
    pub ws: ws::Sender,
    /// An arc clone of my local vector clocks.
    pub clocks: Arc<Mutex<HashMap<String, u32>>>,
    /// My address/name.
    pub me: String,
}

//TODO: When connected to another client, add it to the `clocks`.
impl ws::Handler for MessageHandler {
    fn on_message(&mut self, msg: ws::Message) -> ws::Result<()> {
        match msg {
            ws::Message::Binary(vector) => {
                let encoded_msg = &*vector.into_boxed_slice();
                let message: PeerMessage = decode(encoded_msg).unwrap();

                // Checkings if all clients have everyone's clocks
                let mut clocks = self.clocks.lock().unwrap();
                for (key, val) in message.clocks.iter() {
                    let value = val.clone();
                    let keyer = key.clone();
                    if !clocks.contains_key(&keyer) {
                        clocks.insert(String::from(keyer), value);
                    }
                }

                info!("Peer {} with clocks: {:?} got message: {}",
                        message.sender, message.clocks, message.message);
                //message_checking(&mut clocks, message.clone());
                Ok(())
            },
            ws::Message::Text(string) => {
                info!("Received peer's name. Adding {} to the client list", string);
                let mut clocks = self.clocks.lock().unwrap();
                clocks.insert(string, 0u32);
                Ok(())
            }
        }
    }
    fn on_close(&mut self, code: ws::CloseCode, reason: &str) {
        if reason.is_empty() {
            info!("Client disconnected with code: {:?}", code); //This works: CloseCode::Abnormal
        } else {
            info!("Client disconnected with code: {:?} and reason: {}", code, reason);
        }
    }
    fn on_error(&mut self, err: ws::Error) {
        warn!("Error family robinson! {:?}", err.kind);
    }
}

pub struct MessageFactory {
    vclocks: Arc<Mutex<HashMap<String, u32>>>,
    me: String,
}

impl MessageFactory {
    pub fn build(vclocks: Arc<Mutex<HashMap<String, u32>>>) -> MessageFactory {
        MessageFactory {
            vclocks: vclocks,
            me: String::from("undefined"),
        }
    }
    pub fn me(&self, me: &str) -> MessageFactory {
        MessageFactory {
            vclocks: self.vclocks.clone(),
            me: String::from(me),
        }
    }
}

impl ws::Factory for MessageFactory {
    type Handler = MessageHandler;

    fn connection_made(&mut self, ws: ws::Sender) -> Self::Handler {
        MessageHandler {
            ws: ws,
            clocks: self.vclocks.clone(),
            me: self.me.clone(),
        }
    }

    fn server_connected(&mut self, ws: ws::Sender) -> Self::Handler {
        let _ = ws.send(self.me.clone());
        MessageHandler {
            ws: ws,
            clocks: self.vclocks.clone(),
            me: self.me.clone(),
        }
    }

    fn client_connected(&mut self, ws: ws::Sender) -> Self::Handler {
        let _ = ws.send(self.me.clone());
        MessageHandler {
            ws: ws,
            clocks: self.vclocks.clone(),
            me: self.me.clone(),
        }
    }
}
<commit_msg>Using unwrap to avoid hiding possible crashes<commit_after>extern crate ws;

use message::PeerMessage;
use std::collections::hash_map::HashMap;
use std::sync::{Arc, Mutex};

use bincode::rustc_serialize::decode;

/// Connection handler for incoming messages.
pub struct MessageHandler {
    /// Sender that is used to communicate for the handler.
    pub ws: ws::Sender,
    /// An arc clone of my local vector clocks.
    pub clocks: Arc<Mutex<HashMap<String, u32>>>,
    /// My address/name.
    pub me: String,
}

//TODO: When connected to another client, add it to the `clocks`.
impl ws::Handler for MessageHandler {
    fn on_message(&mut self, msg: ws::Message) -> ws::Result<()> {
        match msg {
            ws::Message::Binary(vector) => {
                let encoded_msg = &*vector.into_boxed_slice();
                let message: PeerMessage = decode(encoded_msg).unwrap();

                // Checkings if all clients have everyone's clocks
                let mut clocks = self.clocks.lock().unwrap();
                for (key, val) in message.clocks.iter() {
                    let value = val.clone();
                    let keyer = key.clone();
                    if !clocks.contains_key(&keyer) {
                        clocks.insert(String::from(keyer), value);
                    }
                }

                info!("Peer {} with clocks: {:?} got message: {}",
                        message.sender, message.clocks, message.message);
                //message_checking(&mut clocks, message.clone());
                Ok(())
            },
            ws::Message::Text(string) => {
                info!("Received peer's name. Adding {} to the client list", string);
                let mut clocks = self.clocks.lock().unwrap();
                clocks.insert(string, 0u32);
                Ok(())
            }
        }
    }
    fn on_close(&mut self, code: ws::CloseCode, reason: &str) {
        if reason.is_empty() {
            info!("Client disconnected with code: {:?}", code); //This works: CloseCode::Abnormal
        } else {
            info!("Client disconnected with code: {:?} and reason: {}", code, reason);
        }
    }
    fn on_error(&mut self, err: ws::Error) {
        warn!("Error family robinson! {:?}", err.kind);
    }
}

pub struct MessageFactory {
    vclocks: Arc<Mutex<HashMap<String, u32>>>,
    me: String,
}

impl MessageFactory {
    pub fn build(vclocks: Arc<Mutex<HashMap<String, u32>>>) -> MessageFactory {
        MessageFactory {
            vclocks: vclocks,
            me: String::from("undefined"),
        }
    }
    pub fn me(&self, me: &str) -> MessageFactory {
        MessageFactory {
            vclocks: self.vclocks.clone(),
            me: String::from(me),
        }
    }
}

impl ws::Factory for MessageFactory {
    type Handler = MessageHandler;

    fn connection_made(&mut self, ws: ws::Sender) -> Self::Handler {
        MessageHandler {
            ws: ws,
            clocks: self.vclocks.clone(),
            me: self.me.clone(),
        }
    }

    fn server_connected(&mut self, ws: ws::Sender) -> Self::Handler {
        ws.send(self.me.clone()).unwrap();
        MessageHandler {
            ws: ws,
            clocks: self.vclocks.clone(),
            me: self.me.clone(),
        }
    }

    fn client_connected(&mut self, ws: ws::Sender) -> Self::Handler {
        ws.send(self.me.clone()).unwrap();
        MessageHandler {
            ws: ws,
            clocks: self.vclocks.clone(),
            me: self.me.clone(),
        }
    }
}
<|endoftext|>


<commit_before>use std::any::Any;
use std::fs::{self, File, Metadata};
use std::io::{self, Write, Error, ErrorKind};
use std::marker::PhantomData;
use std::path::{Path, PathBuf};
use std::process::Command;

use conduit_mime_types::Types;

use ::response::Response;
use ::request::Request;

pub struct FileMode;
pub struct DirectoryMode;

pub trait Handler {
    fn handle_request(&self, req: &mut Request, res: &mut Response);
}

#[derive(Debug)]
pub struct ServerHandler<M: Any> {
    root: PathBuf,
    mimetypes: Types,
    _kind: PhantomData<M>,
}

impl<M: Any> ServerHandler<M> {
    pub fn new(root: &PathBuf) -> ServerHandler<M> {
        let mimetypes = match Types::new() {
            Ok(types) => types,
            Err(error) => panic!(error),
        };

        ServerHandler {
            root: root.to_owned(),
            mimetypes: mimetypes,
            _kind: PhantomData
        }
    }

    fn get_resource_and_metadata(&self, req: &Request)
            -> Result<(PathBuf, Metadata), Error> {

        let path = req.path();
        let path = &path[1..path.len()];

        let resource = self.root.join(path);
        let metadata = try!(fs::metadata(&resource));

        Ok((resource, metadata))
    }

    fn send_file(&self, resource: &PathBuf, metadata: &Metadata, res: &mut Response) {
        let mut f = File::open(&resource).unwrap();
        let mime = self.mimetypes.mime_for_path(Path::new(&resource));

        res.with_header("Content-Type", mime)
            .with_header("Content-Length", &metadata.len().to_string());

        let res = res.start().unwrap();
        io::copy(&mut f, res).unwrap();

        // let mut buf: [u8; 4096] = [0; 4096];
        // loop {
        //     match f.read(&mut buf) {
        //         Ok(bytes_read) => {
        //             if bytes_read == 0 {
        //                 break;
        //             }
        //             res.write(&buf[0..bytes_read]).unwrap();
        //         },
        //         Err(e) => panic!(e)
        //     }
        // }
    }

    fn send_not_found(&self, res: &mut Response) {
        res.with_status(404, "Not Found");
        let res = res.start().unwrap();
        res.write("404 - Not Found".as_bytes()).unwrap();
        res.flush().ok().expect("Failed to send error response");
    }

    fn send_error(&self, res: &mut Response, status: i32, description: &str) {
        res.with_status(status, description);
        let res = res.start().unwrap();
        res.write(format!("{} - {}", status, description).as_bytes()).unwrap();
        res.flush().ok().expect("Failed to send error response");
    }
}

impl Handler for ServerHandler<FileMode> {
    fn handle_request(&self, req: &mut Request, res: &mut Response) {
        let (resource, metadata) = match self.get_resource_and_metadata(req) {
            Ok(result) => result,
            Err(e) => {
                if e.kind() == ErrorKind::NotFound {
                    self.send_not_found(res);
                } else {
                    self.send_error(res, 500, "Internal Server Error");
                }
                return;
            }
        };

        if !metadata.is_file() {
            self.send_not_found(res);
            return;
        }

        self.send_file(&resource, &metadata, res);
    }
}

impl Handler for ServerHandler<DirectoryMode> {
    fn handle_request(&self, req: &mut Request, res: &mut Response) {
        let (resource, metadata) = match self.get_resource_and_metadata(req) {
            Ok(result) => result,
            Err(e) => {
                if e.kind() == ErrorKind::NotFound {
                    self.send_not_found(res);
                } else {
                    self.send_error(res, 500, "Internal Server Error");
                }
                return;
            }
        };

        if metadata.is_file() {
            self.send_file(&resource, &metadata, res);
            return;
        }

        let output = Command::new("ls")
            .arg(&resource)
            .output()
            .unwrap_or_else(|e| panic!(format!("Failed to list dir: {}", e)));

        let s: String;
        if output.status.success() {
            s = String::from_utf8_lossy(&output.stdout).as_ref().to_owned();
        } else {
            s = String::from_utf8_lossy(&output.stderr).as_ref().to_owned();
            panic!("rustc failed and stderr was:\n{}", s);
        }

        let mut path = req.path().to_owned();
        if path.len() == 1 && path == "/" {
            path.clear();
        }
        else if path.len() > 1 && !path.ends_with("/") {
            path.push('/');
        }

        res.with_header("Content-Type", "text/html");

        let res = res.start().unwrap();

        res.write("
".as_bytes()).unwrap();
        for name in s.split('\n') {
            if name.len() == 0 { continue }
            let mut name = name.to_owned();

            let metadata = fs::metadata(Path::new(&resource).join(&name)).unwrap();

            if metadata.is_dir() {
                name = format!("{}/", name);
            }

            res.write(format!("
{1}
", path, name).as_bytes()).unwrap();
        }
        res.write("
".as_bytes()).unwrap();
        res.flush().unwrap();
    }
}
<commit_msg>set charset for html response<commit_after>use std::any::Any;
use std::fs::{self, File, Metadata};
use std::io::{self, Write, Error, ErrorKind};
use std::marker::PhantomData;
use std::path::{Path, PathBuf};
use std::process::Command;

use conduit_mime_types::Types;

use ::response::Response;
use ::request::Request;

pub struct FileMode;
pub struct DirectoryMode;

pub trait Handler {
    fn handle_request(&self, req: &mut Request, res: &mut Response);
}

#[derive(Debug)]
pub struct ServerHandler<M: Any> {
    root: PathBuf,
    mimetypes: Types,
    _kind: PhantomData<M>,
}

impl<M: Any> ServerHandler<M> {
    pub fn new(root: &PathBuf) -> ServerHandler<M> {
        let mimetypes = match Types::new() {
            Ok(types) => types,
            Err(error) => panic!(error),
        };

        ServerHandler {
            root: root.to_owned(),
            mimetypes: mimetypes,
            _kind: PhantomData
        }
    }

    fn get_resource_and_metadata(&self, req: &Request)
            -> Result<(PathBuf, Metadata), Error> {

        let path = req.path();
        let path = &path[1..path.len()];

        let resource = self.root.join(path);
        let metadata = try!(fs::metadata(&resource));

        Ok((resource, metadata))
    }

    fn send_file(&self, resource: &PathBuf, metadata: &Metadata, res: &mut Response) {
        let mut f = File::open(&resource).unwrap();
        let mime = self.mimetypes.mime_for_path(Path::new(&resource));

        res.with_header("Content-Type", mime)
            .with_header("Content-Length", &metadata.len().to_string());

        let res = res.start().unwrap();
        io::copy(&mut f, res).unwrap();

        // let mut buf: [u8; 4096] = [0; 4096];
        // loop {
        //     match f.read(&mut buf) {
        //         Ok(bytes_read) => {
        //             if bytes_read == 0 {
        //                 break;
        //             }
        //             res.write(&buf[0..bytes_read]).unwrap();
        //         },
        //         Err(e) => panic!(e)
        //     }
        // }
    }

    fn send_not_found(&self, res: &mut Response) {
        res.with_status(404, "Not Found");
        let res = res.start().unwrap();
        res.write("404 - Not Found".as_bytes()).unwrap();
        res.flush().ok().expect("Failed to send error response");
    }

    fn send_error(&self, res: &mut Response, status: i32, description: &str) {
        res.with_status(status, description);
        let res = res.start().unwrap();
        res.write(format!("{} - {}", status, description).as_bytes()).unwrap();
        res.flush().ok().expect("Failed to send error response");
    }
}

impl Handler for ServerHandler<FileMode> {
    fn handle_request(&self, req: &mut Request, res: &mut Response) {
        let (resource, metadata) = match self.get_resource_and_metadata(req) {
            Ok(result) => result,
            Err(e) => {
                if e.kind() == ErrorKind::NotFound {
                    self.send_not_found(res);
                } else {
                    self.send_error(res, 500, "Internal Server Error");
                }
                return;
            }
        };

        if !metadata.is_file() {
            self.send_not_found(res);
            return;
        }

        self.send_file(&resource, &metadata, res);
    }
}

impl Handler for ServerHandler<DirectoryMode> {
    fn handle_request(&self, req: &mut Request, res: &mut Response) {
        let (resource, metadata) = match self.get_resource_and_metadata(req) {
            Ok(result) => result,
            Err(e) => {
                if e.kind() == ErrorKind::NotFound {
                    self.send_not_found(res);
                } else {
                    self.send_error(res, 500, "Internal Server Error");
                }
                return;
            }
        };

        if metadata.is_file() {
            self.send_file(&resource, &metadata, res);
            return;
        }

        let output = Command::new("ls")
            .arg(&resource)
            .output()
            .unwrap_or_else(|e| panic!(format!("Failed to list dir: {}", e)));

        let s: String;
        if output.status.success() {
            s = String::from_utf8_lossy(&output.stdout).as_ref().to_owned();
        } else {
            s = String::from_utf8_lossy(&output.stderr).as_ref().to_owned();
            panic!("rustc failed and stderr was:\n{}", s);
        }

        let mut path = req.path().to_owned();
        if path.len() == 1 && path == "/" {
            path.clear();
        }
        else if path.len() > 1 && !path.ends_with("/") {
            path.push('/');
        }

        res.with_header("Content-Type", "text/html; charset=utf-8");

        let res = res.start().unwrap();

        res.write("
".as_bytes()).unwrap();
        for name in s.split('\n') {
            if name.len() == 0 { continue }
            let mut name = name.to_owned();

            let metadata = fs::metadata(Path::new(&resource).join(&name)).unwrap();

            if metadata.is_dir() {
                name = format!("{}/", name);
            }

            res.write(format!("
{1}
", path, name).as_bytes()).unwrap();
        }
        res.write("
".as_bytes()).unwrap();
        res.flush().unwrap();
    }
}
<|endoftext|>


<commit_before>use std::env;
use std::io::*;
use std::net::*;

fn main() {
    let addr = env::args().nth(1).unwrap_or("127.0.0.1:12345".to_string());
    let addr = addr.parse::<SocketAddr>().unwrap();
    let l = TcpListener::bind(&addr).unwrap();
    println!("listening...");
    let b = [0; 64 * 1024];
    loop {
        let mut c = l.accept().unwrap().0;
        println!("got a connection, writing some data to it");
        while c.write(&b).is_ok() {}
        println!("connection is gone, listening for another");
    }
}
<commit_msg>Print out connection rates in the server<commit_after>use std::env;
use std::io::*;
use std::net::*;
use std::sync::atomic::*;
use std::thread;
use std::time::Duration;

static AMT: AtomicUsize = ATOMIC_USIZE_INIT;

fn main() {
    let addr = env::args().nth(1).unwrap_or("127.0.0.1:12345".to_string());
    let addr = addr.parse::<SocketAddr>().unwrap();
    let l = TcpListener::bind(&addr).unwrap();
    println!("listening...");
    let b = [0; 64 * 1024];
    thread::spawn(|| {
        loop {
            thread::sleep(Duration::new(1, 0));
            println!("{}", AMT.swap(0, Ordering::SeqCst));
        }
    });
    loop {
        let mut c = l.accept().unwrap().0;
        println!("got a connection, writing some data to it");
        while let Ok(n) = c.write(&b) {
            AMT.fetch_add(n, Ordering::SeqCst);
        }
        println!("connection is gone, listening for another");
    }
}
<|endoftext|>


<commit_before>use opentelemetry::{api, exporter::trace};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, SystemTime};

pub(crate) mod annotation;
pub(crate) mod endpoint;
pub(crate) mod span;

use endpoint::Endpoint;

/// Converts `api::Event` into an `annotation::Annotation`
impl Into<annotation::Annotation> for api::Event {
    fn into(self) -> annotation::Annotation {
        let timestamp = self
            .timestamp
            .duration_since(SystemTime::UNIX_EPOCH)
            .unwrap_or_else(|_| Duration::from_secs(0))
            .as_micros() as u64;

        annotation::Annotation::builder()
            .timestamp(timestamp)
            .value(self.name)
            .build()
    }
}

/// Converts `api::SpanKind` into an `Option<span::Kind>`
fn into_zipkin_span_kind(kind: api::SpanKind) -> Option<span::Kind> {
    match kind {
        api::SpanKind::Client => Some(span::Kind::Client),
        api::SpanKind::Server => Some(span::Kind::Server),
        api::SpanKind::Producer => Some(span::Kind::Producer),
        api::SpanKind::Consumer => Some(span::Kind::Consumer),
        api::SpanKind::Internal => None,
    }
}

/// Converts a `trace::SpanData` to a `span::SpanData` for a given `ExporterConfig`, which can then
/// be ingested into a Zipkin collector.
pub(crate) fn into_zipkin_span(
    local_endpoint: Endpoint,
    span_data: &Arc<trace::SpanData>,
) -> span::Span {
    let mut user_defined_span_kind = false;
    let tags = map_from_kvs(
        span_data
            .attributes
            .iter()
            .map(|(k, v)| {
                if k == &api::Key::new("span.kind") {
                    user_defined_span_kind = true;
                }
                api::KeyValue::new(k.clone(), v.clone())
            })
            .chain(
                span_data
                    .resource
                    .iter()
                    .map(|(k, v)| api::KeyValue::new(k.clone(), v.clone())),
            ),
    );

    span::Span::builder()
        .trace_id(span_data.span_context.trace_id().to_hex())
        .parent_id(span_data.parent_span_id.to_hex())
        .id(span_data.span_context.span_id().to_hex())
        .name(span_data.name.clone())
        .kind(if user_defined_span_kind {
            None
        } else {
            into_zipkin_span_kind(span_data.span_kind.clone())
        })
        .timestamp(
            span_data
                .start_time
                .duration_since(SystemTime::UNIX_EPOCH)
                .unwrap_or_else(|_| Duration::from_secs(0))
                .as_micros() as u64,
        )
        .duration(
            span_data
                .end_time
                .duration_since(span_data.start_time)
                .unwrap_or_else(|_| Duration::from_secs(0))
                .as_micros() as u64,
        )
        .local_endpoint(local_endpoint)
        .annotations(
            span_data
                .message_events
                .iter()
                .cloned()
                .map(Into::into)
                .collect(),
        )
        .tags(tags)
        .build()
}

fn map_from_kvs<T>(kvs: T) -> HashMap<String, String>
where
    T: IntoIterator<Item = api::KeyValue>,
{
    let mut map: HashMap<String, String> = HashMap::new();
    for kv in kvs {
        map.insert(kv.key.into(), kv.value.into());
    }
    map
}
<commit_msg>add otel.status_code and otel.status_message tags in Zipkin exporter (#236)<commit_after>use opentelemetry::{api, exporter::trace};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, SystemTime};

pub(crate) mod annotation;
pub(crate) mod endpoint;
pub(crate) mod span;

use endpoint::Endpoint;

/// Converts `api::Event` into an `annotation::Annotation`
impl Into<annotation::Annotation> for api::Event {
    fn into(self) -> annotation::Annotation {
        let timestamp = self
            .timestamp
            .duration_since(SystemTime::UNIX_EPOCH)
            .unwrap_or_else(|_| Duration::from_secs(0))
            .as_micros() as u64;

        annotation::Annotation::builder()
            .timestamp(timestamp)
            .value(self.name)
            .build()
    }
}

/// Converts StatusCode to str
fn from_statuscode_to_str(status_code: api::StatusCode) -> &'static str {
    match status_code {
        api::StatusCode::OK => "OK",
        api::StatusCode::Canceled => "CANCELLED",
        api::StatusCode::Unknown => "UNKNOWN",
        api::StatusCode::InvalidArgument => "INVALID_ARGUMENT",
        api::StatusCode::DeadlineExceeded => "DEADLINE_EXCEEDED",
        api::StatusCode::NotFound => "NOT_FOUND",
        api::StatusCode::AlreadyExists => "ALREADY_EXISTS",
        api::StatusCode::PermissionDenied => "PERMISSION_DENIED",
        api::StatusCode::ResourceExhausted => "RESOURSE_EXHAUSTED",
        api::StatusCode::FailedPrecondition => "FAILED_PRECONDITION",
        api::StatusCode::Aborted => "ABORTED",
        api::StatusCode::OutOfRange => "OUT_OF_RANGE",
        api::StatusCode::Unimplemented => "UNINPLEMENTED",
        api::StatusCode::Internal => "INTERNAL",
        api::StatusCode::Unavailable => "UNAVAILABLE",
        api::StatusCode::DataLoss => "DATA_LOSS",
        api::StatusCode::Unauthenticated => "UNAUTHENTICATED",
    }
}

/// Converts `api::SpanKind` into an `Option<span::Kind>`
fn into_zipkin_span_kind(kind: api::SpanKind) -> Option<span::Kind> {
    match kind {
        api::SpanKind::Client => Some(span::Kind::Client),
        api::SpanKind::Server => Some(span::Kind::Server),
        api::SpanKind::Producer => Some(span::Kind::Producer),
        api::SpanKind::Consumer => Some(span::Kind::Consumer),
        api::SpanKind::Internal => None,
    }
}

/// Converts a `trace::SpanData` to a `span::SpanData` for a given `ExporterConfig`, which can then
/// be ingested into a Zipkin collector.
pub(crate) fn into_zipkin_span(
    local_endpoint: Endpoint,
    span_data: &Arc<trace::SpanData>,
) -> span::Span {
    let mut user_defined_span_kind = false;
    let mut tags = map_from_kvs(
        span_data
            .attributes
            .iter()
            .map(|(k, v)| {
                if k == &api::Key::new("span.kind") {
                    user_defined_span_kind = true;
                }
                api::KeyValue::new(k.clone(), v.clone())
            })
            .chain(
                span_data
                    .resource
                    .iter()
                    .map(|(k, v)| api::KeyValue::new(k.clone(), v.clone())),
            ),
    );

    tags.insert(
        "otel.status_code".into(),
        from_statuscode_to_str(span_data.status_code.clone()).into(),
    );
    tags.insert(
        "otel.status_description".into(),
        span_data.status_message.clone(),
    );

    span::Span::builder()
        .trace_id(span_data.span_context.trace_id().to_hex())
        .parent_id(span_data.parent_span_id.to_hex())
        .id(span_data.span_context.span_id().to_hex())
        .name(span_data.name.clone())
        .kind(if user_defined_span_kind {
            None
        } else {
            into_zipkin_span_kind(span_data.span_kind.clone())
        })
        .timestamp(
            span_data
                .start_time
                .duration_since(SystemTime::UNIX_EPOCH)
                .unwrap_or_else(|_| Duration::from_secs(0))
                .as_micros() as u64,
        )
        .duration(
            span_data
                .end_time
                .duration_since(span_data.start_time)
                .unwrap_or_else(|_| Duration::from_secs(0))
                .as_micros() as u64,
        )
        .local_endpoint(local_endpoint)
        .annotations(
            span_data
                .message_events
                .iter()
                .cloned()
                .map(Into::into)
                .collect(),
        )
        .tags(tags)
        .build()
}

fn map_from_kvs<T>(kvs: T) -> HashMap<String, String>
where
    T: IntoIterator<Item = api::KeyValue>,
{
    let mut map: HashMap<String, String> = HashMap::new();
    for kv in kvs {
        map.insert(kv.key.into(), kv.value.into());
    }
    map
}
<|endoftext|>


<commit_before>use crates_index_diff::{Change, Index};

fn main() {
    let index = Index::from_path_or_cloned(
        std::env::args_os()
            .nth(1)
            .expect("First argument is the location of the crates.io index"),
    )
    .unwrap();
    let changes = index.changes("e36aee0~1", "e36aee0").unwrap();
    for change in changes {
        dbg!(&change);
        match &change {
            Change::Added(cv) if cv.name == "dl" => {}
            Change::Yanked(cv) if cv.name == "dl" => {}
            Change::Deleted { name, .. } if name == "dl" => {}
            _ => continue,
        }

        eprintln!("{:?}", change);
    }
}
<commit_msg>remove accidentally added example<commit_after><|endoftext|>


<commit_before>use memory::FrameAllocator;
use memory::paging::ENTRY_COUNT;
use memory::paging::entry::*;
use core::ops::{Index, IndexMut};
use core::marker::PhantomData;

pub const P4: *mut Table<Level4> = 0xffffffff_fffff000 as *mut _;

pub struct Table<L: TableLevel> {
    entries: [Entry; ENTRY_COUNT],
    _phantom: PhantomData<L>,
}

impl<L> Table<L> where L: TableLevel
{
    pub fn zero(&mut self) {
        for entry in self.entries.iter_mut() {
            entry.set_unused();
        }
    }
}

impl<L> Table<L> where L: HierachicalLevel
{
    pub fn next_table(&self, index: usize) -> Option<&Table<L::NextLevel>> {
        self.next_table_address(index).map(|t| unsafe { &*(t as *const _) })
    }

    pub fn next_table_mut(&mut self, index: usize) -> Option<&mut Table<L::NextLevel>> {
        self.next_table_address(index).map(|t| unsafe { &mut *(t as *mut _) })
    }

    pub fn next_table_create<A>(&mut self,
                                index: usize,
                                allocator: &mut A)
                                -> &mut Table<L::NextLevel>
        where A: FrameAllocator
    {
        if let None = self.next_table_address(index) {
            assert!(!self.entries[index].flags().contains(HUGE_PAGE),
                    "mapping code does not support huge pages");
            let frame = allocator.allocate_frame().expect("no frames available");
            self.entries[index].set(frame, PRESENT | WRITABLE);
            self.next_table_mut(index).unwrap().zero();
        }
        self.next_table_mut(index).unwrap()
    }

    fn next_table_address(&self, index: usize) -> Option<usize> {
        use memory::paging::Page;

        let entry_flags = self[index].flags();
        if entry_flags.contains(PRESENT) && !entry_flags.contains(HUGE_PAGE) {
            let table_page = Page::containing_address(self as *const _ as usize);
            assert!(table_page.number >= 0o_777_000_000_000);
            let next_table_page = Page {
                number: ((table_page.number << 9) & 0o_777_777_777_777) | index,
            };
            Some(next_table_page.start_address())
        } else {
            None
        }
    }
}

impl<L> Index<usize> for Table<L> where L: TableLevel
{
    type Output = Entry;

    fn index(&self, index: usize) -> &Entry {
        &self.entries[index]
    }
}

impl<L> IndexMut<usize> for Table<L> where L: TableLevel
{
    fn index_mut(&mut self, index: usize) -> &mut Entry {
        &mut self.entries[index]
    }
}

pub trait TableLevel {}

pub enum Level4 {}
#[allow(dead_code)]
enum Level3 {}
#[allow(dead_code)]
enum Level2 {}
#[allow(dead_code)]
enum Level1 {}

impl TableLevel for Level4 {}
impl TableLevel for Level3 {}
impl TableLevel for Level2 {}
impl TableLevel for Level1 {}

trait HierachicalLevel: TableLevel {
    type NextLevel: TableLevel;
}

impl HierachicalLevel for Level4 {
    type NextLevel = Level3;
}

impl HierachicalLevel for Level3 {
    type NextLevel = Level2;
}

impl HierachicalLevel for Level2 {
    type NextLevel = Level1;
}
<commit_msg>Calculate next_table_address through directly through addresses<commit_after>use memory::FrameAllocator;
use memory::paging::ENTRY_COUNT;
use memory::paging::entry::*;
use core::ops::{Index, IndexMut};
use core::marker::PhantomData;

pub const P4: *mut Table<Level4> = 0xffffffff_fffff000 as *mut _;

pub struct Table<L: TableLevel> {
    entries: [Entry; ENTRY_COUNT],
    _phantom: PhantomData<L>,
}

impl<L> Table<L> where L: TableLevel
{
    pub fn zero(&mut self) {
        for entry in self.entries.iter_mut() {
            entry.set_unused();
        }
    }
}

impl<L> Table<L> where L: HierachicalLevel
{
    pub fn next_table(&self, index: usize) -> Option<&Table<L::NextLevel>> {
        self.next_table_address(index).map(|t| unsafe { &*(t as *const _) })
    }

    pub fn next_table_mut(&mut self, index: usize) -> Option<&mut Table<L::NextLevel>> {
        self.next_table_address(index).map(|t| unsafe { &mut *(t as *mut _) })
    }

    pub fn next_table_create<A>(&mut self,
                                index: usize,
                                allocator: &mut A)
                                -> &mut Table<L::NextLevel>
        where A: FrameAllocator
    {
        if let None = self.next_table_address(index) {
            assert!(!self.entries[index].flags().contains(HUGE_PAGE),
                    "mapping code does not support huge pages");
            let frame = allocator.allocate_frame().expect("no frames available");
            self.entries[index].set(frame, PRESENT | WRITABLE);
            self.next_table_mut(index).unwrap().zero();
        }
        self.next_table_mut(index).unwrap()
    }

    fn next_table_address(&self, index: usize) -> Option<usize> {
        let entry_flags = self[index].flags();
        if entry_flags.contains(PRESENT) && !entry_flags.contains(HUGE_PAGE) {
            let table_address = self as *const _ as usize;
            Some((table_address << 9) | (index << 12))
        } else {
            None
        }
    }
}

impl<L> Index<usize> for Table<L> where L: TableLevel
{
    type Output = Entry;

    fn index(&self, index: usize) -> &Entry {
        &self.entries[index]
    }
}

impl<L> IndexMut<usize> for Table<L> where L: TableLevel
{
    fn index_mut(&mut self, index: usize) -> &mut Entry {
        &mut self.entries[index]
    }
}

pub trait TableLevel {}

pub enum Level4 {}
#[allow(dead_code)]
enum Level3 {}
#[allow(dead_code)]
enum Level2 {}
#[allow(dead_code)]
enum Level1 {}

impl TableLevel for Level4 {}
impl TableLevel for Level3 {}
impl TableLevel for Level2 {}
impl TableLevel for Level1 {}

trait HierachicalLevel: TableLevel {
    type NextLevel: TableLevel;
}

impl HierachicalLevel for Level4 {
    type NextLevel = Level3;
}

impl HierachicalLevel for Level3 {
    type NextLevel = Level2;
}

impl HierachicalLevel for Level2 {
    type NextLevel = Level1;
}
<|endoftext|>


<commit_before>//
//  SOS: the Stupid Operating System
//  by Hawk Weisman (hi@hawkweisman.me)
//
//  Copyright (c) 2015 Hawk Weisman
//  Released under the terms of the MIT license. See `LICENSE` in the root
//  directory of this repository for more information.
//
use vga::{Terminal, Palette, Color};
use spin::Mutex;

/// The system's global VGA terminal
pub static CONSOLE: Mutex<Terminal>
    = Mutex::new(unsafe { Terminal::new(
         Palette::new(Color::LightGreen, Color::Black )
       , 0xB8000
    )});
<commit_msg>[kernel] Terminal default color made light grey rather than green This is so that green text can be used for "okay" markers in the init log whenever I start actually working on issue #6.<commit_after>//
//  SOS: the Stupid Operating System
//  by Hawk Weisman (hi@hawkweisman.me)
//
//  Copyright (c) 2015 Hawk Weisman
//  Released under the terms of the MIT license. See `LICENSE` in the root
//  directory of this repository for more information.
//
use vga::{Terminal, Palette, Color};
use spin::Mutex;

/// The system's global VGA terminal
pub static CONSOLE: Mutex<Terminal>
    = Mutex::new(unsafe { Terminal::new(
         Palette::new(Color::LightGrey, Color::Black )
       , 0xB8000
    )});
<|endoftext|>