<commit_before>//! Simple hierarchical profiler
use std::{
    cell::RefCell,
    collections::{BTreeMap, HashSet},
    io::{stderr, Write},
    sync::{
        atomic::{AtomicBool, Ordering},
        RwLock,
    },
    time::{Duration, Instant},
};

use once_cell::sync::Lazy;

/// Filtering syntax
/// env RA_PROFILE=*             // dump everything
/// env RA_PROFILE=foo|bar|baz   // enabled only selected entries
/// env RA_PROFILE=*@3>10        // dump everything, up to depth 3, if it takes more than 10 ms
pub fn init() {
    let spec = std::env::var("RA_PROFILE").unwrap_or_default();
    init_from(&spec);
}

pub fn init_from(spec: &str) {
    let filter = if spec.is_empty() { Filter::disabled() } else { Filter::from_spec(spec) };
    set_filter(filter);
}

pub type Label = &'static str;

/// This function starts a profiling scope in the current execution stack with a given description.
/// It returns a Profile structure and measure elapsed time between this method invocation and Profile structure drop.
/// It supports nested profiling scopes in case when this function invoked multiple times at the execution stack. In this case the profiling information will be nested at the output.
/// Profiling information is being printed in the stderr.
///
/// # Example
/// ```
/// use ra_prof::{profile, set_filter, Filter};
///
/// let f = Filter::from_spec("profile1|profile2@2");
/// set_filter(f);
/// profiling_function1();
///
/// fn profiling_function1() {
///     let _p = profile("profile1");
///     profiling_function2();
/// }
///
/// fn profiling_function2() {
///     let _p = profile("profile2");
/// }
/// ```
/// This will print in the stderr the following:
/// ```text
///  0ms - profile
///      0ms - profile2
/// ```
pub fn profile(label: Label) -> Profiler {
    assert!(!label.is_empty());
    if !PROFILING_ENABLED.load(Ordering::Relaxed) {
        return Profiler { label: None, detail: None };
    }

    PROFILE_STACK.with(|stack| {
        let mut stack = stack.borrow_mut();
        if stack.starts.is_empty() {
            if let Ok(f) = FILTER.try_read() {
                if f.version > stack.filter_data.version {
                    stack.filter_data = f.clone();
                }
            };
        }
        if stack.starts.len() > stack.filter_data.depth {
            return Profiler { label: None, detail: None };
        }
        let allowed = &stack.filter_data.allowed;
        if stack.starts.is_empty() && !allowed.is_empty() && !allowed.contains(label) {
            return Profiler { label: None, detail: None };
        }

        stack.starts.push(Instant::now());
        Profiler { label: Some(label), detail: None }
    })
}

pub struct Profiler {
    label: Option<Label>,
    detail: Option<String>,
}

impl Profiler {
    pub fn detail(mut self, detail: impl FnOnce() -> String) -> Profiler {
        if self.label.is_some() {
            self.detail = Some(detail())
        }
        self
    }
}

/// Set profiling filter. It specifies descriptions allowed to profile.
/// This is helpful when call stack has too many nested profiling scopes.
/// Additionally filter can specify maximum depth of profiling scopes nesting.
///
/// #Example
/// ```
/// use ra_prof::{set_filter, Filter};
/// let f = Filter::from_spec("profile1|profile2@2");
/// set_filter(f);
/// ```
fn set_filter(f: Filter) {
    PROFILING_ENABLED.store(f.depth > 0, Ordering::SeqCst);
    let set: HashSet<_> = f.allowed.iter().cloned().collect();
    let mut old = FILTER.write().unwrap();
    let filter_data = FilterData {
        depth: f.depth,
        allowed: set,
        longer_than: f.longer_than,
        version: old.version + 1,
    };
    *old = filter_data;
}

struct Filter {
    depth: usize,
    allowed: Vec<String>,
    longer_than: Duration,
}

impl Filter {
    fn from_spec(mut spec: &str) -> Filter {
        let longer_than = if let Some(idx) = spec.rfind('>') {
            let longer_than = spec[idx + 1..].parse().expect("invalid profile longer_than");
            spec = &spec[..idx];
            Duration::from_millis(longer_than)
        } else {
            Duration::new(0, 0)
        };

        let depth = if let Some(idx) = spec.rfind('@') {
            let depth: usize = spec[idx + 1..].parse().expect("invalid profile depth");
            spec = &spec[..idx];
            depth
        } else {
            999
        };
        let allowed =
            if spec == "*" { Vec::new() } else { spec.split('|').map(String::from).collect() };
        Filter::new(depth, allowed, longer_than)
    }

    pub fn disabled() -> Filter {
        Filter::new(0, Vec::new(), Duration::new(0, 0))
    }

    pub fn new(depth: usize, allowed: Vec<String>, longer_than: Duration) -> Filter {
        Filter { depth, allowed, longer_than }
    }
}

struct ProfileStack {
    starts: Vec<Instant>,
    messages: Vec<Message>,
    filter_data: FilterData,
}

struct Message {
    level: usize,
    duration: Duration,
    label: Label,
    detail: Option<String>,
}

impl ProfileStack {
    fn new() -> ProfileStack {
        ProfileStack { starts: Vec::new(), messages: Vec::new(), filter_data: Default::default() }
    }
}

#[derive(Default, Clone)]
struct FilterData {
    depth: usize,
    version: usize,
    allowed: HashSet<String>,
    longer_than: Duration,
}

static PROFILING_ENABLED: AtomicBool = AtomicBool::new(false);

static FILTER: Lazy<RwLock<FilterData>> = Lazy::new(Default::default);

thread_local!(static PROFILE_STACK: RefCell<ProfileStack> = RefCell::new(ProfileStack::new()));

impl Drop for Profiler {
    fn drop(&mut self) {
        match self {
            Profiler { label: Some(label), detail } => {
                PROFILE_STACK.with(|stack| {
                    let mut stack = stack.borrow_mut();
                    let start = stack.starts.pop().unwrap();
                    let duration = start.elapsed();
                    let level = stack.starts.len();
                    stack.messages.push(Message { level, duration, label, detail: detail.take() });
                    if level == 0 {
                        let stdout = stderr();
                        let longer_than = stack.filter_data.longer_than;
                        // Convert to millis for comparison to avoid problems with rounding
                        // (otherwise we could print `0ms` despite user's `>0` filter when
                        // `duration` is just a few nanos).
                        if duration.as_millis() > longer_than.as_millis() {
                            print(&stack.messages, longer_than, &mut stdout.lock());
                        }
                        stack.messages.clear();
                    }
                });
            }
            Profiler { label: None, .. } => (),
        }
    }
}

fn print(msgs: &[Message], longer_than: Duration, out: &mut impl Write) {
    if msgs.is_empty() {
        return;
    }
    let children_map = idx_to_children(msgs);
    let root_idx = msgs.len() - 1;
    print_for_idx(root_idx, &children_map, msgs, longer_than, out);
}

fn print_for_idx(
    current_idx: usize,
    children_map: &[Vec<usize>],
    msgs: &[Message],
    longer_than: Duration,
    out: &mut impl Write,
) {
    let current = &msgs[current_idx];
    let current_indent = "    ".repeat(current.level);
    let detail = current.detail.as_ref().map(|it| format!(" @ {}", it)).unwrap_or_default();
    writeln!(
        out,
        "{}{:5}ms - {}{}",
        current_indent,
        current.duration.as_millis(),
        current.label,
        detail,
    )
    .expect("printing profiling info");

    let longer_than_millis = longer_than.as_millis();
    let children_indices = &children_map[current_idx];
    let mut accounted_for = Duration::default();
    let mut short_children = BTreeMap::new(); // Use `BTreeMap` to get deterministic output.

    for child_idx in children_indices.iter() {
        let child = &msgs[*child_idx];
        if child.duration.as_millis() > longer_than_millis {
            print_for_idx(*child_idx, children_map, msgs, longer_than, out);
        } else {
            let pair = short_children.entry(child.label).or_insert((Duration::default(), 0));
            pair.0 += child.duration;
            pair.1 += 1;
        }
        accounted_for += child.duration;
    }

    for (child_msg, (duration, count)) in short_children.iter() {
        let millis = duration.as_millis();
        writeln!(out, "    {}{:5}ms - {} ({} calls)", current_indent, millis, child_msg, count)
            .expect("printing profiling info");
    }

    let unaccounted_millis = (current.duration - accounted_for).as_millis();
    if !children_indices.is_empty()
        && unaccounted_millis > 0
        && unaccounted_millis > longer_than_millis
    {
        writeln!(out, "    {}{:5}ms - ???", current_indent, unaccounted_millis)
            .expect("printing profiling info");
    }
}

/// Returns a mapping from an index in the `msgs` to the vector with the indices of its children.
///
/// This assumes that the entries in `msgs` are in the order of when the calls to `profile` finish.
/// In other words, a postorder of the call graph. In particular, the root is the last element of
/// `msgs`.
fn idx_to_children(msgs: &[Message]) -> Vec<Vec<usize>> {
    // Initialize with the index of the root; `msgs` and `ancestors` should be never empty.
    assert!(!msgs.is_empty());
    let mut ancestors = vec![msgs.len() - 1];
    let mut result: Vec<Vec<usize>> = vec![vec![]; msgs.len()];
    for (idx, msg) in msgs[..msgs.len() - 1].iter().enumerate().rev() {
        // We need to find the parent of the current message, i.e., the last ancestor that has a
        // level lower than the current message.
        while msgs[*ancestors.last().unwrap()].level >= msg.level {
            ancestors.pop();
        }
        result[*ancestors.last().unwrap()].push(idx);
        ancestors.push(idx);
    }
    // Note that above we visited all children from the last to the first one. Let's reverse vectors
    // to get the more natural order where the first element is the first child.
    for vec in result.iter_mut() {
        vec.reverse();
    }
    result
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_basic_profile() {
        let s = vec!["profile1".to_string(), "profile2".to_string()];
        let f = Filter::new(2, s, Duration::new(0, 0));
        set_filter(f);
        profiling_function1();
    }

    fn profiling_function1() {
        let _p = profile("profile1");
        profiling_function2();
    }

    fn profiling_function2() {
        let _p = profile("profile2");
    }

    #[test]
    fn test_longer_than() {
        let mut result = vec![];
        let msgs = vec![
            Message { level: 1, duration: Duration::from_nanos(3), label: "bar", detail: None },
            Message { level: 1, duration: Duration::from_nanos(2), label: "bar", detail: None },
            Message { level: 0, duration: Duration::from_millis(1), label: "foo", detail: None },
        ];
        print(&msgs, Duration::from_millis(0), &mut result);
        // The calls to `bar` are so short that they'll be rounded to 0ms and should get collapsed
        // when printing.
        assert_eq!(
            std::str::from_utf8(&result).unwrap(),
            "    1ms - foo\n        0ms - bar (2 calls)\n"
        );
    }

    #[test]
    fn test_unaccounted_for_topmost() {
        let mut result = vec![];
        let msgs = vec![
            Message { level: 1, duration: Duration::from_millis(2), label: "bar", detail: None },
            Message { level: 0, duration: Duration::from_millis(5), label: "foo", detail: None },
        ];
        print(&msgs, Duration::from_millis(0), &mut result);
        assert_eq!(
            std::str::from_utf8(&result).unwrap().lines().collect::<Vec<_>>(),
            vec![
                "    5ms - foo",
                "        2ms - bar",
                "        3ms - ???",
                // Dummy comment to improve formatting
            ]
        );
    }

    #[test]
    fn test_unaccounted_for_multiple_levels() {
        let mut result = vec![];
        let msgs = vec![
            Message { level: 2, duration: Duration::from_millis(3), label: "baz", detail: None },
            Message { level: 1, duration: Duration::from_millis(5), label: "bar", detail: None },
            Message { level: 2, duration: Duration::from_millis(2), label: "baz", detail: None },
            Message { level: 1, duration: Duration::from_millis(4), label: "bar", detail: None },
            Message { level: 0, duration: Duration::from_millis(9), label: "foo", detail: None },
        ];
        print(&msgs, Duration::from_millis(0), &mut result);
        assert_eq!(
            std::str::from_utf8(&result).unwrap().lines().collect::<Vec<_>>(),
            vec![
                "    9ms - foo",
                "        5ms - bar",
                "            3ms - baz",
                "            2ms - ???",
                "        4ms - bar",
                "            2ms - baz",
                "            2ms - ???",
            ]
        );
    }
}
<commit_msg>Simplify hprof<commit_after>//! Simple hierarchical profiler
use std::{
    cell::RefCell,
    collections::{BTreeMap, HashSet},
    io::{stderr, Write},
    sync::{
        atomic::{AtomicBool, Ordering},
        RwLock,
    },
    time::{Duration, Instant},
};

use once_cell::sync::Lazy;

/// Filtering syntax
/// env RA_PROFILE=*             // dump everything
/// env RA_PROFILE=foo|bar|baz   // enabled only selected entries
/// env RA_PROFILE=*@3>10        // dump everything, up to depth 3, if it takes more than 10 ms
pub fn init() {
    let spec = std::env::var("RA_PROFILE").unwrap_or_default();
    init_from(&spec);
}

pub fn init_from(spec: &str) {
    let filter = if spec.is_empty() { Filter::disabled() } else { Filter::from_spec(spec) };
    filter.install();
}

pub type Label = &'static str;

/// This function starts a profiling scope in the current execution stack with a given description.
/// It returns a Profile structure and measure elapsed time between this method invocation and Profile structure drop.
/// It supports nested profiling scopes in case when this function invoked multiple times at the execution stack. In this case the profiling information will be nested at the output.
/// Profiling information is being printed in the stderr.
///
/// # Example
/// ```
/// use ra_prof::{profile, set_filter, Filter};
///
/// let f = Filter::from_spec("profile1|profile2@2");
/// set_filter(f);
/// profiling_function1();
///
/// fn profiling_function1() {
///     let _p = profile("profile1");
///     profiling_function2();
/// }
///
/// fn profiling_function2() {
///     let _p = profile("profile2");
/// }
/// ```
/// This will print in the stderr the following:
/// ```text
///  0ms - profile
///      0ms - profile2
/// ```
pub fn profile(label: Label) -> Profiler {
    assert!(!label.is_empty());
    let enabled = PROFILING_ENABLED.load(Ordering::Relaxed)
        && PROFILE_STACK.with(|stack| stack.borrow_mut().push(label));
    let label = if enabled { Some(label) } else { None };
    Profiler { label, detail: None }
}

pub struct Profiler {
    label: Option<Label>,
    detail: Option<String>,
}

impl Profiler {
    pub fn detail(mut self, detail: impl FnOnce() -> String) -> Profiler {
        if self.label.is_some() {
            self.detail = Some(detail())
        }
        self
    }
}

static PROFILING_ENABLED: AtomicBool = AtomicBool::new(false);
static FILTER: Lazy<RwLock<Filter>> = Lazy::new(Default::default);
thread_local!(static PROFILE_STACK: RefCell<ProfileStack> = RefCell::new(ProfileStack::new()));

#[derive(Default, Clone, Debug)]
struct Filter {
    depth: usize,
    allowed: HashSet<String>,
    longer_than: Duration,
    version: usize,
}

impl Filter {
    fn new(depth: usize, allowed: HashSet<String>, longer_than: Duration) -> Filter {
        Filter { depth, allowed, longer_than, version: 0 }
    }

    fn disabled() -> Filter {
        Filter::default()
    }

    fn from_spec(mut spec: &str) -> Filter {
        let longer_than = if let Some(idx) = spec.rfind('>') {
            let longer_than = spec[idx + 1..].parse().expect("invalid profile longer_than");
            spec = &spec[..idx];
            Duration::from_millis(longer_than)
        } else {
            Duration::new(0, 0)
        };

        let depth = if let Some(idx) = spec.rfind('@') {
            let depth: usize = spec[idx + 1..].parse().expect("invalid profile depth");
            spec = &spec[..idx];
            depth
        } else {
            999
        };
        let allowed =
            if spec == "*" { HashSet::new() } else { spec.split('|').map(String::from).collect() };
        Filter::new(depth, allowed, longer_than)
    }

    fn install(mut self) {
        PROFILING_ENABLED.store(self.depth > 0, Ordering::SeqCst);
        let mut old = FILTER.write().unwrap();
        self.version = old.version + 1;
        *old = self;
    }
}

struct ProfileStack {
    starts: Vec<Instant>,
    messages: Vec<Message>,
    filter: Filter,
}

struct Message {
    level: usize,
    duration: Duration,
    label: Label,
    detail: Option<String>,
}

impl ProfileStack {
    fn new() -> ProfileStack {
        ProfileStack { starts: Vec::new(), messages: Vec::new(), filter: Default::default() }
    }

    fn push(&mut self, label: Label) -> bool {
        if self.starts.is_empty() {
            if let Ok(f) = FILTER.try_read() {
                if f.version > self.filter.version {
                    self.filter = f.clone();
                }
            };
        }
        if self.starts.len() > self.filter.depth {
            return false;
        }
        let allowed = &self.filter.allowed;
        if self.starts.is_empty() && !allowed.is_empty() && !allowed.contains(label) {
            return false;
        }

        self.starts.push(Instant::now());
        true
    }

    pub fn pop(&mut self, label: Label, detail: Option<String>) {
        let start = self.starts.pop().unwrap();
        let duration = start.elapsed();
        let level = self.starts.len();
        self.messages.push(Message { level, duration, label, detail });
        if level == 0 {
            let stdout = stderr();
            let longer_than = self.filter.longer_than;
            // Convert to millis for comparison to avoid problems with rounding
            // (otherwise we could print `0ms` despite user's `>0` filter when
            // `duration` is just a few nanos).
            if duration.as_millis() > longer_than.as_millis() {
                print(&self.messages, longer_than, &mut stdout.lock());
            }
            self.messages.clear();
        }
    }
}

impl Drop for Profiler {
    fn drop(&mut self) {
        match self {
            Profiler { label: Some(label), detail } => {
                PROFILE_STACK.with(|stack| {
                    stack.borrow_mut().pop(label, detail.take());
                });
            }
            Profiler { label: None, .. } => (),
        }
    }
}

fn print(msgs: &[Message], longer_than: Duration, out: &mut impl Write) {
    if msgs.is_empty() {
        return;
    }
    let children_map = idx_to_children(msgs);
    let root_idx = msgs.len() - 1;
    print_for_idx(root_idx, &children_map, msgs, longer_than, out);
}

fn print_for_idx(
    current_idx: usize,
    children_map: &[Vec<usize>],
    msgs: &[Message],
    longer_than: Duration,
    out: &mut impl Write,
) {
    let current = &msgs[current_idx];
    let current_indent = "    ".repeat(current.level);
    let detail = current.detail.as_ref().map(|it| format!(" @ {}", it)).unwrap_or_default();
    writeln!(
        out,
        "{}{:5}ms - {}{}",
        current_indent,
        current.duration.as_millis(),
        current.label,
        detail,
    )
    .expect("printing profiling info");

    let longer_than_millis = longer_than.as_millis();
    let children_indices = &children_map[current_idx];
    let mut accounted_for = Duration::default();
    let mut short_children = BTreeMap::new(); // Use `BTreeMap` to get deterministic output.

    for child_idx in children_indices.iter() {
        let child = &msgs[*child_idx];
        if child.duration.as_millis() > longer_than_millis {
            print_for_idx(*child_idx, children_map, msgs, longer_than, out);
        } else {
            let pair = short_children.entry(child.label).or_insert((Duration::default(), 0));
            pair.0 += child.duration;
            pair.1 += 1;
        }
        accounted_for += child.duration;
    }

    for (child_msg, (duration, count)) in short_children.iter() {
        let millis = duration.as_millis();
        writeln!(out, "    {}{:5}ms - {} ({} calls)", current_indent, millis, child_msg, count)
            .expect("printing profiling info");
    }

    let unaccounted_millis = (current.duration - accounted_for).as_millis();
    if !children_indices.is_empty()
        && unaccounted_millis > 0
        && unaccounted_millis > longer_than_millis
    {
        writeln!(out, "    {}{:5}ms - ???", current_indent, unaccounted_millis)
            .expect("printing profiling info");
    }
}

/// Returns a mapping from an index in the `msgs` to the vector with the indices of its children.
///
/// This assumes that the entries in `msgs` are in the order of when the calls to `profile` finish.
/// In other words, a postorder of the call graph. In particular, the root is the last element of
/// `msgs`.
fn idx_to_children(msgs: &[Message]) -> Vec<Vec<usize>> {
    // Initialize with the index of the root; `msgs` and `ancestors` should be never empty.
    assert!(!msgs.is_empty());
    let mut ancestors = vec![msgs.len() - 1];
    let mut result: Vec<Vec<usize>> = vec![vec![]; msgs.len()];
    for (idx, msg) in msgs[..msgs.len() - 1].iter().enumerate().rev() {
        // We need to find the parent of the current message, i.e., the last ancestor that has a
        // level lower than the current message.
        while msgs[*ancestors.last().unwrap()].level >= msg.level {
            ancestors.pop();
        }
        result[*ancestors.last().unwrap()].push(idx);
        ancestors.push(idx);
    }
    // Note that above we visited all children from the last to the first one. Let's reverse vectors
    // to get the more natural order where the first element is the first child.
    for vec in result.iter_mut() {
        vec.reverse();
    }
    result
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_basic_profile() {
        let s = vec!["profile1".to_string(), "profile2".to_string()];
        let f = Filter::new(2, s, Duration::new(0, 0));
        set_filter(f);
        profiling_function1();
    }

    fn profiling_function1() {
        let _p = profile("profile1");
        profiling_function2();
    }

    fn profiling_function2() {
        let _p = profile("profile2");
    }

    #[test]
    fn test_longer_than() {
        let mut result = vec![];
        let msgs = vec![
            Message { level: 1, duration: Duration::from_nanos(3), label: "bar", detail: None },
            Message { level: 1, duration: Duration::from_nanos(2), label: "bar", detail: None },
            Message { level: 0, duration: Duration::from_millis(1), label: "foo", detail: None },
        ];
        print(&msgs, Duration::from_millis(0), &mut result);
        // The calls to `bar` are so short that they'll be rounded to 0ms and should get collapsed
        // when printing.
        assert_eq!(
            std::str::from_utf8(&result).unwrap(),
            "    1ms - foo\n        0ms - bar (2 calls)\n"
        );
    }

    #[test]
    fn test_unaccounted_for_topmost() {
        let mut result = vec![];
        let msgs = vec![
            Message { level: 1, duration: Duration::from_millis(2), label: "bar", detail: None },
            Message { level: 0, duration: Duration::from_millis(5), label: "foo", detail: None },
        ];
        print(&msgs, Duration::from_millis(0), &mut result);
        assert_eq!(
            std::str::from_utf8(&result).unwrap().lines().collect::<Vec<_>>(),
            vec![
                "    5ms - foo",
                "        2ms - bar",
                "        3ms - ???",
                // Dummy comment to improve formatting
            ]
        );
    }

    #[test]
    fn test_unaccounted_for_multiple_levels() {
        let mut result = vec![];
        let msgs = vec![
            Message { level: 2, duration: Duration::from_millis(3), label: "baz", detail: None },
            Message { level: 1, duration: Duration::from_millis(5), label: "bar", detail: None },
            Message { level: 2, duration: Duration::from_millis(2), label: "baz", detail: None },
            Message { level: 1, duration: Duration::from_millis(4), label: "bar", detail: None },
            Message { level: 0, duration: Duration::from_millis(9), label: "foo", detail: None },
        ];
        print(&msgs, Duration::from_millis(0), &mut result);
        assert_eq!(
            std::str::from_utf8(&result).unwrap().lines().collect::<Vec<_>>(),
            vec![
                "    9ms - foo",
                "        5ms - bar",
                "            3ms - baz",
                "            2ms - ???",
                "        4ms - bar",
                "            2ms - baz",
                "            2ms - ???",
            ]
        );
    }
}
<|endoftext|>


<commit_before>// build-pass (FIXME(62277): could be check-pass?)
// compiler-flags: -g

pub struct Dst {
    pub a: (),
    pub b: (),
    pub data: [u8],
}

pub unsafe fn borrow(bytes: &[u8]) -> &Dst {
    let dst: &Dst = std::mem::transmute((bytes.as_ptr(), bytes.len()));
    dst
}

fn main() {}
<commit_msg>Fix typo passing flags to rustc<commit_after>// build-pass (FIXME(62277): could be check-pass?)
// compile-flags: -g

pub struct Dst {
    pub a: (),
    pub b: (),
    pub data: [u8],
}

pub unsafe fn borrow(bytes: &[u8]) -> &Dst {
    let dst: &Dst = std::mem::transmute((bytes.as_ptr(), bytes.len()));
    dst
}

fn main() {}
<|endoftext|>


<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Interfaces to the operating system provided random number
//! generators.

use rand::Rng;
use ops::Drop;

#[cfg(unix)]
use rand::reader::ReaderRng;
#[cfg(unix)]
use io::File;

#[cfg(windows)]
use cast;
#[cfg(windows)]
use libc::{c_long, DWORD, BYTE};
#[cfg(windows)]
type HCRYPTPROV = c_long;
// the extern functions imported from the runtime on Windows are
// implemented so that they either succeed or abort(), so we can just
// assume they work when we call them.

/// A random number generator that retrieves randomness straight from
/// the operating system. Platform sources:
///
/// - Unix-like systems (Linux, Android, Mac OSX): read directly from
///   `/dev/urandom`.
/// - Windows: calls `CryptGenRandom`, using the default cryptographic
///   service provider with the `PROV_RSA_FULL` type.
///
/// This does not block.
#[cfg(unix)]
pub struct OSRng {
    priv inner: ReaderRng<File>
}
/// A random number generator that retrieves randomness straight from
/// the operating system. Platform sources:
///
/// - Unix-like systems (Linux, Android, Mac OSX): read directly from
///   `/dev/urandom`.
/// - Windows: calls `CryptGenRandom`, using the default cryptographic
///   service provider with the `PROV_RSA_FULL` type.
///
/// This does not block.
#[cfg(windows)]
pub struct OSRng {
    priv hcryptprov: HCRYPTPROV
}

impl OSRng {
    /// Create a new `OSRng`.
    #[cfg(unix)]
    pub fn new() -> OSRng {
        use path::Path;
        let reader = File::open(&Path::new("/dev/urandom"));
        let reader = reader.expect("Error opening /dev/urandom");
        let reader_rng = ReaderRng::new(reader);

        OSRng { inner: reader_rng }
    }

    /// Create a new `OSRng`.
    #[cfg(windows)]
    pub fn new() -> OSRng {
        extern { fn rust_win32_rand_acquire(phProv: *mut HCRYPTPROV); }

        let mut hcp = 0;
        unsafe {rust_win32_rand_acquire(&mut hcp)};

        OSRng { hcryptprov: hcp }
    }
}

#[cfg(unix)]
impl Rng for OSRng {
    fn next_u32(&mut self) -> u32 {
        self.inner.next_u32()
    }
    fn next_u64(&mut self) -> u64 {
        self.inner.next_u64()
    }
    fn fill_bytes(&mut self, v: &mut [u8]) {
        self.inner.fill_bytes(v)
    }
}

#[cfg(windows)]
impl Rng for OSRng {
    fn next_u32(&mut self) -> u32 {
        let mut v = [0u8, .. 4];
        self.fill_bytes(v);
        unsafe { cast::transmute(v) }
    }
    fn next_u64(&mut self) -> u64 {
        let mut v = [0u8, .. 8];
        self.fill_bytes(v);
        unsafe { cast::transmute(v) }
    }
    fn fill_bytes(&mut self, v: &mut [u8]) {
        extern {
            fn rust_win32_rand_gen(hProv: HCRYPTPROV, dwLen: DWORD,
                                   pbBuffer: *mut BYTE);
        }

        unsafe {rust_win32_rand_gen(self.hcryptprov, v.len() as DWORD, v.as_mut_ptr())}
    }
}

impl Drop for OSRng {
    #[cfg(unix)]
    fn drop(&mut self) {
        // ensure that OSRng is not implicitly copyable on all
        // platforms, for consistency.
    }

    #[cfg(windows)]
    fn drop(&mut self) {
        extern { fn rust_win32_rand_release(hProv: HCRYPTPROV); }

        unsafe {rust_win32_rand_release(self.hcryptprov)}
    }
}

#[cfg(test)]
mod test {
    use prelude::*;
    use super::*;
    use rand::Rng;
    use task;

    #[test]
    fn test_os_rng() {
        let mut r = OSRng::new();

        r.next_u32();
        r.next_u64();

        let mut v = [0u8, .. 1000];
        r.fill_bytes(v);
    }

    #[test]
    fn test_os_rng_tasks() {

        let mut chans = ~[];
        for _ in range(0, 20) {
            let (p, c) = Chan::new();
            chans.push(c);
            do task::spawn {
                // wait until all the tasks are ready to go.
                p.recv();

                // deschedule to attempt to interleave things as much
                // as possible (XXX: is this a good test?)
                let mut r = OSRng::new();
                task::deschedule();
                let mut v = [0u8, .. 1000];

                for _ in range(0, 100) {
                    r.next_u32();
                    task::deschedule();
                    r.next_u64();
                    task::deschedule();
                    r.fill_bytes(v);
                    task::deschedule();
                }
            }
        }

        // start all the tasks
        for c in chans.iter() {
            c.send(())
        }
    }
}
<commit_msg>Fix remaining cases of leaking imports<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Interfaces to the operating system provided random number
//! generators.

use rand::Rng;
use ops::Drop;

#[cfg(unix)]
use rand::reader::ReaderRng;
#[cfg(unix)]
use io::File;

#[cfg(windows)]
use cast;
#[cfg(windows)]
use libc::{c_long, DWORD, BYTE};
#[cfg(windows)]
type HCRYPTPROV = c_long;
// the extern functions imported from the runtime on Windows are
// implemented so that they either succeed or abort(), so we can just
// assume they work when we call them.

/// A random number generator that retrieves randomness straight from
/// the operating system. Platform sources:
///
/// - Unix-like systems (Linux, Android, Mac OSX): read directly from
///   `/dev/urandom`.
/// - Windows: calls `CryptGenRandom`, using the default cryptographic
///   service provider with the `PROV_RSA_FULL` type.
///
/// This does not block.
#[cfg(unix)]
pub struct OSRng {
    priv inner: ReaderRng<File>
}
/// A random number generator that retrieves randomness straight from
/// the operating system. Platform sources:
///
/// - Unix-like systems (Linux, Android, Mac OSX): read directly from
///   `/dev/urandom`.
/// - Windows: calls `CryptGenRandom`, using the default cryptographic
///   service provider with the `PROV_RSA_FULL` type.
///
/// This does not block.
#[cfg(windows)]
pub struct OSRng {
    priv hcryptprov: HCRYPTPROV
}

impl OSRng {
    /// Create a new `OSRng`.
    #[cfg(unix)]
    pub fn new() -> OSRng {
        use path::Path;
        let reader = File::open(&Path::new("/dev/urandom"));
        let reader = reader.expect("Error opening /dev/urandom");
        let reader_rng = ReaderRng::new(reader);

        OSRng { inner: reader_rng }
    }

    /// Create a new `OSRng`.
    #[cfg(windows)]
    pub fn new() -> OSRng {
        extern { fn rust_win32_rand_acquire(phProv: *mut HCRYPTPROV); }

        let mut hcp = 0;
        unsafe {rust_win32_rand_acquire(&mut hcp)};

        OSRng { hcryptprov: hcp }
    }
}

#[cfg(unix)]
impl Rng for OSRng {
    fn next_u32(&mut self) -> u32 {
        self.inner.next_u32()
    }
    fn next_u64(&mut self) -> u64 {
        self.inner.next_u64()
    }
    fn fill_bytes(&mut self, v: &mut [u8]) {
        self.inner.fill_bytes(v)
    }
}

#[cfg(windows)]
impl Rng for OSRng {
    fn next_u32(&mut self) -> u32 {
        let mut v = [0u8, .. 4];
        self.fill_bytes(v);
        unsafe { cast::transmute(v) }
    }
    fn next_u64(&mut self) -> u64 {
        let mut v = [0u8, .. 8];
        self.fill_bytes(v);
        unsafe { cast::transmute(v) }
    }
    fn fill_bytes(&mut self, v: &mut [u8]) {
        use container::Container;
        use vec::MutableVector;

        extern {
            fn rust_win32_rand_gen(hProv: HCRYPTPROV, dwLen: DWORD,
                                   pbBuffer: *mut BYTE);
        }

        unsafe {rust_win32_rand_gen(self.hcryptprov, v.len() as DWORD, v.as_mut_ptr())}
    }
}

impl Drop for OSRng {
    #[cfg(unix)]
    fn drop(&mut self) {
        // ensure that OSRng is not implicitly copyable on all
        // platforms, for consistency.
    }

    #[cfg(windows)]
    fn drop(&mut self) {
        extern { fn rust_win32_rand_release(hProv: HCRYPTPROV); }

        unsafe {rust_win32_rand_release(self.hcryptprov)}
    }
}

#[cfg(test)]
mod test {
    use prelude::*;
    use super::*;
    use rand::Rng;
    use task;

    #[test]
    fn test_os_rng() {
        let mut r = OSRng::new();

        r.next_u32();
        r.next_u64();

        let mut v = [0u8, .. 1000];
        r.fill_bytes(v);
    }

    #[test]
    fn test_os_rng_tasks() {

        let mut chans = ~[];
        for _ in range(0, 20) {
            let (p, c) = Chan::new();
            chans.push(c);
            do task::spawn {
                // wait until all the tasks are ready to go.
                p.recv();

                // deschedule to attempt to interleave things as much
                // as possible (XXX: is this a good test?)
                let mut r = OSRng::new();
                task::deschedule();
                let mut v = [0u8, .. 1000];

                for _ in range(0, 100) {
                    r.next_u32();
                    task::deschedule();
                    r.next_u64();
                    task::deschedule();
                    r.fill_bytes(v);
                    task::deschedule();
                }
            }
        }

        // start all the tasks
        for c in chans.iter() {
            c.send(())
        }
    }
}
<|endoftext|>


<commit_before>extern crate moho;
extern crate sdl2_image;
extern crate sdl2;
extern crate glm;

use self::moho::input_manager::*;

use self::sdl2::keyboard::Keycode;
use self::sdl2::mouse::Mouse;
use self::sdl2::render::Renderer;
use self::sdl2_image::LoadTexture;

use std::error::Error;
use std::path::Path;

use meteor::Meteor;
use drawable::Drawable;

pub struct MasterSmasher<'a> {
    meteor: Meteor,
    background: Drawable,
    input_manager: InputManager<SdlEventStreamGenerator>,
    renderer: Renderer<'a>,
}

impl<'a> MasterSmasher<'a> {
    pub fn new() -> Result<Self, Box<Error>> {
        const WINDOW_HEIGHT: u32 = 600;
        const WINDOW_WIDTH: u32 = 800;

        let (renderer, input_manager) =
            try!(moho::init("Master Smasher", WINDOW_WIDTH, WINDOW_HEIGHT));
        let background_path = Path::new("resources/background_game.png");
        let meteor_path = Path::new("resources/meteor.png");

        let background = Drawable::new(try!(renderer.load_texture(background_path)));
        let meteor = Meteor::new(try!(renderer.load_texture(meteor_path)),
                                 glm::uvec2(WINDOW_WIDTH, WINDOW_HEIGHT));

        Ok(MasterSmasher {
            meteor: meteor,
            background: background,
            input_manager: input_manager,
            renderer: renderer,
        })
    }

    pub fn run(&mut self) -> Result<(), Box<Error>> {
        'running: loop {
            self.input_manager.update();

            if self.input_manager.is_key_down(Keycode::Escape) {
                break;
            }

            if self.input_manager.is_mouse_down(Mouse::Left) {
                if !self.meteor.is_launched() {
                    self.meteor.launch(self.input_manager.mouse_coords());
                }
            }

            self.meteor.update();
            try!(self.draw());
        }

        Ok(())
    }

    fn draw(&mut self) -> Result<(), Box<Error>> {
        self.renderer.clear();
        try!(self.background.draw(&mut self.renderer));
        try!(self.meteor.draw(&mut self.renderer));
        self.renderer.present();
        Ok(())
    }
}
<commit_msg>launch whenever mouse did a click instead of checking if it's pressed<commit_after>extern crate moho;
extern crate sdl2_image;
extern crate sdl2;
extern crate glm;

use self::moho::input_manager::*;

use self::sdl2::keyboard::Keycode;
use self::sdl2::mouse::Mouse;
use self::sdl2::render::Renderer;
use self::sdl2_image::LoadTexture;

use std::error::Error;
use std::path::Path;

use meteor::Meteor;
use drawable::Drawable;

pub struct MasterSmasher<'a> {
    meteor: Meteor,
    background: Drawable,
    input_manager: InputManager<SdlEventStreamGenerator>,
    renderer: Renderer<'a>,
}

impl<'a> MasterSmasher<'a> {
    pub fn new() -> Result<Self, Box<Error>> {
        const WINDOW_HEIGHT: u32 = 600;
        const WINDOW_WIDTH: u32 = 800;

        let (renderer, input_manager) =
            try!(moho::init("Master Smasher", WINDOW_WIDTH, WINDOW_HEIGHT));
        let background_path = Path::new("resources/background_game.png");
        let meteor_path = Path::new("resources/meteor.png");

        let background = Drawable::new(try!(renderer.load_texture(background_path)));
        let meteor = Meteor::new(try!(renderer.load_texture(meteor_path)),
                                 glm::uvec2(WINDOW_WIDTH, WINDOW_HEIGHT));

        Ok(MasterSmasher {
            meteor: meteor,
            background: background,
            input_manager: input_manager,
            renderer: renderer,
        })
    }

    pub fn run(&mut self) -> Result<(), Box<Error>> {
        'running: loop {
            self.input_manager.update();

            if self.input_manager.is_key_down(Keycode::Escape) {
                break;
            }

            if self.input_manager.did_click_mouse(Mouse::Left) {
                if !self.meteor.is_launched() {
                    self.meteor.launch(self.input_manager.mouse_coords());
                }
            }

            self.meteor.update();
            try!(self.draw());
        }

        Ok(())
    }

    fn draw(&mut self) -> Result<(), Box<Error>> {
        self.renderer.clear();
        try!(self.background.draw(&mut self.renderer));
        try!(self.meteor.draw(&mut self.renderer));
        self.renderer.present();
        Ok(())
    }
}
<|endoftext|>


<commit_before>extern crate cc;
extern crate metadeps;

use std::env;
use std::ffi::OsString;
use std::fs;
use std::path::PathBuf;
use std::process::Command;

macro_rules! t {
    ($e:expr) => (match $e {
        Ok(t) => t,
        Err(e) => panic!("{} return the error {}", stringify!($e), e),
    })
}

fn main() {
    // Use the installed libcares if it is available.
    if metadeps::probe().is_ok() {
        return;
    }

    // Rerun if the c-ares source code has changed.
    println!("cargo:rerun-if-changed=c-ares");

    // MSVC builds are different.
    let target = env::var("TARGET").unwrap();
    if target.contains("msvc") {
        build_msvc(&target);
        return;
    }

    let dst = PathBuf::from(env::var_os("OUT_DIR").unwrap());
    let build = dst.join("build");
    let _ = fs::create_dir(&build);

    // Prepare.
    let src = env::current_dir().unwrap();
    run(Command::new("sh")
        .current_dir(&src.join("c-ares"))
        .arg("buildconf"));

    // Configure.
    let cfg = cc::Build::new();
    let compiler = cfg.get_compiler();
    let mut cflags = OsString::new();
    for arg in compiler.args() {
        cflags.push(arg);
        cflags.push(" ");
    }

    let mut cmd = Command::new("sh");
    cmd.env("CFLAGS", &cflags)
        .env("CC", compiler.path())
        .current_dir(&build)
        .arg(format!("{}", src.join("c-ares/configure").display()))
        .arg("--enable-static=yes")
        .arg("--enable-shared=no")
        .arg("--enable-optimize")
        .arg(format!("--prefix={}", dst.display()));

    // This code fragment copied from curl-rust... c-ares and curl come from
    // the same developer so are usually pretty similar, and this seems to
    // work.
    //
    // NOTE GNU terminology
    // BUILD = machine where we are (cross) compiling
    // HOST = machine where the compiled binary will be used
    // TARGET = only relevant when compiling compilers
    let host = env::var("HOST").unwrap();
    if target != host && (!target.contains("windows") || !host.contains("windows")) {
        if target.contains("windows") {
            cmd.arg(format!("--host={}", host));
            cmd.arg(format!("--target={}", target));
        } else {
            cmd.arg(format!("--build={}", host));
            cmd.arg(format!("--host={}", target));
        }
    }
    run(&mut cmd);

    // Compile.
    run(Command::new(make())
        .arg(format!("-j{}", env::var("NUM_JOBS").unwrap()))
        .current_dir(&build));

    // Link to compiled library.
    println!("cargo:rustc-link-search={}/.libs", build.display());
    println!("cargo:rustc-link-lib=static=cares");
}

fn run(cmd: &mut Command) {
    println!("running: {:?}", cmd);
    assert!(t!(cmd.status()).success());
}

fn make() -> &'static str {
    if cfg!(target_os = "freebsd") {
        "gmake"
    } else {
        "make"
    }
}

fn nmake(target: &str) -> Command {
    // cargo messes with the environment in a way that nmake does not like -
    // see https://github.com/rust-lang/cargo/issues/4156.  Explicitly remove
    // the unwanted variables.
    let mut cmd = cc::windows_registry::find(target, "nmake.exe").unwrap();
    cmd.env_remove("MAKEFLAGS").env_remove("MFLAGS");
    cmd
}

fn build_msvc(target: &str) {
    // Prepare.
    let src = env::current_dir().unwrap();
    let c_ares_dir = &src.join("c-ares");
    run(Command::new("cmd")
        .current_dir(c_ares_dir)
        .arg("/c")
        .arg("buildconf.bat"));

    // Compile.
    let mut cmd = nmake(target);
    cmd.current_dir(c_ares_dir);
    cmd.args(&["/f", "Makefile.msvc", "CFG=lib-release", "c-ares"]);
    run(&mut cmd);

    // Install library.
    let dst = PathBuf::from(env::var_os("OUT_DIR").unwrap());
    let build = dst.join("build");
    let mut cmd = nmake(target);
    cmd.current_dir(c_ares_dir);
    cmd.args(&["/f", "Makefile.msvc", "/a", "CFG=lib-release", "install"]);
    cmd.env("INSTALL_DIR", format!("{}", build.display()));
    run(&mut cmd);

    // Link to compiled library.
    println!("cargo:rustc-link-search={}/lib", build.display());
    println!("cargo:rustc-link-lib=static=libcares");
}
<commit_msg>rustfmt<commit_after>extern crate cc;
extern crate metadeps;

use std::env;
use std::ffi::OsString;
use std::fs;
use std::path::PathBuf;
use std::process::Command;

macro_rules! t {
    ($e:expr) => (match $e {
        Ok(t) => t,
        Err(e) => panic!("{} return the error {}", stringify!($e), e),
    })
}

fn main() {
    // Use the installed libcares if it is available.
    if metadeps::probe().is_ok() {
        return;
    }

    // Rerun if the c-ares source code has changed.
    println!("cargo:rerun-if-changed=c-ares");

    // MSVC builds are different.
    let target = env::var("TARGET").unwrap();
    if target.contains("msvc") {
        build_msvc(&target);
        return;
    }

    let dst = PathBuf::from(env::var_os("OUT_DIR").unwrap());
    let build = dst.join("build");
    let _ = fs::create_dir(&build);

    // Prepare.
    let src = env::current_dir().unwrap();
    run(
        Command::new("sh")
            .current_dir(&src.join("c-ares"))
            .arg("buildconf"),
    );

    // Configure.
    let cfg = cc::Build::new();
    let compiler = cfg.get_compiler();
    let mut cflags = OsString::new();
    for arg in compiler.args() {
        cflags.push(arg);
        cflags.push(" ");
    }

    let mut cmd = Command::new("sh");
    cmd.env("CFLAGS", &cflags)
        .env("CC", compiler.path())
        .current_dir(&build)
        .arg(format!("{}", src.join("c-ares/configure").display()))
        .arg("--enable-static=yes")
        .arg("--enable-shared=no")
        .arg("--enable-optimize")
        .arg(format!("--prefix={}", dst.display()));

    // This code fragment copied from curl-rust... c-ares and curl come from
    // the same developer so are usually pretty similar, and this seems to
    // work.
    //
    // NOTE GNU terminology
    // BUILD = machine where we are (cross) compiling
    // HOST = machine where the compiled binary will be used
    // TARGET = only relevant when compiling compilers
    let host = env::var("HOST").unwrap();
    if target != host && (!target.contains("windows") || !host.contains("windows")) {
        if target.contains("windows") {
            cmd.arg(format!("--host={}", host));
            cmd.arg(format!("--target={}", target));
        } else {
            cmd.arg(format!("--build={}", host));
            cmd.arg(format!("--host={}", target));
        }
    }
    run(&mut cmd);

    // Compile.
    run(
        Command::new(make())
            .arg(format!("-j{}", env::var("NUM_JOBS").unwrap()))
            .current_dir(&build),
    );

    // Link to compiled library.
    println!("cargo:rustc-link-search={}/.libs", build.display());
    println!("cargo:rustc-link-lib=static=cares");
}

fn run(cmd: &mut Command) {
    println!("running: {:?}", cmd);
    assert!(t!(cmd.status()).success());
}

fn make() -> &'static str {
    if cfg!(target_os = "freebsd") {
        "gmake"
    } else {
        "make"
    }
}

fn nmake(target: &str) -> Command {
    // cargo messes with the environment in a way that nmake does not like -
    // see https://github.com/rust-lang/cargo/issues/4156.  Explicitly remove
    // the unwanted variables.
    let mut cmd = cc::windows_registry::find(target, "nmake.exe").unwrap();
    cmd.env_remove("MAKEFLAGS").env_remove("MFLAGS");
    cmd
}

fn build_msvc(target: &str) {
    // Prepare.
    let src = env::current_dir().unwrap();
    let c_ares_dir = &src.join("c-ares");
    run(
        Command::new("cmd")
            .current_dir(c_ares_dir)
            .arg("/c")
            .arg("buildconf.bat"),
    );

    // Compile.
    let mut cmd = nmake(target);
    cmd.current_dir(c_ares_dir);
    cmd.args(&["/f", "Makefile.msvc", "CFG=lib-release", "c-ares"]);
    run(&mut cmd);

    // Install library.
    let dst = PathBuf::from(env::var_os("OUT_DIR").unwrap());
    let build = dst.join("build");
    let mut cmd = nmake(target);
    cmd.current_dir(c_ares_dir);
    cmd.args(&["/f", "Makefile.msvc", "/a", "CFG=lib-release", "install"]);
    cmd.env("INSTALL_DIR", format!("{}", build.display()));
    run(&mut cmd);

    // Link to compiled library.
    println!("cargo:rustc-link-search={}/lib", build.display());
    println!("cargo:rustc-link-lib=static=libcares");
}
<|endoftext|>


<commit_before>use rand::{thread_rng, Rng};

use network::Backend;
use sozu_command::config::LoadBalancingAlgorithms;

use std::{ rc::Rc, cell::RefCell };
use std::fmt::Debug;

pub trait LoadBalancingAlgorithm: Debug {
  fn next_available_backend(&mut self, backends: &Vec<Rc<RefCell<Backend>>>) -> Option<Rc<RefCell<Backend>>>;
}

#[derive(Debug)]
pub struct RoundRobinAlgorithm {
  pub next_backend: u32,
}

impl LoadBalancingAlgorithm for RoundRobinAlgorithm {

  fn next_available_backend(&mut self , backends: &Vec<Rc<RefCell<Backend>>>) -> Option<Rc<RefCell<Backend>>> {
    let res = backends.get(self.next_backend as usize)
                      .map(|backend| (*backend).clone());

    self.next_backend = (self.next_backend + 1) % backends.len() as u32;
    res
  }

}

impl RoundRobinAlgorithm {

  fn new() -> Self {
    Self {
      next_backend: 0,
    }
  }

}

#[derive(Debug)]
pub struct RandomAlgorithm;

impl LoadBalancingAlgorithm for RandomAlgorithm {

  fn next_available_backend(&mut self, backends: &Vec<Rc<RefCell<Backend>>>) -> Option<Rc<RefCell<Backend>>> {
    let mut rng = thread_rng();

    rng.choose(backends)
      .map(|backend| (*backend).clone())
  }

}

#[derive(Debug)]
pub struct LeastConnectionsAlgorithm;

impl LoadBalancingAlgorithm for LeastConnectionsAlgorithm {

  fn next_available_backend(&mut self, backends: &Vec<Rc<RefCell<Backend>>>) -> Option<Rc<RefCell<Backend>>> {
    backends
      .iter()
      .min_by_key(|backend| backend.borrow().active_connections)
      .map(|backend| (*backend).clone())
  }

}

#[cfg(test)]
mod test {
  use super::*;
  use std::net::{IpAddr, Ipv4Addr, SocketAddr};
  use network::BackendStatus;
  use network::retry::{RetryPolicyWrapper, ExponentialBackoffPolicy};

  fn create_backend(id: String, connections: Option<usize>) -> Backend {
    Backend {
      sticky_id: None,
      backend_id: id,
      address: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080),
      status: BackendStatus::Normal,
      retry_policy: RetryPolicyWrapper::ExponentialBackoff(ExponentialBackoffPolicy::new(1)),
      active_connections: connections.unwrap_or(0),
      failures: 0,
      load_balancing_parameters: None,
    }
  }

  #[test]
  fn it_should_find_the_backend_with_least_connections() {
    let backend_with_least_connection = Rc::new(RefCell::new(create_backend("yolo".to_string(), Some(1))));

    let backends = vec![
      Rc::new(RefCell::new(create_backend("nolo".to_string(), Some(10)))),
      Rc::new(RefCell::new(create_backend("philo".to_string(), Some(20)))),
      backend_with_least_connection.clone(),
    ];

    let mut least_connection_algorithm = LeastConnectionsAlgorithm{};

    let backend_res = least_connection_algorithm.next_available_backend(&backends).unwrap();
    let backend = backend_res.borrow();

    assert!(*backend == *backend_with_least_connection.borrow());
  }

  #[test]
  fn it_shouldnt_find_backend_with_least_connections_when_list_is_empty() {
    let backends = vec![];

    let mut least_connection_algorithm = LeastConnectionsAlgorithm{};

    let backend = least_connection_algorithm.next_available_backend(&backends);
    assert!(backend.is_none());
  }
}
<commit_msg>RoundRobinAlgorithm: fix OOB read in next_available_backend when a backend was removed<commit_after>use rand::{thread_rng, Rng};

use network::Backend;
use sozu_command::config::LoadBalancingAlgorithms;

use std::{ rc::Rc, cell::RefCell };
use std::fmt::Debug;

pub trait LoadBalancingAlgorithm: Debug {
  fn next_available_backend(&mut self, backends: &Vec<Rc<RefCell<Backend>>>) -> Option<Rc<RefCell<Backend>>>;
}

#[derive(Debug)]
pub struct RoundRobinAlgorithm {
  pub next_backend: u32,
}

impl LoadBalancingAlgorithm for RoundRobinAlgorithm {

  fn next_available_backend(&mut self , backends: &Vec<Rc<RefCell<Backend>>>) -> Option<Rc<RefCell<Backend>>> {
    let res = backends.get(self.next_backend as usize % backends.len())
                      .map(|backend| (*backend).clone());

    self.next_backend = (self.next_backend + 1) % backends.len() as u32;
    res
  }

}

impl RoundRobinAlgorithm {

  fn new() -> Self {
    Self {
      next_backend: 0,
    }
  }

}

#[derive(Debug)]
pub struct RandomAlgorithm;

impl LoadBalancingAlgorithm for RandomAlgorithm {

  fn next_available_backend(&mut self, backends: &Vec<Rc<RefCell<Backend>>>) -> Option<Rc<RefCell<Backend>>> {
    let mut rng = thread_rng();

    rng.choose(backends)
      .map(|backend| (*backend).clone())
  }

}

#[derive(Debug)]
pub struct LeastConnectionsAlgorithm;

impl LoadBalancingAlgorithm for LeastConnectionsAlgorithm {

  fn next_available_backend(&mut self, backends: &Vec<Rc<RefCell<Backend>>>) -> Option<Rc<RefCell<Backend>>> {
    backends
      .iter()
      .min_by_key(|backend| backend.borrow().active_connections)
      .map(|backend| (*backend).clone())
  }

}

#[cfg(test)]
mod test {
  use super::*;
  use std::net::{IpAddr, Ipv4Addr, SocketAddr};
  use network::BackendStatus;
  use network::retry::{RetryPolicyWrapper, ExponentialBackoffPolicy};

  fn create_backend(id: String, connections: Option<usize>) -> Backend {
    Backend {
      sticky_id: None,
      backend_id: id,
      address: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080),
      status: BackendStatus::Normal,
      retry_policy: RetryPolicyWrapper::ExponentialBackoff(ExponentialBackoffPolicy::new(1)),
      active_connections: connections.unwrap_or(0),
      failures: 0,
      load_balancing_parameters: None,
    }
  }

  #[test]
  fn it_should_find_the_backend_with_least_connections() {
    let backend_with_least_connection = Rc::new(RefCell::new(create_backend("yolo".to_string(), Some(1))));

    let backends = vec![
      Rc::new(RefCell::new(create_backend("nolo".to_string(), Some(10)))),
      Rc::new(RefCell::new(create_backend("philo".to_string(), Some(20)))),
      backend_with_least_connection.clone(),
    ];

    let mut least_connection_algorithm = LeastConnectionsAlgorithm{};

    let backend_res = least_connection_algorithm.next_available_backend(&backends).unwrap();
    let backend = backend_res.borrow();

    assert!(*backend == *backend_with_least_connection.borrow());
  }

  #[test]
  fn it_shouldnt_find_backend_with_least_connections_when_list_is_empty() {
    let backends = vec![];

    let mut least_connection_algorithm = LeastConnectionsAlgorithm{};

    let backend = least_connection_algorithm.next_available_backend(&backends);
    assert!(backend.is_none());
  }

  #[test]
  fn it_should_find_backend_with_roundrobin_when_some_backends_were_removed() {
    let mut backends = vec![
      Rc::new(RefCell::new(create_backend("toto".to_string(), None))),
      Rc::new(RefCell::new(create_backend("voto".to_string(), None))),
      Rc::new(RefCell::new(create_backend("yoto".to_string(), None)))
    ];

    let mut roundrobin = RoundRobinAlgorithm { next_backend: 1 };
    let backend = roundrobin.next_available_backend(&backends);
    assert_eq!(backend.as_ref(), backends.get(1));

    backends.remove(1);

    let backend2 = roundrobin.next_available_backend(&backends);
    assert_eq!(backend2.as_ref(),  backends.get(0));
  }
}
<|endoftext|>


<commit_before>//! Functions to be used for clap::Arg::validator()
//! to validate arguments

use std::path::PathBuf;
use boolinator::Boolinator;

pub fn is_file(s: String) -> Result<(), String> {
    PathBuf::from(s.clone()).is_file().as_result((), format!("Not a File: {}", s))
}

pub fn is_directory(s: String) -> Result<(), String> {
    PathBuf::from(s.clone()).is_dir().as_result((), format!("Not a Directory: {}", s))
}

pub fn is_integer(s: String) -> Result<(), String> {
    use std::str::FromStr;

    let i : Result<i64, _> = FromStr::from_str(&s);
    i.map(|_| ()).map_err(|_| format!("Not an integer: {}", s))
}

pub fn is_url(s: String) -> Result<(), String> {
    use url::Url;
    Url::parse(&s).map(|_| ()).map_err(|_| format!("Not a URL: {}", s))
}

<commit_msg>Add is_tag clap validator helper<commit_after>//! Functions to be used for clap::Arg::validator()
//! to validate arguments

use std::path::PathBuf;
use boolinator::Boolinator;

pub fn is_file(s: String) -> Result<(), String> {
    PathBuf::from(s.clone()).is_file().as_result((), format!("Not a File: {}", s))
}

pub fn is_directory(s: String) -> Result<(), String> {
    PathBuf::from(s.clone()).is_dir().as_result((), format!("Not a Directory: {}", s))
}

pub fn is_integer(s: String) -> Result<(), String> {
    use std::str::FromStr;

    let i : Result<i64, _> = FromStr::from_str(&s);
    i.map(|_| ()).map_err(|_| format!("Not an integer: {}", s))
}

pub fn is_url(s: String) -> Result<(), String> {
    use url::Url;
    Url::parse(&s).map(|_| ()).map_err(|_| format!("Not a URL: {}", s))
}

pub fn is_tag(s: String) -> Result<(), String> {
    use regex::Regex;
    lazy_static! { static ref TAG_RE : Regex = Regex::new("[:alpha:][:word:]*").unwrap(); }

    TAG_RE
        .is_match(&s)
        .as_result((), format!("Not a valid Tag: '{}' - Valid is [a-zA-Z][0-9a-zA-Z]*", s))
}

<|endoftext|>


<commit_before>use std::env;
use std::io;
use std::mem;
use std::num;
use std::fs;
use std::cmp::Ordering;
use std::collections::HashMap;
use std::io::Read;
use std::io::Write;
use std::io::BufRead;
use std::usize;

struct CooccurRec {
    word1: u32,
    word2: u32,
    val: f32,
}

fn get_filename(file_head: &str, n: i32) -> String {
    return format!("{}_{:>04}.bin", file_head, n);
}

fn get_word(fin: &io::Read) -> Option<String> {
    let mut word: Vec<u8> = vec![];
    let mut byte: [u8; 1];
    while fin.read_exact(&mut byte).is_ok() {
        match byte[0] {
            b' ' | b'\t' | b'\n' => { break; }
            c => { word.push(c); }
        }
    }
    if word.len() > 0 { String::from_utf8(word).ok() } else { None }
}

fn get_cooccurrence(verbose: i32, symmetric: bool, window_size: i32, max_product: usize, overflow_length: usize,
                    vocab_file: &str, file_head: &str) -> i32 {
    macro_rules! progress {
        ($v:expr, $fmt:expr) => (
            if verbose > $v { write!(io::stderr(), $fmt).unwrap(); }
        );
        ($v:expr, $fmt:expr, $($arg:tt)*) => (
            if verbose > $v { write!(io::stderr(), $fmt, $($arg)*).unwrap(); }
        );
    }
    macro_rules! log {
        ($v:expr, $fmt:expr) => (
            if verbose > $v { writeln!(io::stderr(), $fmt).unwrap(); }
        );
        ($v:expr, $fmt:expr, $($arg:tt)*) => (
            if verbose > $v { writeln!(io::stderr(), $fmt, $($arg)*).unwrap(); }
        );
    }

    log!(-1, "COUNTING COOCCURRENCES");
    log!(0, "window size: {}", window_size);
    log!(0, "context: {}", if symmetric { "symmetric" } else { "asymmetric" });
    log!(1, "max product: {}", max_product);
    log!(1, "overflow length: {}", overflow_length);

    let mut vocab_hash: HashMap<String, u32> = HashMap::new();
    {
        log!(1, "Reading vocab from file \"{}\"...", vocab_file);
        let file = io::BufReader::new(fs::File::open(vocab_file).expect("Unable to open vocab file."));
        for line in file.lines() {
            let line = line.expect("vocab file read error.");
            let vec: Vec<&str> = line.split_whitespace().collect();
            let word = vec[0];
            let num = vec[1].parse::<u32>().expect("Parse error.");

            vocab_hash.insert(word.to_string(), num);
        }
        log!(1, "loaded {} words.\nBuilding lookup table...", vocab_hash.len());
    }
    let mut table: Vec<Vec<f32>> = Vec::with_capacity(vocab_hash.len());
    {
        macro_rules! min {
            ($a: expr, $b: expr) => { if $a < $b { $a } else { $b } };
        }
        for a in 0..vocab_hash.len() {
            table.push(vec![0.0_f32 ; min!(max_product / a, vocab_hash.len())]);
        }
    }
    {
        let mut counter = 0i32;
        let stdin = io::BufReader::new(io::stdin());
        'outer: loop {
            progress!(1, "Processing token: {}", counter);
            let mut file = fs::File::open(get_filename(file_head, counter)).expect("File open error.");
            while 0 {
                let flag = get_word(&stdin);
            }
            // sort cr
            // write cr
        }
    }
    0
}

fn main() {
    let mut verbose = 1i32;
    let mut window_size = 15i32;
    let mut symmetric = true;
    let mut memory_limit = 3.0;
    let mut vocab_file = "vocab.txt".to_string();
    let mut file_head = "overflow".to_string();
    let mut max_product: Option<usize> = None;
    let mut overflow_length: Option<usize> = None;
    {
        let args: Vec<String> = env::args().collect();

        if args.len() == 1 {
            println!("Tool to calculate word-word cooccurrence statistics
This program is transportation of GloVe cooccur.
Original program (written in C) author is Jeffrey Pennington (jpennin@stanford.edu)
Transpoted to rust by Takumi Kato (takumi.kt@gmail.com)

Usage options:
\t-verbose <int>
\t\tSet verbosity: 0, 1, or 2 (default)
\t-symmetric <int>
\t\tIf <int> = 0, only use left context; if <int> = 1 (default), use left and right
\t-window-size <int>
\t\tNumber of context words to the left (and to the right, if symmetric = 1); default 15
\t-vocab-file <file>
\t\tFile containing vocabulary (truncated unigram counts, produced by 'vocab_count'); default vocab.txt
\t-memory <float>
\t\tSoft limit for memory consumption, in GB -- based on simple heuristic, so not extremely accurate; default 4.0
\t-max-product <int>
\t\tLimit the size of dense cooccurrence array by specifying the max product <int> of the frequency counts of the two cooccurring words.
\t\tThis value overrides that which is automatically produced by '-memory'. Typically only needs adjustment for use with very large corpora.
\toverflow-file <file>
\t\tFilename, excluding extension, for temporary file; default overflow

Example usage:
./cooccur -verbose 2 -symmetric 0 -window-size 10 -vocab-file vocab.txt -memory 8.0 -overflow-file tempoverflow < corpus.txt > cooccurrences.bin

");
            std::process::exit(0);
        }
        let mut is_skip = true;
        let mut it = args.into_iter().peekable();
        loop {
            let arg = match it.next() {
                Some(s) => s,
                None => break,
            };
            if is_skip {
                is_skip = false;
                continue
            }
            match arg.as_ref() {
                "-verbose" => {
                    verbose = it.peek().and_then(|v| v.parse().ok())
                        .and_then(|x| if 0 <= x && x <= 2 { Some(x) } else { None })
                        .expect("-verbose <int>, verbosity: 0, 1, or 2 (default)");
                    is_skip = true;
                },
                "-symmetric" => {
                    symmetric = it.peek().and_then(|v| v.parse().ok())
                        .and_then(|x| if 0 <= x && x <= 1 { Some(x) } else { None })
                        .expect("symmetric <int>, 0 (only use left context) or 1 (use left and right; default)") == 1;
                    is_skip = true;
                },
                "-window-size" => {
                    window_size = it.peek().and_then(|v| v.parse().ok()).expect("-window-size <int>");
                    is_skip = true;
                },
                "-memory" => {
                    memory_limit = it.peek().and_then(|v| v.parse().ok()).expect("-memory <float>");
                    is_skip = true;
                },
                "-vocab-file" => {
                    vocab_file = it.peek().expect("-vocab-file <file>").to_string();
                    is_skip = true;
                },
                "-overflow" => {
                    file_head = it.peek().expect("-overflow <file>").to_string();
                    is_skip = true;
                },
                "-max-product" => {
                    max_product = Some(it.peek().and_then(|v| v.parse().ok()).expect("-max-product <int>"));
                    is_skip = true;
                },
                "-overflow-length" => {
                    overflow_length = Some(it.peek().and_then(|v| v.parse().ok()).expect("-overflow-length <int>"));
                    is_skip = true;
                },
                &_ => {},
            }
            let rlimit = 0.85 * memory_limit * 1073741824_f64 / mem::size_of::<f64>() as f64;
            let mut n = 1e5_f64;
            while (rlimit - n * (n.ln() + 0.1544313298)).abs() > 1e-3 { n = rlimit / (n.ln() + 0.1544313298) }
            max_product = max_product.or_else(|| Some(n as usize));
            overflow_length = overflow_length.or_else(|| Some((rlimit / 6.0) as usize));
        }
        //std::process::exit(get_counts(verbose, max_vocab, min_count));
    }
}
<commit_msg>update cooccur (still not working)<commit_after>use std::env;
use std::io;
use std::mem;
use std::num;
use std::fs;
use std::cmp::Ordering;
use std::collections::HashMap;
use std::io::Read;
use std::io::Write;
use std::io::BufRead;
use std::usize;

struct CooccurRec {
    word1: u32,
    word2: u32,
    val: f32,
}

fn get_filename(file_head: &str, n: i32) -> String {
    return format!("{}_{:>04}.bin", file_head, n);
}

fn get_word(fin: &io::Read) -> Option<String> {
    let mut word: Vec<u8> = vec![];
    let mut byte: [u8; 1];
    while fin.read_exact(&mut byte).is_ok() {
        match byte[0] {
            b' ' | b'\t' | b'\n' => { break; }
            c => { word.push(c); }
        }
    }
    if word.len() > 0 { String::from_utf8(word).ok() } else { None }
}

fn get_cooccurrence(verbose: i32, symmetric: bool, window_size: i32, max_product: usize, overflow_length: usize,
                    vocab_file: &str, file_head: &str) -> i32 {
    macro_rules! progress {
        ($v:expr, $fmt:expr) => (
            if verbose > $v { write!(io::stderr(), $fmt).unwrap(); }
        );
        ($v:expr, $fmt:expr, $($arg:tt)*) => (
            if verbose > $v { write!(io::stderr(), $fmt, $($arg)*).unwrap(); }
        );
    }
    macro_rules! log {
        ($v:expr, $fmt:expr) => (
            if verbose > $v { writeln!(io::stderr(), $fmt).unwrap(); }
        );
        ($v:expr, $fmt:expr, $($arg:tt)*) => (
            if verbose > $v { writeln!(io::stderr(), $fmt, $($arg)*).unwrap(); }
        );
    }
    macro_rules! min {
        ($a: expr, $b: expr) => { if $a <= $b { $a } else { $b } };
    }
    macro_rules! max {
        ($a: expr, $b: expr) => { if $a >= $b { $a } else { $b } };
    }

    log!(-1, "COUNTING COOCCURRENCES");
    log!(0, "window size: {}", window_size);
    log!(0, "context: {}", if symmetric { "symmetric" } else { "asymmetric" });
    log!(1, "max product: {}", max_product);
    log!(1, "overflow length: {}", overflow_length);

    let mut vocab_hash: HashMap<String, i64> = HashMap::new();
    {
        log!(1, "Reading vocab from file \"{}\"...", vocab_file);
        let file = io::BufReader::new(fs::File::open(vocab_file).expect("Unable to open vocab file."));
        let mut vocab_rank = 1i64;
        for line in file.lines() {
            let line = line.expect("vocab file read error.");
            let vec: Vec<&str> = line.split_whitespace().collect();
            let word = vec[0];
            //let num = vec[1].parse::<u32>().expect("Parse error.");

            vocab_hash.insert(word.to_string(), vocab_rank);
            vocab_rank += 1;
        }
        log!(1, "loaded {} words.\nBuilding lookup table...", vocab_hash.len());
    }
    let mut table: Vec<Vec<f32>> = Vec::with_capacity(vocab_hash.len());
    {
        for a in 0..vocab_hash.len() {
            table.push(vec![0.0_f32 ; min!(max_product / a, vocab_hash.len())]);
        }
    }
    {
        let mut fcounter = 0i32;
        let mut wcounter = 0i64;
        let mut cr: Vec<CooccurRec> = Vec::with_capacity(overflow_length + 1);
        let mut history = vec![0i64 ; window_size];
        let stdin = io::BufReader::new(io::stdin());
        'outer: loop {  // for each file.
            progress!(1, "Processing token: 0");
            let mut file = fs::File::open(get_filename(file_head, fcounter)).expect("File open error.");
            loop {  // for each line.
                let mut n_words = 0;
                while let Some(word) = get_word(&stdin) {
                    wcounter += 1;
                    if counter % 100000 == 0 { progress!(1, "\x1b[19G{}", wcounter); }
                    if let Some(w2) = vocab_hash.get(word) {
                        for k in max!(wcounter - window_size, 0) .. (wcounter - 1).rev() {
                            let w1 = history[k % window_size];
                            if w1 < max_product / w2 {  // Product is small enough to store in a full array
                                table[w1 - 1][w2 - 2] += (1.0 / (wcounter - k) as f64) as f32;
                                if symmetric { table[w2 - 1][w1 - 2] += (1.0 / (wcounter - k) as f64) as f32; }
                            }
                            else {  // Product is too big, data is likely to be sparse. Store these entries in a temporary buffer to be sorted, mered (accumulated), and written to file when it gets full.
                                cr.push(CooccurRec { word1: w1, word2: w2, val: get_val(?)});
                                if symmetric { cr.push(CooccurRec { word1: w2, word2: w1, val: get_val(?)}); }
                            }
                            history[wcounter % window_size] = w2;
                            // TODO: write here...
                        }
                    }
                }
                if n_words == 0 { break; }
            }
            // TODO:
            // sort cr
            // write cr
        }
        // TODO: write out ...
    }
    0
}

fn main() {
    let mut verbose = 1i32;
    let mut window_size = 15i32;
    let mut symmetric = true;
    let mut memory_limit = 3.0;
    let mut vocab_file = "vocab.txt".to_string();
    let mut file_head = "overflow".to_string();
    let mut max_product: Option<usize> = None;
    let mut overflow_length: Option<usize> = None;
    {
        let args: Vec<String> = env::args().collect();

        if args.len() == 1 {
            println!("Tool to calculate word-word cooccurrence statistics
This program is transportation of GloVe cooccur.
Original program (written in C) author is Jeffrey Pennington (jpennin@stanford.edu)
Transpoted to rust by Takumi Kato (takumi.kt@gmail.com)

Usage options:
\t-verbose <int>
\t\tSet verbosity: 0, 1, or 2 (default)
\t-symmetric <int>
\t\tIf <int> = 0, only use left context; if <int> = 1 (default), use left and right
\t-window-size <int>
\t\tNumber of context words to the left (and to the right, if symmetric = 1); default 15
\t-vocab-file <file>
\t\tFile containing vocabulary (truncated unigram counts, produced by 'vocab_count'); default vocab.txt
\t-memory <float>
\t\tSoft limit for memory consumption, in GB -- based on simple heuristic, so not extremely accurate; default 4.0
\t-max-product <int>
\t\tLimit the size of dense cooccurrence array by specifying the max product <int> of the frequency counts of the two cooccurring words.
\t\tThis value overrides that which is automatically produced by '-memory'. Typically only needs adjustment for use with very large corpora.
\toverflow-file <file>
\t\tFilename, excluding extension, for temporary file; default overflow

Example usage:
./cooccur -verbose 2 -symmetric 0 -window-size 10 -vocab-file vocab.txt -memory 8.0 -overflow-file tempoverflow < corpus.txt > cooccurrences.bin

");
            std::process::exit(0);
        }
        let mut is_skip = true;
        let mut it = args.into_iter().peekable();
        loop {
            let arg = match it.next() {
                Some(s) => s,
                None => break,
            };
            if is_skip {
                is_skip = false;
                continue
            }
            match arg.as_ref() {
                "-verbose" => {
                    verbose = it.peek().and_then(|v| v.parse().ok())
                        .and_then(|x| if 0 <= x && x <= 2 { Some(x) } else { None })
                        .expect("-verbose <int>, verbosity: 0, 1, or 2 (default)");
                    is_skip = true;
                },
                "-symmetric" => {
                    symmetric = it.peek().and_then(|v| v.parse().ok())
                        .and_then(|x| if 0 <= x && x <= 1 { Some(x) } else { None })
                        .expect("symmetric <int>, 0 (only use left context) or 1 (use left and right; default)") == 1;
                    is_skip = true;
                },
                "-window-size" => {
                    window_size = it.peek().and_then(|v| v.parse().ok()).expect("-window-size <int>");
                    is_skip = true;
                },
                "-memory" => {
                    memory_limit = it.peek().and_then(|v| v.parse().ok()).expect("-memory <float>");
                    is_skip = true;
                },
                "-vocab-file" => {
                    vocab_file = it.peek().expect("-vocab-file <file>").to_string();
                    is_skip = true;
                },
                "-overflow" => {
                    file_head = it.peek().expect("-overflow <file>").to_string();
                    is_skip = true;
                },
                "-max-product" => {
                    max_product = Some(it.peek().and_then(|v| v.parse().ok()).expect("-max-product <int>"));
                    is_skip = true;
                },
                "-overflow-length" => {
                    overflow_length = Some(it.peek().and_then(|v| v.parse().ok()).expect("-overflow-length <int>"));
                    is_skip = true;
                },
                &_ => {},
            }
            let rlimit = 0.85 * memory_limit * 1073741824_f64 / mem::size_of::<f64>() as f64;
            let mut n = 1e5_f64;
            while (rlimit - n * (n.ln() + 0.1544313298)).abs() > 1e-3 { n = rlimit / (n.ln() + 0.1544313298) }
            max_product = max_product.or_else(|| Some(n as usize));
            overflow_length = overflow_length.or_else(|| Some((rlimit / 6.0) as usize));
        }
        //std::process::exit(get_counts(verbose, max_vocab, min_count));
    }
}
<|endoftext|>


<commit_before>use std::collections::HashSet;
use std::ops::Deref;
use std::rc::Rc;

use serde_json;
use url::Url;
use url_serde;

use resman::ResMan;
use super::routegraph::RouteGraph;
use super::adjlist::AdjList;

#[derive(Debug)]
pub enum Error {
    /// No effect matches the metadata requested.
    NoMatchingEffect(EffectId),
}

/// Alias for a `Result` with our error type.
pub type ResultE<T> = Result<T, Error>;

/// Serializable info needed to look up an effect.
#[derive(Clone, Debug)]
#[derive(Serialize, Deserialize)]
pub struct EffectId {
    /// Canonical name of the effect
    name: String,
    /// Hash of the effect's definition file, or None if the effect is primitive
    sha256: Option<[u8; 32]>,
    // TODO: consider a smallset.
    // TODO: use #[serde(with = "url_serde")]
    /// List of URLs where the Effect can be obtained
    urls: HashSet<url_serde::Serde<Url>>,
}

/// All information that will be loaded from disk/network to describe how to
/// synthesize the Effect.
#[derive(Serialize, Deserialize)]
pub struct EffectDesc {
    meta: EffectMeta,
    adjlist: AdjList,
}

/// Validated version of EffectDesc. Guaranteed to be synthesizable.
pub struct Effect {
    meta: EffectMeta,
    // TODO: Effects are immutable, so we can make this an AdjList and store
    // the connectivity information separately.
    // option, because effect MAY be primitive.
    graph: Option<RouteGraph>,
}

#[derive(Clone, Debug)]
#[derive(Serialize, Deserialize)]
pub struct EffectMeta {
    /// Canonical name of the effect
    name: String,
    /// List of URLs where the Effect can be obtained
    urls: HashSet<url_serde::Serde<Url>>,
}

impl Effect {
    pub fn are_slots_connected(&self, from_slot: u32, from_ch: u8, to_slot: u32, to_ch: u8) -> bool {
        match self.graph {
            Some(ref g) => g.are_slots_connected(from_slot, from_ch, to_slot, to_ch),
            // For primitive effects, we assume ALL slots are connected.
            None => true,
        }
    }
    pub fn id(&self) -> EffectId {
        EffectId {
            name: self.meta.name.clone(),
            sha256: None,
            urls: self.meta.urls.clone(),
        }
    }
    /// Given the effect's information, and an interface by which to load
    /// resources, return an actual Effect.
    pub fn from_id(id: EffectId, resman: &ResMan) -> ResultE<Rc<Self>> {
        // For primitive effects, don't attempt to locate their descriptions (they don't exist)
        if id.is_primitive() {
            let me = Self {
                meta: EffectMeta {
                    name: id.name,
                    urls: id.urls,
                },
                graph: None,
            };
            return Ok(Rc::new(me));
        }
        // Locate descriptions for non-primitive effects
        for reader in resman.find_effect(&id) {
            // Try to deserialize to an effect description
            let desc: Result<EffectDesc, serde_json::Error> = serde_json::from_reader(reader);
            match desc {
                Ok(desc) => match RouteGraph::from_adjlist(desc.adjlist, resman) {
                    Ok(graph) => {
                        let me = Self {
                            meta: desc.meta,
                            graph: Some(graph),
                        };
                        // TODO: implement some form of caching
                        return Ok(Rc::new(me));
                    },
                    Err(error) => {
                        println!("Warning: RouteGraph::from_adjlist failed: {:?}", error)
                    }
                },
                Err(error) => println!("Warning: unable to deserialize EffectDesc: {:?}", error)
            }
        }
        // No matching effects
        Err(Error::NoMatchingEffect(id))
    }
    pub fn routegraph(&self) -> &Option<RouteGraph> {
        &self.graph
    }
}

impl EffectId {
    pub fn new<U>(name: String, sha256: Option<[u8; 32]>, urls: U) -> Self
        where U: IntoIterator<Item=Url>
    {
        Self {
            name,
            sha256,
            urls: urls.into_iter().map(|url| url_serde::Serde(url)).collect(),
        }
    }
    pub fn sha256(&self) -> &Option<[u8; 32]> {
        &self.sha256
    }
    /// Returns true if the effect cannot be decomposed.
    /// This is determined by the effect providing a SINGLE url, with the primitive:// Schema
    pub fn is_primitive(&self) -> bool {
        self.urls.len() == 1 && self.urls.iter().all(|url| {
            url.scheme() == "primitive"
        })
    }
    pub fn get_primitive_url(&self) -> Option<&Url> {
        if self.is_primitive() {
            self.urls.iter().next().map(|url| url.deref())
        } else {
            None
        }
    }
}

impl EffectDesc {
    pub fn new(meta: EffectMeta, adjlist: AdjList) -> Self {
        Self{ meta, adjlist }
    }
    pub fn id(&self) -> EffectId {
        // TODO: need to calculate sha
        EffectId {
            name: self.meta.name.clone(),
            sha256: None,
            urls: self.meta.urls.clone(),
        }
    }
}

impl EffectMeta {
    pub fn new<U>(name: String, urls: U) -> Self
        where U: IntoIterator<Item=Url>
    {
        Self {
            name,
            urls: urls.into_iter().map(|url| url_serde::Serde(url)).collect(),
        }
    }
}

impl PartialEq for EffectMeta {
    // Equality implemented in a way where we can easily check things like
    //   "Is this the same primitive Delay effect this renderer knows how to implement?"
    fn eq(&self, other: &EffectMeta) -> bool {
        self.name == other.name &&
            self.urls == other.urls
    }
}
impl Eq for EffectMeta {}

impl PartialEq for Effect {
    fn eq(&self, other: &Effect) -> bool {
        self.meta == other.meta
    }
}
impl Eq for Effect {}

<commit_msg>Always calculate SHAs for Effect and EffectDesc<commit_after>use std::collections::HashSet;
use std::io::Cursor;
use std::ops::Deref;
use std::rc::Rc;

use digest::digest_reader;
use serde_json;
use sha2::Sha256;
use url::Url;
use url_serde;

use resman::ResMan;
use super::routegraph::RouteGraph;
use super::adjlist::AdjList;

#[derive(Debug)]
pub enum Error {
    /// No effect matches the metadata requested.
    NoMatchingEffect(EffectId),
}

/// Alias for a `Result` with our error type.
pub type ResultE<T> = Result<T, Error>;

/// Serializable info needed to look up an effect.
#[derive(Clone, Debug)]
#[derive(Serialize, Deserialize)]
pub struct EffectId {
    /// Canonical name of the effect
    name: String,
    /// Hash of the effect's definition file, or None if the effect is primitive
    sha256: Option<[u8; 32]>,
    // TODO: consider a smallset.
    // TODO: use #[serde(with = "url_serde")]
    /// List of URLs where the Effect can be obtained
    urls: HashSet<url_serde::Serde<Url>>,
}

/// All information that will be loaded from disk/network to describe how to
/// synthesize the Effect.
#[derive(Serialize, Deserialize)]
pub struct EffectDesc {
    meta: EffectMeta,
    adjlist: AdjList,
}

/// Validated version of EffectDesc. Guaranteed to be synthesizable.
pub struct Effect {
    id: EffectId,
    meta: EffectMeta,
    // TODO: Effects are immutable, so we can make this an AdjList and store
    // the connectivity information separately.
    // option, because effect MAY be primitive.
    graph: Option<RouteGraph>,
}

#[derive(Clone, Debug)]
#[derive(Serialize, Deserialize)]
pub struct EffectMeta {
    /// Canonical name of the effect
    name: String,
    /// List of URLs where the Effect can be obtained
    urls: HashSet<url_serde::Serde<Url>>,
}

impl Effect {
    pub fn are_slots_connected(&self, from_slot: u32, from_ch: u8, to_slot: u32, to_ch: u8) -> bool {
        match self.graph {
            Some(ref g) => g.are_slots_connected(from_slot, from_ch, to_slot, to_ch),
            // For primitive effects, we assume ALL slots are connected.
            None => true,
        }
    }
    pub fn id(&self) -> EffectId {
        EffectId {
            name: self.meta.name.clone(),
            sha256: None,
            urls: self.meta.urls.clone(),
        }
    }
    /// Given the effect's information, and an interface by which to load
    /// resources, return an actual Effect.
    pub fn from_id(id: EffectId, resman: &ResMan) -> ResultE<Rc<Self>> {
        // For primitive effects, don't attempt to locate their descriptions (they don't exist)
        if id.is_primitive() {
            let me = Self {
                meta: EffectMeta {
                    name: id.name.clone(),
                    urls: id.urls.clone(),
                },
                id: id,
                graph: None,
            };
            return Ok(Rc::new(me));
        }
        // Locate descriptions for non-primitive effects
        for reader in resman.find_effect(&id) {
            // Try to deserialize to an effect description
            let desc: Result<EffectDesc, serde_json::Error> = serde_json::from_reader(reader);
            match desc {
                Ok(desc) => {
                    // TODO: since we've matched the effect, we only need to recalculate
                    // the id if the original search had missing hashes.
                    let id = desc.id();
                    match RouteGraph::from_adjlist(desc.adjlist, resman) {
                        Ok(graph) => {
                            let me = Self {
                                id,
                                meta: desc.meta,
                                graph: Some(graph),
                            };
                            // TODO: implement some form of caching
                            return Ok(Rc::new(me));
                        },
                        Err(error) => {
                            println!("Warning: RouteGraph::from_adjlist failed: {:?}", error)
                        }
                    }
                },
                Err(error) => println!("Warning: unable to deserialize EffectDesc: {:?}", error)
            }
        }
        // No matching effects
        Err(Error::NoMatchingEffect(id))
    }
    pub fn routegraph(&self) -> &Option<RouteGraph> {
        &self.graph
    }
}

impl EffectId {
    pub fn new<U>(name: String, sha256: Option<[u8; 32]>, urls: U) -> Self
        where U: IntoIterator<Item=Url>
    {
        Self {
            name,
            sha256,
            urls: urls.into_iter().map(|url| url_serde::Serde(url)).collect(),
        }
    }
    pub fn sha256(&self) -> &Option<[u8; 32]> {
        &self.sha256
    }
    /// Returns true if the effect cannot be decomposed.
    /// This is determined by the effect providing a SINGLE url, with the primitive:// Schema
    pub fn is_primitive(&self) -> bool {
        self.urls.len() == 1 && self.urls.iter().all(|url| {
            url.scheme() == "primitive"
        })
    }
    pub fn get_primitive_url(&self) -> Option<&Url> {
        if self.is_primitive() {
            self.urls.iter().next().map(|url| url.deref())
        } else {
            None
        }
    }
}

impl EffectDesc {
    pub fn new(meta: EffectMeta, adjlist: AdjList) -> Self {
        Self{ meta, adjlist }
    }
    pub fn id(&self) -> EffectId {
        // TODO: calcular sha using a smaller buffer
        let as_vec = serde_json::to_vec(self).unwrap();
        let result = digest_reader::<Sha256>(&mut Cursor::new(as_vec)).unwrap();
        let mut hash: [u8; 32] = Default::default();
        hash.copy_from_slice(result.as_slice());
        EffectId {
            name: self.meta.name.clone(),
            sha256: Some(hash),
            urls: self.meta.urls.clone(),
        }
    }
}

impl EffectMeta {
    pub fn new<U>(name: String, urls: U) -> Self
        where U: IntoIterator<Item=Url>
    {
        Self {
            name,
            urls: urls.into_iter().map(|url| url_serde::Serde(url)).collect(),
        }
    }
}

impl PartialEq for EffectMeta {
    // Equality implemented in a way where we can easily check things like
    //   "Is this the same primitive Delay effect this renderer knows how to implement?"
    fn eq(&self, other: &EffectMeta) -> bool {
        self.name == other.name &&
            self.urls == other.urls
    }
}
impl Eq for EffectMeta {}

impl PartialEq for Effect {
    fn eq(&self, other: &Effect) -> bool {
        self.meta == other.meta
    }
}
impl Eq for Effect {}

<|endoftext|>