<commit_before>// Copyright 2016 Adam Perry. Dual-licensed MIT and Apache 2.0 (see LICENSE files for details).

use std::collections::BTreeMap;
use std::io::Read;
use std::thread::sleep;
use std::time::Duration;
use std::u32;

use chrono::{DateTime, UTC};
use hyper;
use hyper::client::{RedirectPolicy, RequestBuilder, Response};
use hyper::header::Headers;
use serde::Deserialize;
use serde_json;

use config::CONFIG;
use error::{DashError, DashResult};
use github::models::{CommentFromJson, IssueFromJson, PullRequestFromJson, PullRequestUrls};

pub const BASE_URL: &'static str = "https://api.github.com";

pub const DELAY: u64 = 300;

type ParameterMap = BTreeMap<&'static str, String>;

header! { (Auth, "Authorization") => [String] }
header! { (UA, "User-Agent") => [String] }
header! { (TZ, "Time-Zone") => [String] }
header! { (Accept, "Accept") => [String] }
header! { (RateLimitRemaining, "X-RateLimit-Remaining") => [u32] }
header! { (RateLimitReset, "X-RateLimit-Reset") => [i64] }
header! { (Link, "Link") => [String] }

const PER_PAGE: u32 = 100;

#[derive(Debug)]
pub struct Client {
    token: String,
    ua: String,
    client: hyper::Client,
    rate_limit: u32,
    rate_limit_timeout: DateTime<UTC>,
}

impl Client {
    pub fn new() -> Self {
        let mut client = hyper::Client::new();
        client.set_redirect_policy(RedirectPolicy::FollowAll);

        Client {
            token: CONFIG.github_access_token.clone(),
            ua: CONFIG.github_user_agent.clone(),
            client: client,
            rate_limit: u32::MAX,
            rate_limit_timeout: UTC::now(),
        }
    }

    pub fn org_repos(&self, org: &str) -> DashResult<Vec<String>> {
        let url = format!("{}/orgs/{}/repos", BASE_URL, org);

        let vals: Vec<serde_json::Value> = try!(self.get_models(&url, &ParameterMap::new()));

        let mut repos = Vec::new();
        for v in vals {

            let v = match v.as_object() {
                Some(v) => v,
                None => return Err(DashError::Misc),
            };

            let repo = match v.get("name") {
                Some(n) => {
                    match n.as_string() {
                        Some(s) => format!("{}/{}", org, s),
                        None => return Err(DashError::Misc),
                    }
                }
                None => return Err(DashError::Misc),
            };

            repos.push(repo);

        }
        Ok(repos)
    }

    pub fn issues_since(&self, repo: &str, start: DateTime<UTC>) -> DashResult<Vec<IssueFromJson>> {

        let url = format!("{}/repos/{}/issues", BASE_URL, repo);
        let mut params = ParameterMap::new();

        params.insert("state", "all".to_string());
        params.insert("since", format!("{:?}", start));
        params.insert("state", "all".to_string());
        params.insert("per_page", format!("{}", PER_PAGE));
        params.insert("direction", "asc".to_string());

        // make the request
        self.get_models(&url, &params)
    }

    pub fn comments_since(&self,
                          repo: &str,
                          start: DateTime<UTC>)
                          -> DashResult<Vec<CommentFromJson>> {
        let url = format!("{}/repos/{}/issues/comments", BASE_URL, repo);
        let mut params = ParameterMap::new();

        params.insert("sort", "created".to_string());
        params.insert("direction", "asc".to_string());
        params.insert("since", format!("{:?}", start));
        params.insert("per_page", format!("{}", PER_PAGE));

        self.get_models(&url, &params)
    }

    fn get_models<M: Deserialize>(&self,
                                  start_url: &str,
                                  params: &ParameterMap)
                                  -> DashResult<Vec<M>> {
        let mut res = try!(self.get(start_url, Some(&params)));

        // let's try deserializing!
        let mut buf = String::new();
        try!(res.read_to_string(&mut buf));

        let mut models = try!(serde_json::from_str::<Vec<M>>(&buf));

        let mut next_url = Self::next_page(&res.headers);
        while next_url.is_some() {
            // TODO figure out a better rate limit
            sleep(Duration::from_millis(DELAY));
            let url = next_url.unwrap();
            let mut next_res = try!(self.get(&url, None));

            buf.clear();
            try!(next_res.read_to_string(&mut buf));

            models.extend(try!(serde_json::from_str::<Vec<M>>(&buf)));

            next_url = Self::next_page(&next_res.headers);
        }

        Ok(models)
    }

    pub fn fetch_pull_request(&self, pr_info: &PullRequestUrls) -> DashResult<PullRequestFromJson> {
        let url = pr_info.get("url");

        if let Some(url) = url {
            let mut res = try!(self.get(url, None));
            let mut buf = String::new();
            try!(res.read_to_string(&mut buf));

            Ok(try!(serde_json::from_str::<PullRequestFromJson>(&buf)))
        } else {
            Err(DashError::Misc)
        }
    }

    fn next_page(h: &Headers) -> Option<String> {
        if let Some(lh) = h.get::<Link>() {
            for link in (**lh).split(",").map(|s| s.trim()) {

                let tokens = link.split(";").map(|s| s.trim()).collect::<Vec<_>>();

                if tokens.len() != 2 {
                    continue;
                }

                if tokens[1] == "rel=\"next\"" {
                    let url = tokens[0].trim_left_matches('<').trim_right_matches('>').to_string();
                    return Some(url);
                }
            }
        }

        None
    }

    pub fn new_comment(&self,
                       repo: &str,
                       issue_num: i32,
                       text: &str)
                       -> DashResult<CommentFromJson> {
        let url = format!("{}/repos/{}/issues/{}/comments", BASE_URL, repo, issue_num);

        let mut obj = BTreeMap::new();
        obj.insert("body", text);

        let payload = serde_json::to_string(&obj)?;

        // FIXME propagate an error if it's a 404 or other error

        let mut body = String::new();
        self.post(&url, &payload)?.read_to_string(&mut body)?;

        let comment = serde_json::from_str::<CommentFromJson>(&body)?;

        Ok(comment)
    }

    pub fn edit_comment(&self,
                        repo: &str,
                        comment_num: i32,
                        text: &str)
                        -> DashResult<CommentFromJson> {
        let url = format!("{}/repos/{}/issues/comments/{}", BASE_URL, repo, comment_num);

        let mut obj = BTreeMap::new();
        obj.insert("body", text);

        let payload = serde_json::to_string(&obj)?;

        // FIXME propagate an error if it's a 404 or other error

        let mut body = String::new();
        self.patch(&url, &payload)?.read_to_string(&mut body)?;

        let comment = serde_json::from_str::<CommentFromJson>(&body)?;

        Ok(comment)
    }

    pub fn get_comment(&self, repo: &str, comment_num: i32) -> DashResult<CommentFromJson> {
        let url = format!("{}/repos/{}/issues/comments/{}", BASE_URL, repo, comment_num);

        let mut body = String::new();
        self.get(&url, None)?.read_to_string(&mut body)?;

        let comment = serde_json::from_str::<CommentFromJson>(&body)?;

        Ok(comment)
    }

    fn patch<'a>(&self, url: &str, payload: &str) -> Result<Response, hyper::error::Error> {
        self.set_headers(self.client.patch(url).body(payload)).send()
    }

    fn post<'a>(&self, url: &str, payload: &str) -> Result<Response, hyper::error::Error> {
        self.set_headers(self.client.post(url).body(payload)).send()
    }

    fn get<'a>(&self,
               url: &'a str,
               params: Option<&ParameterMap>)
               -> Result<Response, hyper::error::Error> {

        let qp_string = match params {
            Some(p) => {
                let mut qp = String::from("?");
                for (k, v) in p.iter() {
                    if qp.len() > 1 {
                        qp.push('&');
                    }
                    qp.push_str(&format!("{}={}", k, v));
                }
                qp
            }
            None => "".to_string(),
        };

        let url = format!("{}{}", url, qp_string);

        debug!("GETing: {}", &url);

        self.set_headers(self.client.get(&url))
            .send()
    }

    fn set_headers<'a>(&self, req: RequestBuilder<'a>) -> RequestBuilder<'a> {
        req.header(Auth(format!("token {}", &self.token)))
            .header(UA(self.ua.clone()))
            .header(TZ("UTC".to_string()))
            .header(Accept("application/vnd.github.v3".to_string()))
            .header(hyper::header::Connection::close())
    }
}
<commit_msg>Adding logging for responses from GH which fail to parse.<commit_after>// Copyright 2016 Adam Perry. Dual-licensed MIT and Apache 2.0 (see LICENSE files for details).

use std::collections::BTreeMap;
use std::io::Read;
use std::thread::sleep;
use std::time::Duration;
use std::u32;

use chrono::{DateTime, UTC};
use hyper;
use hyper::client::{RedirectPolicy, RequestBuilder, Response};
use hyper::header::Headers;
use serde::Deserialize;
use serde_json;

use config::CONFIG;
use error::{DashError, DashResult};
use github::models::{CommentFromJson, IssueFromJson, PullRequestFromJson, PullRequestUrls};

pub const BASE_URL: &'static str = "https://api.github.com";

pub const DELAY: u64 = 300;

type ParameterMap = BTreeMap<&'static str, String>;

header! { (Auth, "Authorization") => [String] }
header! { (UA, "User-Agent") => [String] }
header! { (TZ, "Time-Zone") => [String] }
header! { (Accept, "Accept") => [String] }
header! { (RateLimitRemaining, "X-RateLimit-Remaining") => [u32] }
header! { (RateLimitReset, "X-RateLimit-Reset") => [i64] }
header! { (Link, "Link") => [String] }

const PER_PAGE: u32 = 100;

#[derive(Debug)]
pub struct Client {
    token: String,
    ua: String,
    client: hyper::Client,
    rate_limit: u32,
    rate_limit_timeout: DateTime<UTC>,
}

impl Client {
    pub fn new() -> Self {
        let mut client = hyper::Client::new();
        client.set_redirect_policy(RedirectPolicy::FollowAll);

        Client {
            token: CONFIG.github_access_token.clone(),
            ua: CONFIG.github_user_agent.clone(),
            client: client,
            rate_limit: u32::MAX,
            rate_limit_timeout: UTC::now(),
        }
    }

    pub fn org_repos(&self, org: &str) -> DashResult<Vec<String>> {
        let url = format!("{}/orgs/{}/repos", BASE_URL, org);

        let vals: Vec<serde_json::Value> = try!(self.get_models(&url, &ParameterMap::new()));

        let mut repos = Vec::new();
        for v in vals {

            let v = match v.as_object() {
                Some(v) => v,
                None => return Err(DashError::Misc),
            };

            let repo = match v.get("name") {
                Some(n) => {
                    match n.as_string() {
                        Some(s) => format!("{}/{}", org, s),
                        None => return Err(DashError::Misc),
                    }
                }
                None => return Err(DashError::Misc),
            };

            repos.push(repo);

        }
        Ok(repos)
    }

    pub fn issues_since(&self, repo: &str, start: DateTime<UTC>) -> DashResult<Vec<IssueFromJson>> {

        let url = format!("{}/repos/{}/issues", BASE_URL, repo);
        let mut params = ParameterMap::new();

        params.insert("state", "all".to_string());
        params.insert("since", format!("{:?}", start));
        params.insert("state", "all".to_string());
        params.insert("per_page", format!("{}", PER_PAGE));
        params.insert("direction", "asc".to_string());

        // make the request
        self.get_models(&url, &params)
    }

    pub fn comments_since(&self,
                          repo: &str,
                          start: DateTime<UTC>)
                          -> DashResult<Vec<CommentFromJson>> {
        let url = format!("{}/repos/{}/issues/comments", BASE_URL, repo);
        let mut params = ParameterMap::new();

        params.insert("sort", "created".to_string());
        params.insert("direction", "asc".to_string());
        params.insert("since", format!("{:?}", start));
        params.insert("per_page", format!("{}", PER_PAGE));

        self.get_models(&url, &params)
    }

    fn get_models<M: Deserialize>(&self,
                                  start_url: &str,
                                  params: &ParameterMap)
                                  -> DashResult<Vec<M>> {
        let mut res = try!(self.get(start_url, Some(&params)));

        // let's try deserializing!
        let mut buf = String::new();
        try!(res.read_to_string(&mut buf));

        let mut models = match serde_json::from_str::<Vec<M>>(&buf) {
            Ok(m) => m,
            Err(why) => {
                error!("Unable to parse from JSON ({:?}): {}", &why, buf);
                return Err(why.into());
            }
        };

        let mut next_url = Self::next_page(&res.headers);
        while next_url.is_some() {
            // TODO figure out a better rate limit
            sleep(Duration::from_millis(DELAY));
            let url = next_url.unwrap();
            let mut next_res = try!(self.get(&url, None));

            buf.clear();
            try!(next_res.read_to_string(&mut buf));

            models.extend(match serde_json::from_str::<Vec<M>>(&buf) {
                Ok(m) => m,
                Err(why) => {
                    error!("Unable to parse from JSON ({:?}): {}", &why, buf);
                    return Err(why.into());
                }
            });

            next_url = Self::next_page(&next_res.headers);
        }

        Ok(models)
    }

    pub fn fetch_pull_request(&self, pr_info: &PullRequestUrls) -> DashResult<PullRequestFromJson> {
        let url = pr_info.get("url");

        if let Some(url) = url {
            let mut res = try!(self.get(url, None));
            let mut buf = String::new();
            try!(res.read_to_string(&mut buf));

            Ok(try!(serde_json::from_str::<PullRequestFromJson>(&buf)))
        } else {
            Err(DashError::Misc)
        }
    }

    fn next_page(h: &Headers) -> Option<String> {
        if let Some(lh) = h.get::<Link>() {
            for link in (**lh).split(",").map(|s| s.trim()) {

                let tokens = link.split(";").map(|s| s.trim()).collect::<Vec<_>>();

                if tokens.len() != 2 {
                    continue;
                }

                if tokens[1] == "rel=\"next\"" {
                    let url = tokens[0].trim_left_matches('<').trim_right_matches('>').to_string();
                    return Some(url);
                }
            }
        }

        None
    }

    pub fn new_comment(&self,
                       repo: &str,
                       issue_num: i32,
                       text: &str)
                       -> DashResult<CommentFromJson> {
        let url = format!("{}/repos/{}/issues/{}/comments", BASE_URL, repo, issue_num);

        let mut obj = BTreeMap::new();
        obj.insert("body", text);

        let payload = serde_json::to_string(&obj)?;

        // FIXME propagate an error if it's a 404 or other error

        let mut body = String::new();
        self.post(&url, &payload)?.read_to_string(&mut body)?;

        let comment = serde_json::from_str::<CommentFromJson>(&body)?;

        Ok(comment)
    }

    pub fn edit_comment(&self,
                        repo: &str,
                        comment_num: i32,
                        text: &str)
                        -> DashResult<CommentFromJson> {
        let url = format!("{}/repos/{}/issues/comments/{}", BASE_URL, repo, comment_num);

        let mut obj = BTreeMap::new();
        obj.insert("body", text);

        let payload = serde_json::to_string(&obj)?;

        // FIXME propagate an error if it's a 404 or other error

        let mut body = String::new();
        self.patch(&url, &payload)?.read_to_string(&mut body)?;

        let comment = serde_json::from_str::<CommentFromJson>(&body)?;

        Ok(comment)
    }

    pub fn get_comment(&self, repo: &str, comment_num: i32) -> DashResult<CommentFromJson> {
        let url = format!("{}/repos/{}/issues/comments/{}", BASE_URL, repo, comment_num);

        let mut body = String::new();
        self.get(&url, None)?.read_to_string(&mut body)?;

        let comment = serde_json::from_str::<CommentFromJson>(&body)?;

        Ok(comment)
    }

    fn patch<'a>(&self, url: &str, payload: &str) -> Result<Response, hyper::error::Error> {
        self.set_headers(self.client.patch(url).body(payload)).send()
    }

    fn post<'a>(&self, url: &str, payload: &str) -> Result<Response, hyper::error::Error> {
        self.set_headers(self.client.post(url).body(payload)).send()
    }

    fn get<'a>(&self,
               url: &'a str,
               params: Option<&ParameterMap>)
               -> Result<Response, hyper::error::Error> {

        let qp_string = match params {
            Some(p) => {
                let mut qp = String::from("?");
                for (k, v) in p.iter() {
                    if qp.len() > 1 {
                        qp.push('&');
                    }
                    qp.push_str(&format!("{}={}", k, v));
                }
                qp
            }
            None => "".to_string(),
        };

        let url = format!("{}{}", url, qp_string);

        debug!("GETing: {}", &url);

        self.set_headers(self.client.get(&url))
            .send()
    }

    fn set_headers<'a>(&self, req: RequestBuilder<'a>) -> RequestBuilder<'a> {
        req.header(Auth(format!("token {}", &self.token)))
            .header(UA(self.ua.clone()))
            .header(TZ("UTC".to_string()))
            .header(Accept("application/vnd.github.v3".to_string()))
            .header(hyper::header::Connection::close())
    }
}
<|endoftext|>


<commit_before>use super::btree::{BPlusTree, RTCursor as BPlusTreeCursor};
use super::*;
use client::AsyncClient;
use index::btree::LevelTree;
use index::btree::NodeCellRef;
use itertools::Itertools;
use parking_lot::RwLock;
use ram::segs::MAX_SEGMENT_SIZE;
use std::collections::BTreeSet;
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use std::{mem, ptr};

pub const LEVEL_ELEMENTS_MULTIPLIER: usize = 10;
pub const LEVEL_PAGE_DIFF_MULTIPLIER: usize = 10;

const LEVEL_M_MAX_ELEMENTS_COUNT: usize = LEVEL_M * LEVEL_M * LEVEL_M;
const LEVEL_M: usize = 24;
const LEVEL_1: usize = LEVEL_M * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_2: usize = LEVEL_1 * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_3: usize = LEVEL_2 * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_4: usize = LEVEL_3 * LEVEL_PAGE_DIFF_MULTIPLIER;

type LevelTrees = Vec<Box<LevelTree>>;
pub type Ptr = NodeCellRef;
pub type Key = EntryKey;

with_levels! {
    LM, LEVEL_M;
    L1, LEVEL_1;
    L2, LEVEL_2;
    L3, LEVEL_3;
    // L4, LEVEL_4; // See https://github.com/rust-lang/rust/issues/58164
}

pub struct LSMTree {
    trees: LevelTrees,
    // use Vec here for convenience
    max_sizes: Vec<usize>,
}

unsafe impl Send for LSMTree {}
unsafe impl Sync for LSMTree {}

impl LSMTree {
    pub fn new(neb_client: &Arc<AsyncClient>) -> Arc<Self> {
        debug!("Initializing LSM-tree...");
        let (trees, max_sizes) = init_lsm_level_trees(neb_client);
        debug!("Initialized LSM-tree");
        let lsm_tree = Arc::new(LSMTree { trees, max_sizes });
        let tree_clone = lsm_tree.clone();
        thread::Builder::new().name("LSM-Tree Sentinel".to_string()).spawn(move || {
            tree_clone.sentinel();
        });
        lsm_tree
    }

    pub fn insert(&self, mut key: EntryKey, id: &Id) {
        key_with_id(&mut key, id);
        self.trees[0].insert_into(&key)
    }

    pub fn remove(&self, mut key: EntryKey, id: &Id) -> bool {
        key_with_id(&mut key, id);
        self.trees
            .iter()
            .map(|tree| tree.mark_key_deleted(&key))
            .collect_vec() // collect here to prevent short circuit
            .into_iter()
            .any(|d| d)
    }

    pub fn seek(&self, mut key: EntryKey, ordering: Ordering) -> LSMTreeCursor {
        match ordering {
            Ordering::Forward => key_with_id(&mut key, &Id::unit_id()),
            Ordering::Backward => key_with_id(&mut key, &Id::new(::std::u64::MAX, ::std::u64::MAX)),
        };
        let mut cursors: Vec<Box<Cursor>> = vec![];
        for tree in &self.trees {
            cursors.push(tree.seek_for(&key, ordering));
        }
        return LSMTreeCursor::new(cursors);
    }

    fn sentinel(&self) {
        loop {
            for i in 0..self.trees.len() - 1 {
                debug!("Checking tree merge {}", i);
                let lower = &*self.trees[i];
                let upper = &*self.trees[i + 1];
                if lower.count() > self.max_sizes[i] {
                    lower.merge_to(upper);
                }
            }
            thread::sleep(Duration::from_millis(500));
        }
    }

    pub fn level_sizes(&self) -> Vec<usize> {
        self.trees.iter().map(|t| t.count()).collect()
    }

    pub fn len(&self) -> usize {
        self.trees.iter().map(|tree| tree.count()).sum::<usize>()
    }
}

pub struct LSMTreeCursor {
    level_cursors: Vec<Box<Cursor>>,
}

impl LSMTreeCursor {
    fn new(cursors: Vec<Box<Cursor>>) -> Self {
        LSMTreeCursor {
            level_cursors: cursors,
        }
    }
}

impl Cursor for LSMTreeCursor {
    fn next(&mut self) -> bool {
        if let Some(prev_key) = self.current().map(|k| k.to_owned()) {
            let prob = self.level_cursors
                .iter_mut()
                .filter(|c| c.current().is_some())
                .min_by(|a, b| a.current().unwrap().cmp(b.current().unwrap()))
                .map(|c| (c.next(), c))
                .map(|(has_next, c)| if !has_next || c.current().unwrap() <= &prev_key {
                    None
                } else {
                    Some(has_next)
                })
                .unwrap_or(Some(false));
            if let Some(res) = prob {
                res
            } else {
                self.next()
            }
        } else {
            false
        }
    }

    fn current(&self) -> Option<&EntryKey> {
        self.level_cursors.iter().filter_map(|c| c.current()).min()
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use byteorder::BigEndian;
    use byteorder::WriteBytesExt;
    use client;
    use futures::prelude::*;
    use rand::distributions::Uniform;
    use rand::thread_rng;
    use rand::Rng;
    use rayon::prelude::*;
    use server::NebServer;
    use server::ServerOptions;
    use std::env;
    use std::io::Cursor as StdCursor;

    fn u64_to_slice(n: u64) -> [u8; 8] {
        let mut key_slice = [0u8; 8];
        {
            let mut cursor = StdCursor::new(&mut key_slice[..]);
            cursor.write_u64::<BigEndian>(n);
        };
        key_slice
    }

    fn dump_trees(lsm_tree: &LSMTree, name: &str) {
        for i in 0..lsm_tree.trees.len() {
            lsm_tree.trees[i].dump(&format!("{}_lsm_{}_dump.json", name, i));
        }
    }

    #[test]
    pub fn insertions() {
        env_logger::init();
        let server_group = "sstable_index_init";
        let server_addr = String::from("127.0.0.1:5700");
        let server = NebServer::new_from_opts(
            &ServerOptions {
                chunk_count: 1,
                memory_size: 3 * 1024 * 1024 * 1024,
                backup_storage: None,
                wal_storage: None,
            },
            &server_addr,
            &server_group,
        );
        let client = Arc::new(
            client::AsyncClient::new(&server.rpc, &vec![server_addr], server_group).unwrap(),
        );
        client.new_schema_with_id(btree::page_schema()).wait();
        let tree = LSMTree::new(&client);
        let num = env::var("LSM_TREE_TEST_ITEMS")
            // this value cannot do anything useful to the test
            // must arrange a long-term test to cover every levels
            .unwrap_or("1000".to_string())
            .parse::<u64>()
            .unwrap();

        let tree_clone = tree.clone();
        debug!("Testing LSM-tree");
        thread::spawn(move || loop {
            thread::sleep(Duration::from_secs(10));
            let tree_len = tree_clone.len();
            debug!(
                "LSM-Tree now have {}/{} elements, total {:.2}%",
                tree_len,
                num,
                tree_len as f32 / num as f32 * 100.0
            );
        });

        (0..num).collect::<Vec<_>>().par_iter().for_each(|i| {
            let i = *i;
            let id = Id::new(0, i);
            let key_slice = u64_to_slice(i);
            let key = SmallVec::from_slice(&key_slice);
            tree.insert(key, &id);
        });

        if tree.len() > LEVEL_M_MAX_ELEMENTS_COUNT {
            debug!("Level trees sizes are {:?}, wait for continuous merge", tree.level_sizes());
            thread::sleep(Duration::from_secs(30));
            debug!("Level trees sizes are {:?}", tree.level_sizes());
        }
        debug!("Start validations");

        dump_trees(&*tree, "after_insertion");

        (0..num).collect::<Vec<_>>().iter().for_each(|i| {
            let mut rng = rand::rngs::OsRng::new().unwrap();
            let die_range = Uniform::new_inclusive(1, 6);
            let mut roll_die = rng.sample_iter(&die_range);
            let i = *i;
            let id = Id::new(0, i);
            let key_slice = u64_to_slice(i);
            let mut key = SmallVec::from_slice(&key_slice);
            debug!("checking: {}", i);
            let mut cursor = tree.seek(key.clone(), Ordering::Forward);
            key_with_id(&mut key, &id);
            assert_eq!(cursor.current(), Some(&key), "{}", i);
            if roll_die.next().unwrap() == 6 {
                for j in i..num {
                    let id = Id::new(0, j);
                    let key_slice = u64_to_slice(j);
                    let mut key = SmallVec::from_slice(&key_slice);
                    key_with_id(&mut key, &id);
                    assert_eq!(cursor.current(), Some(&key), "{}/{}", i, j);
                    assert_eq!(cursor.next(), j != num - 1, "{}/{}", i, j);
                }
            }
        });
    }
}
<commit_msg>allow start lsm tree sentinel manually<commit_after>use super::btree::{BPlusTree, RTCursor as BPlusTreeCursor};
use super::*;
use client::AsyncClient;
use index::btree::LevelTree;
use index::btree::NodeCellRef;
use itertools::Itertools;
use parking_lot::RwLock;
use ram::segs::MAX_SEGMENT_SIZE;
use std::collections::BTreeSet;
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use std::{mem, ptr};

pub const LEVEL_ELEMENTS_MULTIPLIER: usize = 10;
pub const LEVEL_PAGE_DIFF_MULTIPLIER: usize = 10;

const LEVEL_M_MAX_ELEMENTS_COUNT: usize = LEVEL_M * LEVEL_M * LEVEL_M;
const LEVEL_M: usize = 24;
const LEVEL_1: usize = LEVEL_M * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_2: usize = LEVEL_1 * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_3: usize = LEVEL_2 * LEVEL_PAGE_DIFF_MULTIPLIER;
const LEVEL_4: usize = LEVEL_3 * LEVEL_PAGE_DIFF_MULTIPLIER;

type LevelTrees = Vec<Box<LevelTree>>;
pub type Ptr = NodeCellRef;
pub type Key = EntryKey;

with_levels! {
    LM, LEVEL_M;
    L1, LEVEL_1;
    L2, LEVEL_2;
    L3, LEVEL_3;
    // L4, LEVEL_4; // See https://github.com/rust-lang/rust/issues/58164
}

pub struct LSMTree {
    trees: LevelTrees,
    // use Vec here for convenience
    max_sizes: Vec<usize>,
}

unsafe impl Send for LSMTree {}
unsafe impl Sync for LSMTree {}

impl LSMTree {
    pub fn new(neb_client: &Arc<AsyncClient>) -> Arc<Self> {
        debug!("Initializing LSM-tree...");
        let (trees, max_sizes) = init_lsm_level_trees(neb_client);
        debug!("Initialized LSM-tree");
        Arc::new(LSMTree { trees, max_sizes })
    }

    pub fn insert(&self, mut key: EntryKey, id: &Id) {
        key_with_id(&mut key, id);
        self.trees[0].insert_into(&key)
    }

    pub fn remove(&self, mut key: EntryKey, id: &Id) -> bool {
        key_with_id(&mut key, id);
        self.trees
            .iter()
            .map(|tree| tree.mark_key_deleted(&key))
            .collect_vec() // collect here to prevent short circuit
            .into_iter()
            .any(|d| d)
    }

    pub fn seek(&self, mut key: EntryKey, ordering: Ordering) -> LSMTreeCursor {
        match ordering {
            Ordering::Forward => key_with_id(&mut key, &Id::unit_id()),
            Ordering::Backward => key_with_id(&mut key, &Id::new(::std::u64::MAX, ::std::u64::MAX)),
        };
        let mut cursors: Vec<Box<Cursor>> = vec![];
        for tree in &self.trees {
            cursors.push(tree.seek_for(&key, ordering));
        }
        return LSMTreeCursor::new(cursors);
    }

    pub fn check_and_merge(&self) {
        for i in 0..self.trees.len() - 1 {
            debug!("Checking tree merge {}", i);
            let lower = &*self.trees[i];
            let upper = &*self.trees[i + 1];
            if lower.count() > self.max_sizes[i] {
                lower.merge_to(upper);
            }
        }
    }

    fn sentinel(this: &Arc<Self>) {
        let this = this.clone();
        thread::Builder::new().name("LSM-Tree Sentinel".to_string()).spawn(move || {
            loop {
                this.check_and_merge();
                thread::sleep(Duration::from_millis(500));
            }
        });
    }

    pub fn level_sizes(&self) -> Vec<usize> {
        self.trees.iter().map(|t| t.count()).collect()
    }

    pub fn len(&self) -> usize {
        self.trees.iter().map(|tree| tree.count()).sum::<usize>()
    }
}

pub struct LSMTreeCursor {
    level_cursors: Vec<Box<Cursor>>,
}

impl LSMTreeCursor {
    fn new(cursors: Vec<Box<Cursor>>) -> Self {
        LSMTreeCursor {
            level_cursors: cursors,
        }
    }
}

impl Cursor for LSMTreeCursor {
    fn next(&mut self) -> bool {
        if let Some(prev_key) = self.current().map(|k| k.to_owned()) {
            let prob = self.level_cursors
                .iter_mut()
                .filter(|c| c.current().is_some())
                .min_by(|a, b| a.current().unwrap().cmp(b.current().unwrap()))
                .map(|c| (c.next(), c))
                .map(|(has_next, c)| if !has_next || c.current().unwrap() <= &prev_key {
                    None
                } else {
                    Some(has_next)
                })
                .unwrap_or(Some(false));
            if let Some(res) = prob {
                res
            } else {
                self.next()
            }
        } else {
            false
        }
    }

    fn current(&self) -> Option<&EntryKey> {
        self.level_cursors.iter().filter_map(|c| c.current()).min()
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use byteorder::BigEndian;
    use byteorder::WriteBytesExt;
    use client;
    use futures::prelude::*;
    use rand::distributions::Uniform;
    use rand::thread_rng;
    use rand::Rng;
    use rayon::prelude::*;
    use server::NebServer;
    use server::ServerOptions;
    use std::env;
    use std::io::Cursor as StdCursor;

    fn u64_to_slice(n: u64) -> [u8; 8] {
        let mut key_slice = [0u8; 8];
        {
            let mut cursor = StdCursor::new(&mut key_slice[..]);
            cursor.write_u64::<BigEndian>(n);
        };
        key_slice
    }

    fn dump_trees(lsm_tree: &LSMTree, name: &str) {
        for i in 0..lsm_tree.trees.len() {
            lsm_tree.trees[i].dump(&format!("{}_lsm_{}_dump.json", name, i));
        }
    }

    #[test]
    pub fn insertions() {
        env_logger::init();
        let server_group = "sstable_index_init";
        let server_addr = String::from("127.0.0.1:5700");
        let server = NebServer::new_from_opts(
            &ServerOptions {
                chunk_count: 1,
                memory_size: 3 * 1024 * 1024 * 1024,
                backup_storage: None,
                wal_storage: None,
            },
            &server_addr,
            &server_group,
        );
        let client = Arc::new(
            client::AsyncClient::new(&server.rpc, &vec![server_addr], server_group).unwrap(),
        );
        client.new_schema_with_id(btree::page_schema()).wait();
        let tree = LSMTree::new(&client);
        let num = env::var("LSM_TREE_TEST_ITEMS")
            // this value cannot do anything useful to the test
            // must arrange a long-term test to cover every levels
            .unwrap_or("1000".to_string())
            .parse::<u64>()
            .unwrap();

        let tree_clone = tree.clone();
        debug!("Testing LSM-tree");
        thread::spawn(move || loop {
            thread::sleep(Duration::from_secs(10));
            let tree_len = tree_clone.len();
            debug!(
                "LSM-Tree now have {}/{} elements, total {:.2}%",
                tree_len,
                num,
                tree_len as f32 / num as f32 * 100.0
            );
        });

        (0..num).collect::<Vec<_>>().par_iter().for_each(|i| {
            let i = *i;
            let id = Id::new(0, i);
            let key_slice = u64_to_slice(i);
            let key = SmallVec::from_slice(&key_slice);
            tree.insert(key, &id);
        });

        if tree.len() > LEVEL_M_MAX_ELEMENTS_COUNT {
            debug!("Level trees sizes are {:?}, wait for continuous merge", tree.level_sizes());
            thread::sleep(Duration::from_secs(30));
            debug!("Level trees sizes are {:?}", tree.level_sizes());
        }
        debug!("Start validations");

        dump_trees(&*tree, "before_insertion");
        tree.check_and_merge();
        dump_trees(&*tree, "after_insertion");

        (0..num).collect::<Vec<_>>().iter().for_each(|i| {
            let mut rng = rand::rngs::OsRng::new().unwrap();
            let die_range = Uniform::new_inclusive(1, 15);
            let mut roll_die = rng.sample_iter(&die_range);
            let i = *i;
            let id = Id::new(0, i);
            let key_slice = u64_to_slice(i);
            let mut key = SmallVec::from_slice(&key_slice);
            debug!("checking: {}", i);
            let mut cursor = tree.seek(key.clone(), Ordering::Forward);
            key_with_id(&mut key, &id);
            assert_eq!(cursor.current(), Some(&key), "{}", i);
            if roll_die.next().unwrap() == 15 {
                for j in i..num {
                    let id = Id::new(0, j);
                    let key_slice = u64_to_slice(j);
                    let mut key = SmallVec::from_slice(&key_slice);
                    key_with_id(&mut key, &id);
                    assert_eq!(cursor.current(), Some(&key), "{}/{}", i, j);
                    assert_eq!(cursor.next(), j != num - 1, "{}/{}", i, j);
                }
            }
        });
    }
}
<|endoftext|>


<commit_before>use stats::median;
use request_response_matcher::*;

pub mod percentile;

#[derive(PartialEq)]
#[derive(Debug)]
pub struct RequestLogAnalyzerResult {
    pub count: usize,
    pub max: usize,
    pub min: usize,
    pub avg: usize,
    pub median: usize,
    pub percentile90: usize,
}

pub fn analyze(request_response_pairs: &Vec<RequestResponsePair>) -> Option<RequestLogAnalyzerResult> {
    if request_response_pairs.len() == 0 {
        return None;
    }

    let times: Vec<i64> = request_response_pairs.iter()
        .map(|rr: &RequestResponsePair| -> i64 {rr.response.response_time.num_milliseconds() })
        .collect();

    let sum: usize = times.iter().sum::<i64>() as usize;
    let avg: usize = sum / times.len();

    let max: usize = *times.iter().max().unwrap() as usize;
    let min: usize = *times.iter().min().unwrap() as usize;

    let percentile90: usize = percentile::percentile(&times, 0.9) as usize;

    let median = median(times.into_iter()).unwrap() as usize;

    Some(RequestLogAnalyzerResult {
        count: request_response_pairs.len().into(),
        max: max,
        min: min,
        avg: avg,
        median: median,
        percentile90: percentile90,
    })
}

#[cfg(test)]
mod tests {
    use log_parser::log_events::*;
    use request_response_matcher::*;
    use super::*;

    #[test]
    fn test_analyze() {
        let request_response_pairs = vec![
            RequestResponsePair {
                request: Request::new_from_log_line(&"08/Apr/2016:09:57:47 +0200 [001] -> GET /content/some/page.html HTTP/1.1".to_string()).unwrap(),
                response: Response::new_from_log_line(&"08/Apr/2016:09:57:47 +0200 [001] <- 200 text/html 1ms".to_string()).unwrap(),
            },
            RequestResponsePair {
                request: Request::new_from_log_line(&"08/Apr/2016:09:58:47 +0200 [02] -> GET /content/some/other.html HTTP/1.1".to_string()).unwrap(),
                response: Response::new_from_log_line(&"08/Apr/2016:09:58:47 +0200 [02] <- 200 text/html 10ms".to_string()).unwrap(),
            },
            RequestResponsePair {
                request: Request::new_from_log_line(&"08/Apr/2016:10:58:47 +0200 [03] -> GET /content/some/third.html HTTP/1.1".to_string()).unwrap(),
                response: Response::new_from_log_line(&"08/Apr/2016:10:58:47 +0200 [03] <- 200 text/html 100ms".to_string()).unwrap(),
            },
        ];

        let result = analyze(&request_response_pairs);

        let expected = Some(RequestLogAnalyzerResult {
            count: 3,
            max: 100,
            min: 1,
            avg: 37,
            median: 10,
            percentile90: 100,
        });

        assert_eq!(result, expected);
    }

    #[test]
    fn test_analyze_empty() {
        let request_response_pairs = vec![];

        let result = analyze(&request_response_pairs);

        let expected = None;

        assert_eq!(result, expected);
    }
}
<commit_msg>Analyze ms times only, not full pairs<commit_after>use stats::median;
use request_response_matcher::*;

pub mod percentile;

#[derive(PartialEq)]
#[derive(Debug)]
pub struct RequestLogAnalyzerResult {
    pub count: usize,
    pub max: usize,
    pub min: usize,
    pub avg: usize,
    pub median: usize,
    pub percentile90: usize,
}

pub fn analyze(times: &Vec<i64>) -> Option<RequestLogAnalyzerResult> {
    if times.len() == 0 {
        return None;
    }

    let sum: usize = times.iter().sum::<i64>() as usize;
    let avg: usize = sum / times.len();

    let max: usize = *times.iter().max().unwrap() as usize;
    let min: usize = *times.iter().min().unwrap() as usize;

    let percentile90: usize = percentile::percentile(&times, 0.9) as usize;

    let median = median(times.iter().cloned()).unwrap() as usize;

    Some(RequestLogAnalyzerResult {
        count: times.len().into(),
        max: max,
        min: min,
        avg: avg,
        median: median,
        percentile90: percentile90,
    })
}

#[cfg(test)]
mod tests {
    use log_parser::log_events::*;
    use request_response_matcher::*;
    use super::*;

    #[test]
    fn test_analyze() {
        let times: Vec<i64> = vec![1, 10, 100];

        let result = analyze(&times);

        let expected = Some(RequestLogAnalyzerResult {
            count: 3,
            max: 100,
            min: 1,
            avg: 37,
            median: 10,
            percentile90: 100,
        });

        assert_eq!(result, expected);
    }

    #[test]
    fn test_analyze_empty() {
        let times = vec![];

        let result = analyze(&times);

        let expected = None;

        assert_eq!(result, expected);
    }
}
<|endoftext|>


<commit_before>use std::util;
use std::num::One;

pub fn fibonacci<T: One>() -> Fibonacci<T> {
    fibonacci_with_init(One::one(), One::one())
}

pub fn fibonacci_with_init<T>(a0: T, a1: T) -> Fibonacci<T> {
    Fibonacci { current: a0, next: a1 }
}

struct Fibonacci<T> { current: T, next: T }

impl<T: Add<T,T>> Iterator<T> for Fibonacci<T> {
    #[inline]
    fn next(&mut self) -> Option<T> {
        let new_next    = self.current + self.next;
        let new_current = util::replace(&mut self.next, new_next);
        let retval = util::replace(&mut self.current,   new_current);
        Some(retval)
    }
}

pub fn triangle<T: One + Add<T, T>>() -> Triangle<T> {
    let one: T = One::one();
    Triangle { diff: one + one, next: one }
}

struct Triangle<T> { diff: T, next: T }

impl<T: Add<T, T> + One> Iterator<T> for Triangle<T> {
    #[inline]
    fn next(&mut self) -> Option<T> {
        let new_next = self.next + self.diff;
        self.diff = self.diff + One::one();
        Some(util::replace(&mut self.next, new_next))
    }
}

pub fn prim_pythagorean(m: uint) -> PrimPythagoreanIterator {
    let n0 = if m.is_even() { 1 } else { 2 };
    PrimPythagoreanIterator { m: m, n: n0 }
}

struct PrimPythagoreanIterator { priv m: uint, priv n: uint }

impl Iterator<(uint, uint, uint)> for PrimPythagoreanIterator {
    fn next(&mut self) -> Option<(uint, uint, uint)> {
        let m = self.m;
        while self.n < m {
            let n = self.n;
            self.n += 2;

            if m.gcd(&n) != 1 { continue }

            let (m2, n2)  = (m * m, n * n);
            let (a, b, c) = (m2 - n2, 2 * m * n, m2 + n2);
            if a < b {
                return(Some((a, b, c)))
            } else {
                return(Some((b, a, c)))
            }
        };
        None
    }
}

#[cfg(test)]
mod test {
    #[test]
    fn test_fibonacci() {
        let fib = ~[ 1u, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233 ];
        assert_eq!(super::fibonacci::<uint>().take(fib.len()).to_owned_vec(), fib);

        let fib = ~[ 0u, 0, 0, 0, 0, 0, 0];
        assert_eq!(super::fibonacci_with_init(0u, 0).take(fib.len()).to_owned_vec(), fib);

        let fib = ~[ 1u, 5, 6, 11, 17, 28, 45, 73, 118, 191, 309, 500];
        assert_eq!(super::fibonacci_with_init(1u, 5).take(fib.len()).to_owned_vec(), fib);
    }

    #[test]
    fn test_triangle() {
        let tri = ~[1u, 3, 6, 10, 15, 21];
        let gen = super::triangle::<uint>().take(tri.len()).to_owned_vec();
        assert_eq!(gen, tri);
    }

    #[test]
    fn test_prim_pythagorean_iterator() {
        fn check(m: uint, v: ~[(uint, uint, uint)]) {
            assert_eq!(super::prim_pythagorean(m).to_owned_vec(), v);
        }

        check(2, ~[(3, 4, 5)]);
        check(3, ~[(5, 12, 13)]);
        check(4, ~[(8, 15, 17), (7, 24, 25)]);
    }
}
<commit_msg>remove unnecessary-parens<commit_after>use std::util;
use std::num::One;

pub fn fibonacci<T: One>() -> Fibonacci<T> {
    fibonacci_with_init(One::one(), One::one())
}

pub fn fibonacci_with_init<T>(a0: T, a1: T) -> Fibonacci<T> {
    Fibonacci { current: a0, next: a1 }
}

struct Fibonacci<T> { current: T, next: T }

impl<T: Add<T,T>> Iterator<T> for Fibonacci<T> {
    #[inline]
    fn next(&mut self) -> Option<T> {
        let new_next    = self.current + self.next;
        let new_current = util::replace(&mut self.next, new_next);
        let retval = util::replace(&mut self.current,   new_current);
        Some(retval)
    }
}

pub fn triangle<T: One + Add<T, T>>() -> Triangle<T> {
    let one: T = One::one();
    Triangle { diff: one + one, next: one }
}

struct Triangle<T> { diff: T, next: T }

impl<T: Add<T, T> + One> Iterator<T> for Triangle<T> {
    #[inline]
    fn next(&mut self) -> Option<T> {
        let new_next = self.next + self.diff;
        self.diff = self.diff + One::one();
        Some(util::replace(&mut self.next, new_next))
    }
}

pub fn prim_pythagorean(m: uint) -> PrimPythagoreanIterator {
    let n0 = if m.is_even() { 1 } else { 2 };
    PrimPythagoreanIterator { m: m, n: n0 }
}

struct PrimPythagoreanIterator { priv m: uint, priv n: uint }

impl Iterator<(uint, uint, uint)> for PrimPythagoreanIterator {
    fn next(&mut self) -> Option<(uint, uint, uint)> {
        let m = self.m;
        while self.n < m {
            let n = self.n;
            self.n += 2;

            if m.gcd(&n) != 1 { continue }

            let (m2, n2)  = (m * m, n * n);
            let (a, b, c) = (m2 - n2, 2 * m * n, m2 + n2);
            if a < b {
                return Some((a, b, c))
            } else {
                return Some((b, a, c))
            }
        };
        None
    }
}

#[cfg(test)]
mod test {
    #[test]
    fn test_fibonacci() {
        let fib = ~[ 1u, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233 ];
        assert_eq!(super::fibonacci::<uint>().take(fib.len()).to_owned_vec(), fib);

        let fib = ~[ 0u, 0, 0, 0, 0, 0, 0];
        assert_eq!(super::fibonacci_with_init(0u, 0).take(fib.len()).to_owned_vec(), fib);

        let fib = ~[ 1u, 5, 6, 11, 17, 28, 45, 73, 118, 191, 309, 500];
        assert_eq!(super::fibonacci_with_init(1u, 5).take(fib.len()).to_owned_vec(), fib);
    }

    #[test]
    fn test_triangle() {
        let tri = ~[1u, 3, 6, 10, 15, 21];
        let gen = super::triangle::<uint>().take(tri.len()).to_owned_vec();
        assert_eq!(gen, tri);
    }

    #[test]
    fn test_prim_pythagorean_iterator() {
        fn check(m: uint, v: ~[(uint, uint, uint)]) {
            assert_eq!(super::prim_pythagorean(m).to_owned_vec(), v);
        }

        check(2, ~[(3, 4, 5)]);
        check(3, ~[(5, 12, 13)]);
        check(4, ~[(8, 15, 17), (7, 24, 25)]);
    }
}
<|endoftext|>


<commit_before>//!
//! Provides `udev` related functionality for automated device scanning.
//!
//! This module mainly provides the [`UdevBackend`](::backend::udev::UdevBackend), which constantly monitors available DRM devices
//! and notifies a user supplied [`UdevHandler`](::backend::udev::UdevHandler) of any changes.
//!
//! Additionally this contains some utility functions related to scanning.
//!
//! See also `anvil/src/udev.rs` for pure hardware backed example of a compositor utilizing this
//! backend.

use nix::sys::stat::{dev_t, stat};
use std::{
    collections::HashSet,
    ffi::OsString,
    io::Result as IoResult,
    os::unix::io::{AsRawFd, RawFd},
    path::{Path, PathBuf},
};
use udev::{Enumerator, EventType, MonitorBuilder, MonitorSocket};

use calloop::{
    generic::{Generic, SourceFd},
    mio::Interest,
    InsertError, LoopHandle, Source,
};

/// Backend to monitor available drm devices.
///
/// Provides a way to automatically scan for available gpus and notifies the
/// given handler of any changes. Can be used to provide hot-plug functionality for gpus and
/// attached monitors.
pub struct UdevBackend<T: UdevHandler + 'static> {
    devices: HashSet<dev_t>,
    monitor: MonitorSocket,
    handler: T,
    logger: ::slog::Logger,
}

impl<T: UdevHandler + 'static> AsRawFd for UdevBackend<T> {
    fn as_raw_fd(&self) -> RawFd {
        self.monitor.as_raw_fd()
    }
}

impl<T: UdevHandler + 'static> UdevBackend<T> {
    /// Creates a new [`UdevBackend`]
    ///
    /// ## Arguments
    /// `handler` - User-provided handler to respond to any detected changes
    /// `seat`    -
    /// `logger`  - slog Logger to be used by the backend and its `DrmDevices`.
    pub fn new<L, S: AsRef<str>>(mut handler: T, seat: S, logger: L) -> IoResult<UdevBackend<T>>
    where
        L: Into<Option<::slog::Logger>>,
    {
        let log = crate::slog_or_stdlog(logger).new(o!("smithay_module" => "backend_udev"));

        let devices = all_gpus(seat)?
            .into_iter()
            // Create devices
            .flat_map(|path| match stat(&path) {
                Ok(stat) => {
                    handler.device_added(stat.st_rdev, path);
                    Some(stat.st_rdev)
                }
                Err(err) => {
                    warn!(log, "Unable to get id of {:?}, Error: {:?}. Skipping", path, err);
                    None
                }
            })
            .collect();

        let monitor = MonitorBuilder::new()?.match_subsystem("drm")?.listen()?;

        Ok(UdevBackend {
            devices,
            monitor,
            handler,
            logger: log,
        })
    }
}

impl<T: UdevHandler + 'static> Drop for UdevBackend<T> {
    fn drop(&mut self) {
        for device in &self.devices {
            self.handler.device_removed(*device);
        }
    }
}

/// calloop event source associated with the Udev backend
pub type UdevSource<T> = Generic<SourceFd<UdevBackend<T>>>;

/// Binds a [`UdevBackend`] to a given [`EventLoop`](calloop::EventLoop).
///
/// Allows the backend to receive kernel events and thus to drive the [`UdevHandler`].
/// No runtime functionality can be provided without using this function.
pub fn udev_backend_bind<T: UdevHandler + 'static, Data: 'static>(
    udev: UdevBackend<T>,
    handle: &LoopHandle<Data>,
) -> Result<Source<UdevSource<T>>, InsertError<UdevSource<T>>> {
    let mut source = Generic::from_fd_source(udev);
    source.set_interest(Interest::READABLE);

    handle.insert_source(source, |evt, _| {
        evt.source.borrow_mut().0.process_events();
    })
}

impl<T: UdevHandler + 'static> UdevBackend<T> {
    fn process_events(&mut self) {
        let monitor = self.monitor.clone();
        for event in monitor {
            match event.event_type() {
                // New device
                EventType::Add => {
                    info!(self.logger, "Device Added");
                    if let (Some(path), Some(devnum)) = (event.devnode(), event.devnum()) {
                        if self.devices.insert(devnum) {
                            self.handler.device_added(devnum, path.to_path_buf());
                        }
                    }
                }
                // Device removed
                EventType::Remove => {
                    info!(self.logger, "Device Remove");
                    if let Some(devnum) = event.devnum() {
                        if self.devices.remove(&devnum) {
                            self.handler.device_removed(devnum);
                        }
                    }
                }
                // New connector
                EventType::Change => {
                    info!(self.logger, "Device Changed");
                    if let Some(devnum) = event.devnum() {
                        info!(self.logger, "Devnum: {:b}", devnum);
                        if self.devices.contains(&devnum) {
                            self.handler.device_changed(devnum);
                        } else {
                            info!(self.logger, "changed, but device not tracked by backend");
                        };
                    } else {
                        info!(self.logger, "changed, but no devnum");
                    }
                }
                _ => {}
            }
        }
    }
}

/// Handler for the [`UdevBackend`], allows to open, close and update drm devices as they change during runtime.
pub trait UdevHandler {
    /// Called when a new device is detected.
    fn device_added(&mut self, device: dev_t, path: PathBuf);
    /// Called when an open device is changed.
    ///
    /// This usually indicates that some connectors did become available or were unplugged. The handler
    /// should scan again for connected monitors and mode switch accordingly.
    fn device_changed(&mut self, device: dev_t);
    /// Called when a device was removed.
    fn device_removed(&mut self, device: dev_t);
}

/// Returns the path of the primary GPU device if any
///
/// Might be used for filtering in [`UdevHandler::device_added`] or for manual
/// [`LegacyDrmDevice`](::backend::drm::legacy::LegacyDrmDevice) initialization.
pub fn primary_gpu<S: AsRef<str>>(seat: S) -> IoResult<Option<PathBuf>> {
    let mut enumerator = Enumerator::new()?;
    enumerator.match_subsystem("drm")?;
    enumerator.match_sysname("card[0-9]*")?;

    let mut result = None;
    for device in enumerator.scan_devices()? {
        if device
            .property_value("ID_SEAT")
            .map(|x| x.to_os_string())
            .unwrap_or_else(|| OsString::from("seat0"))
            == *seat.as_ref()
        {
            if let Some(pci) = device.parent_with_subsystem(Path::new("pci"))? {
                if let Some(id) = pci.attribute_value("boot_vga") {
                    if id == "1" {
                        result = Some(device);
                    }
                }
            } else if result.is_none() {
                result = Some(device);
            }
        }
    }
    Ok(result.and_then(|device| device.devnode().map(PathBuf::from)))
}

/// Returns the paths of all available GPU devices
///
/// Might be used for manual  [`LegacyDrmDevice`](::backend::drm::legacy::LegacyDrmDevice)
/// initialization.
pub fn all_gpus<S: AsRef<str>>(seat: S) -> IoResult<Vec<PathBuf>> {
    let mut enumerator = Enumerator::new()?;
    enumerator.match_subsystem("drm")?;
    enumerator.match_sysname("card[0-9]*")?;
    Ok(enumerator
        .scan_devices()?
        .filter(|device| {
            device
                .property_value("ID_SEAT")
                .map(|x| x.to_os_string())
                .unwrap_or_else(|| OsString::from("seat0"))
                == *seat.as_ref()
        })
        .flat_map(|device| device.devnode().map(PathBuf::from))
        .collect())
}
<commit_msg>Fix primary_gpu<commit_after>//!
//! Provides `udev` related functionality for automated device scanning.
//!
//! This module mainly provides the [`UdevBackend`](::backend::udev::UdevBackend), which constantly monitors available DRM devices
//! and notifies a user supplied [`UdevHandler`](::backend::udev::UdevHandler) of any changes.
//!
//! Additionally this contains some utility functions related to scanning.
//!
//! See also `anvil/src/udev.rs` for pure hardware backed example of a compositor utilizing this
//! backend.

use nix::sys::stat::{dev_t, stat};
use std::{
    collections::HashSet,
    ffi::OsString,
    io::Result as IoResult,
    os::unix::io::{AsRawFd, RawFd},
    path::{Path, PathBuf},
};
use udev::{Enumerator, EventType, MonitorBuilder, MonitorSocket};

use calloop::{
    generic::{Generic, SourceFd},
    mio::Interest,
    InsertError, LoopHandle, Source,
};

/// Backend to monitor available drm devices.
///
/// Provides a way to automatically scan for available gpus and notifies the
/// given handler of any changes. Can be used to provide hot-plug functionality for gpus and
/// attached monitors.
pub struct UdevBackend<T: UdevHandler + 'static> {
    devices: HashSet<dev_t>,
    monitor: MonitorSocket,
    handler: T,
    logger: ::slog::Logger,
}

impl<T: UdevHandler + 'static> AsRawFd for UdevBackend<T> {
    fn as_raw_fd(&self) -> RawFd {
        self.monitor.as_raw_fd()
    }
}

impl<T: UdevHandler + 'static> UdevBackend<T> {
    /// Creates a new [`UdevBackend`]
    ///
    /// ## Arguments
    /// `handler` - User-provided handler to respond to any detected changes
    /// `seat`    -
    /// `logger`  - slog Logger to be used by the backend and its `DrmDevices`.
    pub fn new<L, S: AsRef<str>>(mut handler: T, seat: S, logger: L) -> IoResult<UdevBackend<T>>
    where
        L: Into<Option<::slog::Logger>>,
    {
        let log = crate::slog_or_stdlog(logger).new(o!("smithay_module" => "backend_udev"));

        let devices = all_gpus(seat)?
            .into_iter()
            // Create devices
            .flat_map(|path| match stat(&path) {
                Ok(stat) => {
                    handler.device_added(stat.st_rdev, path);
                    Some(stat.st_rdev)
                }
                Err(err) => {
                    warn!(log, "Unable to get id of {:?}, Error: {:?}. Skipping", path, err);
                    None
                }
            })
            .collect();

        let monitor = MonitorBuilder::new()?.match_subsystem("drm")?.listen()?;

        Ok(UdevBackend {
            devices,
            monitor,
            handler,
            logger: log,
        })
    }
}

impl<T: UdevHandler + 'static> Drop for UdevBackend<T> {
    fn drop(&mut self) {
        for device in &self.devices {
            self.handler.device_removed(*device);
        }
    }
}

/// calloop event source associated with the Udev backend
pub type UdevSource<T> = Generic<SourceFd<UdevBackend<T>>>;

/// Binds a [`UdevBackend`] to a given [`EventLoop`](calloop::EventLoop).
///
/// Allows the backend to receive kernel events and thus to drive the [`UdevHandler`].
/// No runtime functionality can be provided without using this function.
pub fn udev_backend_bind<T: UdevHandler + 'static, Data: 'static>(
    udev: UdevBackend<T>,
    handle: &LoopHandle<Data>,
) -> Result<Source<UdevSource<T>>, InsertError<UdevSource<T>>> {
    let mut source = Generic::from_fd_source(udev);
    source.set_interest(Interest::READABLE);

    handle.insert_source(source, |evt, _| {
        evt.source.borrow_mut().0.process_events();
    })
}

impl<T: UdevHandler + 'static> UdevBackend<T> {
    fn process_events(&mut self) {
        let monitor = self.monitor.clone();
        for event in monitor {
            match event.event_type() {
                // New device
                EventType::Add => {
                    info!(self.logger, "Device Added");
                    if let (Some(path), Some(devnum)) = (event.devnode(), event.devnum()) {
                        if self.devices.insert(devnum) {
                            self.handler.device_added(devnum, path.to_path_buf());
                        }
                    }
                }
                // Device removed
                EventType::Remove => {
                    info!(self.logger, "Device Remove");
                    if let Some(devnum) = event.devnum() {
                        if self.devices.remove(&devnum) {
                            self.handler.device_removed(devnum);
                        }
                    }
                }
                // New connector
                EventType::Change => {
                    info!(self.logger, "Device Changed");
                    if let Some(devnum) = event.devnum() {
                        info!(self.logger, "Devnum: {:b}", devnum);
                        if self.devices.contains(&devnum) {
                            self.handler.device_changed(devnum);
                        } else {
                            info!(self.logger, "changed, but device not tracked by backend");
                        };
                    } else {
                        info!(self.logger, "changed, but no devnum");
                    }
                }
                _ => {}
            }
        }
    }
}

/// Handler for the [`UdevBackend`], allows to open, close and update drm devices as they change during runtime.
pub trait UdevHandler {
    /// Called when a new device is detected.
    fn device_added(&mut self, device: dev_t, path: PathBuf);
    /// Called when an open device is changed.
    ///
    /// This usually indicates that some connectors did become available or were unplugged. The handler
    /// should scan again for connected monitors and mode switch accordingly.
    fn device_changed(&mut self, device: dev_t);
    /// Called when a device was removed.
    fn device_removed(&mut self, device: dev_t);
}

/// Returns the path of the primary GPU device if any
///
/// Might be used for filtering in [`UdevHandler::device_added`] or for manual
/// [`LegacyDrmDevice`](::backend::drm::legacy::LegacyDrmDevice) initialization.
pub fn primary_gpu<S: AsRef<str>>(seat: S) -> IoResult<Option<PathBuf>> {
    let mut enumerator = Enumerator::new()?;
    enumerator.match_subsystem("drm")?;
    enumerator.match_sysname("card[0-9]*")?;

    if let Some(path) = enumerator
        .scan_devices()?
        .filter_map(|device| {
            if device
                .property_value("ID_SEAT")
                .map(|x| x.to_os_string())
                .unwrap_or_else(|| OsString::from("seat0"))
                == *seat.as_ref()
            {
                if let Ok(Some(pci)) = device.parent_with_subsystem(Path::new("pci")) {
                    if let Some(id) = pci.attribute_value("boot_vga") {
                        if id == "1" {
                            return Some(device);
                        }
                    }
                }
            };
            None
        })
        .flat_map(|device| device.devnode().map(PathBuf::from))
        .next()
    {
        Ok(Some(path))
    } else {
        all_gpus(seat).map(|all| all.into_iter().next())
    }
}

/// Returns the paths of all available GPU devices
///
/// Might be used for manual  [`LegacyDrmDevice`](::backend::drm::legacy::LegacyDrmDevice)
/// initialization.
pub fn all_gpus<S: AsRef<str>>(seat: S) -> IoResult<Vec<PathBuf>> {
    let mut enumerator = Enumerator::new()?;
    enumerator.match_subsystem("drm")?;
    enumerator.match_sysname("card[0-9]*")?;
    Ok(enumerator
        .scan_devices()?
        .filter(|device| {
            device
                .property_value("ID_SEAT")
                .map(|x| x.to_os_string())
                .unwrap_or_else(|| OsString::from("seat0"))
                == *seat.as_ref()
        })
        .flat_map(|device| device.devnode().map(PathBuf::from))
        .collect())
}
<|endoftext|>


<commit_before>// Build script.
//
// This program is run by Cargo when building lib/codegen. It is used to generate Rust code from
// the language definitions in the lib/codegen/meta directory.
//
// Environment:
//
// OUT_DIR
//     Directory where generated files should be placed.
//
// TARGET
//     Target triple provided by Cargo.
//
// CRANELIFT_TARGETS (Optional)
//     A setting for conditional compilation of isa targets. Possible values can be "native" or
//     known isa targets separated by ','.
//
// The build script expects to be run from the directory where this build.rs file lives. The
// current directory is used to find the sources.

extern crate cranelift_codegen_meta as meta;

use meta::isa::Isa;
use std::env;
use std::process;

use std::time::Instant;

fn main() {
    let start_time = Instant::now();

    let out_dir = env::var("OUT_DIR").expect("The OUT_DIR environment variable must be set");
    let target_triple = env::var("TARGET").expect("The TARGET environment variable must be set");
    let cranelift_targets = env::var("CRANELIFT_TARGETS").ok();
    let cranelift_targets = cranelift_targets.as_ref().map(|s| s.as_ref());
    let python = identify_python();

    // Configure isa targets cfg.
    match isa_targets(cranelift_targets, &target_triple) {
        Ok(isa_targets) => {
            for isa in &isa_targets {
                println!("cargo:rustc-cfg=build_{}", isa.to_string());
            }
        }
        Err(err) => {
            eprintln!("Error: {}", err);
            process::exit(1);
        }
    }

    let cur_dir = env::current_dir().expect("Can't access current working directory");
    let crate_dir = cur_dir.as_path();

    // Make sure we rebuild if this build script changes.
    // I guess that won't happen if you have non-UTF8 bytes in your path names.
    // The `build.py` script prints out its own dependencies.
    println!(
        "cargo:rerun-if-changed={}",
        crate_dir.join("build.rs").to_str().unwrap()
    );

    // Scripts are in `$crate_dir/meta-python`.
    let meta_dir = crate_dir.join("meta-python");
    let build_script = meta_dir.join("build.py");

    // Launch build script with Python. We'll just find python in the path.
    // Use -B to disable .pyc files, because they cause trouble for vendoring
    // scripts, and this is a build step that isn't run very often anyway.
    let status = process::Command::new(python)
        .current_dir(crate_dir)
        .arg("-B")
        .arg(build_script)
        .arg("--out-dir")
        .arg(out_dir.clone())
        .status()
        .expect("Failed to launch second-level build script; is python installed?");
    if !status.success() {
        process::exit(status.code().unwrap());
    }

    // DEVELOPMENT:
    // ------------------------------------------------------------------------
    // Now that the Python build process is complete, generate files that are
    // emitted by the `meta` crate.
    // ------------------------------------------------------------------------
    let isas = meta::isa::define_all();

    if let Err(err) = meta::gen_types::generate("types.rs", &out_dir) {
        eprintln!("Error: {}", err);
        process::exit(1);
    }

    for isa in isas {
        if let Err(err) = meta::gen_registers::generate(isa, "registers", &out_dir) {
            eprintln!("Error: {}", err);
            process::exit(1);
        }
    }

    println!(
        "cargo:warning=Cranelift meta-build step took {:?}",
        Instant::now() - start_time
    );
    println!(
        "cargo:warning=Meta-build script generated files in {}",
        out_dir
    );
}

fn identify_python() -> &'static str {
    for python in &["python", "python3", "python2.7"] {
        if process::Command::new(python)
            .arg("--version")
            .status()
            .is_ok()
        {
            return python;
        }
    }
    panic!("The Cranelift build requires Python (version 2.7 or version 3)");
}

/// Returns isa targets to configure conditional compilation.
fn isa_targets(cranelift_targets: Option<&str>, target_triple: &str) -> Result<Vec<Isa>, String> {
    match cranelift_targets {
        Some("native") => Isa::from_arch(target_triple.split('-').next().unwrap())
            .map(|isa| vec![isa])
            .ok_or_else(|| {
                format!(
                    "no supported isa found for target triple `{}`",
                    target_triple
                )
            }),
        Some(targets) => {
            let unknown_isa_targets = targets
                .split(',')
                .filter(|target| Isa::new(target).is_none())
                .collect::<Vec<_>>();
            let isa_targets = targets.split(',').flat_map(Isa::new).collect::<Vec<_>>();
            match (unknown_isa_targets.is_empty(), isa_targets.is_empty()) {
                (true, true) => Ok(Isa::all().to_vec()),
                (true, _) => Ok(isa_targets),
                (_, _) => Err(format!(
                    "unknown isa targets: `{}`",
                    unknown_isa_targets.join(", ")
                )),
            }
        }
        None => Ok(Isa::all().to_vec()),
    }
}
<commit_msg>Remove the build time and directory from cranelift-codegen-meta's output.<commit_after>// Build script.
//
// This program is run by Cargo when building lib/codegen. It is used to generate Rust code from
// the language definitions in the lib/codegen/meta directory.
//
// Environment:
//
// OUT_DIR
//     Directory where generated files should be placed.
//
// TARGET
//     Target triple provided by Cargo.
//
// CRANELIFT_TARGETS (Optional)
//     A setting for conditional compilation of isa targets. Possible values can be "native" or
//     known isa targets separated by ','.
//
// The build script expects to be run from the directory where this build.rs file lives. The
// current directory is used to find the sources.

extern crate cranelift_codegen_meta as meta;

use meta::isa::Isa;
use std::env;
use std::process;

fn main() {
    let out_dir = env::var("OUT_DIR").expect("The OUT_DIR environment variable must be set");
    let target_triple = env::var("TARGET").expect("The TARGET environment variable must be set");
    let cranelift_targets = env::var("CRANELIFT_TARGETS").ok();
    let cranelift_targets = cranelift_targets.as_ref().map(|s| s.as_ref());
    let python = identify_python();

    // Configure isa targets cfg.
    match isa_targets(cranelift_targets, &target_triple) {
        Ok(isa_targets) => {
            for isa in &isa_targets {
                println!("cargo:rustc-cfg=build_{}", isa.to_string());
            }
        }
        Err(err) => {
            eprintln!("Error: {}", err);
            process::exit(1);
        }
    }

    let cur_dir = env::current_dir().expect("Can't access current working directory");
    let crate_dir = cur_dir.as_path();

    // Make sure we rebuild if this build script changes.
    // I guess that won't happen if you have non-UTF8 bytes in your path names.
    // The `build.py` script prints out its own dependencies.
    println!(
        "cargo:rerun-if-changed={}",
        crate_dir.join("build.rs").to_str().unwrap()
    );

    // Scripts are in `$crate_dir/meta-python`.
    let meta_dir = crate_dir.join("meta-python");
    let build_script = meta_dir.join("build.py");

    // Launch build script with Python. We'll just find python in the path.
    // Use -B to disable .pyc files, because they cause trouble for vendoring
    // scripts, and this is a build step that isn't run very often anyway.
    let status = process::Command::new(python)
        .current_dir(crate_dir)
        .arg("-B")
        .arg(build_script)
        .arg("--out-dir")
        .arg(out_dir.clone())
        .status()
        .expect("Failed to launch second-level build script; is python installed?");
    if !status.success() {
        process::exit(status.code().unwrap());
    }

    // DEVELOPMENT:
    // ------------------------------------------------------------------------
    // Now that the Python build process is complete, generate files that are
    // emitted by the `meta` crate.
    // ------------------------------------------------------------------------
    let isas = meta::isa::define_all();

    if let Err(err) = meta::gen_types::generate("types.rs", &out_dir) {
        eprintln!("Error: {}", err);
        process::exit(1);
    }

    for isa in isas {
        if let Err(err) = meta::gen_registers::generate(isa, "registers", &out_dir) {
            eprintln!("Error: {}", err);
            process::exit(1);
        }
    }
}

fn identify_python() -> &'static str {
    for python in &["python", "python3", "python2.7"] {
        if process::Command::new(python)
            .arg("--version")
            .status()
            .is_ok()
        {
            return python;
        }
    }
    panic!("The Cranelift build requires Python (version 2.7 or version 3)");
}

/// Returns isa targets to configure conditional compilation.
fn isa_targets(cranelift_targets: Option<&str>, target_triple: &str) -> Result<Vec<Isa>, String> {
    match cranelift_targets {
        Some("native") => Isa::from_arch(target_triple.split('-').next().unwrap())
            .map(|isa| vec![isa])
            .ok_or_else(|| {
                format!(
                    "no supported isa found for target triple `{}`",
                    target_triple
                )
            }),
        Some(targets) => {
            let unknown_isa_targets = targets
                .split(',')
                .filter(|target| Isa::new(target).is_none())
                .collect::<Vec<_>>();
            let isa_targets = targets.split(',').flat_map(Isa::new).collect::<Vec<_>>();
            match (unknown_isa_targets.is_empty(), isa_targets.is_empty()) {
                (true, true) => Ok(Isa::all().to_vec()),
                (true, _) => Ok(isa_targets),
                (_, _) => Err(format!(
                    "unknown isa targets: `{}`",
                    unknown_isa_targets.join(", ")
                )),
            }
        }
        None => Ok(Isa::all().to_vec()),
    }
}
<|endoftext|>


<commit_before>#[warn(non_camel_case_types)]
#[warn(non_snake_case)]
#[warn(unused_qualifications)]
#[warn(non_upper_case_globals)]
#[warn(missing_docs)]

extern crate eulerrust;
use std::collections::HashSet;

fn get_max_rem(a : u64) -> u64 {
	let mut set = HashSet::new();
	let asq = a*a;

	let mut alo = 1;
	let mut ahi = 1;

	for n in 1..(2*asq){
		alo *= a-1;
		alo %= asq;
		ahi *= a+1;
		ahi %= asq;

		if n % 2 == 0 {continue;}

		let r = (alo + ahi) % asq;
		//~ println!("a: {}, n: {}, r: {}", a, n, r);
		if set.contains(&r){ break;}
		set.insert(r);
	}
	set.iter().fold(0, |a, &b| {if a > b {a} else {b}})
}

#[test]
fn test_max_rem(){
	assert_eq!(get_max_rem(7), 42);
}

#[test]
fn test_max_rem_long(){
	let mut sum = 0;
	for a in 3..1001 {
		sum += get_max_rem(a);
	}
	assert_eq!(sum, 333082500);
}

pub fn main(){
	let mut sum = 0;
	for a in 3..1001 {
		sum += get_max_rem(a);
	}

	println!("sum: {}", sum);
}
<commit_msg>Added integer hints for type resolving (necessary now that u32/i32 is default)<commit_after>#[warn(non_camel_case_types)]
#[warn(non_snake_case)]
#[warn(unused_qualifications)]
#[warn(non_upper_case_globals)]
#[warn(missing_docs)]

extern crate eulerrust;
use std::collections::HashSet;

fn get_max_rem(a : u64) -> u64 {
	let mut set = HashSet::new();
	let asq = a*a;

	let mut alo = 1;
	let mut ahi = 1;

	for n in 1..(2*asq){
		alo *= a-1;
		alo %= asq;
		ahi *= a+1;
		ahi %= asq;

		if n % 2 == 0 {continue;}

		let r = (alo + ahi) % asq;
		//~ println!("a: {}, n: {}, r: {}", a, n, r);
		if set.contains(&r){ break;}
		set.insert(r);
	}
	set.iter().fold(0, |a, &b| {if a > b {a} else {b}})
}

#[test]
fn test_max_rem(){
	assert_eq!(get_max_rem(7), 42);
}

#[test]
fn test_max_rem_long(){
	let mut sum = 0;
	for a in 3u64..1001u64 {
		sum += get_max_rem(a);
	}
	assert_eq!(sum, 333082500);
}

pub fn main(){
	let mut sum = 0;
	for a in 3u64..1001u64 {
		sum += get_max_rem(a);
	}

	println!("sum: {}", sum);
}
<|endoftext|>