<commit_before>//! Implements some higher-level searching facilities.

use std::cmp::min;
use std::sync::Arc;
use std::time::{SystemTime, Duration};
use basetypes::*;
use tt::*;
use position::*;
use search::*;

/// Contains information about the current progress of a search.
pub struct SearchStatus {
    /// The starting time for the search.
    pub started_at: SystemTime,

    /// The starting (root) position for the search.
    pub position: Position,

    /// `true` if the search has stopped, `false` otherwise.
    pub done: bool,

    /// The reached search depth.
    pub depth: u8,

    /// The number of legal moves in the starting (root) position.
    pub legal_moves_count: usize,

    /// The best variations found so far. The first move in each
    /// variation will be different.
    pub variations: Vec<Variation>,

    /// The duration of the search in milliseconds.
    pub duration_millis: u64,

    /// The number of analyzed nodes.
    pub searched_nodes: NodeCount,

    /// Average number of analyzed nodes per second.
    pub nps: NodeCount,
}

/// A thread that executes consecutive searches in different starting
/// positions.
pub struct SearchThread {
    tt: Arc<Tt>,
    status: SearchStatus,
    searcher: DeepeningSearcher<MultipvSearcher<AlphabetaSearcher>>,
}

impl SearchThread {
    /// Creates a new instance.
    pub fn new(tt: Arc<Tt>) -> SearchThread {
        SearchThread {
            tt: tt.clone(),
            status: SearchStatus {
                started_at: SystemTime::now(),
                position: Position::from_fen(START_POSITION_FEN).ok().unwrap(),
                done: true,
                depth: 0,
                variations: vec![Variation {
                                     value: VALUE_MIN,
                                     bound: BOUND_LOWER,
                                     moves: vec![],
                                 }],
                legal_moves_count: 20,
                searched_nodes: 0,
                duration_millis: 0,
                nps: 0,
            },
            searcher: DeepeningSearcher::new(tt),
        }
    }

    /// Terminates the currently running search (if any), starts a new
    /// search, updates the status.
    ///
    /// `position` is the starting position for the new
    /// search. `variation_count` specifies how many best lines of
    /// play to calculate (the first move in each line will be
    /// different). `searchmoves` restricts the analysis to the
    /// supplied list of moves only (no restrictions if the suppied
    /// list is empty). The move format is long algebraic
    /// notation. Examples: e2e4, e7e5, e1g1 (white short castling),
    /// e7e8q (for promotion).
    #[allow(unused_variables)]
    pub fn search(&mut self,
                  position: &Position,
                  variation_count: usize,
                  mut searchmoves: Vec<String>) {
        let mut legal_moves = position.legal_moves();
        let legal_moves_count = legal_moves.len();

        // Remove all legal moves not allowed by `searchmoves`.
        if !searchmoves.is_empty() {
            searchmoves.sort();
            let mut moves = vec![];
            for m in legal_moves.iter() {
                if searchmoves.binary_search(&m.notation()).is_ok() {
                    moves.push(*m);
                }
            }
            if !moves.is_empty() {
                legal_moves = moves;
            }
        };

        self.stop();
        self.searcher.start_search(SearchParams {
            search_id: 0,
            position: position.clone(),
            depth: MAX_DEPTH,
            lower_bound: VALUE_MIN,
            upper_bound: VALUE_MAX,
            searchmoves: legal_moves,
            variation_count: variation_count,
        });

        self.status = SearchStatus {
            started_at: SystemTime::now(),
            position: position.clone(),
            done: false,
            depth: 0,
            legal_moves_count: legal_moves_count,
            variations: vec![Variation {
                                 value: VALUE_MIN,
                                 bound: BOUND_LOWER,
                                 moves: vec![],
                             }], // TODO: Set good initial value (vec![best_move]).
            searched_nodes: 0,
            duration_millis: 0,
            ..self.status
        };
    }

    /// Terminates the currently running search (if any), updates the
    /// status.
    pub fn stop(&mut self) {
        self.searcher.terminate_search();
        while !self.status().done {
            self.wait_status_update(Duration::from_millis(1000));
        }
    }

    /// Returns the status of the current search.
    ///
    /// **Important note:** Consecutive calls to this method will
    /// return the same unchanged result. Only after calling `search`,
    /// `stop`, or `wait_status_update`, the result returned by
    /// `status` may change.
    #[inline(always)]
    pub fn status(&self) -> &SearchStatus {
        &self.status
    }

    /// Waits for a search status update, timing out after a specified
    /// duration or earlier.
    pub fn wait_status_update(&mut self, duration: Duration) {
        self.searcher.wait_report(duration);
        while let Ok(r) = self.searcher.try_recv_report() {
            self.process_report(r)
        }
    }

    /// A helper method. It updates the current status according to
    /// the received report message.
    fn process_report(&mut self, report: Report) {
        let duration = self.status.started_at.elapsed().unwrap();
        self.status.duration_millis = 1000 * duration.as_secs() +
                                      (duration.subsec_nanos() / 1000000) as u64;
        self.status.searched_nodes = report.searched_nodes;
        self.status.nps = 1000 * (self.status.nps + self.status.searched_nodes) /
                          (1000 + self.status.duration_millis);
        if self.status.depth < report.depth {
            self.status.depth = report.depth;
            self.status.variations = vec![extract_pv(&self.tt,
                                                     &self.status.position,
                                                     report.depth)];
        }
        self.status.done = report.done;
    }
}

/// Decides when the search must be terminated.
pub struct TimeManager {
    move_time_millis: u64, // move time in milliseconds
    must_play: bool,
}

impl TimeManager {
    /// Creates a new instance.
    ///
    /// `position` gives the current position. `pondering_is_allowed`
    /// tells if the engine is allowed to use opponent's time for
    /// thinking. `wtime_millis`, `btime_millis`, `winc_millis`, and
    /// `binc_millis` specify the remaining time in milliseconds, and
    /// the number of milliseconds with which the remaining time will
    /// be incremented on each move (for black and white). `movestogo`
    /// specifies the number of moves to the next time control.
    #[allow(unused_variables)]
    pub fn new(position: &Position,
               pondering_is_allowed: bool,
               wtime_millis: Option<u64>,
               btime_millis: Option<u64>,
               winc_millis: Option<u64>,
               binc_millis: Option<u64>,
               movestogo: Option<u64>)
               -> TimeManager {
        // TODO: We ignore "pondering_is_allowed".

        let (time, inc) = if position.to_move() == WHITE {
            (wtime_millis, winc_millis.unwrap_or(0))
        } else {
            (btime_millis, binc_millis.unwrap_or(0))
        };
        let time = time.unwrap_or(0);
        let movestogo = movestogo.unwrap_or(40);
        let movetime = (time + inc * movestogo) / movestogo;
        TimeManager {
            move_time_millis: min(movetime, time / 2),
            must_play: false,
        }
    }

    /// Registers the current search status with the time manager.
    pub fn update_status(&mut self, new_status: &SearchStatus) {
        // TODO: Implement smarter time management.
        self.must_play = new_status.duration_millis >= self.move_time_millis;
    }

    /// Decides if the search must be terminated.
    #[inline]
    pub fn must_play(&self) -> bool {
        self.must_play
    }
}
<commit_msg>improve SearchStatus<commit_after>//! Implements some higher-level searching facilities.

use std::cmp::min;
use std::sync::Arc;
use std::time::{SystemTime, Duration};
use basetypes::*;
use tt::*;
use position::*;
use search::*;

/// Contains information about the current progress of a search.
pub struct SearchStatus {
    /// The starting time for the search.
    pub started_at: SystemTime,

    /// The root position for the search.
    pub position: Position,

    /// `true` if the search is done, `false` otherwise.
    pub done: bool,

    /// The search depth completed so far.
    pub depth: u8,

    /// The number of first moves (from the root position) that are
    /// being considered.
    pub searchmoves_count: usize,

    /// The best variations found so far, sorted by descending first
    /// move strength. (The first move in each variation will be
    /// different.)
    pub variations: Vec<Variation>,

    /// The duration of the search in milliseconds.
    pub duration_millis: u64,

    /// The number of analyzed nodes.
    pub searched_nodes: NodeCount,

    /// Average number of analyzed nodes per second.
    pub nps: NodeCount,
}

/// A thread that executes consecutive searches in different starting
/// positions.
pub struct SearchThread {
    tt: Arc<Tt>,
    status: SearchStatus,
    searcher: DeepeningSearcher<MultipvSearcher<AlphabetaSearcher>>,
}

impl SearchThread {
    /// Creates a new instance.
    pub fn new(tt: Arc<Tt>) -> SearchThread {
        SearchThread {
            tt: tt.clone(),
            status: SearchStatus {
                started_at: SystemTime::now(),
                position: Position::from_fen(START_POSITION_FEN).ok().unwrap(),
                done: true,
                depth: 0,
                variations: vec![Variation {
                                     value: VALUE_MIN,
                                     bound: BOUND_LOWER,
                                     moves: vec![],
                                 }],
                searchmoves_count: 20,
                searched_nodes: 0,
                duration_millis: 0,
                nps: 0,
            },
            searcher: DeepeningSearcher::new(tt),
        }
    }

    /// Terminates the currently running search (if any), starts a new
    /// search, updates the status.
    ///
    /// `position` is the starting position for the new
    /// search. `variation_count` specifies how many best lines of
    /// play to calculate (the first move in each line will be
    /// different). `searchmoves` restricts the analysis to the
    /// supplied list of moves only (no restrictions if the suppied
    /// list is empty). The move format is long algebraic
    /// notation. Examples: e2e4, e7e5, e1g1 (white short castling),
    /// e7e8q (for promotion).
    pub fn search(&mut self,
                  position: &Position,
                  variation_count: usize,
                  mut searchmoves: Vec<String>) {
        // Validate `searchmoves`.
        let mut moves = vec![];
        let legal_moves = position.legal_moves();
        if !searchmoves.is_empty() {
            searchmoves.sort();
            for m in legal_moves.iter() {
                if searchmoves.binary_search(&m.notation()).is_ok() {
                    moves.push(*m);
                }
            }
        };
        let searchmoves = if moves.is_empty() {
            legal_moves
        } else {
            moves
        };
        let searchmoves_count = searchmoves.len();

        // Terminate the currently running search.
        self.stop();

        // Start a new search.
        self.searcher.start_search(SearchParams {
            search_id: 0,
            position: position.clone(),
            depth: MAX_DEPTH,
            lower_bound: VALUE_MIN,
            upper_bound: VALUE_MAX,
            searchmoves: searchmoves,
            variation_count: variation_count,
        });

        // Update the status.
        self.status = SearchStatus {
            started_at: SystemTime::now(),
            position: position.clone(),
            done: false,
            depth: 0,
            searchmoves_count: searchmoves_count,
            variations: vec![Variation {
                                 value: VALUE_MIN,
                                 bound: BOUND_LOWER,
                                 moves: vec![],
                             }], // TODO: Set good initial value (vec![best_move]).
            searched_nodes: 0,
            duration_millis: 0,
            ..self.status
        };
    }

    /// Terminates the currently running search (if any), updates the
    /// status.
    pub fn stop(&mut self) {
        self.searcher.terminate_search();
        while !self.status().done {
            self.wait_status_update(Duration::from_millis(1000));
        }
    }

    /// Returns the status of the current search.
    ///
    /// **Important note:** Consecutive calls to this method will
    /// return the same unchanged result. Only after calling `search`,
    /// `stop`, or `wait_status_update`, the result returned by
    /// `status` may change.
    #[inline(always)]
    pub fn status(&self) -> &SearchStatus {
        &self.status
    }

    /// Waits for a search status update, timing out after a specified
    /// duration or earlier.
    pub fn wait_status_update(&mut self, duration: Duration) {
        self.searcher.wait_report(duration);
        while let Ok(r) = self.searcher.try_recv_report() {
            self.process_report(r)
        }
    }

    /// A helper method. It updates the current status according to
    /// the received report message.
    fn process_report(&mut self, report: Report) {
        let duration = self.status.started_at.elapsed().unwrap();
        self.status.duration_millis = 1000 * duration.as_secs() +
                                      (duration.subsec_nanos() / 1000000) as u64;
        self.status.searched_nodes = report.searched_nodes;
        self.status.nps = 1000 * (self.status.nps + self.status.searched_nodes) /
                          (1000 + self.status.duration_millis);
        if self.status.depth < report.depth {
            self.status.depth = report.depth;
            self.status.variations = vec![extract_pv(&self.tt,
                                                     &self.status.position,
                                                     report.depth)];
        }
        self.status.done = report.done;
    }
}

/// Decides when the search must be terminated.
pub struct TimeManager {
    move_time_millis: u64, // move time in milliseconds
    must_play: bool,
}

impl TimeManager {
    /// Creates a new instance.
    ///
    /// `position` gives the current position. `pondering_is_allowed`
    /// tells if the engine is allowed to use opponent's time for
    /// thinking. `wtime_millis`, `btime_millis`, `winc_millis`, and
    /// `binc_millis` specify the remaining time in milliseconds, and
    /// the number of milliseconds with which the remaining time will
    /// be incremented on each move (for black and white). `movestogo`
    /// specifies the number of moves to the next time control.
    #[allow(unused_variables)]
    pub fn new(position: &Position,
               pondering_is_allowed: bool,
               wtime_millis: Option<u64>,
               btime_millis: Option<u64>,
               winc_millis: Option<u64>,
               binc_millis: Option<u64>,
               movestogo: Option<u64>)
               -> TimeManager {
        // TODO: We ignore "pondering_is_allowed".

        let (time, inc) = if position.to_move() == WHITE {
            (wtime_millis, winc_millis.unwrap_or(0))
        } else {
            (btime_millis, binc_millis.unwrap_or(0))
        };
        let time = time.unwrap_or(0);
        let movestogo = movestogo.unwrap_or(40);
        let movetime = (time + inc * movestogo) / movestogo;
        TimeManager {
            move_time_millis: min(movetime, time / 2),
            must_play: false,
        }
    }

    /// Registers the current search status with the time manager.
    pub fn update_status(&mut self, new_status: &SearchStatus) {
        // TODO: Implement smarter time management.
        self.must_play = new_status.duration_millis >= self.move_time_millis;
    }

    /// Decides if the search must be terminated.
    #[inline]
    pub fn must_play(&self) -> bool {
        self.must_play
    }
}
<|endoftext|>


<commit_before>use crate::backend::{Backend, SupportsOnConflictClause};
use crate::expression::SqlLiteral;
use crate::query_builder::*;
use crate::query_source::Column;
use crate::result::QueryResult;

#[cfg(feature = "postgres")]
pub use pg::on_constraint::*;

#[doc(hidden)]
pub trait OnConflictTarget<Table> {}

#[doc(hidden)]
#[derive(Debug, Clone, Copy)]
pub struct NoConflictTarget;

impl<DB> QueryFragment<DB> for NoConflictTarget
where
    DB: Backend + SupportsOnConflictClause,
{
    fn walk_ast(&self, _: AstPass<DB>) -> QueryResult<()> {
        Ok(())
    }
}

impl<Table> OnConflictTarget<Table> for NoConflictTarget {}

#[doc(hidden)]
#[derive(Debug, Clone, Copy)]
pub struct ConflictTarget<T>(pub T);

impl<DB, T> QueryFragment<DB> for ConflictTarget<T>
where
    DB: Backend + SupportsOnConflictClause,
    T: Column,
{
    fn walk_ast(&self, mut out: AstPass<DB>) -> QueryResult<()> {
        out.push_sql(" (");
        out.push_identifier(T::NAME)?;
        out.push_sql(")");
        Ok(())
    }
}

impl<T> OnConflictTarget<T::Table> for ConflictTarget<T>
where
    T: Column,
{
}

impl<DB, ST> QueryFragment<DB> for ConflictTarget<SqlLiteral<ST>>
where
    DB: Backend + SupportsOnConflictClause,
    SqlLiteral<ST>: QueryFragment<DB>,
{
    fn walk_ast(&self, mut out: AstPass<DB>) -> QueryResult<()> {
        out.push_sql(" ");
        self.0.walk_ast(out.reborrow())?;
        Ok(())
    }
}

impl<Tab, ST> OnConflictTarget<Tab> for ConflictTarget<SqlLiteral<ST>> {}

macro_rules! on_conflict_tuples {
    ($($col:ident),+) => {
        impl<DB, T, $($col),+> QueryFragment<DB> for ConflictTarget<(T, $($col),+)> where
            DB: Backend + SupportsOnConflictClause,
            T: Column,
            $($col: Column<Table=T::Table>,)+
        {
            fn walk_ast(&self, mut out: AstPass<DB>) -> QueryResult<()> {
                out.push_sql(" (");
                out.push_identifier(T::NAME)?;
                $(
                    out.push_sql(", ");
                    out.push_identifier($col::NAME)?;
                )+
                out.push_sql(")");
                Ok(())
            }
        }

        impl<T, $($col),+> OnConflictTarget<T::Table> for ConflictTarget<(T, $($col),+)> where
            T: Column,
            $($col: Column<Table=T::Table>,)+
        {
        }
    }
}

on_conflict_tuples!(U);
on_conflict_tuples!(U, V);
on_conflict_tuples!(U, V, W);
on_conflict_tuples!(U, V, W, X);
on_conflict_tuples!(U, V, W, X, Y);
on_conflict_tuples!(U, V, W, X, Y, Z);
<commit_msg>rustfmt<commit_after>use crate::backend::{Backend, SupportsOnConflictClause};
use crate::expression::SqlLiteral;
use crate::query_builder::*;
use crate::query_source::Column;
use crate::result::QueryResult;

#[cfg(feature = "postgres")]
pub use pg::on_constraint::*;

#[doc(hidden)]
pub trait OnConflictTarget<Table> {}

#[doc(hidden)]
#[derive(Debug, Clone, Copy)]
pub struct NoConflictTarget;

impl<DB> QueryFragment<DB> for NoConflictTarget
where
    DB: Backend + SupportsOnConflictClause,
{
    fn walk_ast(&self, _: AstPass<DB>) -> QueryResult<()> {
        Ok(())
    }
}

impl<Table> OnConflictTarget<Table> for NoConflictTarget {}

#[doc(hidden)]
#[derive(Debug, Clone, Copy)]
pub struct ConflictTarget<T>(pub T);

impl<DB, T> QueryFragment<DB> for ConflictTarget<T>
where
    DB: Backend + SupportsOnConflictClause,
    T: Column,
{
    fn walk_ast(&self, mut out: AstPass<DB>) -> QueryResult<()> {
        out.push_sql(" (");
        out.push_identifier(T::NAME)?;
        out.push_sql(")");
        Ok(())
    }
}

impl<T> OnConflictTarget<T::Table> for ConflictTarget<T> where T: Column {}

impl<DB, ST> QueryFragment<DB> for ConflictTarget<SqlLiteral<ST>>
where
    DB: Backend + SupportsOnConflictClause,
    SqlLiteral<ST>: QueryFragment<DB>,
{
    fn walk_ast(&self, mut out: AstPass<DB>) -> QueryResult<()> {
        out.push_sql(" ");
        self.0.walk_ast(out.reborrow())?;
        Ok(())
    }
}

impl<Tab, ST> OnConflictTarget<Tab> for ConflictTarget<SqlLiteral<ST>> {}

macro_rules! on_conflict_tuples {
    ($($col:ident),+) => {
        impl<DB, T, $($col),+> QueryFragment<DB> for ConflictTarget<(T, $($col),+)> where
            DB: Backend + SupportsOnConflictClause,
            T: Column,
            $($col: Column<Table=T::Table>,)+
        {
            fn walk_ast(&self, mut out: AstPass<DB>) -> QueryResult<()> {
                out.push_sql(" (");
                out.push_identifier(T::NAME)?;
                $(
                    out.push_sql(", ");
                    out.push_identifier($col::NAME)?;
                )+
                out.push_sql(")");
                Ok(())
            }
        }

        impl<T, $($col),+> OnConflictTarget<T::Table> for ConflictTarget<(T, $($col),+)> where
            T: Column,
            $($col: Column<Table=T::Table>,)+
        {
        }
    }
}

on_conflict_tuples!(U);
on_conflict_tuples!(U, V);
on_conflict_tuples!(U, V, W);
on_conflict_tuples!(U, V, W, X);
on_conflict_tuples!(U, V, W, X, Y);
on_conflict_tuples!(U, V, W, X, Y, Z);
<|endoftext|>


<commit_before>mod data;

use std::io::Read;

use super::regex::Regex;

use super::hyper::Client;
use super::hyper::header::Connection;

use super::rustc_serialize::json;
use super::rustc_serialize::json::Json;

use super::flate2::read::GzDecoder;

use self::data::StackExchangeAnswer;

pub struct StackExchangeApi {
    api: &'static str,
    protocol: &'static str,
    version: &'static str,
    client: Client,
}

impl StackExchangeApi {
    pub fn new() -> Self {
        StackExchangeApi {
            api: "api.stackexchange.com",
            protocol: "https",
            version: "2.2",
            client: Client::new(),
        }
    }

    pub fn answers(&self, url: &str) -> Result<Vec<StackExchangeAnswer>, String> {
        let quiestion_id: &str;
        let site: &str;

        if url.contains("stackoverflow.com") {
            let re = Regex::new(r".*stackoverflow.com/questions/(?P<q_id>\d+)/").unwrap();
            let captures = re.captures(&url).unwrap();
            quiestion_id = captures.name("q_id").unwrap();
            site = "stackoverflow".into();
        } else if url.contains("stackexchange.com") {
            let re = Regex::new(r".*//(?P<site>.*).stackexchange.com/questions/(?P<q_id>\d+)/")
                         .unwrap();
            let captures = re.captures(&url).unwrap();
            quiestion_id = captures.name("q_id").unwrap();
            site = captures.name("site").unwrap();
        } else {
            return Err("Result is not related to StackExchange.".into());
        }

        let quiestion_id: u64 = quiestion_id.parse::<u64>().unwrap();

        Ok(self.query(site, quiestion_id, 20))
    }

    fn request(&self, url: &str) -> String {
        let mut res = self.client
                          .get(url)
                          .header(Connection::close())
                          .send()
                          .unwrap();

        let mut buffer = Vec::new();
        res.read_to_end(&mut buffer).unwrap();

        let mut gzip_decoder = GzDecoder::new(buffer.as_slice()).unwrap();
        let mut body = String::new();
        gzip_decoder.read_to_string(&mut body).unwrap();

        body
    }

    fn from_json(&self, json_str: &str) -> Vec<StackExchangeAnswer> {
        let json = Json::from_str(&json_str).unwrap();
        let data = json.as_object().unwrap();
        let items = data.get("items").unwrap().as_array().unwrap();

        let answers: Vec<StackExchangeAnswer> = items.iter()
                                                     .map(|i| {
                                                         json::decode(&i.to_string()).unwrap()
                                                     })
                                                     .collect();

        answers
    }

    fn query_meta(&self,
                  site: &str,
                  quiestion_id: u64,
                  max_count: usize)
                  -> Vec<StackExchangeAnswer> {
        let url = format!("{}://{}/{}/questions/{}/answers?site={}",
                          self.protocol,
                          self.api,
                          self.version,
                          quiestion_id,
                          site);

        let response = self.request(&url);
        let mut answers = self.from_json(&response);
        answers.sort_by_key(|a| -a.score);

        let chosen_answers = answers.drain(0..)
                                    .filter(|a| a.score > 0)
                                    .take(max_count)
                                    .collect();

        chosen_answers
    }

    fn query(&self, site: &str, quiestion_id: u64, max_count: usize) -> Vec<StackExchangeAnswer> {
        let answers_meta = self.query_meta(site, quiestion_id, max_count);

        let answer_ids = answers_meta.iter()
                                     .map(|a| a.answer_id.to_string())
                                     .collect::<Vec<_>>()
                                     .join(";");

        let url = format!("{}://{}/{}/answers/{}/?site={}&filter=withbody",
                          self.protocol,
                          self.api,
                          self.version,
                          answer_ids,
                          site);

        let response = self.request(&url);
        let mut answers = self.from_json(&response);
        answers.sort_by_key(|a| -a.score);

        answers
    }
}
<commit_msg>Removed unused arguments.<commit_after>mod data;

use std::io::Read;

use super::regex::Regex;

use super::hyper::Client;
use super::hyper::header::Connection;

use super::rustc_serialize::json;
use super::rustc_serialize::json::Json;

use super::flate2::read::GzDecoder;

use self::data::StackExchangeAnswer;

pub struct StackExchangeApi {
    api: &'static str,
    protocol: &'static str,
    version: &'static str,
    client: Client,
}

impl StackExchangeApi {
    pub fn new() -> Self {
        StackExchangeApi {
            api: "api.stackexchange.com",
            protocol: "https",
            version: "2.2",
            client: Client::new(),
        }
    }

    pub fn answers(&self, url: &str) -> Result<Vec<StackExchangeAnswer>, String> {
        let quiestion_id: &str;
        let site: &str;

        if url.contains("stackoverflow.com") {
            let re = Regex::new(r".*stackoverflow.com/questions/(?P<q_id>\d+)/").unwrap();
            let captures = re.captures(&url).unwrap();
            quiestion_id = captures.name("q_id").unwrap();
            site = "stackoverflow".into();
        } else if url.contains("stackexchange.com") {
            let re = Regex::new(r".*//(?P<site>.*).stackexchange.com/questions/(?P<q_id>\d+)/")
                         .unwrap();
            let captures = re.captures(&url).unwrap();
            quiestion_id = captures.name("q_id").unwrap();
            site = captures.name("site").unwrap();
        } else {
            return Err("Result is not related to StackExchange.".into());
        }

        let quiestion_id: u64 = quiestion_id.parse::<u64>().unwrap();

        Ok(self.query(site, quiestion_id))
    }

    fn request(&self, url: &str) -> String {
        let mut res = self.client
                          .get(url)
                          .header(Connection::close())
                          .send()
                          .unwrap();

        let mut buffer = Vec::new();
        res.read_to_end(&mut buffer).unwrap();

        let mut gzip_decoder = GzDecoder::new(buffer.as_slice()).unwrap();
        let mut body = String::new();
        gzip_decoder.read_to_string(&mut body).unwrap();

        body
    }

    fn from_json(&self, json_str: &str) -> Vec<StackExchangeAnswer> {
        let json = Json::from_str(&json_str).unwrap();
        let data = json.as_object().unwrap();
        let items = data.get("items").unwrap().as_array().unwrap();

        let answers: Vec<StackExchangeAnswer> = items.iter()
                                                     .map(|i| {
                                                         json::decode(&i.to_string()).unwrap()
                                                     })
                                                     .collect();

        answers
    }

    fn query_meta(&self,
                  site: &str,
                  quiestion_id: u64)
                  -> Vec<StackExchangeAnswer> {
        let url = format!("{}://{}/{}/questions/{}/answers?site={}",
                          self.protocol,
                          self.api,
                          self.version,
                          quiestion_id,
                          site);

        let response = self.request(&url);
        let mut answers = self.from_json(&response);

        let chosen_answers: Vec<_> = answers.drain(0..)
                                    .filter(|a| a.score > 0)
                                    .collect();
        chosen_answers
    }

    fn query(&self, site: &str, quiestion_id: u64) -> Vec<StackExchangeAnswer> {
        let answers_meta = self.query_meta(site, quiestion_id);

        let answer_ids = answers_meta.iter()
                                     .map(|a| a.answer_id.to_string())
                                     .collect::<Vec<_>>()
                                     .join(";");

        let url = format!("{}://{}/{}/answers/{}/?site={}&filter=withbody",
                          self.protocol,
                          self.api,
                          self.version,
                          answer_ids,
                          site);

        let response = self.request(&url);
        let answers = self.from_json(&response);

        answers
    }
}
<|endoftext|>


<commit_before>//! This module contains the implementation of a classic `PairHMM` as described in
//! Durbin, R., Eddy, S., Krogh, A., & Mitchison, G. (1998). Biological Sequence Analysis.
//! Current Topics in Genome Analysis 2008. http://doi.org/10.1017/CBO9780511790492.
//! It also contains a modified variant `HomopolyPairHMM` with additional homopolymer states suited
//! for dealing with homopolymer runs in sequencing as often encountered in Oxford Nanopore
//! sequencing data.
//!
//! Traits defined in this module apply to both `PairHMM` and `HomopolyPairHMM`.
//!
//! # Examples
//! ```
//! use approx::assert_relative_eq;
//! use bio::stats::pairhmm::{
//!     EmissionParameters, GapParameters, PairHMM, StartEndGapParameters, XYEmission,
//! };
//! use bio::stats::{LogProb, Prob};
//! use num_traits::Zero;
//!
//! // Two sequences for which we'd like to know if they are likely related.
//! let x = b"AAAA";
//! let y = b"AAAT";
//!
//! // For this example, we disallow gaps, so all probabilities are zero here.
//! struct GapParams;
//! impl GapParameters for GapParams {
//!     fn prob_gap_x(&self) -> LogProb {
//!         LogProb::zero()
//!     }
//!     fn prob_gap_y(&self) -> LogProb {
//!         LogProb::zero()
//!     }
//!     fn prob_gap_x_extend(&self) -> LogProb {
//!         LogProb::zero()
//!     }
//!     fn prob_gap_y_extend(&self) -> LogProb {
//!         LogProb::zero()
//!     }
//! }
//! let gap_params = GapParams;
//!
//! // The PairHMM instance stores the gap params, since these are constant.
//! let mut pairhmm = PairHMM::new(&gap_params);
//!
//! // However, emission parameters depend on the actual sequences
//! struct EmissionParams {
//!     x: &'static [u8],
//!     y: &'static [u8],
//! }
//!
//! const PROB_SUBSTITUTION: f64 = 0.1;
//! const PROB_NO_SUBSTITUION: f64 = 1. - PROB_SUBSTITUTION;
//! impl EmissionParameters for EmissionParams {
//!     fn prob_emit_xy(&self, i: usize, j: usize) -> XYEmission {
//!         if self.x[i] == self.y[j] {
//!             // if two bases match, emit a Match!
//!             XYEmission::Match(LogProb::from(Prob(PROB_NO_SUBSTITUION)))
//!         } else {
//!             // otherwise emit a Mismatch!
//!             // Note that the probability here is `mismatch / 3`, since probabilities should sum
//!             // to 1 and there are 3 possible mismatch configurations
//!             XYEmission::Mismatch(LogProb::from(Prob(PROB_SUBSTITUTION / 3.)))
//!         }
//!     }
//!
//!     // In this example, emitting x[i] is as likely as not observing a mismatch.
//!     // In more complex cases, this might e.g. depend on base qualities reported by the sequencer
//!     fn prob_emit_x(&self, i: usize) -> LogProb {
//!         LogProb::from(Prob(PROB_NO_SUBSTITUION))
//!     }
//!     fn prob_emit_y(&self, j: usize) -> LogProb {
//!         LogProb::from(Prob(PROB_NO_SUBSTITUION))
//!     }
//!
//!     fn len_x(&self) -> usize {
//!         self.x.len()
//!     }
//!     fn len_y(&self) -> usize {
//!         self.y.len()
//!     }
//! }
//!
//! // Since we want to do global alignment here, disallow free start and end gaps in x.
//! struct GlobalAlignmentMode;
//! impl StartEndGapParameters for GlobalAlignmentMode {
//!     fn free_start_gap_x(&self) -> bool {
//!         false
//!     }
//!     fn free_end_gap_x(&self) -> bool {
//!         false
//!     }
//! }
//!
//! // Finally calculate the probability of relatedness between x and y!
//! let prob_related = pairhmm.prob_related(&EmissionParams { x, y }, &GlobalAlignmentMode, None);
//!
//! // … and compare it to a rough estimation
//! let prob_expected = LogProb::from(Prob(PROB_NO_SUBSTITUION.powi(3) * PROB_SUBSTITUTION / 3.));
//! assert_relative_eq!(*prob_related, *prob_expected, epsilon = 1e-5);
//! ```
pub use homopolypairhmm::{HomopolyPairHMM, HopParameters};
pub use pairhmm::PairHMM;

use crate::stats::LogProb;

mod homopolypairhmm;
mod pairhmm;

// traits common to pairhmm implementations

/// Trait for parametrization of `PairHMM` emission behavior.
pub trait EmissionParameters {
    /// Emission probability for `(x[i], y[j])`.
    /// Returns a tuple with probability and a boolean indicating whether emissions match
    /// (e.g., are the same DNA alphabet letter).
    fn prob_emit_xy(&self, i: usize, j: usize) -> XYEmission;

    /// Emission probability for `(x[i], -)`.
    fn prob_emit_x(&self, i: usize) -> LogProb;

    /// Emission probability for `(-, y[j])`.
    fn prob_emit_y(&self, j: usize) -> LogProb;

    fn len_x(&self) -> usize;

    fn len_y(&self) -> usize;
}
/// Trait needed for the `HomopolyPairHMM`, because its implementation details
/// depend on the actual bases to distinguish between Match states.
pub trait Emission {
    /// Base emitted at `i` in sequence `x`.
    /// Should be one of b'A', b'C', b'G' or b'T'.
    fn emission_x(&self, i: usize) -> u8;
    /// Base emitted at `i` in sequence `y`.
    /// Should be one of b'A', b'C', b'G' or b'T'.
    fn emission_y(&self, j: usize) -> u8;
}

/// Trait for parametrization of `PairHMM` gap behavior.
pub trait GapParameters {
    /// Probability to open gap in x.
    fn prob_gap_x(&self) -> LogProb;

    /// Probability to open gap in y.
    fn prob_gap_y(&self) -> LogProb;

    /// Probability to extend gap in x.
    fn prob_gap_x_extend(&self) -> LogProb;

    /// Probability to extend gap in y.
    fn prob_gap_y_extend(&self) -> LogProb;
}

/// Trait for parametrization of `PairHMM` start and end gap behavior.
/// This trait can be used to implement global and semiglobal alignments.
///
/// * global: methods return `false` and `LogProb::ln_zero()`.
/// * semiglobal: methods return `true` and `LogProb::ln_one()`.
pub trait StartEndGapParameters {
    /// Probability to start at `x[i]`. This can be left unchanged if you use `free_start_gap_x` and
    /// `free_end_gap_x`.
    #[inline]
    #[allow(unused_variables)]
    fn prob_start_gap_x(&self, i: usize) -> LogProb {
        if self.free_start_gap_x() {
            LogProb::ln_one()
        } else {
            // For global alignment, this has to return 0.0.
            LogProb::ln_zero()
        }
    }

    /// Allow free start gap in x.
    fn free_start_gap_x(&self) -> bool;

    /// Allow free end gap in x.
    fn free_end_gap_x(&self) -> bool;
}

#[derive(Debug)]
pub enum XYEmission {
    Match(LogProb),
    Mismatch(LogProb),
}

impl XYEmission {
    pub fn prob(&self) -> LogProb {
        match *self {
            XYEmission::Match(p) => p,
            XYEmission::Mismatch(p) => p,
        }
    }

    pub fn is_match(&self) -> bool {
        match *self {
            XYEmission::Match(_) => true,
            XYEmission::Mismatch(_) => false,
        }
    }
}
<commit_msg>pub use BaseSpecificHopParameters, (#483)<commit_after>//! This module contains the implementation of a classic `PairHMM` as described in
//! Durbin, R., Eddy, S., Krogh, A., & Mitchison, G. (1998). Biological Sequence Analysis.
//! Current Topics in Genome Analysis 2008. http://doi.org/10.1017/CBO9780511790492.
//! It also contains a modified variant `HomopolyPairHMM` with additional homopolymer states suited
//! for dealing with homopolymer runs in sequencing as often encountered in Oxford Nanopore
//! sequencing data.
//!
//! Traits defined in this module apply to both `PairHMM` and `HomopolyPairHMM`.
//!
//! # Examples
//! ```
//! use approx::assert_relative_eq;
//! use bio::stats::pairhmm::{
//!     EmissionParameters, GapParameters, PairHMM, StartEndGapParameters, XYEmission,
//! };
//! use bio::stats::{LogProb, Prob};
//! use num_traits::Zero;
//!
//! // Two sequences for which we'd like to know if they are likely related.
//! let x = b"AAAA";
//! let y = b"AAAT";
//!
//! // For this example, we disallow gaps, so all probabilities are zero here.
//! struct GapParams;
//! impl GapParameters for GapParams {
//!     fn prob_gap_x(&self) -> LogProb {
//!         LogProb::zero()
//!     }
//!     fn prob_gap_y(&self) -> LogProb {
//!         LogProb::zero()
//!     }
//!     fn prob_gap_x_extend(&self) -> LogProb {
//!         LogProb::zero()
//!     }
//!     fn prob_gap_y_extend(&self) -> LogProb {
//!         LogProb::zero()
//!     }
//! }
//! let gap_params = GapParams;
//!
//! // The PairHMM instance stores the gap params, since these are constant.
//! let mut pairhmm = PairHMM::new(&gap_params);
//!
//! // However, emission parameters depend on the actual sequences
//! struct EmissionParams {
//!     x: &'static [u8],
//!     y: &'static [u8],
//! }
//!
//! const PROB_SUBSTITUTION: f64 = 0.1;
//! const PROB_NO_SUBSTITUION: f64 = 1. - PROB_SUBSTITUTION;
//! impl EmissionParameters for EmissionParams {
//!     fn prob_emit_xy(&self, i: usize, j: usize) -> XYEmission {
//!         if self.x[i] == self.y[j] {
//!             // if two bases match, emit a Match!
//!             XYEmission::Match(LogProb::from(Prob(PROB_NO_SUBSTITUION)))
//!         } else {
//!             // otherwise emit a Mismatch!
//!             // Note that the probability here is `mismatch / 3`, since probabilities should sum
//!             // to 1 and there are 3 possible mismatch configurations
//!             XYEmission::Mismatch(LogProb::from(Prob(PROB_SUBSTITUTION / 3.)))
//!         }
//!     }
//!
//!     // In this example, emitting x[i] is as likely as not observing a mismatch.
//!     // In more complex cases, this might e.g. depend on base qualities reported by the sequencer
//!     fn prob_emit_x(&self, i: usize) -> LogProb {
//!         LogProb::from(Prob(PROB_NO_SUBSTITUION))
//!     }
//!     fn prob_emit_y(&self, j: usize) -> LogProb {
//!         LogProb::from(Prob(PROB_NO_SUBSTITUION))
//!     }
//!
//!     fn len_x(&self) -> usize {
//!         self.x.len()
//!     }
//!     fn len_y(&self) -> usize {
//!         self.y.len()
//!     }
//! }
//!
//! // Since we want to do global alignment here, disallow free start and end gaps in x.
//! struct GlobalAlignmentMode;
//! impl StartEndGapParameters for GlobalAlignmentMode {
//!     fn free_start_gap_x(&self) -> bool {
//!         false
//!     }
//!     fn free_end_gap_x(&self) -> bool {
//!         false
//!     }
//! }
//!
//! // Finally calculate the probability of relatedness between x and y!
//! let prob_related = pairhmm.prob_related(&EmissionParams { x, y }, &GlobalAlignmentMode, None);
//!
//! // … and compare it to a rough estimation
//! let prob_expected = LogProb::from(Prob(PROB_NO_SUBSTITUION.powi(3) * PROB_SUBSTITUTION / 3.));
//! assert_relative_eq!(*prob_related, *prob_expected, epsilon = 1e-5);
//! ```
pub use homopolypairhmm::{BaseSpecificHopParameters, HomopolyPairHMM, HopParameters};
pub use pairhmm::PairHMM;

use crate::stats::LogProb;

mod homopolypairhmm;
mod pairhmm;

// traits common to pairhmm implementations

/// Trait for parametrization of `PairHMM` emission behavior.
pub trait EmissionParameters {
    /// Emission probability for `(x[i], y[j])`.
    /// Returns a tuple with probability and a boolean indicating whether emissions match
    /// (e.g., are the same DNA alphabet letter).
    fn prob_emit_xy(&self, i: usize, j: usize) -> XYEmission;

    /// Emission probability for `(x[i], -)`.
    fn prob_emit_x(&self, i: usize) -> LogProb;

    /// Emission probability for `(-, y[j])`.
    fn prob_emit_y(&self, j: usize) -> LogProb;

    fn len_x(&self) -> usize;

    fn len_y(&self) -> usize;
}
/// Trait needed for the `HomopolyPairHMM`, because its implementation details
/// depend on the actual bases to distinguish between Match states.
pub trait Emission {
    /// Base emitted at `i` in sequence `x`.
    /// Should be one of b'A', b'C', b'G' or b'T'.
    fn emission_x(&self, i: usize) -> u8;
    /// Base emitted at `i` in sequence `y`.
    /// Should be one of b'A', b'C', b'G' or b'T'.
    fn emission_y(&self, j: usize) -> u8;
}

/// Trait for parametrization of `PairHMM` gap behavior.
pub trait GapParameters {
    /// Probability to open gap in x.
    fn prob_gap_x(&self) -> LogProb;

    /// Probability to open gap in y.
    fn prob_gap_y(&self) -> LogProb;

    /// Probability to extend gap in x.
    fn prob_gap_x_extend(&self) -> LogProb;

    /// Probability to extend gap in y.
    fn prob_gap_y_extend(&self) -> LogProb;
}

/// Trait for parametrization of `PairHMM` start and end gap behavior.
/// This trait can be used to implement global and semiglobal alignments.
///
/// * global: methods return `false` and `LogProb::ln_zero()`.
/// * semiglobal: methods return `true` and `LogProb::ln_one()`.
pub trait StartEndGapParameters {
    /// Probability to start at `x[i]`. This can be left unchanged if you use `free_start_gap_x` and
    /// `free_end_gap_x`.
    #[inline]
    #[allow(unused_variables)]
    fn prob_start_gap_x(&self, i: usize) -> LogProb {
        if self.free_start_gap_x() {
            LogProb::ln_one()
        } else {
            // For global alignment, this has to return 0.0.
            LogProb::ln_zero()
        }
    }

    /// Allow free start gap in x.
    fn free_start_gap_x(&self) -> bool;

    /// Allow free end gap in x.
    fn free_end_gap_x(&self) -> bool;
}

#[derive(Debug)]
pub enum XYEmission {
    Match(LogProb),
    Mismatch(LogProb),
}

impl XYEmission {
    pub fn prob(&self) -> LogProb {
        match *self {
            XYEmission::Match(p) => p,
            XYEmission::Mismatch(p) => p,
        }
    }

    pub fn is_match(&self) -> bool {
        match *self {
            XYEmission::Match(_) => true,
            XYEmission::Mismatch(_) => false,
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2018 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use super::engine::{Engine, Error as EngineError, ScanMode, StatisticsSummary};
use super::metrics::*;
use super::mvcc::{MvccReader, MvccTxn};
use super::{Callback, Error, Key, Result};
use kvproto::kvrpcpb::Context;
use std::fmt::{self, Display, Formatter};
use std::mem;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use util::time::{duration_to_sec, SlowTimer};
use util::worker::{self, Builder, Runnable, ScheduleError, Worker};

// TODO: make it configurable.
pub const GC_BATCH_SIZE: usize = 512;

/// After the GC scan of a key, output a message to the log if there are at least this many
/// versions of the key.
const GC_LOG_FOUND_VERSION_THRESHOLD: usize = 30;

/// After the GC delete versions of a key, output a message to the log if at least this many
/// versions are deleted.
const GC_LOG_DELETED_VERSION_THRESHOLD: usize = 30;

pub const GC_MAX_PENDING_TASKS: usize = 2;
const GC_SNAPSHOT_TIMEOUT_SECS: u64 = 10;
const GC_TASK_SLOW_SECONDS: u64 = 30;

struct GCTask {
    pub ctx: Context,
    pub safe_point: u64,
    pub callback: Callback<()>,
}

impl Display for GCTask {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        let epoch = format!("{:?}", self.ctx.region_epoch.as_ref());
        f.debug_struct("GCTask")
            .field("region", &self.ctx.get_region_id())
            .field("epoch", &epoch)
            .field("safe_point", &self.safe_point)
            .finish()
    }
}

/// `GCRunner` is used to perform GC on the engine
struct GCRunner<E: Engine> {
    engine: E,
    ratio_threshold: f64,

    stats: StatisticsSummary,
}

impl<E: Engine> GCRunner<E> {
    pub fn new(engine: E, ratio_threshold: f64) -> GCRunner<E> {
        GCRunner {
            engine,
            ratio_threshold,

            stats: StatisticsSummary::default(),
        }
    }

    fn get_snapshot(&self, ctx: &mut Context) -> Result<E::Snap> {
        let timeout = Duration::from_secs(GC_SNAPSHOT_TIMEOUT_SECS);
        match wait_op!(|cb| self.engine.async_snapshot(ctx, cb), timeout) {
            Some((cb_ctx, Ok(snapshot))) => {
                if let Some(term) = cb_ctx.term {
                    ctx.set_term(term);
                }
                Ok(snapshot)
            }
            Some((_, Err(e))) => Err(e),
            None => Err(EngineError::Timeout(timeout)),
        }.map_err(Error::from)
    }

    /// Scan keys in the region. Returns scanned keys if any, and a key indicating scan progress
    fn scan_keys(
        &mut self,
        ctx: &mut Context,
        safe_point: u64,
        from: Option<Key>,
        limit: usize,
    ) -> Result<(Vec<Key>, Option<Key>)> {
        let snapshot = self.get_snapshot(ctx)?;
        let mut reader = MvccReader::new(
            snapshot,
            Some(ScanMode::Forward),
            !ctx.get_not_fill_cache(),
            None,
            None,
            ctx.get_isolation_level(),
        );

        // If from.is_some(), it must not be the first scan of the region.
        // So we must continue doing GC.
        let skip_gc = from.is_none() && !reader.need_gc(safe_point, self.ratio_threshold);
        let res = if skip_gc {
            KV_GC_SKIPPED_COUNTER.inc();
            Ok((vec![], None))
        } else {
            reader
                .scan_keys(from.clone(), limit)
                .map_err(Error::from)
                .and_then(|(keys, next)| {
                    if keys.is_empty() {
                        assert!(next.is_none());
                        if from.is_none() {
                            KV_GC_EMPTY_RANGE_COUNTER.inc();
                        }
                    }
                    Ok((keys, next))
                })
        };
        self.stats.add_statistics(reader.get_statistics());
        res
    }

    /// Clean up outdated data.
    fn gc_keys(
        &mut self,
        ctx: &mut Context,
        safe_point: u64,
        keys: Vec<Key>,
        mut next_scan_key: Option<Key>,
    ) -> Result<Option<Key>> {
        let snapshot = self.get_snapshot(ctx)?;
        let mut txn = MvccTxn::new(snapshot, 0, !ctx.get_not_fill_cache()).unwrap();
        for k in keys {
            // TODO: Duplicated code in scheduler.rs
            let gc_info = txn.gc(k.clone(), safe_point)?;

            if gc_info.found_versions >= GC_LOG_FOUND_VERSION_THRESHOLD {
                info!(
                    "[region {}] GC found at least {} versions for key {}",
                    ctx.get_region_id(),
                    gc_info.found_versions,
                    k
                );
            }
            // TODO: we may delete only part of the versions in a batch, which may not beyond
            // the logging threshold `GC_LOG_DELETED_VERSION_THRESHOLD`.
            if gc_info.deleted_versions as usize >= GC_LOG_DELETED_VERSION_THRESHOLD {
                info!(
                    "[region {}] GC deleted {} versions for key {}",
                    ctx.get_region_id(),
                    gc_info.deleted_versions,
                    k
                );
            }

            if !gc_info.is_completed {
                next_scan_key = Some(k);
                break;
            }
        }
        self.stats.add_statistics(&txn.take_statistics());

        let modifies = txn.into_modifies();
        if !modifies.is_empty() {
            self.engine.write(ctx, modifies)?;
        }
        Ok(next_scan_key)
    }

    pub fn gc(&mut self, ctx: &mut Context, safe_point: u64) -> Result<()> {
        debug!(
            "doing gc on region {}, safe_point {}",
            ctx.get_region_id(),
            safe_point
        );

        let mut next_key = None;
        loop {
            let (keys, next) = self
                .scan_keys(ctx, safe_point, next_key, GC_BATCH_SIZE)
                .map_err(|e| {
                    warn!("gc scan_keys failed on region {}: {:?}", safe_point, &e);
                    e
                })?;
            if keys.is_empty() {
                break;
            }

            next_key = self.gc_keys(ctx, safe_point, keys, next).map_err(|e| {
                warn!("gc_keys failed on region {}: {:?}", safe_point, &e);
                e
            })?;
            if next_key.is_none() {
                break;
            }
        }

        debug!(
            "gc on region {}, safe_point {} has finished",
            ctx.get_region_id(),
            safe_point
        );
        Ok(())
    }
}

impl<E: Engine> Runnable<GCTask> for GCRunner<E> {
    fn run(&mut self, mut task: GCTask) {
        GC_GCTASK_COUNTER.inc();

        let timer = SlowTimer::from_secs(GC_TASK_SLOW_SECONDS);
        let result = self.gc(&mut task.ctx, task.safe_point);
        GC_DURATION_HISTOGRAM.observe(duration_to_sec(timer.elapsed()));
        slow_log!(timer, "{}", task);

        if result.is_err() {
            GC_GCTASK_FAIL_COUNTER.inc();
        }
        (task.callback)(result);
    }

    // The default implementation of `run_batch` prints a warning to log when it takes over 1 second
    // to handle a task. It's not proper here, so override it to remove the log.
    fn run_batch(&mut self, tasks: &mut Vec<GCTask>) {
        for task in tasks.drain(..) {
            self.run(task);
        }
    }

    fn on_tick(&mut self) {
        let stats = mem::replace(&mut self.stats, StatisticsSummary::default());
        for (cf, details) in stats.stat.details() {
            for (tag, count) in details {
                GC_KEYS_COUNTER_VEC
                    .with_label_values(&[cf, tag])
                    .inc_by(count as i64);
            }
        }
    }
}

/// `GCWorker` is used to schedule GC operations
#[derive(Clone)]
pub struct GCWorker<E: Engine> {
    engine: E,
    ratio_threshold: f64,
    worker: Arc<Mutex<Worker<GCTask>>>,
    worker_scheduler: worker::Scheduler<GCTask>,
}

impl<E: Engine> GCWorker<E> {
    pub fn new(engine: E, ratio_threshold: f64) -> GCWorker<E> {
        let worker = Arc::new(Mutex::new(
            Builder::new("gc-worker")
                .pending_capacity(GC_MAX_PENDING_TASKS)
                .create(),
        ));
        let worker_scheduler = worker.lock().unwrap().scheduler();
        GCWorker {
            engine,
            ratio_threshold,
            worker,
            worker_scheduler,
        }
    }

    pub fn start(&self) -> Result<()> {
        let runner = GCRunner::new(self.engine.clone(), self.ratio_threshold);
        self.worker
            .lock()
            .unwrap()
            .start(runner)
            .map_err(|e| box_err!("failed to start gc_worker, err: {:?}", e))
    }

    pub fn stop(&self) -> Result<()> {
        let h = self.worker.lock().unwrap().stop().unwrap();
        if let Err(e) = h.join() {
            Err(box_err!("failed to join gc_worker handle, err: {:?}", e))
        } else {
            Ok(())
        }
    }

    pub fn async_gc(&self, ctx: Context, safe_point: u64, callback: Callback<()>) -> Result<()> {
        self.worker_scheduler
            .schedule(GCTask {
                ctx,
                safe_point,
                callback,
            })
            .or_else(|e| match e {
                ScheduleError::Full(task) => {
                    GC_TOO_BUSY_COUNTER.inc();
                    (task.callback)(Err(Error::GCWorkerTooBusy));
                    Ok(())
                }
                _ => Err(box_err!("failed to schedule gc task: {:?}", e)),
            })
    }
}
<commit_msg>reduce key clone in gc (#3558)<commit_after>// Copyright 2018 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use super::engine::{Engine, Error as EngineError, ScanMode, StatisticsSummary};
use super::metrics::*;
use super::mvcc::{MvccReader, MvccTxn};
use super::{Callback, Error, Key, Result};
use kvproto::kvrpcpb::Context;
use std::fmt::{self, Display, Formatter};
use std::mem;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use util::time::{duration_to_sec, SlowTimer};
use util::worker::{self, Builder, Runnable, ScheduleError, Worker};

// TODO: make it configurable.
pub const GC_BATCH_SIZE: usize = 512;

/// After the GC scan of a key, output a message to the log if there are at least this many
/// versions of the key.
const GC_LOG_FOUND_VERSION_THRESHOLD: usize = 30;

/// After the GC delete versions of a key, output a message to the log if at least this many
/// versions are deleted.
const GC_LOG_DELETED_VERSION_THRESHOLD: usize = 30;

pub const GC_MAX_PENDING_TASKS: usize = 2;
const GC_SNAPSHOT_TIMEOUT_SECS: u64 = 10;
const GC_TASK_SLOW_SECONDS: u64 = 30;

struct GCTask {
    pub ctx: Context,
    pub safe_point: u64,
    pub callback: Callback<()>,
}

impl Display for GCTask {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        let epoch = format!("{:?}", self.ctx.region_epoch.as_ref());
        f.debug_struct("GCTask")
            .field("region", &self.ctx.get_region_id())
            .field("epoch", &epoch)
            .field("safe_point", &self.safe_point)
            .finish()
    }
}

/// `GCRunner` is used to perform GC on the engine
struct GCRunner<E: Engine> {
    engine: E,
    ratio_threshold: f64,

    stats: StatisticsSummary,
}

impl<E: Engine> GCRunner<E> {
    pub fn new(engine: E, ratio_threshold: f64) -> GCRunner<E> {
        GCRunner {
            engine,
            ratio_threshold,

            stats: StatisticsSummary::default(),
        }
    }

    fn get_snapshot(&self, ctx: &mut Context) -> Result<E::Snap> {
        let timeout = Duration::from_secs(GC_SNAPSHOT_TIMEOUT_SECS);
        match wait_op!(|cb| self.engine.async_snapshot(ctx, cb), timeout) {
            Some((cb_ctx, Ok(snapshot))) => {
                if let Some(term) = cb_ctx.term {
                    ctx.set_term(term);
                }
                Ok(snapshot)
            }
            Some((_, Err(e))) => Err(e),
            None => Err(EngineError::Timeout(timeout)),
        }.map_err(Error::from)
    }

    /// Scan keys in the region. Returns scanned keys if any, and a key indicating scan progress
    fn scan_keys(
        &mut self,
        ctx: &mut Context,
        safe_point: u64,
        from: Option<Key>,
        limit: usize,
    ) -> Result<(Vec<Key>, Option<Key>)> {
        let snapshot = self.get_snapshot(ctx)?;
        let mut reader = MvccReader::new(
            snapshot,
            Some(ScanMode::Forward),
            !ctx.get_not_fill_cache(),
            None,
            None,
            ctx.get_isolation_level(),
        );

        let is_range_start = from.is_none();

        // range start gc with from == None, and this is an optimization to
        // skip gc before scanning all data.
        let skip_gc = is_range_start && !reader.need_gc(safe_point, self.ratio_threshold);
        let res = if skip_gc {
            KV_GC_SKIPPED_COUNTER.inc();
            Ok((vec![], None))
        } else {
            reader
                .scan_keys(from, limit)
                .map_err(Error::from)
                .and_then(|(keys, next)| {
                    if keys.is_empty() {
                        assert!(next.is_none());
                        if is_range_start {
                            KV_GC_EMPTY_RANGE_COUNTER.inc();
                        }
                    }
                    Ok((keys, next))
                })
        };
        self.stats.add_statistics(reader.get_statistics());
        res
    }

    /// Clean up outdated data.
    fn gc_keys(
        &mut self,
        ctx: &mut Context,
        safe_point: u64,
        keys: Vec<Key>,
        mut next_scan_key: Option<Key>,
    ) -> Result<Option<Key>> {
        let snapshot = self.get_snapshot(ctx)?;
        let mut txn = MvccTxn::new(snapshot, 0, !ctx.get_not_fill_cache()).unwrap();
        for k in keys {
            let gc_info = txn.gc(k.clone(), safe_point)?;

            if gc_info.found_versions >= GC_LOG_FOUND_VERSION_THRESHOLD {
                info!(
                    "[region {}] GC found at least {} versions for key {}",
                    ctx.get_region_id(),
                    gc_info.found_versions,
                    k
                );
            }
            // TODO: we may delete only part of the versions in a batch, which may not beyond
            // the logging threshold `GC_LOG_DELETED_VERSION_THRESHOLD`.
            if gc_info.deleted_versions as usize >= GC_LOG_DELETED_VERSION_THRESHOLD {
                info!(
                    "[region {}] GC deleted {} versions for key {}",
                    ctx.get_region_id(),
                    gc_info.deleted_versions,
                    k
                );
            }

            if !gc_info.is_completed {
                next_scan_key = Some(k);
                break;
            }
        }
        self.stats.add_statistics(&txn.take_statistics());

        let modifies = txn.into_modifies();
        if !modifies.is_empty() {
            self.engine.write(ctx, modifies)?;
        }
        Ok(next_scan_key)
    }

    pub fn gc(&mut self, ctx: &mut Context, safe_point: u64) -> Result<()> {
        debug!(
            "doing gc on region {}, safe_point {}",
            ctx.get_region_id(),
            safe_point
        );

        let mut next_key = None;
        loop {
            let (keys, next) = self
                .scan_keys(ctx, safe_point, next_key, GC_BATCH_SIZE)
                .map_err(|e| {
                    warn!("gc scan_keys failed on region {}: {:?}", safe_point, &e);
                    e
                })?;
            if keys.is_empty() {
                break;
            }

            next_key = self.gc_keys(ctx, safe_point, keys, next).map_err(|e| {
                warn!("gc_keys failed on region {}: {:?}", safe_point, &e);
                e
            })?;
            if next_key.is_none() {
                break;
            }
        }

        debug!(
            "gc on region {}, safe_point {} has finished",
            ctx.get_region_id(),
            safe_point
        );
        Ok(())
    }
}

impl<E: Engine> Runnable<GCTask> for GCRunner<E> {
    fn run(&mut self, mut task: GCTask) {
        GC_GCTASK_COUNTER.inc();

        let timer = SlowTimer::from_secs(GC_TASK_SLOW_SECONDS);
        let result = self.gc(&mut task.ctx, task.safe_point);
        GC_DURATION_HISTOGRAM.observe(duration_to_sec(timer.elapsed()));
        slow_log!(timer, "{}", task);

        if result.is_err() {
            GC_GCTASK_FAIL_COUNTER.inc();
        }
        (task.callback)(result);
    }

    // The default implementation of `run_batch` prints a warning to log when it takes over 1 second
    // to handle a task. It's not proper here, so override it to remove the log.
    fn run_batch(&mut self, tasks: &mut Vec<GCTask>) {
        for task in tasks.drain(..) {
            self.run(task);
        }
    }

    fn on_tick(&mut self) {
        let stats = mem::replace(&mut self.stats, StatisticsSummary::default());
        for (cf, details) in stats.stat.details() {
            for (tag, count) in details {
                GC_KEYS_COUNTER_VEC
                    .with_label_values(&[cf, tag])
                    .inc_by(count as i64);
            }
        }
    }
}

/// `GCWorker` is used to schedule GC operations
#[derive(Clone)]
pub struct GCWorker<E: Engine> {
    engine: E,
    ratio_threshold: f64,
    worker: Arc<Mutex<Worker<GCTask>>>,
    worker_scheduler: worker::Scheduler<GCTask>,
}

impl<E: Engine> GCWorker<E> {
    pub fn new(engine: E, ratio_threshold: f64) -> GCWorker<E> {
        let worker = Arc::new(Mutex::new(
            Builder::new("gc-worker")
                .pending_capacity(GC_MAX_PENDING_TASKS)
                .create(),
        ));
        let worker_scheduler = worker.lock().unwrap().scheduler();
        GCWorker {
            engine,
            ratio_threshold,
            worker,
            worker_scheduler,
        }
    }

    pub fn start(&self) -> Result<()> {
        let runner = GCRunner::new(self.engine.clone(), self.ratio_threshold);
        self.worker
            .lock()
            .unwrap()
            .start(runner)
            .map_err(|e| box_err!("failed to start gc_worker, err: {:?}", e))
    }

    pub fn stop(&self) -> Result<()> {
        let h = self.worker.lock().unwrap().stop().unwrap();
        if let Err(e) = h.join() {
            Err(box_err!("failed to join gc_worker handle, err: {:?}", e))
        } else {
            Ok(())
        }
    }

    pub fn async_gc(&self, ctx: Context, safe_point: u64, callback: Callback<()>) -> Result<()> {
        self.worker_scheduler
            .schedule(GCTask {
                ctx,
                safe_point,
                callback,
            })
            .or_else(|e| match e {
                ScheduleError::Full(task) => {
                    GC_TOO_BUSY_COUNTER.inc();
                    (task.callback)(Err(Error::GCWorkerTooBusy));
                    Ok(())
                }
                _ => Err(box_err!("failed to schedule gc task: {:?}", e)),
            })
    }
}
<|endoftext|>


<commit_before>// Dogfood cannot run on Windows
#![cfg(not(windows))]
#![feature(once_cell)]

use std::lazy::SyncLazy;
use std::path::PathBuf;
use std::process::Command;

mod cargo;

static CLIPPY_PATH: SyncLazy<PathBuf> = SyncLazy::new(|| cargo::TARGET_LIB.join("cargo-clippy"));

#[test]
fn dogfood_clippy() {
    // run clippy on itself and fail the test if lint warnings are reported
    if cargo::is_rustc_test_suite() {
        return;
    }
    let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));

    let mut command = Command::new(&*CLIPPY_PATH);
    command
        .current_dir(root_dir)
        .env("CLIPPY_DOGFOOD", "1")
        .env("CARGO_INCREMENTAL", "0")
        .arg("clippy-preview")
        .arg("--all-targets")
        .arg("--all-features")
        .args(&["-p", "clippy_lints", "-p", "clippy_utils", "-p", "rustc_tools_util"])
        .arg("--")
        .args(&["-D", "clippy::all"])
        .args(&["-D", "clippy::pedantic"])
        .arg("-Cdebuginfo=0"); // disable debuginfo to generate less data in the target dir

    // internal lints only exist if we build with the internal-lints feature
    if cfg!(feature = "internal-lints") {
        command.args(&["-D", "clippy::internal"]);
    }

    let output = command.output().unwrap();

    println!("status: {}", output.status);
    println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
    println!("stderr: {}", String::from_utf8_lossy(&output.stderr));

    assert!(output.status.success());
}

#[test]
fn dogfood_subprojects() {
    fn test_no_deps_ignores_path_deps_in_workspaces() {
        if cargo::is_rustc_test_suite() {
            return;
        }
        let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
        let target_dir = root.join("target").join("dogfood");
        let cwd = root.join("clippy_workspace_tests");

        // Make sure we start with a clean state
        Command::new("cargo")
            .current_dir(&cwd)
            .env("CARGO_TARGET_DIR", &target_dir)
            .arg("clean")
            .args(&["-p", "subcrate"])
            .args(&["-p", "path_dep"])
            .output()
            .unwrap();

        // `path_dep` is a path dependency of `subcrate` that would trigger a denied lint.
        // Make sure that with the `--no-deps` argument Clippy does not run on `path_dep`.
        let output = Command::new(&*CLIPPY_PATH)
            .current_dir(&cwd)
            .env("CLIPPY_DOGFOOD", "1")
            .env("CARGO_INCREMENTAL", "0")
            .arg("clippy")
            .args(&["-p", "subcrate"])
            .arg("--")
            .arg("--no-deps")
            .arg("-Cdebuginfo=0") // disable debuginfo to generate less data in the target dir
            .args(&["--cfg", r#"feature="primary_package_test""#])
            .output()
            .unwrap();
        println!("status: {}", output.status);
        println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
        println!("stderr: {}", String::from_utf8_lossy(&output.stderr));

        assert!(output.status.success());

        let lint_path_dep = || {
            // Test that without the `--no-deps` argument, `path_dep` is linted.
            let output = Command::new(&*CLIPPY_PATH)
                .current_dir(&cwd)
                .env("CLIPPY_DOGFOOD", "1")
                .env("CARGO_INCREMENTAL", "0")
                .arg("clippy")
                .args(&["-p", "subcrate"])
                .arg("--")
                .arg("-Cdebuginfo=0") // disable debuginfo to generate less data in the target dir
                .args(&["--cfg", r#"feature="primary_package_test""#])
                .output()
                .unwrap();
            println!("status: {}", output.status);
            println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
            println!("stderr: {}", String::from_utf8_lossy(&output.stderr));

            assert!(!output.status.success());
            assert!(
                String::from_utf8(output.stderr)
                    .unwrap()
                    .contains("error: empty `loop {}` wastes CPU cycles")
            );
        };

        // Make sure Cargo is aware of the removal of `--no-deps`.
        lint_path_dep();

        let successful_build = || {
            let output = Command::new(&*CLIPPY_PATH)
                .current_dir(&cwd)
                .env("CLIPPY_DOGFOOD", "1")
                .env("CARGO_INCREMENTAL", "0")
                .arg("clippy")
                .args(&["-p", "subcrate"])
                .arg("--")
                .arg("-Cdebuginfo=0") // disable debuginfo to generate less data in the target dir
                .output()
                .unwrap();
            println!("status: {}", output.status);
            println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
            println!("stderr: {}", String::from_utf8_lossy(&output.stderr));

            assert!(output.status.success());

            output
        };

        // Trigger a sucessful build, so Cargo would like to cache the build result.
        successful_build();

        // Make sure there's no spurious rebuild when nothing changes.
        let stderr = String::from_utf8(successful_build().stderr).unwrap();
        assert!(!stderr.contains("Compiling"));
        assert!(!stderr.contains("Checking"));
        assert!(stderr.contains("Finished"));

        // Make sure Cargo is aware of the new `--cfg` flag.
        lint_path_dep();
    }

    // run clippy on remaining subprojects and fail the test if lint warnings are reported
    if cargo::is_rustc_test_suite() {
        return;
    }
    let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));

    // NOTE: `path_dep` crate is omitted on purpose here
    for d in &[
        "clippy_workspace_tests",
        "clippy_workspace_tests/src",
        "clippy_workspace_tests/subcrate",
        "clippy_workspace_tests/subcrate/src",
        "clippy_dev",
        "rustc_tools_util",
    ] {
        let output = Command::new(&*CLIPPY_PATH)
            .current_dir(root_dir.join(d))
            .env("CLIPPY_DOGFOOD", "1")
            .env("CARGO_INCREMENTAL", "0")
            .arg("clippy")
            .arg("--")
            .args(&["-D", "clippy::all"])
            .args(&["-D", "clippy::pedantic"])
            .arg("-Cdebuginfo=0") // disable debuginfo to generate less data in the target dir
            .output()
            .unwrap();
        println!("status: {}", output.status);
        println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
        println!("stderr: {}", String::from_utf8_lossy(&output.stderr));

        assert!(output.status.success());
    }

    // NOTE: Since tests run in parallel we can't run cargo commands on the same workspace at the
    // same time, so we test this immediately after the dogfood for workspaces.
    test_no_deps_ignores_path_deps_in_workspaces();
}
<commit_msg>Move `test_no_deps_ignores_path_deps_in_workspaces()` out of `dogfood_subprojects()`<commit_after>// Dogfood cannot run on Windows
#![cfg(not(windows))]
#![feature(once_cell)]

use std::lazy::SyncLazy;
use std::path::PathBuf;
use std::process::Command;

mod cargo;

static CLIPPY_PATH: SyncLazy<PathBuf> = SyncLazy::new(|| cargo::TARGET_LIB.join("cargo-clippy"));

#[test]
fn dogfood_clippy() {
    // run clippy on itself and fail the test if lint warnings are reported
    if cargo::is_rustc_test_suite() {
        return;
    }
    let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));

    let mut command = Command::new(&*CLIPPY_PATH);
    command
        .current_dir(root_dir)
        .env("CLIPPY_DOGFOOD", "1")
        .env("CARGO_INCREMENTAL", "0")
        .arg("clippy-preview")
        .arg("--all-targets")
        .arg("--all-features")
        .args(&["-p", "clippy_lints", "-p", "clippy_utils", "-p", "rustc_tools_util"])
        .arg("--")
        .args(&["-D", "clippy::all"])
        .args(&["-D", "clippy::pedantic"])
        .arg("-Cdebuginfo=0"); // disable debuginfo to generate less data in the target dir

    // internal lints only exist if we build with the internal-lints feature
    if cfg!(feature = "internal-lints") {
        command.args(&["-D", "clippy::internal"]);
    }

    let output = command.output().unwrap();

    println!("status: {}", output.status);
    println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
    println!("stderr: {}", String::from_utf8_lossy(&output.stderr));

    assert!(output.status.success());
}

fn test_no_deps_ignores_path_deps_in_workspaces() {
    if cargo::is_rustc_test_suite() {
        return;
    }
    let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
    let target_dir = root.join("target").join("dogfood");
    let cwd = root.join("clippy_workspace_tests");

    // Make sure we start with a clean state
    Command::new("cargo")
        .current_dir(&cwd)
        .env("CARGO_TARGET_DIR", &target_dir)
        .arg("clean")
        .args(&["-p", "subcrate"])
        .args(&["-p", "path_dep"])
        .output()
        .unwrap();

    // `path_dep` is a path dependency of `subcrate` that would trigger a denied lint.
    // Make sure that with the `--no-deps` argument Clippy does not run on `path_dep`.
    let output = Command::new(&*CLIPPY_PATH)
        .current_dir(&cwd)
        .env("CLIPPY_DOGFOOD", "1")
        .env("CARGO_INCREMENTAL", "0")
        .arg("clippy")
        .args(&["-p", "subcrate"])
        .arg("--")
        .arg("--no-deps")
        .arg("-Cdebuginfo=0") // disable debuginfo to generate less data in the target dir
        .args(&["--cfg", r#"feature="primary_package_test""#])
        .output()
        .unwrap();
    println!("status: {}", output.status);
    println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
    println!("stderr: {}", String::from_utf8_lossy(&output.stderr));

    assert!(output.status.success());

    let lint_path_dep = || {
        // Test that without the `--no-deps` argument, `path_dep` is linted.
        let output = Command::new(&*CLIPPY_PATH)
            .current_dir(&cwd)
            .env("CLIPPY_DOGFOOD", "1")
            .env("CARGO_INCREMENTAL", "0")
            .arg("clippy")
            .args(&["-p", "subcrate"])
            .arg("--")
            .arg("-Cdebuginfo=0") // disable debuginfo to generate less data in the target dir
            .args(&["--cfg", r#"feature="primary_package_test""#])
            .output()
            .unwrap();
        println!("status: {}", output.status);
        println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
        println!("stderr: {}", String::from_utf8_lossy(&output.stderr));

        assert!(!output.status.success());
        assert!(
            String::from_utf8(output.stderr)
                .unwrap()
                .contains("error: empty `loop {}` wastes CPU cycles")
        );
    };

    // Make sure Cargo is aware of the removal of `--no-deps`.
    lint_path_dep();

    let successful_build = || {
        let output = Command::new(&*CLIPPY_PATH)
            .current_dir(&cwd)
            .env("CLIPPY_DOGFOOD", "1")
            .env("CARGO_INCREMENTAL", "0")
            .arg("clippy")
            .args(&["-p", "subcrate"])
            .arg("--")
            .arg("-Cdebuginfo=0") // disable debuginfo to generate less data in the target dir
            .output()
            .unwrap();
        println!("status: {}", output.status);
        println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
        println!("stderr: {}", String::from_utf8_lossy(&output.stderr));

        assert!(output.status.success());

        output
    };

    // Trigger a sucessful build, so Cargo would like to cache the build result.
    successful_build();

    // Make sure there's no spurious rebuild when nothing changes.
    let stderr = String::from_utf8(successful_build().stderr).unwrap();
    assert!(!stderr.contains("Compiling"));
    assert!(!stderr.contains("Checking"));
    assert!(stderr.contains("Finished"));

    // Make sure Cargo is aware of the new `--cfg` flag.
    lint_path_dep();
}

#[test]
fn dogfood_subprojects() {
    // run clippy on remaining subprojects and fail the test if lint warnings are reported
    if cargo::is_rustc_test_suite() {
        return;
    }
    let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));

    // NOTE: `path_dep` crate is omitted on purpose here
    for d in &[
        "clippy_workspace_tests",
        "clippy_workspace_tests/src",
        "clippy_workspace_tests/subcrate",
        "clippy_workspace_tests/subcrate/src",
        "clippy_dev",
        "rustc_tools_util",
    ] {
        let output = Command::new(&*CLIPPY_PATH)
            .current_dir(root_dir.join(d))
            .env("CLIPPY_DOGFOOD", "1")
            .env("CARGO_INCREMENTAL", "0")
            .arg("clippy")
            .arg("--")
            .args(&["-D", "clippy::all"])
            .args(&["-D", "clippy::pedantic"])
            .arg("-Cdebuginfo=0") // disable debuginfo to generate less data in the target dir
            .output()
            .unwrap();
        println!("status: {}", output.status);
        println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
        println!("stderr: {}", String::from_utf8_lossy(&output.stderr));

        assert!(output.status.success());
    }

    // NOTE: Since tests run in parallel we can't run cargo commands on the same workspace at the
    // same time, so we test this immediately after the dogfood for workspaces.
    test_no_deps_ignores_path_deps_in_workspaces();
}
<|endoftext|>


<commit_before>use hyper;
use serde;

// use Error;
// use error::TransportErrorKind;
// use hyper;
// use serde;
// use serde_json;
// use std;
// use std::io::prelude::*;
//
// #[derive(Debug, PartialEq)]
// pub struct Request {
//     method: hyper::method::Method,
//     path: Vec<String>,
//     query: std::collections::HashMap<String, String>,
//     headers: hyper::header::Headers,
//     body: Body,
// }
//
// impl RequestBuilder {
// }
//
// #[derive(Debug, PartialEq)]
// enum Body {
//     None,
//     Blob(Vec<u8>),
//     Json(JsonBytes),
// }
//
// impl Body {
//     fn into_bytes(self) -> Vec<u8> {}
// }
//
// #[derive(Debug)]
// struct JsonBytes(Vec<u8>);
//
// impl PartialEq for JsonBytes {
//     fn eq(&self, other: &Self) -> bool {
//
//         let &JsonBytes(ref ours) = self;
//         let &JsonBytes(ref theirs) = other;
//
//         if ours.is_empty() && theirs.is_empty() {
//             return true;
//         } else if ours.is_empty() || theirs.is_empty() {
//             return false;
//         }
//
//         let ours: serde_json::Value = serde_json::from_reader(&ours[..]).unwrap();
//         let theirs: serde_json::Value = serde_json::from_reader(&theirs[..]).unwrap();
//
//         ours == theirs
//     }
// }
//
// #[cfg(test)]
// mod tests {
//
//     use Error;
//     use hyper;
//     use serde_json;
//     use std;
//     use super::{Body, JsonBytes, Request, RequestBuilder};
//
//     #[test]
//     fn json_bytes_eq_both_empty() {
//         let a = JsonBytes(Vec::new());
//         let b = JsonBytes(Vec::new());
//         assert_eq!(a, b);
//     }
//
//     #[test]
//     fn json_bytes_eq_one_empty() {
//         let a = JsonBytes(Vec::new());
//         let b = JsonBytes(vec!['{' as u8, '}' as u8]);
//         assert!(a != b);
//         assert!(b != a);
//     }
//
//     #[test]
//     fn json_bytes_eq_nonempty_same() {
//         let a = JsonBytes(String::from("[42, 17]").into());
//         let b = JsonBytes(String::from("[42, 17]").into());
//         assert_eq!(a, b);
//     }
//
//     #[test]
//     fn json_bytes_eq_nonempty_different() {
//         let a = JsonBytes(String::from("[42, 17]").into());
//         let b = JsonBytes(String::from(r#"{"foo": 42}"#).into());
//         assert!(a != b);
//     }
//
//     #[test]
//     fn request_builder_default() {
//
//         let path = vec![String::from("foo"), String::from("bar")];
//
//         let expected = Request {
//             method: hyper::Post,
//             path: path.clone(),
//             query: std::collections::HashMap::new(),
//             headers: hyper::header::Headers::new(),
//             body: Body::None,
//         };
//
//         let got = RequestBuilder::new(hyper::Post, path).unwrap();
//         assert_eq!(expected, got);
//     }
//
//     #[test]
//     fn response_builder_default() {
//
//         let expected = Response {
//             status_code: hyper::BadRequest,
//             headers: hyper::header::Headers::new(),
//             body: Body::None,
//         };
//
//         let got = ResponseBuilder::new(hyper::BadRequest).unwrap();
//         assert_eq!(expected, got);
//     }
//
//     #[test]
//     fn response_builder_with_json_body() {
//
//         let expected = Response {
//             status_code: hyper::Ok,
//             headers: {
//                 let mut headers = hyper::header::Headers::new();
//                 headers.set(hyper::header::ContentType(mime!(Application / Json)));
//                 headers
//             },
//             body: Body::Json(JsonBytes(serde_json::to_vec(&serde_json::builder::ObjectBuilder::new()
//                                           .insert("foo", 42)
//                                           .unwrap())
//                       .unwrap())),
//         };
//
//         let got = ResponseBuilder::new(hyper::Ok)
//                       .with_json_body(&serde_json::builder::ObjectBuilder::new()
//                                            .insert("foo", 42)
//                                            .unwrap())
//                       .unwrap();
//
//         assert_eq!(expected, got);
//     }
//
//     #[test]
//     fn response_decode_json_body_ok() {
//
//         let body = serde_json::builder::ObjectBuilder::new()
//                        .insert("foo", 42)
//                        .unwrap();
//
//         let response = ResponseBuilder::new(hyper::Ok)
//                            .with_json_body(&body)
//                            .unwrap();
//
//         let expected = body;
//         let got: serde_json::Value = response.decode_json_body().unwrap();
//         assert_eq!(expected, got);
//     }
//
//     #[test]
//     fn response_decode_json_body_nok_no_content_type() {
//
//         let response = ResponseBuilder::new(hyper::Ok).unwrap();
//         let got = response.decode_json_body::<serde_json::Value>();
//
//         match got {
//             Err(Error::ResponseNotJson(None)) => (),
//             x @ _ => unexpected_result!(x),
//         }
//     }
//
//     #[test]
//     fn response_decode_json_body_nok_content_type_not_json() {
//
//         use hyper::header::ContentType;
//         use mime::{Mime, TopLevel, SubLevel};
//
//         let response = {
//             let mut response = ResponseBuilder::new(hyper::Ok).unwrap();
//             response.headers.set(ContentType(mime!(Text / Plain)));
//             response
//         };
//
//         let got = response.decode_json_body::<serde_json::Value>();
//
//         match got {
//             Err(Error::ResponseNotJson(Some(Mime(TopLevel::Text, SubLevel::Plain, _)))) => (),
//             x @ _ => unexpected_result!(x),
//         }
//     }
// }
<commit_msg>Delete unused source file: transport/request.rs<commit_after><|endoftext|>


<commit_before>use std::mem;
use std::ptr;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::Duration;

use winapi::um::winsock2::{
    fd_set, select, timeval, WSACleanup, WSAStartup, SOCKET_ERROR, WSADATA,
};

use c_ares;

use crate::error::Error;

// The EventLoop will use select() to check on the status of file descriptors that c-ares cares
// about.
pub struct EventLoop {
    pub ares_channel: Arc<Mutex<c_ares::Channel>>,
    quit: Arc<AtomicBool>,
}

impl EventLoop {
    // Create a new event loop.
    pub fn new(options: c_ares::Options) -> Result<Self, Error> {
        // Initialize sockets.
        unsafe {
            let mut wsadata: WSADATA = mem::uninitialized();
            WSAStartup(0x101, &mut wsadata);
        }

        // Create the c-ares channel.
        let ares_channel = c_ares::Channel::with_options(options)?;
        let locked_channel = Arc::new(Mutex::new(ares_channel));

        // Create and return the event loop.
        let event_loop = EventLoop {
            ares_channel: locked_channel,
            quit: Arc::new(AtomicBool::new(false)),
        };
        Ok(event_loop)
    }

    // Run the event loop.
    pub fn run(self) -> Arc<AtomicBool> {
        let stopper = Arc::clone(&self.quit);
        thread::spawn(|| self.event_loop_thread());
        stopper
    }

    // Event loop thread - waits for events, and handles them.
    fn event_loop_thread(self) {
        let mut read_fds: fd_set = unsafe { mem::uninitialized() };
        let mut write_fds: fd_set = unsafe { mem::uninitialized() };

        // Loop round, asking c-ares what it cares about and doing as asked.
        loop {
            read_fds.fd_count = 0;
            write_fds.fd_count = 0;
            let count = self
                .ares_channel
                .lock()
                .unwrap()
                .fds(&mut read_fds, &mut write_fds);

            if count == 0 {
                thread::sleep(Duration::from_millis(100));
            } else {
                let timeout = timeval {
                    tv_sec: 0,
                    tv_usec: 100_000,
                };
                let results =
                    unsafe { select(0, &mut read_fds, &mut write_fds, ptr::null_mut(), &timeout) };

                // Process whatever happened.
                match results {
                    SOCKET_ERROR => panic!("Socket error"),
                    _ => self
                        .ares_channel
                        .lock()
                        .unwrap()
                        .process(&mut read_fds, &mut write_fds),
                }
            }
            if self.quit.load(Ordering::Relaxed) {
                break;
            }
        }
    }
}

impl Drop for EventLoop {
    fn drop(&mut self) {
        unsafe {
            WSACleanup();
        }
    }
}
<commit_msg>MaybeUninit<commit_after>use std::mem;
use std::ptr;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::Duration;

use winapi::um::winsock2::{
    fd_set, select, timeval, WSACleanup, WSAStartup, SOCKET_ERROR, WSADATA,
};

use c_ares;

use crate::error::Error;

// The EventLoop will use select() to check on the status of file descriptors that c-ares cares
// about.
pub struct EventLoop {
    pub ares_channel: Arc<Mutex<c_ares::Channel>>,
    quit: Arc<AtomicBool>,
}

impl EventLoop {
    // Create a new event loop.
    pub fn new(options: c_ares::Options) -> Result<Self, Error> {
        // Initialize sockets.
        unsafe {
            let mut wsadata = mem::MaybeUninit::<WSADATA>::uninit();
            WSAStartup(0x101, wsadata.as_mut_ptr());
        }

        // Create the c-ares channel.
        let ares_channel = c_ares::Channel::with_options(options)?;
        let locked_channel = Arc::new(Mutex::new(ares_channel));

        // Create and return the event loop.
        let event_loop = EventLoop {
            ares_channel: locked_channel,
            quit: Arc::new(AtomicBool::new(false)),
        };
        Ok(event_loop)
    }

    // Run the event loop.
    pub fn run(self) -> Arc<AtomicBool> {
        let stopper = Arc::clone(&self.quit);
        thread::spawn(|| self.event_loop_thread());
        stopper
    }

    // Event loop thread - waits for events, and handles them.
    fn event_loop_thread(self) {
        let mut read_fds: fd_set = unsafe { mem::MaybeUninit::zeroed().assume_init() };
        let mut write_fds: fd_set = unsafe { mem::MaybeUninit::zeroed().assume_init() };

        // Loop round, asking c-ares what it cares about and doing as asked.
        loop {
            read_fds.fd_count = 0;
            write_fds.fd_count = 0;
            let count = self
                .ares_channel
                .lock()
                .unwrap()
                .fds(&mut read_fds, &mut write_fds);

            if count == 0 {
                thread::sleep(Duration::from_millis(100));
            } else {
                let timeout = timeval {
                    tv_sec: 0,
                    tv_usec: 100_000,
                };
                let results =
                    unsafe { select(0, &mut read_fds, &mut write_fds, ptr::null_mut(), &timeout) };

                // Process whatever happened.
                match results {
                    SOCKET_ERROR => panic!("Socket error"),
                    _ => self
                        .ares_channel
                        .lock()
                        .unwrap()
                        .process(&mut read_fds, &mut write_fds),
                }
            }
            if self.quit.load(Ordering::Relaxed) {
                break;
            }
        }
    }
}

impl Drop for EventLoop {
    fn drop(&mut self) {
        unsafe {
            WSACleanup();
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::error::Error;
use std::env;
use std::fs;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;

fn main() {
    let arg = env::args().nth(1).unwrap_or_else(|| {
        println!("Please pass a src directory as the first argument");
        std::process::exit(1);
    });

    match check_directory(&Path::new(&arg)) {
        Ok(()) => println!("passed!"),
        Err(e) => {
            println!("Error: {}", e);
            std::process::exit(1);
        }
    }
}

fn check_directory(dir: &Path) -> Result<(), Box<Error>> {
    for entry in fs::read_dir(dir)? {
        let entry = entry?;
        let path = entry.path();

        if path.is_dir() {
            continue;
        }

        let mut file = File::open(&path)?;
        let mut contents = String::new();
        file.read_to_string(&mut contents)?;

        if contents.contains("#![feature") {
            // attributes.md contains this and it is legitimate
            if !contents.contains("#![feature(feature1, feature2, feature3)]") {
                return Err(From::from(format!("Feature flag found in {:?}", path)));
            }
        }
    }

    Ok(())
}
<commit_msg>Removed copyright notices & made cosmetic improvements.<commit_after>use std::error::Error;
use std::env;
use std::fs;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;

fn main() {
    let arg = env::args().nth(1).unwrap_or_else(|| {
        println!("Please pass a src directory as the first argument");
        std::process::exit(1);
    });

    match check_directory(&Path::new(&arg)) {
        Ok(()) => println!("passed!"),
        Err(e) => {
            println!("Error: {}", e);
            std::process::exit(1);
        }
    }
}

fn check_directory(dir: &Path) -> Result<(), Box<Error>> {
    for entry in fs::read_dir(dir)? {
        let entry = entry?;
        let path = entry.path();

        if path.is_dir() {
            continue;
        }

        let mut file = File::open(&path)?;
        let mut contents = String::new();
        file.read_to_string(&mut contents)?;

        if contents.contains("#![feature") {
            // `attributes.md` contains this and it is legitimate.
            if !contents.contains("#![feature(feature1, feature2, feature3)]") {
                return Err(From::from(format!("Feature flag found in {:?}", path)));
            }
        }
    }

    Ok(())
}
<|endoftext|>