<commit_before>use crate::{
    complete_output, get_compiler,
    util::{CtxtExt, MapErr},
};
use anyhow::{anyhow, bail, Context, Error};
use napi::{CallContext, JsObject, Task};
use once_cell::sync::Lazy;
use serde::Deserialize;
use std::{collections::HashMap, path::PathBuf, sync::Arc};
use swc::{
    config::{util::BoolOrObject, SourceMapsConfig},
    try_with_handler, TransformOutput,
};
use swc_atoms::JsWord;
use swc_bundler::{Bundler, ModuleData, ModuleRecord};
use swc_common::{
    collections::AHashMap, errors::Handler, BytePos, FileName, SourceMap, Span, DUMMY_SP,
};
use swc_ecma_loader::{
    resolvers::{lru::CachingResolver, node::NodeModulesResolver},
    TargetEnv, NODE_BUILTINS,
};
use swc_ecmascript::{
    ast::*,
    parser::{lexer::Lexer, EsConfig, Parser, StringInput, Syntax},
    visit::{noop_visit_type, Node, Visit, VisitWith},
};

#[js_function(1)]
pub fn bundle(cx: CallContext) -> napi::Result<JsObject> {
    let option = cx.get_buffer_as_string(0)?;

    let task = BundleTask {
        c: get_compiler(&cx),
        config: option,
    };

    cx.env.spawn(task).map(|t| t.promise_object())
}

#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
struct BundleOption {
    entry: PathBuf,
}

struct BundleTask {
    c: Arc<swc::Compiler>,
    config: String,
}

impl Task for BundleTask {
    type Output = TransformOutput;

    type JsValue = JsObject;

    fn compute(&mut self) -> napi::Result<Self::Output> {
        let option: BundleOption = crate::util::deserialize_json(&self.config).convert_err()?;

        try_with_handler(self.c.cm.clone(), true, |handler| {
            let builtins = NODE_BUILTINS
                .to_vec()
                .into_iter()
                .map(JsWord::from)
                .collect::<Vec<_>>();

            //
            let mut bundler = Bundler::new(
                &self.c.globals(),
                self.c.cm.clone(),
                CustomLoader {
                    cm: self.c.cm.clone(),
                    handler: &handler,
                },
                make_resolver(),
                swc_bundler::Config {
                    require: false,
                    disable_inliner: false,
                    external_modules: builtins,
                    module: swc_bundler::ModuleType::Es,
                },
                Box::new(CustomHook),
            );

            let mut entries = HashMap::default();
            let path: PathBuf = option.entry.into();
            let path = path
                .canonicalize()
                .context("failed to canonicalize entry file")?;
            entries.insert("main".to_string(), FileName::Real(path));
            let outputs = bundler.bundle(entries)?;

            let output = outputs
                .into_iter()
                .next()
                .ok_or_else(|| anyhow!("swc_bundler::Bundle::bundle returned empty result"))?;

            let source_map_names = {
                let mut v = SourceMapIdentCollector {
                    names: Default::default(),
                };

                output
                    .module
                    .visit_with(&Invalid { span: DUMMY_SP }, &mut v);

                v.names
            };

            let code = self.c.print(
                &output.module,
                None,
                None,
                true,
                EsVersion::Es5,
                SourceMapsConfig::Bool(true),
                &source_map_names,
                None,
                false,
                Some(BoolOrObject::Bool(true)),
            )?;

            Ok(code)
        })
        .convert_err()
    }

    fn resolve(self, env: napi::Env, output: Self::Output) -> napi::Result<Self::JsValue> {
        complete_output(&env, output)
    }
}

type Resolver = Arc<CachingResolver<NodeModulesResolver>>;

fn make_resolver() -> Resolver {
    static CACHE: Lazy<Resolver> = Lazy::new(|| {
        // TODO: Make target env and alias configurable
        let r = NodeModulesResolver::new(TargetEnv::Node, Default::default());
        let r = CachingResolver::new(256, r);
        Arc::new(r)
    });

    (*CACHE).clone()
}

struct CustomLoader<'a> {
    handler: &'a Handler,
    cm: Arc<SourceMap>,
}

impl swc_bundler::Load for CustomLoader<'_> {
    fn load(&self, f: &FileName) -> Result<ModuleData, Error> {
        let fm = match f {
            FileName::Real(path) => self.cm.load_file(&path)?,
            _ => unreachable!(),
        };

        let lexer = Lexer::new(
            Syntax::Es(EsConfig {
                ..Default::default()
            }),
            EsVersion::Es2020,
            StringInput::from(&*fm),
            None,
        );

        let mut parser = Parser::new_from(lexer);
        let module = parser.parse_module().map_err(|err| {
            err.into_diagnostic(&self.handler).emit();
            anyhow!("failed to parse")
        })?;

        Ok(ModuleData {
            fm,
            module,
            helpers: Default::default(),
        })
    }
}

struct CustomHook;

impl swc_bundler::Hook for CustomHook {
    fn get_import_meta_props(
        &self,
        _span: Span,
        _module_record: &ModuleRecord,
    ) -> Result<Vec<KeyValueProp>, Error> {
        bail!("`import.meta` is not supported yet")
    }
}

pub struct SourceMapIdentCollector {
    names: AHashMap<BytePos, JsWord>,
}

impl Visit for SourceMapIdentCollector {
    noop_visit_type!();

    fn visit_ident(&mut self, ident: &Ident, _: &dyn Node) {
        self.names.insert(ident.span.lo, ident.sym.clone());
    }
}
<commit_msg>Enable require for swc_bundler (#31663)<commit_after>use crate::{
    complete_output, get_compiler,
    util::{CtxtExt, MapErr},
};
use anyhow::{anyhow, bail, Context, Error};
use napi::{CallContext, JsObject, Task};
use once_cell::sync::Lazy;
use serde::Deserialize;
use std::{collections::HashMap, path::PathBuf, sync::Arc};
use swc::{
    config::{util::BoolOrObject, SourceMapsConfig},
    try_with_handler, TransformOutput,
};
use swc_atoms::JsWord;
use swc_bundler::{Bundler, ModuleData, ModuleRecord};
use swc_common::{
    collections::AHashMap, errors::Handler, BytePos, FileName, SourceMap, Span, DUMMY_SP,
};
use swc_ecma_loader::{
    resolvers::{lru::CachingResolver, node::NodeModulesResolver},
    TargetEnv, NODE_BUILTINS,
};
use swc_ecmascript::{
    ast::*,
    parser::{lexer::Lexer, EsConfig, Parser, StringInput, Syntax},
    visit::{noop_visit_type, Node, Visit, VisitWith},
};

#[js_function(1)]
pub fn bundle(cx: CallContext) -> napi::Result<JsObject> {
    let option = cx.get_buffer_as_string(0)?;

    let task = BundleTask {
        c: get_compiler(&cx),
        config: option,
    };

    cx.env.spawn(task).map(|t| t.promise_object())
}

#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
struct BundleOption {
    entry: PathBuf,
}

struct BundleTask {
    c: Arc<swc::Compiler>,
    config: String,
}

impl Task for BundleTask {
    type Output = TransformOutput;

    type JsValue = JsObject;

    fn compute(&mut self) -> napi::Result<Self::Output> {
        let option: BundleOption = crate::util::deserialize_json(&self.config).convert_err()?;

        try_with_handler(self.c.cm.clone(), true, |handler| {
            let builtins = NODE_BUILTINS
                .to_vec()
                .into_iter()
                .map(JsWord::from)
                .collect::<Vec<_>>();

            //
            let mut bundler = Bundler::new(
                &self.c.globals(),
                self.c.cm.clone(),
                CustomLoader {
                    cm: self.c.cm.clone(),
                    handler: &handler,
                },
                make_resolver(),
                swc_bundler::Config {
                    require: true,
                    disable_inliner: false,
                    external_modules: builtins,
                    module: swc_bundler::ModuleType::Es,
                },
                Box::new(CustomHook),
            );

            let mut entries = HashMap::default();
            let path: PathBuf = option.entry.into();
            let path = path
                .canonicalize()
                .context("failed to canonicalize entry file")?;
            entries.insert("main".to_string(), FileName::Real(path));
            let outputs = bundler.bundle(entries)?;

            let output = outputs
                .into_iter()
                .next()
                .ok_or_else(|| anyhow!("swc_bundler::Bundle::bundle returned empty result"))?;

            let source_map_names = {
                let mut v = SourceMapIdentCollector {
                    names: Default::default(),
                };

                output
                    .module
                    .visit_with(&Invalid { span: DUMMY_SP }, &mut v);

                v.names
            };

            let code = self.c.print(
                &output.module,
                None,
                None,
                true,
                EsVersion::Es5,
                SourceMapsConfig::Bool(true),
                &source_map_names,
                None,
                false,
                Some(BoolOrObject::Bool(true)),
            )?;

            Ok(code)
        })
        .convert_err()
    }

    fn resolve(self, env: napi::Env, output: Self::Output) -> napi::Result<Self::JsValue> {
        complete_output(&env, output)
    }
}

type Resolver = Arc<CachingResolver<NodeModulesResolver>>;

fn make_resolver() -> Resolver {
    static CACHE: Lazy<Resolver> = Lazy::new(|| {
        // TODO: Make target env and alias configurable
        let r = NodeModulesResolver::new(TargetEnv::Node, Default::default());
        let r = CachingResolver::new(256, r);
        Arc::new(r)
    });

    (*CACHE).clone()
}

struct CustomLoader<'a> {
    handler: &'a Handler,
    cm: Arc<SourceMap>,
}

impl swc_bundler::Load for CustomLoader<'_> {
    fn load(&self, f: &FileName) -> Result<ModuleData, Error> {
        let fm = match f {
            FileName::Real(path) => self.cm.load_file(&path)?,
            _ => unreachable!(),
        };

        let lexer = Lexer::new(
            Syntax::Es(EsConfig {
                ..Default::default()
            }),
            EsVersion::Es2020,
            StringInput::from(&*fm),
            None,
        );

        let mut parser = Parser::new_from(lexer);
        let module = parser.parse_module().map_err(|err| {
            err.into_diagnostic(&self.handler).emit();
            anyhow!("failed to parse")
        })?;

        Ok(ModuleData {
            fm,
            module,
            helpers: Default::default(),
        })
    }
}

struct CustomHook;

impl swc_bundler::Hook for CustomHook {
    fn get_import_meta_props(
        &self,
        _span: Span,
        _module_record: &ModuleRecord,
    ) -> Result<Vec<KeyValueProp>, Error> {
        bail!("`import.meta` is not supported yet")
    }
}

pub struct SourceMapIdentCollector {
    names: AHashMap<BytePos, JsWord>,
}

impl Visit for SourceMapIdentCollector {
    noop_visit_type!();

    fn visit_ident(&mut self, ident: &Ident, _: &dyn Node) {
        self.names.insert(ident.span.lo, ident.sym.clone());
    }
}
<|endoftext|>


<commit_before>use std::env;
use rustc_serialize::json;
use rustc_serialize::json::{BuilderError, Json};
use curl::{ErrCode, http};
use curl::http::handle::{Method, Request};
use cargo_edit::Manifest;
use regex::Regex;
use std::path::Path;

const REGISTRY_HOST: &'static str = "https://crates.io";

/// Query latest version from crates.io
///
/// The latest version will be returned as a string. This will fail, when
///
/// - there is no Internet connection,
/// - the response from crates.io is an error or in an incorrect format,
/// - or when a crate with the given name does not exist on crates.io.
pub fn get_latest_version(crate_name: &str) -> Result<String, FetchVersionError> {
    if env::var("CARGO_IS_TEST").is_ok() {
        // We are in a simulated reality. Nothing is real here.
        // FIXME: Use actual test handling code.
        return Ok("CURRENT_VERSION_TEST".into());
    }

    let crate_data = try!(fetch_cratesio(&format!("/crates/{}", crate_name)));
    let crate_json = try!(Json::from_str(&crate_data));

    crate_json.as_object()
              .and_then(|c| c.get("crate"))
              .and_then(|c| c.as_object())
              .and_then(|c| c.get("max_version"))
              .and_then(|v| v.as_string())
              .map(|v| v.to_owned())
              .ok_or(FetchVersionError::GetVersion)
}

quick_error! {
    #[derive(Debug)]
    pub enum FetchVersionError {
        CratesIo(err: CratesIoError) {
            from()
            description("crates.io Error")
            display("crates.io Error: {}", err)
            cause(err)
        }
        Json(err: BuilderError) {
            from()
            description("JSON Error")
            display("Error parsing JSON: {}", err)
            cause(err)
        }
        GetVersion { description("get version error") }
    }
}

// ---
// The following was mostly copied from [1] and is therefore
// (c) 2015 Alex Crichton <alex@alexcrichton.com>
//
// [1]: https://github.com/rust-lang/cargo/blob/bd690d8dff83c7b7714f236a08304ee20732382b/src/crates-io/lib.rs
// ---

fn fetch_cratesio(path: &str) -> Result<String, FetchVersionError> {
    let mut http_handle = http::Handle::new();
    let req = Request::new(&mut http_handle, Method::Get)
                  .uri(format!("{}/api/v1{}", REGISTRY_HOST, path))
                  .header("Accept", "application/json")
                  .content_type("application/json");
    handle_cratesio(req.exec()).map_err(From::from)
}

fn handle_cratesio(response: Result<http::Response, ErrCode>) -> Result<String, CratesIoError> {
    let response = try!(response.map_err(CratesIoError::Curl));
    match response.get_code() {
        0 | 200 => {}
        403 => return Err(CratesIoError::Unauthorized),
        404 => return Err(CratesIoError::NotFound),
        _ => return Err(CratesIoError::NotOkResponse(response)),
    }

    let body = match String::from_utf8(response.move_body()) {
        Ok(body) => body,
        Err(..) => return Err(CratesIoError::NonUtf8Body),
    };

    if let Ok(errors) = json::decode::<ApiErrorList>(&body) {
        return Err(CratesIoError::Api(errors.errors.into_iter().map(|s| s.detail).collect()));
    }

    Ok(body)
}

#[derive(RustcDecodable)]
struct ApiErrorList {
    errors: Vec<ApiError>,
}
#[derive(RustcDecodable)]
struct ApiError {
    detail: String,
}

quick_error! {
    #[derive(Debug)]
    pub enum CratesIoError {
        Curl(e: ErrCode) {}
        NotOkResponse(e: http::Response)  {}
        NonUtf8Body  {}
        Api(e: Vec<String>)  {}
        Unauthorized  {}
        NotFound {}
    }
}

quick_error! {
    #[derive(Debug)]
    pub enum FetchGitError {
        FetchGit(err: CratesIoError) {
            from()
            description("fetch error: ")
            display("fetch error: {}", err)
            cause(err)
        }
        ParseRegex { description("parse error: unable to parse git repo url") }
        IncompleteCaptures { description("parse error: the git repo url seems incomplete") }
        LocalCargoToml { description("path error: unable to open Cargo.toml") }
// RemoteCargoToml { description("path error: unable to open Cargo.toml from the provided repo") }
        ParseCargoToml { description("parse error: unable to parse the external Cargo.toml") }
    }
}

/// Query crate name by accessing a github repo Cargo.toml
///
/// The name will be returned as a string. This will fail, when
///
/// - there is no Internet connection,
/// - Cargo.toml is not present in the root of the master branch,
/// - the response from github is an error or in an incorrect format.
pub fn get_crate_name_from_github(repo: &str) -> Result<String, FetchGitError> {
    let re = Regex::new(r"^https://github.com/([-_0-9a-zA-Z]+)/([-_0-9a-zA-Z]+)(/|.git)?$")
                 .unwrap();

    re.captures(repo)
      .ok_or(FetchGitError::ParseRegex)
      .and_then(|cap| {
          match (cap.at(1), cap.at(2)) {
              (Some(ref user), Some(ref repo)) => {
                  let url = format!("https://raw.githubusercontent.com/{}/{}/master/Cargo.toml",
                                    user,
                                    repo);

                  let data: Result<Manifest, _> = get_cargo_toml_from_git_url(&url).and_then(|m| {
                      m.parse()
                       .map_err(|_| FetchGitError::ParseCargoToml)
                  });
                  data.and_then(|ref manifest| get_name_from_manifest(manifest))
              }
              _ => Err(FetchGitError::IncompleteCaptures),
          }
      })
}

/// Query crate name by accessing a gitlab repo Cargo.toml
///
/// The name will be returned as a string. This will fail, when
///
/// - there is no Internet connection,
/// - Cargo.toml is not present in the root of the master branch,
/// - the response from gitlab is an error or in an incorrect format.
pub fn get_crate_name_from_gitlab(repo: &str) -> Result<String, FetchGitError> {
    let re = Regex::new(r"^https://gitlab.com/([-_0-9a-zA-Z]+)/([-_0-9a-zA-Z]+)(/|.git)?$")
                 .unwrap();

    re.captures(repo)
      .ok_or(FetchGitError::ParseRegex)
      .and_then(|cap| {
          match (cap.at(1), cap.at(2)) {
              (Some(ref user), Some(ref repo)) => {
                  let url = format!("https://gitlab.com/{}/{}/raw/master/Cargo.toml", user, repo);

                  let data: Result<Manifest, _> = get_cargo_toml_from_git_url(&url).and_then(|m| {
                      m.parse()
                       .map_err(|_| FetchGitError::ParseCargoToml)
                  });
                  data.and_then(|ref manifest| get_name_from_manifest(manifest))
              }
              _ => Err(FetchGitError::IncompleteCaptures),
          }
      })
}

/// Query crate name by accessing Cargo.toml in a local path
///
/// The name will be returned as a string. This will fail, when
/// Cargo.toml is not present in the root of the path.
pub fn get_crate_name_from_path(path: &str) -> Result<String, FetchGitError> {
    let cargo_file = Path::new(path).join("Cargo.toml");
    Manifest::open(&cargo_file.to_str())
        .map_err(|_| FetchGitError::LocalCargoToml)
        .and_then(|ref manifest| get_name_from_manifest(manifest))
}

fn get_name_from_manifest(manifest: &Manifest) -> Result<String, FetchGitError> {
    manifest.data
            .get("package")
            .and_then(|m| m.lookup("name"))
            .and_then(|name| name.as_str().map(|s| s.to_string()))
            .ok_or(FetchGitError::ParseCargoToml)
}

// FIXME: between the code above and below there is a lot of duplication.
fn get_cargo_toml_from_git_url(url: &str) -> Result<String, FetchGitError> {

    let mut http_handle = http::Handle::new();
    let req = Request::new(&mut http_handle, Method::Get)
                  .uri(url)
                  .header("Accept", "text/plain")
                  .content_type("text/plain");
    handle_git_url(req.exec()).map_err(From::from)
}

fn handle_git_url(response: Result<http::Response, ErrCode>) -> Result<String, CratesIoError> {
    let response = try!(response.map_err(CratesIoError::Curl));
    match response.get_code() {
        0 | 200 => {}
        403 => return Err(CratesIoError::Unauthorized),
        404 => return Err(CratesIoError::NotFound),
        _ => return Err(CratesIoError::NotOkResponse(response)),
    }

    let body = match String::from_utf8(response.move_body()) {
        Ok(body) => body,
        Err(..) => return Err(CratesIoError::NonUtf8Body),
    };

    if let Ok(errors) = json::decode::<ApiErrorList>(&body) {
        return Err(CratesIoError::Api(errors.errors.into_iter().map(|s| s.detail).collect()));
    }

    Ok(body)
}
<commit_msg>fetch code polished and deduplicated<commit_after>use std::env;
use std::path::Path;
use rustc_serialize::json;
use rustc_serialize::json::{BuilderError, Json};
use curl::{ErrCode, http};
use curl::http::handle::{Method, Request};
use cargo_edit::Manifest;
use regex::Regex;

const REGISTRY_HOST: &'static str = "https://crates.io";

/// Query latest version from crates.io
///
/// The latest version will be returned as a string. This will fail, when
///
/// - there is no Internet connection,
/// - the response from crates.io is an error or in an incorrect format,
/// - or when a crate with the given name does not exist on crates.io.
pub fn get_latest_version(crate_name: &str) -> Result<String, FetchVersionError> {
    if env::var("CARGO_IS_TEST").is_ok() {
        // We are in a simulated reality. Nothing is real here.
        // FIXME: Use actual test handling code.
        return Ok("CURRENT_VERSION_TEST".into());
    }

    let crate_data = try!(fetch_cratesio(&format!("/crates/{}", crate_name)));
    let crate_json = try!(Json::from_str(&crate_data));

    crate_json.as_object()
              .and_then(|c| c.get("crate"))
              .and_then(|c| c.as_object())
              .and_then(|c| c.get("max_version"))
              .and_then(|v| v.as_string())
              .map(|v| v.to_owned())
              .ok_or(FetchVersionError::GetVersion)
}

quick_error! {
    #[derive(Debug)]
    pub enum FetchVersionError {
        CratesIo(err: CratesIoError) {
            from()
            description("crates.io Error")
            display("crates.io Error: {}", err)
            cause(err)
        }
        Json(err: BuilderError) {
            from()
            description("JSON Error")
            display("Error parsing JSON: {}", err)
            cause(err)
        }
        GetVersion { description("get version error") }
    }
}

// ---
// The following was mostly copied from [1] and is therefore
// (c) 2015 Alex Crichton <alex@alexcrichton.com>
//
// [1]: https://github.com/rust-lang/cargo/blob/bd690d8dff83c7b7714f236a08304ee20732382b/src/crates-io/lib.rs
// ---

fn fetch_cratesio(path: &str) -> Result<String, FetchVersionError> {
    let mut http_handle = http::Handle::new();
    let req = Request::new(&mut http_handle, Method::Get)
                  .uri(format!("{}/api/v1{}", REGISTRY_HOST, path))
                  .header("Accept", "application/json")
                  .content_type("application/json");
    handle_request(req.exec()).map_err(From::from)
}

fn handle_request(response: Result<http::Response, ErrCode>) -> Result<String, CratesIoError> {
    let response = try!(response.map_err(CratesIoError::Curl));
    match response.get_code() {
        0 | 200 => {}
        403 => return Err(CratesIoError::Unauthorized),
        404 => return Err(CratesIoError::NotFound),
        _ => return Err(CratesIoError::NotOkResponse(response)),
    }

    let body = match String::from_utf8(response.move_body()) {
        Ok(body) => body,
        Err(..) => return Err(CratesIoError::NonUtf8Body),
    };

    if let Ok(errors) = json::decode::<ApiErrorList>(&body) {
        return Err(CratesIoError::Api(errors.errors.into_iter().map(|s| s.detail).collect()));
    }

    Ok(body)
}

#[derive(RustcDecodable)]
struct ApiErrorList {
    errors: Vec<ApiError>,
}
#[derive(RustcDecodable)]
struct ApiError {
    detail: String,
}

quick_error! {
    #[derive(Debug)]
    pub enum CratesIoError {
        Curl(e: ErrCode) {}
        NotOkResponse(e: http::Response)  {}
        NonUtf8Body  {}
        Api(e: Vec<String>)  {}
        Unauthorized  {}
        NotFound {}
    }
}

#[cfg_attr(rustfmt, rustfmt_skip)]
quick_error! {
    #[derive(Debug)]
    pub enum FetchGitError {
        FetchGit(err: CratesIoError) {
            from()
            description("fetch error: ")
            display("fetch error: {}", err)
            cause(err)
        }
        ParseRegex { description("parse error: unable to parse git repo url") }
        IncompleteCaptures { description("parse error: the git repo url seems incomplete") }
        LocalCargoToml { description("path error: unable to open Cargo.toml") }
        // RemoteCargoToml { description("path error: unable to open Cargo.toml from the provided repo") }
        ParseCargoToml { description("parse error: unable to parse the external Cargo.toml") }
    }
}

/// Query crate name by accessing a github repo Cargo.toml
///
/// The name will be returned as a string. This will fail, when
///
/// - there is no Internet connection,
/// - Cargo.toml is not present in the root of the master branch,
/// - the response from github is an error or in an incorrect format.
pub fn get_crate_name_from_github(repo: &str) -> Result<String, FetchGitError> {
    let re = Regex::new(r"^https://github.com/([-_0-9a-zA-Z]+)/([-_0-9a-zA-Z]+)(/|.git)?$")
                 .unwrap();

    re.captures(repo)
      .ok_or(FetchGitError::ParseRegex)
      .and_then(|cap| {
          match (cap.at(1), cap.at(2)) {
              (Some(ref user), Some(ref repo)) => {
                  let url = format!("https://raw.githubusercontent.com/{}/{}/master/Cargo.toml",
                                    user,
                                    repo);

                  let data: Result<Manifest, _> = get_cargo_toml_from_git_url(&url).and_then(|m| {
                      m.parse()
                       .map_err(|_| FetchGitError::ParseCargoToml)
                  });
                  data.and_then(|ref manifest| get_name_from_manifest(manifest))
              }
              _ => Err(FetchGitError::IncompleteCaptures),
          }
      })
}

/// Query crate name by accessing a gitlab repo Cargo.toml
///
/// The name will be returned as a string. This will fail, when
///
/// - there is no Internet connection,
/// - Cargo.toml is not present in the root of the master branch,
/// - the response from gitlab is an error or in an incorrect format.
pub fn get_crate_name_from_gitlab(repo: &str) -> Result<String, FetchGitError> {
    let re = Regex::new(r"^https://gitlab.com/([-_0-9a-zA-Z]+)/([-_0-9a-zA-Z]+)(/|.git)?$")
                 .unwrap();

    re.captures(repo)
      .ok_or(FetchGitError::ParseRegex)
      .and_then(|cap| {
          match (cap.at(1), cap.at(2)) {
              (Some(ref user), Some(ref repo)) => {
                  let url = format!("https://gitlab.com/{}/{}/raw/master/Cargo.toml", user, repo);

                  let data: Result<Manifest, _> = get_cargo_toml_from_git_url(&url).and_then(|m| {
                      m.parse()
                       .map_err(|_| FetchGitError::ParseCargoToml)
                  });
                  data.and_then(|ref manifest| get_name_from_manifest(manifest))
              }
              _ => Err(FetchGitError::IncompleteCaptures),
          }
      })
}

/// Query crate name by accessing Cargo.toml in a local path
///
/// The name will be returned as a string. This will fail, when
/// Cargo.toml is not present in the root of the path.
pub fn get_crate_name_from_path(path: &str) -> Result<String, FetchGitError> {
    let cargo_file = Path::new(path).join("Cargo.toml");
    Manifest::open(&cargo_file.to_str())
        .map_err(|_| FetchGitError::LocalCargoToml)
        .and_then(|ref manifest| get_name_from_manifest(manifest))
}

fn get_name_from_manifest(manifest: &Manifest) -> Result<String, FetchGitError> {
    manifest.data
            .get("package")
            .and_then(|m| m.lookup("name"))
            .and_then(|name| name.as_str().map(|s| s.to_string()))
            .ok_or(FetchGitError::ParseCargoToml)
}

fn get_cargo_toml_from_git_url(url: &str) -> Result<String, FetchGitError> {
    let mut http_handle = http::Handle::new();
    let req = Request::new(&mut http_handle, Method::Get)
                  .uri(url)
                  .header("Accept", "text/plain")
                  .content_type("text/plain");
    handle_request(req.exec()).map_err(From::from)
}
<|endoftext|>


<commit_before>extern crate tempdir;
extern crate hyper;
extern crate grnenvlib;
#[macro_use]
extern crate clap;

use clap::{Arg, App, SubCommand};
use grnenvlib::command;

fn main() {
    default_main();
}

fn cli() -> App<'static, 'static> {
    App::new("grnenv-rs")
        .version(crate_version!())
        .author(crate_authors!())
        .about("A tiny tool for obtain and select multiple Groonga.")
        .subcommand(SubCommand::with_name("init").about("Prepare grnenv-rs."))
        .subcommand(SubCommand::with_name("install")
            .about("Install a given Groonga version and arch")
            .arg(Arg::with_name("VERSION").required(true))
            .arg(Arg::with_name("arch")
                .short("a")
                .long("arch")
                .value_name("ARCH")
                .help("Select architectures. e.g.) x86, x64")
                .takes_value(true)))
        .subcommand(SubCommand::with_name("switch")
            .about("Switch Groonga with given version and arch")
            .arg(Arg::with_name("VERSION").required(true))
            .arg(Arg::with_name("arch")
                .short("a")
                .long("arch")
                .value_name("ARCH")
                .help("Select architectures. e.g.) x86, x64")
                .takes_value(true)))
        .subcommand(SubCommand::with_name("versions").about("Display installed Groonga versions"))
        .subcommand(SubCommand::with_name("uninstall")
            .about("Uninstall a given Groonga version and arch")
            .arg(Arg::with_name("VERSION").required(true))
            .arg(Arg::with_name("arch")
                .short("a")
                .long("arch")
                .value_name("ARCH")
                .help("Select architectures. e.g.) x86, x64")
                .takes_value(true)))
        .subcommand(SubCommand::with_name("list").about("Display installable Groonga versions"))

}

#[cfg(not(windows))]
fn default_main() {
    let matches = cli().get_matches();
    match matches.subcommand() {
        ("init", _) => command::unix::init(),
        ("install", Some(m)) => command::unix::install(m),
        ("switch", Some(m)) => command::unix::switch(m),
        ("versions", _) => command::common::versions(),
        ("uninstall", Some(m)) => command::unix::uninstall(m),
        ("list", _) => command::unix::list(),
        (_, _) => unreachable!(),
    }
}

#[cfg(windows)]
fn default_main() {
    let matches = cli().get_matches();
    match matches.subcommand() {
        ("init", _) => command::windows::init(),
        ("install", Some(m)) => command::windows::install(m),
        ("switch", Some(m)) => command::windows::switch(m),
        ("versions", _) => command::common::versions(),
        ("uninstall", Some(m)) => command::windows::uninstall(m),
        ("list", _) => command::windows::list(),
        (_, _) => unreachable!(),
    }
}
<commit_msg>Add Poc for allowing external subcommand<commit_after>extern crate tempdir;
extern crate hyper;
extern crate grnenvlib;
#[macro_use]
extern crate clap;

use clap::{Arg, App, AppSettings, SubCommand};
use grnenvlib::command;

fn main() {
    default_main();
}

fn cli() -> App<'static, 'static> {
    App::new("grnenv-rs")
        .version(crate_version!())
        .author(crate_authors!())
        .about("A tiny tool for obtain and select multiple Groonga.")
        .setting(AppSettings::AllowExternalSubcommands)
        .subcommand(SubCommand::with_name("init").about("Prepare grnenv-rs."))
        .subcommand(SubCommand::with_name("install")
            .about("Install a given Groonga version and arch")
            .arg(Arg::with_name("VERSION").required(true))
            .arg(Arg::with_name("arch")
                .short("a")
                .long("arch")
                .value_name("ARCH")
                .help("Select architectures. e.g.) x86, x64")
                .takes_value(true)))
        .subcommand(SubCommand::with_name("switch")
            .about("Switch Groonga with given version and arch")
            .arg(Arg::with_name("VERSION").required(true))
            .arg(Arg::with_name("arch")
                .short("a")
                .long("arch")
                .value_name("ARCH")
                .help("Select architectures. e.g.) x86, x64")
                .takes_value(true)))
        .subcommand(SubCommand::with_name("versions").about("Display installed Groonga versions"))
        .subcommand(SubCommand::with_name("uninstall")
            .about("Uninstall a given Groonga version and arch")
            .arg(Arg::with_name("VERSION").required(true))
            .arg(Arg::with_name("arch")
                .short("a")
                .long("arch")
                .value_name("ARCH")
                .help("Select architectures. e.g.) x86, x64")
                .takes_value(true)))
        .subcommand(SubCommand::with_name("list").about("Display installable Groonga versions"))

}

#[cfg(not(windows))]
fn default_main() {
    let matches = cli().get_matches();
    match matches.subcommand() {
        ("init", _) => command::unix::init(),
        ("install", Some(m)) => command::unix::install(m),
        ("switch", Some(m)) => command::unix::switch(m),
        ("versions", _) => command::common::versions(),
        ("uninstall", Some(m)) => command::unix::uninstall(m),
        ("list", _) => command::unix::list(),
        (external, Some(ext_m)) => {
            use std::process;
            let ext_args: Vec<&str> = match ext_m.values_of("") {
                Some(c) => c.collect(),
                None => process::exit(1),
            };
            assert_eq!(external, "subcmd");
            assert_eq!(ext_args, ["--option", "value", "-fff", "--flag"]);
        },
        (_, _) => unreachable!(),
    }
}

#[cfg(windows)]
fn default_main() {
    let matches = cli().get_matches();
    match matches.subcommand() {
        ("init", _) => command::windows::init(),
        ("install", Some(m)) => command::windows::install(m),
        ("switch", Some(m)) => command::windows::switch(m),
        ("versions", _) => command::common::versions(),
        ("uninstall", Some(m)) => command::windows::uninstall(m),
        ("list", _) => command::windows::list(),
        (_, _) => unreachable!(),
    }
}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

extern crate docopt;
#[macro_use]
extern crate log;
extern crate mimir;
extern crate mimirsbrunn;
extern crate rustc_serialize;

use mimir::rubber::Rubber;
use mimirsbrunn::osm_reader::admin::administrative_regions;
use mimirsbrunn::osm_reader::poi::{PoiTypes, pois, default_amenity_types,
                                              default_leisure_types,
                                              compute_poi_weight};
use mimirsbrunn::osm_reader::street::streets;
use mimirsbrunn::osm_reader::parse_osm_pbf;
use mimirsbrunn::admin_geofinder::AdminGeoFinder;

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_level: Vec<u32>,
    flag_city_level: u32,
    flag_connection_string: String,
    flag_import_way: bool,
    flag_import_admin: bool,
    flag_import_poi: bool,
    flag_dataset: String,
}

static USAGE: &'static str =
    "
Usage:
    osm2mimir --help
    osm2mimir --input=<file> \
     [--connection-string=<connection-string>] [--import-way] [--import-admin] [--import-poi] \
     [--dataset=<dataset>] [--city-level=<level>] --level=<level> ...

Options:
    -h, --help              \
     Show this message.
    -i, --input=<file>      OSM PBF file.
    -l, --level=<level>     \
     Admin levels to keep.
    -C, --city-level=<level>
                            City level to \
     calculate weight, [default: 8]
    -w, --import-way        Import ways
    -a, \
     --import-admin      Import admins
    -p, --import-poi        Import POIs
    -c, \
     --connection-string=<connection-string>
                            Elasticsearch \
     parameters, [default: http://localhost:9200/munin]
    -d, --dataset=<dataset>
                            \
     Name of the dataset, [default: fr]
";

fn main() {
    mimir::logger_init().unwrap();
    let args: Args = docopt::Docopt::new(USAGE)
        .and_then(|d| d.decode())
        .unwrap_or_else(|e| e.exit());

    let levels = args.flag_level.iter().cloned().collect();
    let city_level = args.flag_city_level;
    let mut parsed_pbf = parse_osm_pbf(&args.flag_input);
    debug!("creation of indexes");
    let mut rubber = Rubber::new(&args.flag_connection_string);

    info!("creating adminstrative regions");
    let admins = administrative_regions(&mut parsed_pbf, levels);
    let admins_geofinder = admins.iter().cloned().collect::<AdminGeoFinder>();
    {
        info!("Extracting streets from osm");
        let mut streets = streets(&mut parsed_pbf, &admins_geofinder, city_level);

        info!("computing city weight");
        compute_admin_weight(&mut streets);

        info!("computing street weight");
        compute_street_weight(&mut streets, city_level);

    if args.flag_import_way {
        info!("importing streets into Mimir");
        let nb_streets = rubber.index("way", &args.flag_dataset, streets.into_iter())
            .unwrap();
        info!("Nb of indexed street: {}", nb_streets);
    }

    let nb_admins = rubber.index("admin", &args.flag_dataset, admins.iter())
        .unwrap();
    info!("Nb of indexed admin: {}", nb_admins);

    if args.flag_import_poi {
        let mut poi_types = PoiTypes::new();
        poi_types.insert("amenity".to_string(), default_amenity_types());
        poi_types.insert("leisure".to_string(), default_leisure_types());

        info!("Extracting pois from osm");
        let mut pois = pois(&mut parsed_pbf, poi_types, &admins, city_level);

        info!("computing poi weight");
		compute_poi_weight(&mut pois, city_level);

        info!("Importing pois into Mimir");
        let nb_pois = rubber.index("poi", &args.flag_dataset, pois.iter())
            .unwrap();

        info!("Nb of indexed pois: {}", nb_pois);
    }
}
<commit_msg>correction<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

extern crate docopt;
#[macro_use]
extern crate log;
extern crate mimir;
extern crate mimirsbrunn;
extern crate rustc_serialize;

use mimir::rubber::Rubber;
use mimirsbrunn::osm_reader::admin::administrative_regions;
use mimirsbrunn::osm_reader::poi::{PoiTypes, pois, default_amenity_types,
                                              default_leisure_types,
                                              compute_poi_weight};
use mimirsbrunn::osm_reader::street::streets;
use mimirsbrunn::osm_reader::parse_osm_pbf;
use mimirsbrunn::admin_geofinder::AdminGeoFinder;

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_level: Vec<u32>,
    flag_city_level: u32,
    flag_connection_string: String,
    flag_import_way: bool,
    flag_import_admin: bool,
    flag_import_poi: bool,
    flag_dataset: String,
}

static USAGE: &'static str =
    "
Usage:
    osm2mimir --help
    osm2mimir --input=<file> \
     [--connection-string=<connection-string>] [--import-way] [--import-admin] [--import-poi] \
     [--dataset=<dataset>] [--city-level=<level>] --level=<level> ...

Options:
    -h, --help              \
     Show this message.
    -i, --input=<file>      OSM PBF file.
    -l, --level=<level>     \
     Admin levels to keep.
    -C, --city-level=<level>
                            City level to \
     calculate weight, [default: 8]
    -w, --import-way        Import ways
    -a, \
     --import-admin      Import admins
    -p, --import-poi        Import POIs
    -c, \
     --connection-string=<connection-string>
                            Elasticsearch \
     parameters, [default: http://localhost:9200/munin]
    -d, --dataset=<dataset>
                            \
     Name of the dataset, [default: fr]
";

fn main() {
    mimir::logger_init().unwrap();
    let args: Args = docopt::Docopt::new(USAGE)
        .and_then(|d| d.decode())
        .unwrap_or_else(|e| e.exit());

    let levels = args.flag_level.iter().cloned().collect();
    let city_level = args.flag_city_level;
    let mut parsed_pbf = parse_osm_pbf(&args.flag_input);
    debug!("creation of indexes");
    let mut rubber = Rubber::new(&args.flag_connection_string);

    info!("creating adminstrative regions");
    let admins = administrative_regions(&mut parsed_pbf, levels);
    let admins_geofinder = admins.iter().cloned().collect::<AdminGeoFinder>();
    {
        info!("Extracting streets from osm");
        let mut streets = streets(&mut parsed_pbf, &admins_geofinder, city_level);

        info!("computing city weight");
        compute_admin_weight(&mut streets);

        info!("computing street weight");
        compute_street_weight(&mut streets, city_level);

    if args.flag_import_way {
        info!("importing streets into Mimir");
        let nb_streets = rubber.index("way", &args.flag_dataset, streets.into_iter())
            .unwrap();
        info!("Nb of indexed street: {}", nb_streets);
    }

    let nb_admins = rubber.index("admin", &args.flag_dataset, admins.iter())
        .unwrap();
    info!("Nb of indexed admin: {}", nb_admins);

    if args.flag_import_poi {
        let mut poi_types = PoiTypes::new();
        poi_types.insert("amenity".to_string(), default_amenity_types());
        poi_types.insert("leisure".to_string(), default_leisure_types());

        info!("Extracting pois from osm");
        let mut pois = pois(&mut parsed_pbf, poi_types, &admins, city_level);

        info!("computing poi weight");
        compute_poi_weight(&mut pois, city_level);

        info!("Importing pois into Mimir");
        let nb_pois = rubber.index("poi", &args.flag_dataset, pois.iter())
            .unwrap();

        info!("Nb of indexed pois: {}", nb_pois);
    }
}
<|endoftext|>


<commit_before>// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Documentation generation for rustbuild.
//!
//! This module implements generation for all bits and pieces of documentation
//! for the Rust project. This notably includes suites like the rust book, the
//! nomicon, standalone documentation, etc.
//!
//! Everything here is basically just a shim around calling either `rustbook` or
//! `rustdoc`.

use std::fs::{self, File};
use std::io::prelude::*;
use std::io;
use std::path::Path;
use std::process::Command;

use {Build, Compiler, Mode};
use util::{cp_r, symlink_dir};
use build_helper::up_to_date;

/// Invoke `rustbook` as compiled in `stage` for `target` for the doc book
/// `name` into the `out` path.
///
/// This will not actually generate any documentation if the documentation has
/// already been generated.
pub fn rustbook(build: &Build, target: &str, name: &str) {
    let out = build.doc_out(target);
    t!(fs::create_dir_all(&out));

    let out = out.join(name);
    let compiler = Compiler::new(0, &build.config.build);
    let src = build.src.join("src/doc").join(name);
    let index = out.join("index.html");
    let rustbook = build.tool(&compiler, "rustbook");
    if up_to_date(&src, &index) && up_to_date(&rustbook, &index) {
        return
    }
    println!("Rustbook ({}) - {}", target, name);
    let _ = fs::remove_dir_all(&out);
    build.run(build.tool_cmd(&compiler, "rustbook")
                   .arg("build")
                   .arg(&src)
                   .arg("-d")
                   .arg(out));
}

/// Build the book and associated stuff.
///
/// We need to build:
///
/// * Book (first edition)
/// * Book (second edition)
/// * Index page
/// * Redirect pages
pub fn book(build: &Build, target: &str, name: &str) {
    // build book first edition
    rustbook(build, target, &format!("{}/first-edition", name));

    // build book second edition
    rustbook(build, target, &format!("{}/second-edition", name));

    // build the index page
    let index = format!("{}/index.md", name);
    invoke_rustdoc(build, target, &index);

    // build the redirect pages
    for file in t!(fs::read_dir(build.src.join("src/doc/book/redirects"))) {
        let file = t!(file);
        let path = file.path();
        let path = path.to_str().unwrap();

        invoke_rustdoc(build, target, path);
    }
}

fn invoke_rustdoc(build: &Build, target: &str, markdown: &str) {
    let out = build.doc_out(target);

    let compiler = Compiler::new(0, &build.config.build);

    let path = build.src.join("src/doc").join(markdown);

    let rustdoc = build.rustdoc(&compiler);

    let favicon = build.src.join("src/doc/favicon.inc");
    let footer = build.src.join("src/doc/footer.inc");
    t!(fs::copy(build.src.join("src/doc/rust.css"), out.join("rust.css")));

    let version_info = out.join("version_info.html");

    let mut cmd = Command::new(&rustdoc);

    build.add_rustc_lib_path(&compiler, &mut cmd);

    let out = out.join("book");

    cmd.arg("--html-after-content").arg(&footer)
        .arg("--html-before-content").arg(&version_info)
        .arg("--html-in-header").arg(&favicon)
        .arg("--markdown-playground-url")
        .arg("https://play.rust-lang.org/")
        .arg("-o").arg(&out)
        .arg(&path)
        .arg("--markdown-css")
        .arg("rust.css");

    build.run(&mut cmd);
}

/// Generates all standalone documentation as compiled by the rustdoc in `stage`
/// for the `target` into `out`.
///
/// This will list all of `src/doc` looking for markdown files and appropriately
/// perform transformations like substituting `VERSION`, `SHORT_HASH`, and
/// `STAMP` alongw ith providing the various header/footer HTML we've cutomized.
///
/// In the end, this is just a glorified wrapper around rustdoc!
pub fn standalone(build: &Build, target: &str) {
    println!("Documenting standalone ({})", target);
    let out = build.doc_out(target);
    t!(fs::create_dir_all(&out));

    let compiler = Compiler::new(0, &build.config.build);

    let favicon = build.src.join("src/doc/favicon.inc");
    let footer = build.src.join("src/doc/footer.inc");
    let full_toc = build.src.join("src/doc/full-toc.inc");
    t!(fs::copy(build.src.join("src/doc/rust.css"), out.join("rust.css")));

    let version_input = build.src.join("src/doc/version_info.html.template");
    let version_info = out.join("version_info.html");

    if !up_to_date(&version_input, &version_info) {
        let mut info = String::new();
        t!(t!(File::open(&version_input)).read_to_string(&mut info));
        let info = info.replace("VERSION", &build.rust_release())
                       .replace("SHORT_HASH", build.rust_info.sha_short().unwrap_or(""))
                       .replace("STAMP", build.rust_info.sha().unwrap_or(""));
        t!(t!(File::create(&version_info)).write_all(info.as_bytes()));
    }

    for file in t!(fs::read_dir(build.src.join("src/doc"))) {
        let file = t!(file);
        let path = file.path();
        let filename = path.file_name().unwrap().to_str().unwrap();
        if !filename.ends_with(".md") || filename == "README.md" {
            continue
        }

        let html = out.join(filename).with_extension("html");
        let rustdoc = build.rustdoc(&compiler);
        if up_to_date(&path, &html) &&
           up_to_date(&footer, &html) &&
           up_to_date(&favicon, &html) &&
           up_to_date(&full_toc, &html) &&
           up_to_date(&version_info, &html) &&
           up_to_date(&rustdoc, &html) {
            continue
        }

        let mut cmd = Command::new(&rustdoc);
        build.add_rustc_lib_path(&compiler, &mut cmd);
        cmd.arg("--html-after-content").arg(&footer)
           .arg("--html-before-content").arg(&version_info)
           .arg("--html-in-header").arg(&favicon)
           .arg("--markdown-playground-url")
           .arg("https://play.rust-lang.org/")
           .arg("-o").arg(&out)
           .arg(&path);

        if filename == "not_found.md" {
            cmd.arg("--markdown-no-toc")
               .arg("--markdown-css")
               .arg("https://doc.rust-lang.org/rust.css");
        } else {
            cmd.arg("--markdown-css").arg("rust.css");
        }
        build.run(&mut cmd);
    }
}

/// Compile all standard library documentation.
///
/// This will generate all documentation for the standard library and its
/// dependencies. This is largely just a wrapper around `cargo doc`.
pub fn std(build: &Build, stage: u32, target: &str) {
    println!("Documenting stage{} std ({})", stage, target);
    let out = build.doc_out(target);
    t!(fs::create_dir_all(&out));
    let compiler = Compiler::new(stage, &build.config.build);
    let compiler = if build.force_use_stage1(&compiler, target) {
        Compiler::new(1, compiler.host)
    } else {
        compiler
    };
    let out_dir = build.stage_out(&compiler, Mode::Libstd)
                       .join(target).join("doc");
    let rustdoc = build.rustdoc(&compiler);

    // Here what we're doing is creating a *symlink* (directory junction on
    // Windows) to the final output location. This is not done as an
    // optimization but rather for correctness. We've got three trees of
    // documentation, one for std, one for test, and one for rustc. It's then
    // our job to merge them all together.
    //
    // Unfortunately rustbuild doesn't know nearly as well how to merge doc
    // trees as rustdoc does itself, so instead of actually having three
    // separate trees we just have rustdoc output to the same location across
    // all of them.
    //
    // This way rustdoc generates output directly into the output, and rustdoc
    // will also directly handle merging.
    let my_out = build.crate_doc_out(target);
    build.clear_if_dirty(&my_out, &rustdoc);
    t!(symlink_dir_force(&my_out, &out_dir));

    let mut cargo = build.cargo(&compiler, Mode::Libstd, target, "doc");
    cargo.arg("--manifest-path")
         .arg(build.src.join("src/libstd/Cargo.toml"))
         .arg("--features").arg(build.std_features());

    // We don't want to build docs for internal std dependencies unless
    // in compiler-docs mode. When not in that mode, we whitelist the crates
    // for which docs must be built.
    if !build.config.compiler_docs {
        cargo.arg("--no-deps");
        for krate in &["alloc", "collections", "core", "std", "std_unicode"] {
            cargo.arg("-p").arg(krate);
            // Create all crate output directories first to make sure rustdoc uses
            // relative links.
            // FIXME: Cargo should probably do this itself.
            t!(fs::create_dir_all(out_dir.join(krate)));
        }
    }

    build.run(&mut cargo);
    cp_r(&my_out, &out);
}

/// Compile all libtest documentation.
///
/// This will generate all documentation for libtest and its dependencies. This
/// is largely just a wrapper around `cargo doc`.
pub fn test(build: &Build, stage: u32, target: &str) {
    println!("Documenting stage{} test ({})", stage, target);
    let out = build.doc_out(target);
    t!(fs::create_dir_all(&out));
    let compiler = Compiler::new(stage, &build.config.build);
    let compiler = if build.force_use_stage1(&compiler, target) {
        Compiler::new(1, compiler.host)
    } else {
        compiler
    };
    let out_dir = build.stage_out(&compiler, Mode::Libtest)
                       .join(target).join("doc");
    let rustdoc = build.rustdoc(&compiler);

    // See docs in std above for why we symlink
    let my_out = build.crate_doc_out(target);
    build.clear_if_dirty(&my_out, &rustdoc);
    t!(symlink_dir_force(&my_out, &out_dir));

    let mut cargo = build.cargo(&compiler, Mode::Libtest, target, "doc");
    cargo.arg("--manifest-path")
         .arg(build.src.join("src/libtest/Cargo.toml"));
    build.run(&mut cargo);
    cp_r(&my_out, &out);
}

/// Generate all compiler documentation.
///
/// This will generate all documentation for the compiler libraries and their
/// dependencies. This is largely just a wrapper around `cargo doc`.
pub fn rustc(build: &Build, stage: u32, target: &str) {
    println!("Documenting stage{} compiler ({})", stage, target);
    let out = build.doc_out(target);
    t!(fs::create_dir_all(&out));
    let compiler = Compiler::new(stage, &build.config.build);
    let compiler = if build.force_use_stage1(&compiler, target) {
        Compiler::new(1, compiler.host)
    } else {
        compiler
    };
    let out_dir = build.stage_out(&compiler, Mode::Librustc)
                       .join(target).join("doc");
    let rustdoc = build.rustdoc(&compiler);

    // See docs in std above for why we symlink
    let my_out = build.crate_doc_out(target);
    build.clear_if_dirty(&my_out, &rustdoc);
    t!(symlink_dir_force(&my_out, &out_dir));

    let mut cargo = build.cargo(&compiler, Mode::Librustc, target, "doc");
    cargo.arg("--manifest-path")
         .arg(build.src.join("src/rustc/Cargo.toml"))
         .arg("--features").arg(build.rustc_features());

    if build.config.compiler_docs {
        // src/rustc/Cargo.toml contains bin crates called rustc and rustdoc
        // which would otherwise overwrite the docs for the real rustc and
        // rustdoc lib crates.
        cargo.arg("-p").arg("rustc_driver")
             .arg("-p").arg("rustdoc");
    } else {
        // Like with libstd above if compiler docs aren't enabled then we're not
        // documenting internal dependencies, so we have a whitelist.
        cargo.arg("--no-deps");
        for krate in &["proc_macro"] {
            cargo.arg("-p").arg(krate);
        }
    }

    build.run(&mut cargo);
    cp_r(&my_out, &out);
}

/// Generates the HTML rendered error-index by running the
/// `error_index_generator` tool.
pub fn error_index(build: &Build, target: &str) {
    println!("Documenting error index ({})", target);
    let out = build.doc_out(target);
    t!(fs::create_dir_all(&out));
    let compiler = Compiler::new(0, &build.config.build);
    let mut index = build.tool_cmd(&compiler, "error_index_generator");
    index.arg("html");
    index.arg(out.join("error-index.html"));

    // FIXME: shouldn't have to pass this env var
    index.env("CFG_BUILD", &build.config.build);

    build.run(&mut index);
}

fn symlink_dir_force(src: &Path, dst: &Path) -> io::Result<()> {
    if let Ok(m) = fs::symlink_metadata(dst) {
        if m.file_type().is_dir() {
            try!(fs::remove_dir_all(dst));
        } else {
            // handle directory junctions on windows by falling back to
            // `remove_dir`.
            try!(fs::remove_file(dst).or_else(|_| {
                fs::remove_dir(dst)
            }));
        }
    }

    symlink_dir(src, dst)
}
<commit_msg>Fix up some issues.<commit_after>// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Documentation generation for rustbuild.
//!
//! This module implements generation for all bits and pieces of documentation
//! for the Rust project. This notably includes suites like the rust book, the
//! nomicon, standalone documentation, etc.
//!
//! Everything here is basically just a shim around calling either `rustbook` or
//! `rustdoc`.

use std::fs::{self, File};
use std::io::prelude::*;
use std::io;
use std::path::Path;
use std::process::Command;

use {Build, Compiler, Mode};
use util::{cp_r, symlink_dir};
use build_helper::up_to_date;

/// Invoke `rustbook` as compiled in `stage` for `target` for the doc book
/// `name` into the `out` path.
///
/// This will not actually generate any documentation if the documentation has
/// already been generated.
pub fn rustbook(build: &Build, target: &str, name: &str) {
    let out = build.doc_out(target);
    t!(fs::create_dir_all(&out));

    let out = out.join(name);
    let compiler = Compiler::new(0, &build.config.build);
    let src = build.src.join("src/doc").join(name);
    let index = out.join("index.html");
    let rustbook = build.tool(&compiler, "rustbook");
    if up_to_date(&src, &index) && up_to_date(&rustbook, &index) {
        return
    }
    println!("Rustbook ({}) - {}", target, name);
    let _ = fs::remove_dir_all(&out);
    build.run(build.tool_cmd(&compiler, "rustbook")
                   .arg("build")
                   .arg(&src)
                   .arg("-d")
                   .arg(out));
}

/// Build the book and associated stuff.
///
/// We need to build:
///
/// * Book (first edition)
/// * Book (second edition)
/// * Index page
/// * Redirect pages
pub fn book(build: &Build, target: &str, name: &str) {
    // build book first edition
    rustbook(build, target, &format!("{}/first-edition", name));

    // build book second edition
    rustbook(build, target, &format!("{}/second-edition", name));

    // build the index page
    let index = format!("{}/index.md", name);
    println!("Documenting book index ({})", target);
    invoke_rustdoc(build, target, &index);

    // build the redirect pages
    println!("Documenting book redirect pages ({})", target);
    for file in t!(fs::read_dir(build.src.join("src/doc/book/redirects"))) {
        let file = t!(file);
        let path = file.path();
        let path = path.to_str().unwrap();

        invoke_rustdoc(build, target, path);
    }
}

fn invoke_rustdoc(build: &Build, target: &str, markdown: &str) {
    let out = build.doc_out(target);

    let compiler = Compiler::new(0, &build.config.build);

    let path = build.src.join("src/doc").join(markdown);

    let rustdoc = build.rustdoc(&compiler);

    let favicon = build.src.join("src/doc/favicon.inc");
    let footer = build.src.join("src/doc/footer.inc");

    let version_input = build.src.join("src/doc/version_info.html.template");
    let version_info = out.join("version_info.html");

    if !up_to_date(&version_input, &version_info) {
        let mut info = String::new();
        t!(t!(File::open(&version_input)).read_to_string(&mut info));
        let info = info.replace("VERSION", &build.rust_release())
                       .replace("SHORT_HASH", build.rust_info.sha_short().unwrap_or(""))
                       .replace("STAMP", build.rust_info.sha().unwrap_or(""));
        t!(t!(File::create(&version_info)).write_all(info.as_bytes()));
    }

    let mut cmd = Command::new(&rustdoc);

    build.add_rustc_lib_path(&compiler, &mut cmd);

    let out = out.join("book");

    t!(fs::copy(build.src.join("src/doc/rust.css"), out.join("rust.css")));

    cmd.arg("--html-after-content").arg(&footer)
        .arg("--html-before-content").arg(&version_info)
        .arg("--html-in-header").arg(&favicon)
        .arg("--markdown-playground-url")
        .arg("https://play.rust-lang.org/")
        .arg("-o").arg(&out)
        .arg(&path)
        .arg("--markdown-css")
        .arg("rust.css");

    build.run(&mut cmd);
}

/// Generates all standalone documentation as compiled by the rustdoc in `stage`
/// for the `target` into `out`.
///
/// This will list all of `src/doc` looking for markdown files and appropriately
/// perform transformations like substituting `VERSION`, `SHORT_HASH`, and
/// `STAMP` alongw ith providing the various header/footer HTML we've cutomized.
///
/// In the end, this is just a glorified wrapper around rustdoc!
pub fn standalone(build: &Build, target: &str) {
    println!("Documenting standalone ({})", target);
    let out = build.doc_out(target);
    t!(fs::create_dir_all(&out));

    let compiler = Compiler::new(0, &build.config.build);

    let favicon = build.src.join("src/doc/favicon.inc");
    let footer = build.src.join("src/doc/footer.inc");
    let full_toc = build.src.join("src/doc/full-toc.inc");
    t!(fs::copy(build.src.join("src/doc/rust.css"), out.join("rust.css")));

    let version_input = build.src.join("src/doc/version_info.html.template");
    let version_info = out.join("version_info.html");

    if !up_to_date(&version_input, &version_info) {
        let mut info = String::new();
        t!(t!(File::open(&version_input)).read_to_string(&mut info));
        let info = info.replace("VERSION", &build.rust_release())
                       .replace("SHORT_HASH", build.rust_info.sha_short().unwrap_or(""))
                       .replace("STAMP", build.rust_info.sha().unwrap_or(""));
        t!(t!(File::create(&version_info)).write_all(info.as_bytes()));
    }

    for file in t!(fs::read_dir(build.src.join("src/doc"))) {
        let file = t!(file);
        let path = file.path();
        let filename = path.file_name().unwrap().to_str().unwrap();
        if !filename.ends_with(".md") || filename == "README.md" {
            continue
        }

        let html = out.join(filename).with_extension("html");
        let rustdoc = build.rustdoc(&compiler);
        if up_to_date(&path, &html) &&
           up_to_date(&footer, &html) &&
           up_to_date(&favicon, &html) &&
           up_to_date(&full_toc, &html) &&
           up_to_date(&version_info, &html) &&
           up_to_date(&rustdoc, &html) {
            continue
        }

        let mut cmd = Command::new(&rustdoc);
        build.add_rustc_lib_path(&compiler, &mut cmd);
        cmd.arg("--html-after-content").arg(&footer)
           .arg("--html-before-content").arg(&version_info)
           .arg("--html-in-header").arg(&favicon)
           .arg("--markdown-playground-url")
           .arg("https://play.rust-lang.org/")
           .arg("-o").arg(&out)
           .arg(&path);

        if filename == "not_found.md" {
            cmd.arg("--markdown-no-toc")
               .arg("--markdown-css")
               .arg("https://doc.rust-lang.org/rust.css");
        } else {
            cmd.arg("--markdown-css").arg("rust.css");
        }
        build.run(&mut cmd);
    }
}

/// Compile all standard library documentation.
///
/// This will generate all documentation for the standard library and its
/// dependencies. This is largely just a wrapper around `cargo doc`.
pub fn std(build: &Build, stage: u32, target: &str) {
    println!("Documenting stage{} std ({})", stage, target);
    let out = build.doc_out(target);
    t!(fs::create_dir_all(&out));
    let compiler = Compiler::new(stage, &build.config.build);
    let compiler = if build.force_use_stage1(&compiler, target) {
        Compiler::new(1, compiler.host)
    } else {
        compiler
    };
    let out_dir = build.stage_out(&compiler, Mode::Libstd)
                       .join(target).join("doc");
    let rustdoc = build.rustdoc(&compiler);

    // Here what we're doing is creating a *symlink* (directory junction on
    // Windows) to the final output location. This is not done as an
    // optimization but rather for correctness. We've got three trees of
    // documentation, one for std, one for test, and one for rustc. It's then
    // our job to merge them all together.
    //
    // Unfortunately rustbuild doesn't know nearly as well how to merge doc
    // trees as rustdoc does itself, so instead of actually having three
    // separate trees we just have rustdoc output to the same location across
    // all of them.
    //
    // This way rustdoc generates output directly into the output, and rustdoc
    // will also directly handle merging.
    let my_out = build.crate_doc_out(target);
    build.clear_if_dirty(&my_out, &rustdoc);
    t!(symlink_dir_force(&my_out, &out_dir));

    let mut cargo = build.cargo(&compiler, Mode::Libstd, target, "doc");
    cargo.arg("--manifest-path")
         .arg(build.src.join("src/libstd/Cargo.toml"))
         .arg("--features").arg(build.std_features());

    // We don't want to build docs for internal std dependencies unless
    // in compiler-docs mode. When not in that mode, we whitelist the crates
    // for which docs must be built.
    if !build.config.compiler_docs {
        cargo.arg("--no-deps");
        for krate in &["alloc", "collections", "core", "std", "std_unicode"] {
            cargo.arg("-p").arg(krate);
            // Create all crate output directories first to make sure rustdoc uses
            // relative links.
            // FIXME: Cargo should probably do this itself.
            t!(fs::create_dir_all(out_dir.join(krate)));
        }
    }

    build.run(&mut cargo);
    cp_r(&my_out, &out);
}

/// Compile all libtest documentation.
///
/// This will generate all documentation for libtest and its dependencies. This
/// is largely just a wrapper around `cargo doc`.
pub fn test(build: &Build, stage: u32, target: &str) {
    println!("Documenting stage{} test ({})", stage, target);
    let out = build.doc_out(target);
    t!(fs::create_dir_all(&out));
    let compiler = Compiler::new(stage, &build.config.build);
    let compiler = if build.force_use_stage1(&compiler, target) {
        Compiler::new(1, compiler.host)
    } else {
        compiler
    };
    let out_dir = build.stage_out(&compiler, Mode::Libtest)
                       .join(target).join("doc");
    let rustdoc = build.rustdoc(&compiler);

    // See docs in std above for why we symlink
    let my_out = build.crate_doc_out(target);
    build.clear_if_dirty(&my_out, &rustdoc);
    t!(symlink_dir_force(&my_out, &out_dir));

    let mut cargo = build.cargo(&compiler, Mode::Libtest, target, "doc");
    cargo.arg("--manifest-path")
         .arg(build.src.join("src/libtest/Cargo.toml"));
    build.run(&mut cargo);
    cp_r(&my_out, &out);
}

/// Generate all compiler documentation.
///
/// This will generate all documentation for the compiler libraries and their
/// dependencies. This is largely just a wrapper around `cargo doc`.
pub fn rustc(build: &Build, stage: u32, target: &str) {
    println!("Documenting stage{} compiler ({})", stage, target);
    let out = build.doc_out(target);
    t!(fs::create_dir_all(&out));
    let compiler = Compiler::new(stage, &build.config.build);
    let compiler = if build.force_use_stage1(&compiler, target) {
        Compiler::new(1, compiler.host)
    } else {
        compiler
    };
    let out_dir = build.stage_out(&compiler, Mode::Librustc)
                       .join(target).join("doc");
    let rustdoc = build.rustdoc(&compiler);

    // See docs in std above for why we symlink
    let my_out = build.crate_doc_out(target);
    build.clear_if_dirty(&my_out, &rustdoc);
    t!(symlink_dir_force(&my_out, &out_dir));

    let mut cargo = build.cargo(&compiler, Mode::Librustc, target, "doc");
    cargo.arg("--manifest-path")
         .arg(build.src.join("src/rustc/Cargo.toml"))
         .arg("--features").arg(build.rustc_features());

    if build.config.compiler_docs {
        // src/rustc/Cargo.toml contains bin crates called rustc and rustdoc
        // which would otherwise overwrite the docs for the real rustc and
        // rustdoc lib crates.
        cargo.arg("-p").arg("rustc_driver")
             .arg("-p").arg("rustdoc");
    } else {
        // Like with libstd above if compiler docs aren't enabled then we're not
        // documenting internal dependencies, so we have a whitelist.
        cargo.arg("--no-deps");
        for krate in &["proc_macro"] {
            cargo.arg("-p").arg(krate);
        }
    }

    build.run(&mut cargo);
    cp_r(&my_out, &out);
}

/// Generates the HTML rendered error-index by running the
/// `error_index_generator` tool.
pub fn error_index(build: &Build, target: &str) {
    println!("Documenting error index ({})", target);
    let out = build.doc_out(target);
    t!(fs::create_dir_all(&out));
    let compiler = Compiler::new(0, &build.config.build);
    let mut index = build.tool_cmd(&compiler, "error_index_generator");
    index.arg("html");
    index.arg(out.join("error-index.html"));

    // FIXME: shouldn't have to pass this env var
    index.env("CFG_BUILD", &build.config.build);

    build.run(&mut index);
}

fn symlink_dir_force(src: &Path, dst: &Path) -> io::Result<()> {
    if let Ok(m) = fs::symlink_metadata(dst) {
        if m.file_type().is_dir() {
            try!(fs::remove_dir_all(dst));
        } else {
            // handle directory junctions on windows by falling back to
            // `remove_dir`.
            try!(fs::remove_file(dst).or_else(|_| {
                fs::remove_dir(dst)
            }));
        }
    }

    symlink_dir(src, dst)
}
<|endoftext|>


<commit_before>extern crate capnpc;

fn main() {
    ::capnpc::compile(".", &["addressbook.capnp"]).unwrap();
}
<commit_msg>capnpc::compile -> capnpcp::CompilerCommand<commit_after>extern crate capnpc;

fn main() {
    ::capnpc::CompilerCommand::new()
        .file("addressbook.capnp")
        .run()
        .unwrap();
}
<|endoftext|>


<commit_before>/*!
Defines how the documents matching a search query should be processed.
*/

use DocId;
use Result;
use Score;
use SegmentLocalId;
use SegmentReader;

mod count_collector;
pub use self::count_collector::CountCollector;

mod multi_collector;
pub use self::multi_collector::MultiCollector;

mod top_collector;
pub use self::top_collector::TopCollector;

mod facet_collector;
pub use self::facet_collector::FacetCollector;

mod chained_collector;
pub use self::chained_collector::chain;

/// Collectors are in charge of collecting and retaining relevant
/// information from the document found and scored by the query.
///
///
/// For instance,
///
/// - keeping track of the top 10 best documents
/// - computing a breakdown over a fast field
/// - computing the number of documents matching the query
///
/// Queries are in charge of pushing the `DocSet` to the collector.
///
/// As they work on multiple segments, they first inform
/// the collector of a change in a segment and then
/// call the `collect` method to push the document to the collector.
///
/// Temporally, our collector will receive calls
/// - `.set_segment(0, segment_reader_0)`
/// - `.collect(doc0_of_segment_0)`
/// - `.collect(...)`
/// - `.collect(last_doc_of_segment_0)`
/// - `.set_segment(1, segment_reader_1)`
/// - `.collect(doc0_of_segment_1)`
/// - `.collect(...)`
/// - `.collect(last_doc_of_segment_1)`
/// - `...`
/// - `.collect(last_doc_of_last_segment)`
///
/// Segments are not guaranteed to be visited in any specific order.
pub trait Collector {
    /// `set_segment` is called before beginning to enumerate
    /// on this segment.
    fn set_segment(
        &mut self,
        segment_local_id: SegmentLocalId,
        segment: &SegmentReader,
    ) -> Result<()>;
    /// The query pushes the scored document to the collector via this method.
    fn collect(&mut self, doc: DocId, score: Score);

    /// Returns true iff the collector requires to compute scores for documents.
    fn requires_scoring(&self) -> bool;
}

impl<'a, C: Collector> Collector for &'a mut C {
    fn set_segment(
        &mut self,
        segment_local_id: SegmentLocalId,
        segment: &SegmentReader,
    ) -> Result<()> {
        (*self).set_segment(segment_local_id, segment)
    }
    /// The query pushes the scored document to the collector via this method.
    fn collect(&mut self, doc: DocId, score: Score) {
        C::collect(self, doc, score)
    }

    fn requires_scoring(&self) -> bool {
        C::requires_scoring(self)
    }
}

#[cfg(test)]
pub mod tests {

    use super::*;
    use core::SegmentReader;
    use fastfield::BytesFastFieldReader;
    use fastfield::FastFieldReader;
    use schema::Field;
    use DocId;
    use Score;
    use SegmentLocalId;

    /// Stores all of the doc ids.
    /// This collector is only used for tests.
    /// It is unusable in practise, as it does not store
    /// the segment ordinals
    pub struct TestCollector {
        offset: DocId,
        segment_max_doc: DocId,
        docs: Vec<DocId>,
        scores: Vec<Score>,
    }

    impl TestCollector {
        /// Return the exhalist of documents.
        pub fn docs(self) -> Vec<DocId> {
            self.docs
        }

        pub fn scores(self) -> Vec<Score> {
            self.scores
        }
    }

    impl Default for TestCollector {
        fn default() -> TestCollector {
            TestCollector {
                offset: 0,
                segment_max_doc: 0,
                docs: Vec::new(),
                scores: Vec::new(),
            }
        }
    }

    impl Collector for TestCollector {
        fn set_segment(&mut self, _: SegmentLocalId, reader: &SegmentReader) -> Result<()> {
            self.offset += self.segment_max_doc;
            self.segment_max_doc = reader.max_doc();
            Ok(())
        }

        fn collect(&mut self, doc: DocId, score: Score) {
            self.docs.push(doc + self.offset);
            self.scores.push(score);
        }

        fn requires_scoring(&self) -> bool {
            true
        }
    }

    /// Collects in order all of the fast fields for all of the
    /// doc in the `DocSet`
    ///
    /// This collector is mainly useful for tests.
    pub struct FastFieldTestCollector {
        vals: Vec<u64>,
        field: Field,
        ff_reader: Option<FastFieldReader<u64>>,
    }

    impl FastFieldTestCollector {
        pub fn for_field(field: Field) -> FastFieldTestCollector {
            FastFieldTestCollector {
                vals: Vec::new(),
                field,
                ff_reader: None,
            }
        }

        pub fn vals(self) -> Vec<u64> {
            self.vals
        }
    }

    impl Collector for FastFieldTestCollector {
        fn set_segment(&mut self, _: SegmentLocalId, reader: &SegmentReader) -> Result<()> {
            self.ff_reader = Some(reader.fast_field_reader(self.field)?);
            Ok(())
        }

        fn collect(&mut self, doc: DocId, _score: Score) {
            let val = self.ff_reader.as_ref().unwrap().get(doc);
            self.vals.push(val);
        }
        fn requires_scoring(&self) -> bool {
            false
        }
    }

    /// Collects in order all of the fast field bytes for all of the
    /// docs in the `DocSet`
    ///
    /// This collector is mainly useful for tests.
    pub struct BytesFastFieldTestCollector {
        vals: Vec<u8>,
        field: Field,
        ff_reader: Option<BytesFastFieldReader>,
    }

    impl BytesFastFieldTestCollector {
        pub fn for_field(field: Field) -> BytesFastFieldTestCollector {
            BytesFastFieldTestCollector {
                vals: Vec::new(),
                field,
                ff_reader: None,
            }
        }

        pub fn vals(self) -> Vec<u8> {
            self.vals
        }
    }

    impl Collector for BytesFastFieldTestCollector {
        fn set_segment(&mut self, _segment_local_id: u32, segment: &SegmentReader) -> Result<()> {
            self.ff_reader = Some(segment.bytes_fast_field_reader(self.field)?);
            Ok(())
        }

        fn collect(&mut self, doc: u32, _score: f32) {
            let val = self.ff_reader.as_ref().unwrap().get_val(doc);
            self.vals.extend(val);
        }

        fn requires_scoring(&self) -> bool {
            false
        }
    }
}

#[cfg(all(test, feature = "unstable"))]
mod bench {
    use collector::{Collector, CountCollector};
    use test::Bencher;

    #[bench]
    fn build_collector(b: &mut Bencher) {
        b.iter(|| {
            let mut count_collector = CountCollector::default();
            let docs: Vec<u32> = (0..1_000_000).collect();
            for doc in docs {
                count_collector.collect(doc, 1f32);
            }
            count_collector.count()
        });
    }
}
<commit_msg>Exposing the<commit_after>/*!
Defines how the documents matching a search query should be processed.
*/

use DocId;
use Result;
use Score;
use SegmentLocalId;
use SegmentReader;

mod count_collector;
pub use self::count_collector::CountCollector;

mod multi_collector;
pub use self::multi_collector::MultiCollector;

mod top_collector;
pub use self::top_collector::TopCollector;

mod facet_collector;
pub use self::facet_collector::FacetCollector;

mod chained_collector;
pub use self::chained_collector::{chain, ChainedCollector};

/// Collectors are in charge of collecting and retaining relevant
/// information from the document found and scored by the query.
///
///
/// For instance,
///
/// - keeping track of the top 10 best documents
/// - computing a breakdown over a fast field
/// - computing the number of documents matching the query
///
/// Queries are in charge of pushing the `DocSet` to the collector.
///
/// As they work on multiple segments, they first inform
/// the collector of a change in a segment and then
/// call the `collect` method to push the document to the collector.
///
/// Temporally, our collector will receive calls
/// - `.set_segment(0, segment_reader_0)`
/// - `.collect(doc0_of_segment_0)`
/// - `.collect(...)`
/// - `.collect(last_doc_of_segment_0)`
/// - `.set_segment(1, segment_reader_1)`
/// - `.collect(doc0_of_segment_1)`
/// - `.collect(...)`
/// - `.collect(last_doc_of_segment_1)`
/// - `...`
/// - `.collect(last_doc_of_last_segment)`
///
/// Segments are not guaranteed to be visited in any specific order.
pub trait Collector {
    /// `set_segment` is called before beginning to enumerate
    /// on this segment.
    fn set_segment(
        &mut self,
        segment_local_id: SegmentLocalId,
        segment: &SegmentReader,
    ) -> Result<()>;
    /// The query pushes the scored document to the collector via this method.
    fn collect(&mut self, doc: DocId, score: Score);

    /// Returns true iff the collector requires to compute scores for documents.
    fn requires_scoring(&self) -> bool;
}

impl<'a, C: Collector> Collector for &'a mut C {
    fn set_segment(
        &mut self,
        segment_local_id: SegmentLocalId,
        segment: &SegmentReader,
    ) -> Result<()> {
        (*self).set_segment(segment_local_id, segment)
    }
    /// The query pushes the scored document to the collector via this method.
    fn collect(&mut self, doc: DocId, score: Score) {
        C::collect(self, doc, score)
    }

    fn requires_scoring(&self) -> bool {
        C::requires_scoring(self)
    }
}

#[cfg(test)]
pub mod tests {

    use super::*;
    use core::SegmentReader;
    use fastfield::BytesFastFieldReader;
    use fastfield::FastFieldReader;
    use schema::Field;
    use DocId;
    use Score;
    use SegmentLocalId;

    /// Stores all of the doc ids.
    /// This collector is only used for tests.
    /// It is unusable in practise, as it does not store
    /// the segment ordinals
    pub struct TestCollector {
        offset: DocId,
        segment_max_doc: DocId,
        docs: Vec<DocId>,
        scores: Vec<Score>,
    }

    impl TestCollector {
        /// Return the exhalist of documents.
        pub fn docs(self) -> Vec<DocId> {
            self.docs
        }

        pub fn scores(self) -> Vec<Score> {
            self.scores
        }
    }

    impl Default for TestCollector {
        fn default() -> TestCollector {
            TestCollector {
                offset: 0,
                segment_max_doc: 0,
                docs: Vec::new(),
                scores: Vec::new(),
            }
        }
    }

    impl Collector for TestCollector {
        fn set_segment(&mut self, _: SegmentLocalId, reader: &SegmentReader) -> Result<()> {
            self.offset += self.segment_max_doc;
            self.segment_max_doc = reader.max_doc();
            Ok(())
        }

        fn collect(&mut self, doc: DocId, score: Score) {
            self.docs.push(doc + self.offset);
            self.scores.push(score);
        }

        fn requires_scoring(&self) -> bool {
            true
        }
    }

    /// Collects in order all of the fast fields for all of the
    /// doc in the `DocSet`
    ///
    /// This collector is mainly useful for tests.
    pub struct FastFieldTestCollector {
        vals: Vec<u64>,
        field: Field,
        ff_reader: Option<FastFieldReader<u64>>,
    }

    impl FastFieldTestCollector {
        pub fn for_field(field: Field) -> FastFieldTestCollector {
            FastFieldTestCollector {
                vals: Vec::new(),
                field,
                ff_reader: None,
            }
        }

        pub fn vals(self) -> Vec<u64> {
            self.vals
        }
    }

    impl Collector for FastFieldTestCollector {
        fn set_segment(&mut self, _: SegmentLocalId, reader: &SegmentReader) -> Result<()> {
            self.ff_reader = Some(reader.fast_field_reader(self.field)?);
            Ok(())
        }

        fn collect(&mut self, doc: DocId, _score: Score) {
            let val = self.ff_reader.as_ref().unwrap().get(doc);
            self.vals.push(val);
        }
        fn requires_scoring(&self) -> bool {
            false
        }
    }

    /// Collects in order all of the fast field bytes for all of the
    /// docs in the `DocSet`
    ///
    /// This collector is mainly useful for tests.
    pub struct BytesFastFieldTestCollector {
        vals: Vec<u8>,
        field: Field,
        ff_reader: Option<BytesFastFieldReader>,
    }

    impl BytesFastFieldTestCollector {
        pub fn for_field(field: Field) -> BytesFastFieldTestCollector {
            BytesFastFieldTestCollector {
                vals: Vec::new(),
                field,
                ff_reader: None,
            }
        }

        pub fn vals(self) -> Vec<u8> {
            self.vals
        }
    }

    impl Collector for BytesFastFieldTestCollector {
        fn set_segment(&mut self, _segment_local_id: u32, segment: &SegmentReader) -> Result<()> {
            self.ff_reader = Some(segment.bytes_fast_field_reader(self.field)?);
            Ok(())
        }

        fn collect(&mut self, doc: u32, _score: f32) {
            let val = self.ff_reader.as_ref().unwrap().get_val(doc);
            self.vals.extend(val);
        }

        fn requires_scoring(&self) -> bool {
            false
        }
    }
}

#[cfg(all(test, feature = "unstable"))]
mod bench {
    use collector::{Collector, CountCollector};
    use test::Bencher;

    #[bench]
    fn build_collector(b: &mut Bencher) {
        b.iter(|| {
            let mut count_collector = CountCollector::default();
            let docs: Vec<u32> = (0..1_000_000).collect();
            for doc in docs {
                count_collector.collect(doc, 1f32);
            }
            count_collector.count()
        });
    }
}
<|endoftext|>