<commit_before>#[macro_use]
extern crate clap;
extern crate rls_analysis;
extern crate rls_data as data;
extern crate serde;
#[macro_use]
extern crate serde_json;

use data::GlobalCrateId;
use rls_analysis::{AnalysisHost, AnalysisLoader};
use std::collections::HashMap;
use std::fs::{self, File};
use std::path::{Path, PathBuf};

/// A global definition id in a crate.
///
/// FIXME(emilio): This key is kind of slow, because GlobalCrateId contains a
/// String. There's a "disambiguator" field which may be more than enough for
/// our purposes.
#[derive(Clone, Hash, Debug, Eq, PartialEq)]
pub struct DefId(GlobalCrateId, u32);

/// A map from global definition ids to the actual definition.
pub struct Defs {
    map: HashMap<DefId, data::Def>,
}

impl Defs {
    fn new() -> Self {
        Self { map: HashMap::new() }
    }

    fn insert(&mut self, analysis: &data::Analysis, def: &data::Def) {
        let crate_id = analysis.prelude.as_ref().unwrap().crate_id.clone();
        let mut definition = def.clone();
        let crate_independent_qualname =
            format!("{}{}", crate_id.name, def.qualname);
        definition.qualname = crate_independent_qualname;

        let index = definition.id.index;
        let previous = self.map.insert(DefId(crate_id, index), definition);
        assert!(
            previous.is_none(),
            "Found a definition with the same ID twice? {:?}",
            def,
        );
    }

    /// Getter for a given local id, which takes care of converting to a global
    /// ID and returning the definition if present.
    fn get(&self, analysis: &data::Analysis, id: data::Id) -> Option<data::Def> {
        let prelude = analysis.prelude.as_ref().unwrap();
        let krate_id = if id.krate == 0 {
            prelude.crate_id.clone()
        } else {
            // TODO(emilio): This escales with the number of crates in this
            // particular crate, but it's probably not too bad, since it should
            // be a pretty fast linear search.
            let krate = prelude.external_crates.iter().find(|krate| {
                krate.num == id.krate
            });

            let krate = match krate {
                Some(k) => k,
                None => {
                    eprintln!("Crate not found: {:?}", id);
                    return None;
                }
            };

            krate.id.clone()
        };

        let id = DefId(krate_id, id.index);
        self.map.get(&id).cloned()
    }
}

#[derive(Clone)]
pub struct Loader {
    deps_dir: PathBuf,
}

impl Loader {
    pub fn new(deps_dir: PathBuf) -> Self {
        Self { deps_dir }
    }
}

impl AnalysisLoader for Loader {
    fn needs_hard_reload(&self, _: &Path) -> bool {
        true
    }

    fn fresh_host(&self) -> AnalysisHost<Self> {
        AnalysisHost::new_with_loader(self.clone())
    }

    fn set_path_prefix(&mut self, _: &Path) {}

    fn abs_path_prefix(&self) -> Option<PathBuf> {
        None
    }
    fn search_directories(&self) -> Vec<PathBuf> {
        vec![self.deps_dir.clone()]
    }
}

// Searchfox uses 1-indexed lines, 0-indexed columns.
fn span_to_string(span: &data::SpanData) -> String {
    // Rust spans are multi-line... So we just set the start column if it spans
    // multiple rows, searchfox has fallback code to handle this.
    if span.line_start != span.line_end {
        return format!("{}:{}", span.line_start.0, span.column_start.0 - 1);
    }
    if span.column_start == span.column_end {
        return format!("{}:{}", span.line_start.0, span.column_start.0 - 1);
    }
    format!("{}:{}-{}", span.line_start.0, span.column_start.0 - 1, span.column_end.0 - 1)
}

fn visit(
    file: &mut File,
    kind: &'static str,
    location: &data::SpanData,
    qualname: &str,
    context: Option<&str>,
) {
    use serde_json::map::Map;
    use serde_json::value::Value;
    use std::io::Write;

    let mut out = Map::new();
    out.insert("loc".into(), Value::String(span_to_string(location)));
    out.insert("target".into(), json!(1));
    out.insert("kind".into(), Value::String(kind.into()));
    out.insert("pretty".into(), Value::String(qualname.into()));
    out.insert("sym".into(), Value::String(qualname.into()));
    if let Some(context) = context {
        out.insert("context".into(), Value::String(context.into()));
        out.insert("contextsym".into(), Value::String(context.into()));
    }

    let object = serde_json::to_string(&Value::Object(out)).unwrap();
    file.write_all(object.as_bytes()).unwrap();
    write!(file, "\n").unwrap();

    let mut out = Map::new();
    out.insert("loc".into(), Value::String(span_to_string(location)));
    out.insert("source".into(), json!(1));
    out.insert("kind".into(), Value::String(kind.into()));
    out.insert("pretty".into(), Value::String(qualname.into()));
    out.insert("sym".into(), Value::String(qualname.into()));

    let object = serde_json::to_string(&Value::Object(out)).unwrap();
    file.write_all(object.as_bytes()).unwrap();
    write!(file, "\n").unwrap();
}

fn analyze_file(
    file_name: &PathBuf,
    defs: &Defs,
    file_analysis: &data::Analysis,
    src_dir: &Path,
    output_dir: &Path,
) {
    let file = match file_name.strip_prefix(src_dir) {
        Ok(f) => f,
        Err(..) => {
            eprintln!("File not in the source directory: {}", file_name.display());
            return;
        }
    };

    let output_file = output_dir.join(file);
    let mut output_dir = output_file.clone();
    output_dir.pop();
    if let Err(err) = fs::create_dir_all(output_dir) {
        eprintln!(
            "Couldn't create dir for: {}, {:?}",
            output_file.display(),
            err
        );
        return;
    }
    let mut file = match File::create(&output_file) {
        Ok(f) => f,
        Err(err) => {
            eprintln!(
                "Couldn't open output file: {}, {:?}",
                output_file.display(),
                err
            );
            return;
        }
    };

    for import in &file_analysis.imports {
        let id = match import.ref_id {
            Some(id) => id,
            None => {
                eprintln!("Dropping import {}: {}, no ref", import.name, import.value);
                continue;
            }
        };

        let def = match defs.get(file_analysis, id) {
            Some(def) => def,
            None => {
                eprintln!("Dropping import {}: {}, no def for ref {:?}", import.name, import.value, id);
                continue;
            }
        };

        visit(&mut file, "use", &import.span, &def.qualname, None)
    }

    for def in &file_analysis.defs {
        let parent = def.parent
            .and_then(|parent_id| defs.get(file_analysis, parent_id).map(|d| d.qualname));

        let crate_name = &file_analysis.prelude.as_ref().unwrap().crate_id.name;
        let qualname = format!("{}{}", crate_name, def.qualname);
        visit(&mut file, "def", &def.span, &qualname, parent.as_ref().map(|p| &**p))
    }

    for ref_ in &file_analysis.refs {
        let def = match defs.get(file_analysis, ref_.ref_id) {
            Some(d) => d,
            None => {
                eprintln!("Dropping ref {:?}, kind {:?}, no def", ref_.ref_id, ref_.kind);
                continue;
            }
        };
        visit(
            &mut file,
            "use",
            &ref_.span,
            &def.qualname,
            /* context = */ None, // TODO
        )
    }
}

fn analyze_crate(
    analysis: &data::Analysis,
    defs: &Defs,
    src_dir: &Path,
    output_dir: &Path,
) {
    let mut per_file = HashMap::new();

    println!("Analyzing crate: {:?}", analysis.prelude);

    macro_rules! flat_map_per_file {
        ($field:ident) => {
            for item in &analysis.$field {
                let mut file_analysis =
                    per_file.entry(item.span.file_name.clone())
                        .or_insert_with(|| {
                            let prelude = analysis.prelude.clone();
                            let mut analysis = data::Analysis::new(analysis.config.clone());
                            analysis.prelude = prelude;
                            analysis
                        });
                file_analysis.$field.push(item.clone());
            }
        }
    }

    flat_map_per_file!(imports);
    flat_map_per_file!(defs);
    flat_map_per_file!(impls);
    flat_map_per_file!(refs);
    flat_map_per_file!(macro_refs);
    flat_map_per_file!(relations);

    for (mut name, analysis) in per_file.drain() {
        if name.is_relative() {
            name = src_dir.join(name);
        }
        analyze_file(&name, defs, &analysis, src_dir, output_dir);
    }
}

fn main() {
    let matches = app_from_crate!()
        .args_from_usage(
            "<src>    'Points to the source root'
             <input>  'Points to the deps/save-analysis directory'
             <output> 'Points to the directory where searchfox metadata should'",
        )
        .get_matches();

    let src_dir = Path::new(matches.value_of("src").unwrap());
    let input_dir = Path::new(matches.value_of("input").unwrap());
    let output_dir = Path::new(matches.value_of("output").unwrap());

    let loader = Loader::new(PathBuf::from(input_dir));

    let crates = rls_analysis::read_analysis_from_files(&loader, Default::default(), &[]);

    let mut defs = Defs::new();
    for krate in &crates {
        for def in &krate.analysis.defs {
            println!("Indexing def: {:?}", def);
            defs.insert(&krate.analysis, def);
        }
    }

    for krate in crates {
        analyze_crate(&krate.analysis, &defs, &src_dir, &output_dir);
    }
}
<commit_msg>rust-indexer: Fix usage documentation message.<commit_after>#[macro_use]
extern crate clap;
extern crate rls_analysis;
extern crate rls_data as data;
extern crate serde;
#[macro_use]
extern crate serde_json;

use data::GlobalCrateId;
use rls_analysis::{AnalysisHost, AnalysisLoader};
use std::collections::HashMap;
use std::fs::{self, File};
use std::path::{Path, PathBuf};

/// A global definition id in a crate.
///
/// FIXME(emilio): This key is kind of slow, because GlobalCrateId contains a
/// String. There's a "disambiguator" field which may be more than enough for
/// our purposes.
#[derive(Clone, Hash, Debug, Eq, PartialEq)]
pub struct DefId(GlobalCrateId, u32);

/// A map from global definition ids to the actual definition.
pub struct Defs {
    map: HashMap<DefId, data::Def>,
}

impl Defs {
    fn new() -> Self {
        Self { map: HashMap::new() }
    }

    fn insert(&mut self, analysis: &data::Analysis, def: &data::Def) {
        let crate_id = analysis.prelude.as_ref().unwrap().crate_id.clone();
        let mut definition = def.clone();
        let crate_independent_qualname =
            format!("{}{}", crate_id.name, def.qualname);
        definition.qualname = crate_independent_qualname;

        let index = definition.id.index;
        let previous = self.map.insert(DefId(crate_id, index), definition);
        assert!(
            previous.is_none(),
            "Found a definition with the same ID twice? {:?}",
            def,
        );
    }

    /// Getter for a given local id, which takes care of converting to a global
    /// ID and returning the definition if present.
    fn get(&self, analysis: &data::Analysis, id: data::Id) -> Option<data::Def> {
        let prelude = analysis.prelude.as_ref().unwrap();
        let krate_id = if id.krate == 0 {
            prelude.crate_id.clone()
        } else {
            // TODO(emilio): This escales with the number of crates in this
            // particular crate, but it's probably not too bad, since it should
            // be a pretty fast linear search.
            let krate = prelude.external_crates.iter().find(|krate| {
                krate.num == id.krate
            });

            let krate = match krate {
                Some(k) => k,
                None => {
                    eprintln!("Crate not found: {:?}", id);
                    return None;
                }
            };

            krate.id.clone()
        };

        let id = DefId(krate_id, id.index);
        self.map.get(&id).cloned()
    }
}

#[derive(Clone)]
pub struct Loader {
    deps_dir: PathBuf,
}

impl Loader {
    pub fn new(deps_dir: PathBuf) -> Self {
        Self { deps_dir }
    }
}

impl AnalysisLoader for Loader {
    fn needs_hard_reload(&self, _: &Path) -> bool {
        true
    }

    fn fresh_host(&self) -> AnalysisHost<Self> {
        AnalysisHost::new_with_loader(self.clone())
    }

    fn set_path_prefix(&mut self, _: &Path) {}

    fn abs_path_prefix(&self) -> Option<PathBuf> {
        None
    }
    fn search_directories(&self) -> Vec<PathBuf> {
        vec![self.deps_dir.clone()]
    }
}

// Searchfox uses 1-indexed lines, 0-indexed columns.
fn span_to_string(span: &data::SpanData) -> String {
    // Rust spans are multi-line... So we just set the start column if it spans
    // multiple rows, searchfox has fallback code to handle this.
    if span.line_start != span.line_end {
        return format!("{}:{}", span.line_start.0, span.column_start.0 - 1);
    }
    if span.column_start == span.column_end {
        return format!("{}:{}", span.line_start.0, span.column_start.0 - 1);
    }
    format!("{}:{}-{}", span.line_start.0, span.column_start.0 - 1, span.column_end.0 - 1)
}

fn visit(
    file: &mut File,
    kind: &'static str,
    location: &data::SpanData,
    qualname: &str,
    context: Option<&str>,
) {
    use serde_json::map::Map;
    use serde_json::value::Value;
    use std::io::Write;

    let mut out = Map::new();
    out.insert("loc".into(), Value::String(span_to_string(location)));
    out.insert("target".into(), json!(1));
    out.insert("kind".into(), Value::String(kind.into()));
    out.insert("pretty".into(), Value::String(qualname.into()));
    out.insert("sym".into(), Value::String(qualname.into()));
    if let Some(context) = context {
        out.insert("context".into(), Value::String(context.into()));
        out.insert("contextsym".into(), Value::String(context.into()));
    }

    let object = serde_json::to_string(&Value::Object(out)).unwrap();
    file.write_all(object.as_bytes()).unwrap();
    write!(file, "\n").unwrap();

    let mut out = Map::new();
    out.insert("loc".into(), Value::String(span_to_string(location)));
    out.insert("source".into(), json!(1));
    out.insert("kind".into(), Value::String(kind.into()));
    out.insert("pretty".into(), Value::String(qualname.into()));
    out.insert("sym".into(), Value::String(qualname.into()));

    let object = serde_json::to_string(&Value::Object(out)).unwrap();
    file.write_all(object.as_bytes()).unwrap();
    write!(file, "\n").unwrap();
}

fn analyze_file(
    file_name: &PathBuf,
    defs: &Defs,
    file_analysis: &data::Analysis,
    src_dir: &Path,
    output_dir: &Path,
) {
    let file = match file_name.strip_prefix(src_dir) {
        Ok(f) => f,
        Err(..) => {
            eprintln!("File not in the source directory: {}", file_name.display());
            return;
        }
    };

    let output_file = output_dir.join(file);
    let mut output_dir = output_file.clone();
    output_dir.pop();
    if let Err(err) = fs::create_dir_all(output_dir) {
        eprintln!(
            "Couldn't create dir for: {}, {:?}",
            output_file.display(),
            err
        );
        return;
    }
    let mut file = match File::create(&output_file) {
        Ok(f) => f,
        Err(err) => {
            eprintln!(
                "Couldn't open output file: {}, {:?}",
                output_file.display(),
                err
            );
            return;
        }
    };

    for import in &file_analysis.imports {
        let id = match import.ref_id {
            Some(id) => id,
            None => {
                eprintln!("Dropping import {}: {}, no ref", import.name, import.value);
                continue;
            }
        };

        let def = match defs.get(file_analysis, id) {
            Some(def) => def,
            None => {
                eprintln!("Dropping import {}: {}, no def for ref {:?}", import.name, import.value, id);
                continue;
            }
        };

        visit(&mut file, "use", &import.span, &def.qualname, None)
    }

    for def in &file_analysis.defs {
        let parent = def.parent
            .and_then(|parent_id| defs.get(file_analysis, parent_id).map(|d| d.qualname));

        let crate_name = &file_analysis.prelude.as_ref().unwrap().crate_id.name;
        let qualname = format!("{}{}", crate_name, def.qualname);
        visit(&mut file, "def", &def.span, &qualname, parent.as_ref().map(|p| &**p))
    }

    for ref_ in &file_analysis.refs {
        let def = match defs.get(file_analysis, ref_.ref_id) {
            Some(d) => d,
            None => {
                eprintln!("Dropping ref {:?}, kind {:?}, no def", ref_.ref_id, ref_.kind);
                continue;
            }
        };
        visit(
            &mut file,
            "use",
            &ref_.span,
            &def.qualname,
            /* context = */ None, // TODO
        )
    }
}

fn analyze_crate(
    analysis: &data::Analysis,
    defs: &Defs,
    src_dir: &Path,
    output_dir: &Path,
) {
    let mut per_file = HashMap::new();

    println!("Analyzing crate: {:?}", analysis.prelude);

    macro_rules! flat_map_per_file {
        ($field:ident) => {
            for item in &analysis.$field {
                let mut file_analysis =
                    per_file.entry(item.span.file_name.clone())
                        .or_insert_with(|| {
                            let prelude = analysis.prelude.clone();
                            let mut analysis = data::Analysis::new(analysis.config.clone());
                            analysis.prelude = prelude;
                            analysis
                        });
                file_analysis.$field.push(item.clone());
            }
        }
    }

    flat_map_per_file!(imports);
    flat_map_per_file!(defs);
    flat_map_per_file!(impls);
    flat_map_per_file!(refs);
    flat_map_per_file!(macro_refs);
    flat_map_per_file!(relations);

    for (mut name, analysis) in per_file.drain() {
        if name.is_relative() {
            name = src_dir.join(name);
        }
        analyze_file(&name, defs, &analysis, src_dir, output_dir);
    }
}

fn main() {
    let matches = app_from_crate!()
        .args_from_usage(
            "<src>    'Points to the source root'
             <input>  'Points to the deps/save-analysis directory'
             <output> 'Points to the directory where searchfox metadata should go'",
        )
        .get_matches();

    let src_dir = Path::new(matches.value_of("src").unwrap());
    let input_dir = Path::new(matches.value_of("input").unwrap());
    let output_dir = Path::new(matches.value_of("output").unwrap());

    let loader = Loader::new(PathBuf::from(input_dir));

    let crates = rls_analysis::read_analysis_from_files(&loader, Default::default(), &[]);

    let mut defs = Defs::new();
    for krate in &crates {
        for def in &krate.analysis.defs {
            println!("Indexing def: {:?}", def);
            defs.insert(&krate.analysis, def);
        }
    }

    for krate in crates {
        analyze_crate(&krate.analysis, &defs, &src_dir, &output_dir);
    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate clap;
extern crate env_logger;
#[macro_use]
extern crate log;
extern crate rls_analysis;
extern crate rls_data as data;
extern crate serde;
#[macro_use]
extern crate serde_json;

use data::GlobalCrateId;
use data::DefKind;
use rls_analysis::{AnalysisHost, AnalysisLoader, SearchDirectory};
use std::collections::{BTreeSet, HashMap};
use std::io::{BufRead, BufReader};
use std::fs::{self, File};
use std::path::{Path, PathBuf};

/// A global definition id in a crate.
///
/// FIXME(emilio): This key is kind of slow, because GlobalCrateId contains a
/// String. There's a "disambiguator" field which may be more than enough for
/// our purposes.
#[derive(Clone, Hash, Debug, Eq, PartialEq)]
pub struct DefId(GlobalCrateId, u32);

/// A map from global definition ids to the actual definition.
pub struct Defs {
    map: HashMap<DefId, data::Def>,
}

struct TreeInfo<'a> {
    src_dir: &'a Path,
    output_dir: &'a Path,
    objdir: &'a Path,
}

// Given a definition, and the global crate id where that definition is found,
// return a qualified name that identifies the definition unambiguously.
fn crate_independent_qualname(
    def: &data::Def,
    crate_id: &data::GlobalCrateId,
) -> String {
    // For functions with "no_mangle", we just use the name.
    if def.kind == DefKind::Function &&
        def.attributes.iter().any(|attr| attr.value == "no_mangle")
    {
        return def.name.clone();
    }

    format!("{}{}", crate_id.name, def.qualname)
}

impl Defs {
    fn new() -> Self {
        Self { map: HashMap::new() }
    }

    fn insert(&mut self, analysis: &data::Analysis, def: &data::Def) {
        let crate_id = analysis.prelude.as_ref().unwrap().crate_id.clone();
        let mut definition = def.clone();
        definition.qualname = crate_independent_qualname(&def, &crate_id);

        let index = definition.id.index;
        let previous = self.map.insert(DefId(crate_id, index), definition);
        if let Some(previous) = previous {
            // This shouldn't happen, but as of right now it can happen with
            // some builtin definitions when highly generic types are involved.
            // This is probably a rust bug, just ignore it for now.
            debug!(
                "Found a definition with the same ID twice? {:?}, {:?}",
                previous,
                def,
            );
        }
    }

    /// Getter for a given local id, which takes care of converting to a global
    /// ID and returning the definition if present.
    fn get(&self, analysis: &data::Analysis, id: data::Id) -> Option<data::Def> {
        let prelude = analysis.prelude.as_ref().unwrap();
        let krate_id = if id.krate == 0 {
            prelude.crate_id.clone()
        } else {
            // TODO(emilio): This escales with the number of crates in this
            // particular crate, but it's probably not too bad, since it should
            // be a pretty fast linear search.
            let krate = prelude.external_crates.iter().find(|krate| {
                krate.num == id.krate
            });

            let krate = match krate {
                Some(k) => k,
                None => {
                    debug!("Crate not found: {:?}", id);
                    return None;
                }
            };

            krate.id.clone()
        };

        let id = DefId(krate_id, id.index);
        self.map.get(&id).cloned()
    }
}

#[derive(Clone)]
pub struct Loader {
    deps_dirs: Vec<PathBuf>,
}

impl Loader {
    pub fn new(deps_dirs: Vec<PathBuf>) -> Self {
        Self { deps_dirs }
    }
}

impl AnalysisLoader for Loader {
    fn needs_hard_reload(&self, _: &Path) -> bool {
        true
    }

    fn fresh_host(&self) -> AnalysisHost<Self> {
        AnalysisHost::new_with_loader(self.clone())
    }

    fn set_path_prefix(&mut self, _: &Path) {}

    fn abs_path_prefix(&self) -> Option<PathBuf> {
        None
    }
    fn search_directories(&self) -> Vec<SearchDirectory> {
        self.deps_dirs.iter().map(|pb| {
            SearchDirectory {
                path: pb.clone(),
                prefix_rewrite: None,
            }
        }).collect()
    }
}

// Searchfox uses 1-indexed lines, 0-indexed columns.
fn span_to_string(span: &data::SpanData) -> String {
    // Rust spans are multi-line... So we just set the start column if it spans
    // multiple rows, searchfox has fallback code to handle this.
    if span.line_start != span.line_end {
        return format!("{}:{}", span.line_start.0, span.column_start.0 - 1);
    }
    if span.column_start == span.column_end {
        return format!("{}:{}", span.line_start.0, span.column_start.0 - 1);
    }
    format!("{}:{}-{}", span.line_start.0, span.column_start.zero_indexed().0, span.column_end.zero_indexed().0)
}

fn def_kind_to_human(kind: DefKind) -> &'static str {
    match kind {
        DefKind::Enum => "enum",
        DefKind::Local => "local",
        DefKind::ExternType => "extern type",
        DefKind::Const => "constant",
        DefKind::Field => "field",
        DefKind::Function | DefKind::ForeignFunction => "function",
        DefKind::Macro => "macro",
        DefKind::Method => "method",
        DefKind::Mod => "module",
        DefKind::Static | DefKind::ForeignStatic => "static",
        DefKind::Struct => "struct",
        DefKind::Tuple => "tuple",
        DefKind::TupleVariant => "tuple variant",
        DefKind::Union => "union",
        DefKind::Type => "type",
        DefKind::Trait => "trait",
        DefKind::StructVariant => "struct variant",
    }
}

fn visit(
    out_data: &mut BTreeSet<String>,
    kind: &'static str,
    location: &data::SpanData,
    qualname: &str,
    def: &data::Def,
    context: Option<&str>,
) {
    use serde_json::map::Map;
    use serde_json::value::Value;

    let mut out = Map::new();
    out.insert("loc".into(), Value::String(span_to_string(location)));
    out.insert("target".into(), json!(1));
    out.insert("kind".into(), Value::String(kind.into()));
    let pretty = {
        let mut pretty = def_kind_to_human(def.kind).to_owned();
        pretty.push_str(" ");
        pretty.push_str(qualname);

        pretty
    };
    out.insert("pretty".into(), Value::String(pretty.clone()));
    out.insert("sym".into(), Value::String(qualname.into()));
    if let Some(context) = context {
        out.insert("context".into(), Value::String(context.into()));
        out.insert("contextsym".into(), Value::String(context.into()));
    }

    let object = serde_json::to_string(&Value::Object(out)).unwrap();
    out_data.insert(object);

    let mut out = Map::new();
    out.insert("loc".into(), Value::String(span_to_string(location)));
    out.insert("source".into(), json!(1));
    out.insert("kind".into(), Value::String(kind.into()));
    out.insert("pretty".into(), Value::String(pretty));
    out.insert("sym".into(), Value::String(qualname.into()));

    let object = serde_json::to_string(&Value::Object(out)).unwrap();
    out_data.insert(object);
}

fn find_generated_or_src_file(
    file_name: &Path,
    tree_info: &TreeInfo,
) -> Option<PathBuf> {
    if let Ok(generated_path) = file_name.strip_prefix(tree_info.objdir) {
        return Some(Path::new("__GENERATED__").join(generated_path))
    }
    file_name.strip_prefix(tree_info.src_dir).ok().map(From::from)
}

fn read_existing_contents(
    map: &mut BTreeSet<String>,
    file: &Path,
) {
    if let Ok(f) = File::open(file) {
        let mut reader = BufReader::new(f);
        for line in reader.lines() {
            map.insert(line.unwrap());
        }
    }
}

fn analyze_file(
    file_name: &PathBuf,
    defs: &Defs,
    file_analysis: &data::Analysis,
    tree_info: &TreeInfo,
) {
    use std::io::Write;

    let file = match find_generated_or_src_file(file_name, tree_info) {
        Some(f) => f,
        None => {
            error!("File not in the source directory or objdir: {}", file_name.display());
            return;
        }
    };

    let output_file = tree_info.output_dir.join(file);
    let mut dataset = BTreeSet::new();
    read_existing_contents(&mut dataset, &output_file);
    let mut output_dir = output_file.clone();
    output_dir.pop();
    if let Err(err) = fs::create_dir_all(output_dir) {
        error!(
            "Couldn't create dir for: {}, {:?}",
            output_file.display(),
            err
        );
        return;
    }
    let mut file = match File::create(&output_file) {
        Ok(f) => f,
        Err(err) => {
            error!(
                "Couldn't open output file: {}, {:?}",
                output_file.display(),
                err
            );
            return;
        }
    };

    for import in &file_analysis.imports {
        let id = match import.ref_id {
            Some(id) => id,
            None => {
                debug!("Dropping import {} ({:?}): {}, no ref", import.name, import.kind, import.value);
                continue;
            }
        };

        let def = match defs.get(file_analysis, id) {
            Some(def) => def,
            None => {
                debug!("Dropping import {} ({:?}): {}, no def for ref {:?}", import.name, import.kind, import.value, id);
                continue;
            }
        };

        visit(&mut dataset, "use", &import.span, &def.qualname, &def, None)
    }

    for def in &file_analysis.defs {
        let parent =
            def.parent.and_then(|parent_id| defs.get(file_analysis, parent_id));

        if let Some(ref parent) = parent {
            if parent.kind == DefKind::Trait {
                let trait_dependent_name =
                    format!("{}::{}", parent.qualname, def.name);
                visit(
                    &mut dataset,
                    "def",
                    &def.span,
                    &trait_dependent_name,
                    &def,
                    Some(&parent.qualname),
                )
            }
        }

        let crate_id = &file_analysis.prelude.as_ref().unwrap().crate_id;
        let qualname = crate_independent_qualname(&def, crate_id);
        visit(&mut dataset, "def", &def.span, &qualname, &def, parent.as_ref().map(|p| &*p.qualname))
    }

    for ref_ in &file_analysis.refs {
        let def = match defs.get(file_analysis, ref_.ref_id) {
            Some(d) => d,
            None => {
                debug!("Dropping ref {:?}, kind {:?}, no def", ref_.ref_id, ref_.kind);
                continue;
            }
        };
        visit(
            &mut dataset,
            "use",
            &ref_.span,
            &def.qualname,
            &def,
            /* context = */ None, // TODO
        )
    }

    for obj in &dataset {
        file.write_all(obj.as_bytes()).unwrap();
        write!(file, "\n").unwrap();
    }
}

fn analyze_crate(
    analysis: &data::Analysis,
    defs: &Defs,
    tree_info: &TreeInfo,
) {
    let mut per_file = HashMap::new();

    println!("Analyzing crate: {:?}", analysis.prelude);

    macro_rules! flat_map_per_file {
        ($field:ident) => {
            for item in &analysis.$field {
                let mut file_analysis =
                    per_file.entry(item.span.file_name.clone())
                        .or_insert_with(|| {
                            let prelude = analysis.prelude.clone();
                            let mut analysis = data::Analysis::new(analysis.config.clone());
                            analysis.prelude = prelude;
                            analysis
                        });
                file_analysis.$field.push(item.clone());
            }
        }
    }

    flat_map_per_file!(imports);
    flat_map_per_file!(defs);
    flat_map_per_file!(impls);
    flat_map_per_file!(refs);
    flat_map_per_file!(macro_refs);
    flat_map_per_file!(relations);

    for (mut name, analysis) in per_file.drain() {
        if name.is_relative() {
            name = tree_info.src_dir.join(name);
        }
        analyze_file(&name, defs, &analysis, tree_info);
    }
}

fn main() {
    use clap::Arg;
    env_logger::init();
    let matches = app_from_crate!()
        .args_from_usage(
            "<src>      'Points to the source root'
             <output>   'Points to the directory where searchfox metadata should go'
             <objdir>   'Points to the objdir generated files may come from'"
        )
        .arg(Arg::with_name("input")
            .required(false)
            .multiple(true)
            .help("rustc analysis directories")
        )
        .get_matches();

    let src_dir = Path::new(matches.value_of("src").unwrap());
    let output_dir = Path::new(matches.value_of("output").unwrap());
    let objdir = Path::new(matches.value_of("objdir").unwrap());

    let tree_info = TreeInfo { src_dir, output_dir, objdir };

    let input_dirs = match matches.values_of("input") {
        Some(inputs) => inputs.map(PathBuf::from).collect(),
        None => vec![],
    };
    let loader = Loader::new(input_dirs);

    let crates = rls_analysis::read_analysis_from_files(&loader, Default::default(), &[]);

    println!("{:?}", crates.iter().map(|k| &k.id.name).collect::<Vec<_>>());

    let mut defs = Defs::new();
    for krate in &crates {
        for def in &krate.analysis.defs {
            // println!("Indexing def: {:?}", def);
            defs.insert(&krate.analysis, def);
        }
    }

    for krate in crates {
        analyze_crate(&krate.analysis, &defs, &tree_info);
    }
}
<commit_msg>Bug 1474434 - Fix 'pretty' property for rust target records.<commit_after>#[macro_use]
extern crate clap;
extern crate env_logger;
#[macro_use]
extern crate log;
extern crate rls_analysis;
extern crate rls_data as data;
extern crate serde;
#[macro_use]
extern crate serde_json;

use data::GlobalCrateId;
use data::DefKind;
use rls_analysis::{AnalysisHost, AnalysisLoader, SearchDirectory};
use std::collections::{BTreeSet, HashMap};
use std::io::{BufRead, BufReader};
use std::fs::{self, File};
use std::path::{Path, PathBuf};

/// A global definition id in a crate.
///
/// FIXME(emilio): This key is kind of slow, because GlobalCrateId contains a
/// String. There's a "disambiguator" field which may be more than enough for
/// our purposes.
#[derive(Clone, Hash, Debug, Eq, PartialEq)]
pub struct DefId(GlobalCrateId, u32);

/// A map from global definition ids to the actual definition.
pub struct Defs {
    map: HashMap<DefId, data::Def>,
}

struct TreeInfo<'a> {
    src_dir: &'a Path,
    output_dir: &'a Path,
    objdir: &'a Path,
}

// Given a definition, and the global crate id where that definition is found,
// return a qualified name that identifies the definition unambiguously.
fn crate_independent_qualname(
    def: &data::Def,
    crate_id: &data::GlobalCrateId,
) -> String {
    // For functions with "no_mangle", we just use the name.
    if def.kind == DefKind::Function &&
        def.attributes.iter().any(|attr| attr.value == "no_mangle")
    {
        return def.name.clone();
    }

    format!("{}{}", crate_id.name, def.qualname)
}

impl Defs {
    fn new() -> Self {
        Self { map: HashMap::new() }
    }

    fn insert(&mut self, analysis: &data::Analysis, def: &data::Def) {
        let crate_id = analysis.prelude.as_ref().unwrap().crate_id.clone();
        let mut definition = def.clone();
        definition.qualname = crate_independent_qualname(&def, &crate_id);

        let index = definition.id.index;
        let previous = self.map.insert(DefId(crate_id, index), definition);
        if let Some(previous) = previous {
            // This shouldn't happen, but as of right now it can happen with
            // some builtin definitions when highly generic types are involved.
            // This is probably a rust bug, just ignore it for now.
            debug!(
                "Found a definition with the same ID twice? {:?}, {:?}",
                previous,
                def,
            );
        }
    }

    /// Getter for a given local id, which takes care of converting to a global
    /// ID and returning the definition if present.
    fn get(&self, analysis: &data::Analysis, id: data::Id) -> Option<data::Def> {
        let prelude = analysis.prelude.as_ref().unwrap();
        let krate_id = if id.krate == 0 {
            prelude.crate_id.clone()
        } else {
            // TODO(emilio): This escales with the number of crates in this
            // particular crate, but it's probably not too bad, since it should
            // be a pretty fast linear search.
            let krate = prelude.external_crates.iter().find(|krate| {
                krate.num == id.krate
            });

            let krate = match krate {
                Some(k) => k,
                None => {
                    debug!("Crate not found: {:?}", id);
                    return None;
                }
            };

            krate.id.clone()
        };

        let id = DefId(krate_id, id.index);
        self.map.get(&id).cloned()
    }
}

#[derive(Clone)]
pub struct Loader {
    deps_dirs: Vec<PathBuf>,
}

impl Loader {
    pub fn new(deps_dirs: Vec<PathBuf>) -> Self {
        Self { deps_dirs }
    }
}

impl AnalysisLoader for Loader {
    fn needs_hard_reload(&self, _: &Path) -> bool {
        true
    }

    fn fresh_host(&self) -> AnalysisHost<Self> {
        AnalysisHost::new_with_loader(self.clone())
    }

    fn set_path_prefix(&mut self, _: &Path) {}

    fn abs_path_prefix(&self) -> Option<PathBuf> {
        None
    }
    fn search_directories(&self) -> Vec<SearchDirectory> {
        self.deps_dirs.iter().map(|pb| {
            SearchDirectory {
                path: pb.clone(),
                prefix_rewrite: None,
            }
        }).collect()
    }
}

// Searchfox uses 1-indexed lines, 0-indexed columns.
fn span_to_string(span: &data::SpanData) -> String {
    // Rust spans are multi-line... So we just set the start column if it spans
    // multiple rows, searchfox has fallback code to handle this.
    if span.line_start != span.line_end {
        return format!("{}:{}", span.line_start.0, span.column_start.0 - 1);
    }
    if span.column_start == span.column_end {
        return format!("{}:{}", span.line_start.0, span.column_start.0 - 1);
    }
    format!("{}:{}-{}", span.line_start.0, span.column_start.zero_indexed().0, span.column_end.zero_indexed().0)
}

fn def_kind_to_human(kind: DefKind) -> &'static str {
    match kind {
        DefKind::Enum => "enum",
        DefKind::Local => "local",
        DefKind::ExternType => "extern type",
        DefKind::Const => "constant",
        DefKind::Field => "field",
        DefKind::Function | DefKind::ForeignFunction => "function",
        DefKind::Macro => "macro",
        DefKind::Method => "method",
        DefKind::Mod => "module",
        DefKind::Static | DefKind::ForeignStatic => "static",
        DefKind::Struct => "struct",
        DefKind::Tuple => "tuple",
        DefKind::TupleVariant => "tuple variant",
        DefKind::Union => "union",
        DefKind::Type => "type",
        DefKind::Trait => "trait",
        DefKind::StructVariant => "struct variant",
    }
}

fn visit(
    out_data: &mut BTreeSet<String>,
    kind: &'static str,
    location: &data::SpanData,
    qualname: &str,
    def: &data::Def,
    context: Option<&str>,
) {
    use serde_json::map::Map;
    use serde_json::value::Value;

    let mut out = Map::new();
    out.insert("loc".into(), Value::String(span_to_string(location)));
    out.insert("target".into(), json!(1));
    out.insert("kind".into(), Value::String(kind.into()));
    out.insert("pretty".into(), Value::String(qualname.into()));
    out.insert("sym".into(), Value::String(qualname.into()));
    if let Some(context) = context {
        out.insert("context".into(), Value::String(context.into()));
        out.insert("contextsym".into(), Value::String(context.into()));
    }

    let object = serde_json::to_string(&Value::Object(out)).unwrap();
    out_data.insert(object);

    let pretty = {
        let mut pretty = def_kind_to_human(def.kind).to_owned();
        pretty.push_str(" ");
        pretty.push_str(qualname);

        pretty
    };

    let mut out = Map::new();
    out.insert("loc".into(), Value::String(span_to_string(location)));
    out.insert("source".into(), json!(1));
    out.insert("kind".into(), Value::String(kind.into()));
    out.insert("pretty".into(), Value::String(pretty));
    out.insert("sym".into(), Value::String(qualname.into()));

    let object = serde_json::to_string(&Value::Object(out)).unwrap();
    out_data.insert(object);
}

fn find_generated_or_src_file(
    file_name: &Path,
    tree_info: &TreeInfo,
) -> Option<PathBuf> {
    if let Ok(generated_path) = file_name.strip_prefix(tree_info.objdir) {
        return Some(Path::new("__GENERATED__").join(generated_path))
    }
    file_name.strip_prefix(tree_info.src_dir).ok().map(From::from)
}

fn read_existing_contents(
    map: &mut BTreeSet<String>,
    file: &Path,
) {
    if let Ok(f) = File::open(file) {
        let mut reader = BufReader::new(f);
        for line in reader.lines() {
            map.insert(line.unwrap());
        }
    }
}

fn analyze_file(
    file_name: &PathBuf,
    defs: &Defs,
    file_analysis: &data::Analysis,
    tree_info: &TreeInfo,
) {
    use std::io::Write;

    let file = match find_generated_or_src_file(file_name, tree_info) {
        Some(f) => f,
        None => {
            error!("File not in the source directory or objdir: {}", file_name.display());
            return;
        }
    };

    let output_file = tree_info.output_dir.join(file);
    let mut dataset = BTreeSet::new();
    read_existing_contents(&mut dataset, &output_file);
    let mut output_dir = output_file.clone();
    output_dir.pop();
    if let Err(err) = fs::create_dir_all(output_dir) {
        error!(
            "Couldn't create dir for: {}, {:?}",
            output_file.display(),
            err
        );
        return;
    }
    let mut file = match File::create(&output_file) {
        Ok(f) => f,
        Err(err) => {
            error!(
                "Couldn't open output file: {}, {:?}",
                output_file.display(),
                err
            );
            return;
        }
    };

    for import in &file_analysis.imports {
        let id = match import.ref_id {
            Some(id) => id,
            None => {
                debug!("Dropping import {} ({:?}): {}, no ref", import.name, import.kind, import.value);
                continue;
            }
        };

        let def = match defs.get(file_analysis, id) {
            Some(def) => def,
            None => {
                debug!("Dropping import {} ({:?}): {}, no def for ref {:?}", import.name, import.kind, import.value, id);
                continue;
            }
        };

        visit(&mut dataset, "use", &import.span, &def.qualname, &def, None)
    }

    for def in &file_analysis.defs {
        let parent =
            def.parent.and_then(|parent_id| defs.get(file_analysis, parent_id));

        if let Some(ref parent) = parent {
            if parent.kind == DefKind::Trait {
                let trait_dependent_name =
                    format!("{}::{}", parent.qualname, def.name);
                visit(
                    &mut dataset,
                    "def",
                    &def.span,
                    &trait_dependent_name,
                    &def,
                    Some(&parent.qualname),
                )
            }
        }

        let crate_id = &file_analysis.prelude.as_ref().unwrap().crate_id;
        let qualname = crate_independent_qualname(&def, crate_id);
        visit(&mut dataset, "def", &def.span, &qualname, &def, parent.as_ref().map(|p| &*p.qualname))
    }

    for ref_ in &file_analysis.refs {
        let def = match defs.get(file_analysis, ref_.ref_id) {
            Some(d) => d,
            None => {
                debug!("Dropping ref {:?}, kind {:?}, no def", ref_.ref_id, ref_.kind);
                continue;
            }
        };
        visit(
            &mut dataset,
            "use",
            &ref_.span,
            &def.qualname,
            &def,
            /* context = */ None, // TODO
        )
    }

    for obj in &dataset {
        file.write_all(obj.as_bytes()).unwrap();
        write!(file, "\n").unwrap();
    }
}

fn analyze_crate(
    analysis: &data::Analysis,
    defs: &Defs,
    tree_info: &TreeInfo,
) {
    let mut per_file = HashMap::new();

    println!("Analyzing crate: {:?}", analysis.prelude);

    macro_rules! flat_map_per_file {
        ($field:ident) => {
            for item in &analysis.$field {
                let mut file_analysis =
                    per_file.entry(item.span.file_name.clone())
                        .or_insert_with(|| {
                            let prelude = analysis.prelude.clone();
                            let mut analysis = data::Analysis::new(analysis.config.clone());
                            analysis.prelude = prelude;
                            analysis
                        });
                file_analysis.$field.push(item.clone());
            }
        }
    }

    flat_map_per_file!(imports);
    flat_map_per_file!(defs);
    flat_map_per_file!(impls);
    flat_map_per_file!(refs);
    flat_map_per_file!(macro_refs);
    flat_map_per_file!(relations);

    for (mut name, analysis) in per_file.drain() {
        if name.is_relative() {
            name = tree_info.src_dir.join(name);
        }
        analyze_file(&name, defs, &analysis, tree_info);
    }
}

fn main() {
    use clap::Arg;
    env_logger::init();
    let matches = app_from_crate!()
        .args_from_usage(
            "<src>      'Points to the source root'
             <output>   'Points to the directory where searchfox metadata should go'
             <objdir>   'Points to the objdir generated files may come from'"
        )
        .arg(Arg::with_name("input")
            .required(false)
            .multiple(true)
            .help("rustc analysis directories")
        )
        .get_matches();

    let src_dir = Path::new(matches.value_of("src").unwrap());
    let output_dir = Path::new(matches.value_of("output").unwrap());
    let objdir = Path::new(matches.value_of("objdir").unwrap());

    let tree_info = TreeInfo { src_dir, output_dir, objdir };

    let input_dirs = match matches.values_of("input") {
        Some(inputs) => inputs.map(PathBuf::from).collect(),
        None => vec![],
    };
    let loader = Loader::new(input_dirs);

    let crates = rls_analysis::read_analysis_from_files(&loader, Default::default(), &[]);

    println!("{:?}", crates.iter().map(|k| &k.id.name).collect::<Vec<_>>());

    let mut defs = Defs::new();
    for krate in &crates {
        for def in &krate.analysis.defs {
            // println!("Indexing def: {:?}", def);
            defs.insert(&krate.analysis, def);
        }
    }

    for krate in crates {
        analyze_crate(&krate.analysis, &defs, &tree_info);
    }
}
<|endoftext|>


<commit_before>// This file provides functions to create a libxml2 xmlParserCtxtPtr, configured
// to read from a gio::InputStream, and to maintain its loading data in an XmlState.

use gio;
use gio::prelude::*;
use std::borrow::Cow;
use std::cell::{Cell, RefCell};
use std::mem;
use std::ptr;
use std::rc::Rc;
use std::slice;
use std::str;
use std::sync::Once;

use glib::translate::*;

use crate::error::LoadingError;
use crate::property_bag::PropertyBag;
use crate::util::cstr;
use crate::util::utf8_cstr;
use crate::xml::XmlState;
use crate::xml2::*;

fn get_xml2_sax_handler() -> xmlSAXHandler {
    let mut h: xmlSAXHandler = unsafe { mem::zeroed() };

    h.getEntity = Some(sax_get_entity_cb);
    h.entityDecl = Some(sax_entity_decl_cb);
    h.unparsedEntityDecl = Some(sax_unparsed_entity_decl_cb);
    h.getParameterEntity = Some(sax_get_parameter_entity_cb);
    h.characters = Some(sax_characters_cb);
    h.cdataBlock = Some(sax_characters_cb);
    h.startElement = Some(sax_start_element_cb);
    h.endElement = Some(sax_end_element_cb);
    h.processingInstruction = Some(sax_processing_instruction_cb);
    h.serror = Some(rsvg_sax_serror_cb);

    h
}

unsafe extern "C" fn rsvg_sax_serror_cb(user_data: *mut libc::c_void, error: xmlErrorPtr) {
    let xml2_parser = &*(user_data as *mut Xml2Parser);
    let error = error.as_ref().unwrap();

    let level_name = match error.level {
        1 => "warning",
        2 => "error",
        3 => "fatal error",
        _ => "unknown error",
    };

    // "int2" is the column number
    let column = if error.int2 > 0 {
        Cow::Owned(format!(":{}", error.int2))
    } else {
        Cow::Borrowed("")
    };

    let full_error_message = format!(
        "{} code={} ({}) in {}:{}{}: {}",
        level_name,
        error.code,
        error.domain,
        cstr(error.file),
        error.line,
        column,
        cstr(error.message)
    );
    xml2_parser.state.error(ParseFromStreamError::XmlParseError(full_error_message));
}

fn free_xml_parser_and_doc(parser: xmlParserCtxtPtr) {
    // Free the ctxt and its ctxt->myDoc - libxml2 doesn't free them together
    // http://xmlsoft.org/html/libxml-parser.html#xmlFreeParserCtxt
    unsafe {
        if !parser.is_null() {
            let rparser = &mut *parser;

            if !rparser.myDoc.is_null() {
                xmlFreeDoc(rparser.myDoc);
                rparser.myDoc = ptr::null_mut();
            }

            xmlFreeParserCtxt(parser);
        }
    }
}

unsafe extern "C" fn sax_get_entity_cb(
    user_data: *mut libc::c_void,
    name: *const libc::c_char,
) -> xmlEntityPtr {
    let xml2_parser = &*(user_data as *mut Xml2Parser);

    assert!(!name.is_null());
    let name = utf8_cstr(name);

    xml2_parser.state.entity_lookup(name).unwrap_or(ptr::null_mut())
}

unsafe extern "C" fn sax_entity_decl_cb(
    user_data: *mut libc::c_void,
    name: *const libc::c_char,
    type_: libc::c_int,
    _public_id: *const libc::c_char,
    _system_id: *const libc::c_char,
    content: *const libc::c_char,
) {
    let xml2_parser = &*(user_data as *mut Xml2Parser);

    assert!(!name.is_null());

    if type_ != XML_INTERNAL_GENERAL_ENTITY {
        // We don't allow loading external entities; we don't support
        // defining parameter entities in the DTD, and libxml2 should
        // handle internal predefined entities by itself (e.g. "&").
        return;
    }

    let entity = xmlNewEntity(
        ptr::null_mut(),
        name,
        type_,
        ptr::null(),
        ptr::null(),
        content,
    );
    assert!(!entity.is_null());

    let name = utf8_cstr(name);
    xml2_parser.state.entity_insert(name, entity);
}

unsafe extern "C" fn sax_unparsed_entity_decl_cb(
    user_data: *mut libc::c_void,
    name: *const libc::c_char,
    public_id: *const libc::c_char,
    system_id: *const libc::c_char,
    _notation_name: *const libc::c_char,
) {
    sax_entity_decl_cb(
        user_data,
        name,
        XML_INTERNAL_GENERAL_ENTITY,
        public_id,
        system_id,
        ptr::null(),
    );
}

unsafe extern "C" fn sax_start_element_cb(
    user_data: *mut libc::c_void,
    name: *const libc::c_char,
    atts: *const *const libc::c_char,
) {
    let xml2_parser = &*(user_data as *mut Xml2Parser);

    assert!(!name.is_null());
    let name = utf8_cstr(name);

    let pbag = PropertyBag::new_from_key_value_pairs(atts);

    if let Err(e) = xml2_parser.state.start_element(name, &pbag) {
        let _: () = e; // guard in case we change the error type later

        let parser = xml2_parser.parser.get();
        xmlStopParser(parser);
    }
}

unsafe extern "C" fn sax_end_element_cb(user_data: *mut libc::c_void, name: *const libc::c_char) {
    let xml2_parser = &*(user_data as *mut Xml2Parser);

    assert!(!name.is_null());
    let name = utf8_cstr(name);

    xml2_parser.state.end_element(name);
}

unsafe extern "C" fn sax_characters_cb(
    user_data: *mut libc::c_void,
    unterminated_text: *const libc::c_char,
    len: libc::c_int,
) {
    let xml2_parser = &*(user_data as *mut Xml2Parser);

    assert!(!unterminated_text.is_null());
    assert!(len >= 0);

    // libxml2 already validated the incoming string as UTF-8.  Note that
    // it is *not* nul-terminated; this is why we create a byte slice first.
    let bytes = std::slice::from_raw_parts(unterminated_text as *const u8, len as usize);
    let utf8 = str::from_utf8_unchecked(bytes);

    xml2_parser.state.characters(utf8);
}

unsafe extern "C" fn sax_processing_instruction_cb(
    user_data: *mut libc::c_void,
    target: *const libc::c_char,
    data: *const libc::c_char,
) {
    let xml2_parser = &*(user_data as *mut Xml2Parser);

    assert!(!target.is_null());
    let target = utf8_cstr(target);

    let data = if data.is_null() {
        ""
    } else {
        utf8_cstr(data)
    };

    xml2_parser.state.processing_instruction(target, data);
}

unsafe extern "C" fn sax_get_parameter_entity_cb(
    user_data: *mut libc::c_void,
    name: *const libc::c_char,
) -> xmlEntityPtr {
    sax_get_entity_cb(user_data, name)
}

fn set_xml_parse_options(parser: xmlParserCtxtPtr, unlimited_size: bool) {
    let mut options: libc::c_int = XML_PARSE_NONET | XML_PARSE_BIG_LINES;

    if unlimited_size {
        options |= XML_PARSE_HUGE;
    }

    unsafe {
        xmlCtxtUseOptions(parser, options);

        // If false, external entities work, but internal ones don't. if
        // true, internal entities work, but external ones don't. favor
        // internal entities, in order to not cause a regression
        (*parser).replaceEntities = 1;
    }
}

// Struct used as closure data for xmlCreateIOParserCtxt().  In conjunction
// with stream_ctx_read() and stream_ctx_close(), this struct provides the
// I/O callbacks and their context for libxml2.
//
// We call I/O methods on the stream, and as soon as we get an error
// we store it in the gio_error field.  Libxml2 just allows us to
// return -1 from the I/O callbacks in that case; it doesn't actually
// see the error code.
//
// The gio_error field comes from the place that constructs the
// StreamCtx.  That place is later responsible for seeing if the error
// is set; if it is, it means that there was an I/O error.  Otherwise,
// there were no I/O errors but the caller must then ask libxml2 for
// XML parsing errors.
struct StreamCtx {
    stream: gio::InputStream,
    cancellable: Option<gio::Cancellable>,
    gio_error: Rc<RefCell<Option<glib::Error>>>,
}

// read() callback from xmlCreateIOParserCtxt()
unsafe extern "C" fn stream_ctx_read(
    context: *mut libc::c_void,
    buffer: *mut libc::c_char,
    len: libc::c_int,
) -> libc::c_int {
    let ctx = &mut *(context as *mut StreamCtx);

    let mut err_ref = ctx.gio_error.borrow_mut();

    // has the error been set already?
    if err_ref.is_some() {
        return -1;
    }

    let buf: &mut [u8] = slice::from_raw_parts_mut(buffer as *mut u8, len as usize);

    match ctx.stream.read(buf, ctx.cancellable.as_ref()) {
        Ok(size) => size as libc::c_int,

        Err(e) => {
            // Just store the first I/O error we get; ignore subsequent ones.
            *err_ref = Some(e);
            -1
        }
    }
}

// close() callback from xmlCreateIOParserCtxt()
unsafe extern "C" fn stream_ctx_close(context: *mut libc::c_void) -> libc::c_int {
    let ctx = &mut *(context as *mut StreamCtx);

    let ret = match ctx.stream.close(ctx.cancellable.as_ref()) {
        Ok(()) => 0,

        Err(e) => {
            let mut err_ref = ctx.gio_error.borrow_mut();

            // don't overwrite a previous error
            if err_ref.is_none() {
                *err_ref = Some(e);
            }

            -1
        }
    };

    Box::from_raw(ctx);

    ret
}

fn init_libxml2() {
    static ONCE: Once = Once::new();

    ONCE.call_once(|| unsafe {
        xmlInitParser();
    });
}

pub struct Xml2Parser {
    parser: Cell<xmlParserCtxtPtr>,
    state: Rc<XmlState>,
    gio_error: Rc<RefCell<Option<glib::Error>>>,
}

impl Xml2Parser {
    pub fn from_stream(
        state: Rc<XmlState>,
        unlimited_size: bool,
        stream: &gio::InputStream,
        cancellable: Option<&gio::Cancellable>,
    ) -> Result<Box<Xml2Parser>, ParseFromStreamError> {
        init_libxml2();

        // The Xml2Parser we end up creating, if
        // xmlCreateIOParserCtxt() is successful, needs to hold a
        // location to place a GError from within the I/O callbacks
        // stream_ctx_read() and stream_ctx_close().  We put this
        // location in an Rc so that it can outlive the call to
        // xmlCreateIOParserCtxt() in case that fails, since on
        // failure that function frees the StreamCtx.
        let gio_error = Rc::new(RefCell::new(None));

        let ctx = Box::new(StreamCtx {
            stream: stream.clone(),
            cancellable: cancellable.map(|c| c.clone()),
            gio_error: gio_error.clone(),
        });

        let mut sax_handler = get_xml2_sax_handler();

        let mut xml2_parser = Box::new(Xml2Parser {
            parser: Cell::new(ptr::null_mut()),
            state,
            gio_error,
        });

        unsafe {
            let parser = xmlCreateIOParserCtxt(
                &mut sax_handler,
                xml2_parser.as_mut() as *mut _ as *mut _,
                Some(stream_ctx_read),
                Some(stream_ctx_close),
                Box::into_raw(ctx) as *mut _,
                XML_CHAR_ENCODING_NONE,
            );

            if parser.is_null() {
                // on error, xmlCreateIOParserCtxt() frees our ctx via the
                // stream_ctx_close function
                Err(ParseFromStreamError::CouldNotCreateXmlParser)
            } else {
                xml2_parser.parser.set(parser);

                set_xml_parse_options(parser, unlimited_size);

                Ok(xml2_parser)
            }
        }
    }

    pub fn parse(&self) -> Result<(), ParseFromStreamError> {
        unsafe {
            let parser = self.parser.get();

            let xml_parse_success = xmlParseDocument(parser) == 0;

            let mut err_ref = self.gio_error.borrow_mut();

            let io_error = err_ref.take();

            if let Some(io_error) = io_error {
                Err(ParseFromStreamError::IoError(io_error))
            } else if !xml_parse_success {
                let xerr = xmlCtxtGetLastError(parser as *mut _);
                let msg = xml2_error_to_string(xerr);
                Err(ParseFromStreamError::XmlParseError(msg))
            } else {
                Ok(())
            }
        }
    }
}

impl Drop for Xml2Parser {
    fn drop(&mut self) {
        let parser = self.parser.get();
        free_xml_parser_and_doc(parser);
        self.parser.set(ptr::null_mut());
    }
}

fn xml2_error_to_string(xerr: xmlErrorPtr) -> String {
    unsafe {
        if !xerr.is_null() {
            let xerr = &*xerr;

            let file = if xerr.file.is_null() {
                "data".to_string()
            } else {
                from_glib_none(xerr.file)
            };

            let message = if xerr.message.is_null() {
                "-".to_string()
            } else {
                from_glib_none(xerr.message)
            };

            format!(
                "Error domain {} code {} on line {} column {} of {}: {}",
                xerr.domain, xerr.code, xerr.line, xerr.int2, file, message
            )
        } else {
            // The error is not set?  Return a generic message :(
            "Error parsing XML data".to_string()
        }
    }
}

// Error returned when parsing an XML stream
#[derive(Clone)]
pub enum ParseFromStreamError {
    // We couldn't even create the libxml2 parser
    CouldNotCreateXmlParser,

    // GIO error from the I/O callbacks
    IoError(glib::Error),

    // XML parsing error from libxml2
    XmlParseError(String),
}

impl From<ParseFromStreamError> for LoadingError {
    fn from(e: ParseFromStreamError) -> LoadingError {
        match e {
            ParseFromStreamError::CouldNotCreateXmlParser => LoadingError::CouldNotCreateXmlParser,
            ParseFromStreamError::IoError(e) => LoadingError::Glib(e),
            ParseFromStreamError::XmlParseError(s) => LoadingError::XmlParseError(s),
        }
    }
}
<commit_msg>xml2: use "cloned()"<commit_after>// This file provides functions to create a libxml2 xmlParserCtxtPtr, configured
// to read from a gio::InputStream, and to maintain its loading data in an XmlState.

use gio;
use gio::prelude::*;
use std::borrow::Cow;
use std::cell::{Cell, RefCell};
use std::mem;
use std::ptr;
use std::rc::Rc;
use std::slice;
use std::str;
use std::sync::Once;

use glib::translate::*;

use crate::error::LoadingError;
use crate::property_bag::PropertyBag;
use crate::util::cstr;
use crate::util::utf8_cstr;
use crate::xml::XmlState;
use crate::xml2::*;

fn get_xml2_sax_handler() -> xmlSAXHandler {
    let mut h: xmlSAXHandler = unsafe { mem::zeroed() };

    h.getEntity = Some(sax_get_entity_cb);
    h.entityDecl = Some(sax_entity_decl_cb);
    h.unparsedEntityDecl = Some(sax_unparsed_entity_decl_cb);
    h.getParameterEntity = Some(sax_get_parameter_entity_cb);
    h.characters = Some(sax_characters_cb);
    h.cdataBlock = Some(sax_characters_cb);
    h.startElement = Some(sax_start_element_cb);
    h.endElement = Some(sax_end_element_cb);
    h.processingInstruction = Some(sax_processing_instruction_cb);
    h.serror = Some(rsvg_sax_serror_cb);

    h
}

unsafe extern "C" fn rsvg_sax_serror_cb(user_data: *mut libc::c_void, error: xmlErrorPtr) {
    let xml2_parser = &*(user_data as *mut Xml2Parser);
    let error = error.as_ref().unwrap();

    let level_name = match error.level {
        1 => "warning",
        2 => "error",
        3 => "fatal error",
        _ => "unknown error",
    };

    // "int2" is the column number
    let column = if error.int2 > 0 {
        Cow::Owned(format!(":{}", error.int2))
    } else {
        Cow::Borrowed("")
    };

    let full_error_message = format!(
        "{} code={} ({}) in {}:{}{}: {}",
        level_name,
        error.code,
        error.domain,
        cstr(error.file),
        error.line,
        column,
        cstr(error.message)
    );
    xml2_parser.state.error(ParseFromStreamError::XmlParseError(full_error_message));
}

fn free_xml_parser_and_doc(parser: xmlParserCtxtPtr) {
    // Free the ctxt and its ctxt->myDoc - libxml2 doesn't free them together
    // http://xmlsoft.org/html/libxml-parser.html#xmlFreeParserCtxt
    unsafe {
        if !parser.is_null() {
            let rparser = &mut *parser;

            if !rparser.myDoc.is_null() {
                xmlFreeDoc(rparser.myDoc);
                rparser.myDoc = ptr::null_mut();
            }

            xmlFreeParserCtxt(parser);
        }
    }
}

unsafe extern "C" fn sax_get_entity_cb(
    user_data: *mut libc::c_void,
    name: *const libc::c_char,
) -> xmlEntityPtr {
    let xml2_parser = &*(user_data as *mut Xml2Parser);

    assert!(!name.is_null());
    let name = utf8_cstr(name);

    xml2_parser.state.entity_lookup(name).unwrap_or(ptr::null_mut())
}

unsafe extern "C" fn sax_entity_decl_cb(
    user_data: *mut libc::c_void,
    name: *const libc::c_char,
    type_: libc::c_int,
    _public_id: *const libc::c_char,
    _system_id: *const libc::c_char,
    content: *const libc::c_char,
) {
    let xml2_parser = &*(user_data as *mut Xml2Parser);

    assert!(!name.is_null());

    if type_ != XML_INTERNAL_GENERAL_ENTITY {
        // We don't allow loading external entities; we don't support
        // defining parameter entities in the DTD, and libxml2 should
        // handle internal predefined entities by itself (e.g. "&").
        return;
    }

    let entity = xmlNewEntity(
        ptr::null_mut(),
        name,
        type_,
        ptr::null(),
        ptr::null(),
        content,
    );
    assert!(!entity.is_null());

    let name = utf8_cstr(name);
    xml2_parser.state.entity_insert(name, entity);
}

unsafe extern "C" fn sax_unparsed_entity_decl_cb(
    user_data: *mut libc::c_void,
    name: *const libc::c_char,
    public_id: *const libc::c_char,
    system_id: *const libc::c_char,
    _notation_name: *const libc::c_char,
) {
    sax_entity_decl_cb(
        user_data,
        name,
        XML_INTERNAL_GENERAL_ENTITY,
        public_id,
        system_id,
        ptr::null(),
    );
}

unsafe extern "C" fn sax_start_element_cb(
    user_data: *mut libc::c_void,
    name: *const libc::c_char,
    atts: *const *const libc::c_char,
) {
    let xml2_parser = &*(user_data as *mut Xml2Parser);

    assert!(!name.is_null());
    let name = utf8_cstr(name);

    let pbag = PropertyBag::new_from_key_value_pairs(atts);

    if let Err(e) = xml2_parser.state.start_element(name, &pbag) {
        let _: () = e; // guard in case we change the error type later

        let parser = xml2_parser.parser.get();
        xmlStopParser(parser);
    }
}

unsafe extern "C" fn sax_end_element_cb(user_data: *mut libc::c_void, name: *const libc::c_char) {
    let xml2_parser = &*(user_data as *mut Xml2Parser);

    assert!(!name.is_null());
    let name = utf8_cstr(name);

    xml2_parser.state.end_element(name);
}

unsafe extern "C" fn sax_characters_cb(
    user_data: *mut libc::c_void,
    unterminated_text: *const libc::c_char,
    len: libc::c_int,
) {
    let xml2_parser = &*(user_data as *mut Xml2Parser);

    assert!(!unterminated_text.is_null());
    assert!(len >= 0);

    // libxml2 already validated the incoming string as UTF-8.  Note that
    // it is *not* nul-terminated; this is why we create a byte slice first.
    let bytes = std::slice::from_raw_parts(unterminated_text as *const u8, len as usize);
    let utf8 = str::from_utf8_unchecked(bytes);

    xml2_parser.state.characters(utf8);
}

unsafe extern "C" fn sax_processing_instruction_cb(
    user_data: *mut libc::c_void,
    target: *const libc::c_char,
    data: *const libc::c_char,
) {
    let xml2_parser = &*(user_data as *mut Xml2Parser);

    assert!(!target.is_null());
    let target = utf8_cstr(target);

    let data = if data.is_null() {
        ""
    } else {
        utf8_cstr(data)
    };

    xml2_parser.state.processing_instruction(target, data);
}

unsafe extern "C" fn sax_get_parameter_entity_cb(
    user_data: *mut libc::c_void,
    name: *const libc::c_char,
) -> xmlEntityPtr {
    sax_get_entity_cb(user_data, name)
}

fn set_xml_parse_options(parser: xmlParserCtxtPtr, unlimited_size: bool) {
    let mut options: libc::c_int = XML_PARSE_NONET | XML_PARSE_BIG_LINES;

    if unlimited_size {
        options |= XML_PARSE_HUGE;
    }

    unsafe {
        xmlCtxtUseOptions(parser, options);

        // If false, external entities work, but internal ones don't. if
        // true, internal entities work, but external ones don't. favor
        // internal entities, in order to not cause a regression
        (*parser).replaceEntities = 1;
    }
}

// Struct used as closure data for xmlCreateIOParserCtxt().  In conjunction
// with stream_ctx_read() and stream_ctx_close(), this struct provides the
// I/O callbacks and their context for libxml2.
//
// We call I/O methods on the stream, and as soon as we get an error
// we store it in the gio_error field.  Libxml2 just allows us to
// return -1 from the I/O callbacks in that case; it doesn't actually
// see the error code.
//
// The gio_error field comes from the place that constructs the
// StreamCtx.  That place is later responsible for seeing if the error
// is set; if it is, it means that there was an I/O error.  Otherwise,
// there were no I/O errors but the caller must then ask libxml2 for
// XML parsing errors.
struct StreamCtx {
    stream: gio::InputStream,
    cancellable: Option<gio::Cancellable>,
    gio_error: Rc<RefCell<Option<glib::Error>>>,
}

// read() callback from xmlCreateIOParserCtxt()
unsafe extern "C" fn stream_ctx_read(
    context: *mut libc::c_void,
    buffer: *mut libc::c_char,
    len: libc::c_int,
) -> libc::c_int {
    let ctx = &mut *(context as *mut StreamCtx);

    let mut err_ref = ctx.gio_error.borrow_mut();

    // has the error been set already?
    if err_ref.is_some() {
        return -1;
    }

    let buf: &mut [u8] = slice::from_raw_parts_mut(buffer as *mut u8, len as usize);

    match ctx.stream.read(buf, ctx.cancellable.as_ref()) {
        Ok(size) => size as libc::c_int,

        Err(e) => {
            // Just store the first I/O error we get; ignore subsequent ones.
            *err_ref = Some(e);
            -1
        }
    }
}

// close() callback from xmlCreateIOParserCtxt()
unsafe extern "C" fn stream_ctx_close(context: *mut libc::c_void) -> libc::c_int {
    let ctx = &mut *(context as *mut StreamCtx);

    let ret = match ctx.stream.close(ctx.cancellable.as_ref()) {
        Ok(()) => 0,

        Err(e) => {
            let mut err_ref = ctx.gio_error.borrow_mut();

            // don't overwrite a previous error
            if err_ref.is_none() {
                *err_ref = Some(e);
            }

            -1
        }
    };

    Box::from_raw(ctx);

    ret
}

fn init_libxml2() {
    static ONCE: Once = Once::new();

    ONCE.call_once(|| unsafe {
        xmlInitParser();
    });
}

pub struct Xml2Parser {
    parser: Cell<xmlParserCtxtPtr>,
    state: Rc<XmlState>,
    gio_error: Rc<RefCell<Option<glib::Error>>>,
}

impl Xml2Parser {
    pub fn from_stream(
        state: Rc<XmlState>,
        unlimited_size: bool,
        stream: &gio::InputStream,
        cancellable: Option<&gio::Cancellable>,
    ) -> Result<Box<Xml2Parser>, ParseFromStreamError> {
        init_libxml2();

        // The Xml2Parser we end up creating, if
        // xmlCreateIOParserCtxt() is successful, needs to hold a
        // location to place a GError from within the I/O callbacks
        // stream_ctx_read() and stream_ctx_close().  We put this
        // location in an Rc so that it can outlive the call to
        // xmlCreateIOParserCtxt() in case that fails, since on
        // failure that function frees the StreamCtx.
        let gio_error = Rc::new(RefCell::new(None));

        let ctx = Box::new(StreamCtx {
            stream: stream.clone(),
            cancellable: cancellable.cloned(),
            gio_error: gio_error.clone(),
        });

        let mut sax_handler = get_xml2_sax_handler();

        let mut xml2_parser = Box::new(Xml2Parser {
            parser: Cell::new(ptr::null_mut()),
            state,
            gio_error,
        });

        unsafe {
            let parser = xmlCreateIOParserCtxt(
                &mut sax_handler,
                xml2_parser.as_mut() as *mut _ as *mut _,
                Some(stream_ctx_read),
                Some(stream_ctx_close),
                Box::into_raw(ctx) as *mut _,
                XML_CHAR_ENCODING_NONE,
            );

            if parser.is_null() {
                // on error, xmlCreateIOParserCtxt() frees our ctx via the
                // stream_ctx_close function
                Err(ParseFromStreamError::CouldNotCreateXmlParser)
            } else {
                xml2_parser.parser.set(parser);

                set_xml_parse_options(parser, unlimited_size);

                Ok(xml2_parser)
            }
        }
    }

    pub fn parse(&self) -> Result<(), ParseFromStreamError> {
        unsafe {
            let parser = self.parser.get();

            let xml_parse_success = xmlParseDocument(parser) == 0;

            let mut err_ref = self.gio_error.borrow_mut();

            let io_error = err_ref.take();

            if let Some(io_error) = io_error {
                Err(ParseFromStreamError::IoError(io_error))
            } else if !xml_parse_success {
                let xerr = xmlCtxtGetLastError(parser as *mut _);
                let msg = xml2_error_to_string(xerr);
                Err(ParseFromStreamError::XmlParseError(msg))
            } else {
                Ok(())
            }
        }
    }
}

impl Drop for Xml2Parser {
    fn drop(&mut self) {
        let parser = self.parser.get();
        free_xml_parser_and_doc(parser);
        self.parser.set(ptr::null_mut());
    }
}

fn xml2_error_to_string(xerr: xmlErrorPtr) -> String {
    unsafe {
        if !xerr.is_null() {
            let xerr = &*xerr;

            let file = if xerr.file.is_null() {
                "data".to_string()
            } else {
                from_glib_none(xerr.file)
            };

            let message = if xerr.message.is_null() {
                "-".to_string()
            } else {
                from_glib_none(xerr.message)
            };

            format!(
                "Error domain {} code {} on line {} column {} of {}: {}",
                xerr.domain, xerr.code, xerr.line, xerr.int2, file, message
            )
        } else {
            // The error is not set?  Return a generic message :(
            "Error parsing XML data".to_string()
        }
    }
}

// Error returned when parsing an XML stream
#[derive(Clone)]
pub enum ParseFromStreamError {
    // We couldn't even create the libxml2 parser
    CouldNotCreateXmlParser,

    // GIO error from the I/O callbacks
    IoError(glib::Error),

    // XML parsing error from libxml2
    XmlParseError(String),
}

impl From<ParseFromStreamError> for LoadingError {
    fn from(e: ParseFromStreamError) -> LoadingError {
        match e {
            ParseFromStreamError::CouldNotCreateXmlParser => LoadingError::CouldNotCreateXmlParser,
            ParseFromStreamError::IoError(e) => LoadingError::Glib(e),
            ParseFromStreamError::XmlParseError(s) => LoadingError::XmlParseError(s),
        }
    }
}
<|endoftext|>


<commit_before>use std::ops::{Deref, DerefMut};
use std::fmt;
use combine::primitives::SourcePosition;

/// Struct representing a position within a source code file
#[derive(Copy, Clone, PartialEq, Debug)]
pub struct Position {
    pub col: i32,
    pub row: i32,
    pub raw: i32
}

impl Position {

    fn new(col: i32, row: i32) -> Self {
        Position { col: col
                 , row: row
                 , raw: col + row
                 }
    }

    fn from_combine(pos: SourcePosition) -> Self {
        Position { col: pos.column
                 , row: pos.line
                 , raw: pos.column + pos.line
                 }
    }
}

impl fmt::Display for Position {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "line {}, column {}", self.row, self.col)
    }
}

#[derive(Clone, Debug)]
pub struct Positional<T> {
    pub pos: Position,
    pub value: T
}

impl<T> Positional<T> {
    fn at(col: i32, row: i32, value: T) -> Positional<T> {
        Positional { pos: Position::new(col, row)
                   , value: value }
    }
    fn value(&self) -> &T { &self.value }
}

impl<T> fmt::Display for Positional<T>
where T: fmt::Display {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{} at {}", self.value, self.pos)
    }
}

impl<T> PartialEq for Positional<T>
where T: PartialEq {
    fn eq(&self, other: &Positional<T>) -> bool {
        self.value == other.value
    }
}

impl<T> Deref for Positional<T> {
    type Target = T;
    fn deref(&self) -> &T {
        &self.value
    }
}

impl<T> DerefMut for Positional<T> {
    fn deref_mut(&mut self) -> &mut T {
        &mut self.value
    }
}
<commit_msg>Position can now be made from other types<commit_after>use std::ops::{Deref, DerefMut};
use std::fmt;

use std::convert::From;

use combine::primitives::SourcePosition;

/// Struct representing a position within a source code file.
///
/// This represents positions using `i32`s because that's how
/// positions are represented in `combine` (the parsing library
/// that we will use for the Mnemosyne parser). I personally would
/// have used `usize`s...
#[derive(Copy, Clone, PartialEq, Debug)]
pub struct Position {
    pub col: i32,
    pub row: i32,
    pub raw: i32
}

impl Position {

    /// Create a new `Position `at the given column and row.
    #[inline]
    pub fn new(col: i32, row: i32) -> Self {
        Position { col: col
                 , row: row
                 , raw: col + row
                 }
    }

}

impl From<SourcePosition> for Position {
    /// Create a new `Position` from a `combine` `SourcePosition`.
    ///
    /// # Example
    /// ```
    /// let sp = SourcePosition { column: 1, line: 1 };
    /// assert_eq!(Position::from(sp), Position::new(1,1));
    /// ```
    fn from(p: SourcePosition) -> Self { Position::new(p.column, p.line) }
}

impl From<(i32,i32)> for Position {
    /// Create a new `Position` from a tuple of i32s.
    ///
    /// # Example
    /// ```
    /// let tuple: (i32,i32) = (1,1);
    /// assert_eq!(Position::from(tuple), Position::new(1,1));
    /// ```
    fn from((col, row): (i32,i32)) -> Self { Position::new(col,row) }
}

impl fmt::Display for Position {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "line {}, column {}", self.row, self.col)
    }
}

#[derive(Clone, Debug)]
pub struct Positional<T> {
    pub pos: Position,
    pub value: T
}

impl<T> Positional<T> {
    /// Create a new Positional marker at the given position.
    pub fn at(col: i32, row: i32, value: T) -> Positional<T> {
        Positional { pos: Position::new(col, row)
                   , value: value }
    }

    pub fn value(&self) -> &T { &self.value }
}

impl<T> fmt::Display for Positional<T>
where T: fmt::Display {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{} at {}", self.value, self.pos)
    }
}

impl<T> PartialEq for Positional<T>
where T: PartialEq {
    fn eq(&self, other: &Positional<T>) -> bool {
        self.value == other.value
    }
}

impl<T> Deref for Positional<T> {
    type Target = T;
    fn deref(&self) -> &T {
        &self.value
    }
}

impl<T> DerefMut for Positional<T> {
    fn deref_mut(&mut self) -> &mut T {
        &mut self.value
    }
}
<|endoftext|>


<commit_before>extern crate disqrust;
extern crate disque;

use std::time::Duration;
use std::sync::mpsc::{channel, Sender};

use disque::Disque;
use disqrust::{EventLoop, Handler, JobStatus};

#[test]
fn basic() {
    struct MyHandler {
        sender: Sender<Vec<u8>>,
    }

    impl MyHandler {
        fn new(sender: Sender<Vec<u8>>) -> MyHandler {
            MyHandler { sender: sender }
        }
    }

    impl Handler for MyHandler {
        fn process_job(&self, _: &[u8], _: &String, body: Vec<u8>) -> JobStatus {
            self.sender.send(body).unwrap();
            JobStatus::AckJob
        }
        fn process_error(&self, _: &[u8], _: &String, _: u32, _: u32) -> bool {
            panic!("Should not be called")
        }
    }

    let disque = Disque::open("redis://127.0.0.1:7711/").unwrap();
    disque.addjob(b"basicqueue", b"my job", Duration::from_secs(10), None, None, None, None, None, false).unwrap();
    let (tx, rx) = channel();
    let mut el = EventLoop::new(disque, 1, MyHandler::new(tx));
    el.watch_queue(b"basicqueue".to_vec());
    el.run_times(1);
    assert_eq!(rx.try_recv().unwrap(), b"my job".to_vec());
}

#[test]
fn error() {
    struct MyHandler {
        sender: Sender<(u32, u32)>,
    }

    impl MyHandler {
        fn new(sender: Sender<(u32, u32)>) -> MyHandler {
            MyHandler { sender: sender }
        }
    }

    impl Handler for MyHandler {
        fn process_job(&self, _: &[u8], _: &String, _: Vec<u8>) -> JobStatus {
            panic!("Should not be called")
        }
        fn process_error(&self, _: &[u8], _: &String, nack: u32, additional_deliveries: u32) -> bool {
            self.sender.send((nack, additional_deliveries)).unwrap();
            false
        }
    }

    let disque = Disque::open("redis://127.0.0.1:7711/").unwrap();
    let jobid = disque.addjob(b"errorqueue", b"my job", Duration::from_secs(10), None, None, None, None, None, false).unwrap();
    disque.getjob(true, None, &[b"errorqueue"]).unwrap();
    disque.nack(&[jobid.as_bytes()]).unwrap();

    let (tx, rx) = channel();
    let mut el = EventLoop::new(disque, 1, MyHandler::new(tx));
    el.watch_queue(b"errorqueue".to_vec());
    el.run_times(1);
    assert_eq!(rx.try_recv().unwrap(), (1, 0));
}
<commit_msg>Add error andp process test<commit_after>extern crate disqrust;
extern crate disque;

use std::time::Duration;
use std::sync::mpsc::{channel, Sender};

use disque::Disque;
use disqrust::{EventLoop, Handler, JobStatus};

#[test]
fn basic() {
    struct MyHandler {
        sender: Sender<Vec<u8>>,
    }

    impl MyHandler {
        fn new(sender: Sender<Vec<u8>>) -> MyHandler {
            MyHandler { sender: sender }
        }
    }

    impl Handler for MyHandler {
        fn process_job(&self, _: &[u8], _: &String, body: Vec<u8>) -> JobStatus {
            self.sender.send(body).unwrap();
            JobStatus::AckJob
        }
        fn process_error(&self, _: &[u8], _: &String, _: u32, _: u32) -> bool {
            panic!("Should not be called")
        }
    }

    let queue = b"basicqueue";
    let disque = Disque::open("redis://127.0.0.1:7711/").unwrap();
    disque.addjob(queue, b"my job", Duration::from_secs(10), None, None, None, None, None, false).unwrap();
    let (tx, rx) = channel();
    let mut el = EventLoop::new(disque, 1, MyHandler::new(tx));
    el.watch_queue(queue.to_vec());
    el.run_times(1);
    assert_eq!(rx.try_recv().unwrap(), b"my job".to_vec());
}

#[test]
fn error() {
    struct MyHandler {
        sender: Sender<(u32, u32)>,
    }

    impl MyHandler {
        fn new(sender: Sender<(u32, u32)>) -> MyHandler {
            MyHandler { sender: sender }
        }
    }

    impl Handler for MyHandler {
        fn process_job(&self, _: &[u8], _: &String, _: Vec<u8>) -> JobStatus {
            panic!("Should not be called")
        }

        fn process_error(&self, _: &[u8], _: &String, nack: u32, additional_deliveries: u32) -> bool {
            self.sender.send((nack, additional_deliveries)).unwrap();
            false
        }
    }

    let queue = b"errorqueue";
    let disque = Disque::open("redis://127.0.0.1:7711/").unwrap();
    let jobid = disque.addjob(queue, b"my job", Duration::from_secs(10), None, None, None, None, None, false).unwrap();
    disque.getjob(true, None, &[queue]).unwrap();
    disque.nack(&[jobid.as_bytes()]).unwrap();

    let (tx, rx) = channel();
    let mut el = EventLoop::new(disque, 1, MyHandler::new(tx));
    el.watch_queue(queue.to_vec());
    el.run_times(1);
    assert_eq!(rx.try_recv().unwrap(), (1, 0));
}

#[test]
fn error_and_job() {
    struct MyHandler {
        sender: Sender<Option<(u32, u32)>>,
    }

    impl MyHandler {
        fn new(sender: Sender<Option<(u32, u32)>>) -> MyHandler {
            MyHandler { sender: sender }
        }
    }

    impl Handler for MyHandler {
        fn process_job(&self, _: &[u8], _: &String, _: Vec<u8>) -> JobStatus {
            self.sender.send(None).unwrap();
            JobStatus::AckJob
        }

        fn process_error(&self, _: &[u8], _: &String, nack: u32, additional_deliveries: u32) -> bool {
            self.sender.send(Some((nack, additional_deliveries))).unwrap();
            true
        }
    }

    let queue = b"errorjobqueue";
    let disque = Disque::open("redis://127.0.0.1:7711/").unwrap();
    let jobid = disque.addjob(queue, b"my job", Duration::from_secs(10), None, None, None, None, None, false).unwrap();
    disque.getjob(true, None, &[queue]).unwrap();
    disque.nack(&[jobid.as_bytes()]).unwrap();

    let (tx, rx) = channel();
    let mut el = EventLoop::new(disque, 1, MyHandler::new(tx));
    el.watch_queue(queue.to_vec());
    el.run_times(1);
    assert_eq!(rx.try_recv().unwrap(), Some((1, 0)));
    assert_eq!(rx.try_recv().unwrap(), None);
    assert!(rx.try_recv().is_err());
}
<|endoftext|>


<commit_before>// TODO: Clean code, please
// TODO: Check types used. Try to minimize the memory used
// TODO: Add more tests.
// TODO: Better UI. Colors? Num of matches?
// TODO: Try to do the fuzzy search async?
extern crate scout;
extern crate docopt;
extern crate termios;
extern crate termion;

use std::env;

use termion::event::Key;
use termion::input::TermRead;
use termion::{color, style};
use std::io::{self, Read, Write};

const USAGE: &'static str = "
Scout: Small fuzzy finder

This program expects a list of items in the
standard input, so it is better to use it
with pipes.

Usage:
  scout [options]

Options:
  -h --help      Show this screen.
  -v --version   Show version.

Example:
  $ ls | scout
";

fn magic() -> Result<String, io::Error> {
    // Collect initial input
    let mut buffer = String::new();
    try!(io::stdin().read_to_string(&mut buffer));
    let input: Vec<&str> = buffer.split("\n")
        .map(|s| s.trim())
        .filter(|s| !s.is_empty())
        .collect();
    let width = format!("{}", input.len()).len();
    let mut selection = 0; // current selected item

    let mut terminal = scout::Terminal::new();
    let mut result = String::new();
    let mut query: Vec<char> = vec![];

    'event: loop {
        let s: String = query.iter().cloned().collect();
        let query_chars: Vec<char> = query.iter().cloned().collect();
        let choices = scout::explore(&input, &query_chars);

        // Clear the terminal and put the cursor at the beginning
        writeln!(
            &mut terminal,
            "{}{}",
            termion::clear::All,
            termion::cursor::Goto(1, 1),
        ).unwrap();

        // Print all the choices
        for (i, choice) in choices.iter().take(21).cloned().enumerate() {
            // Split the string in different areas
            // to highlight the matching part
            let string = choice.to_string();
            let chars = string.char_indices();
            let mut ended = None;
            let mut line: String = chars.map(|(index, ch)| {
                if index == choice.start() && index < choice.end() {
                    format!("{}{}", color::Fg(color::LightGreen), ch)
                } else if index == choice.end() {
                    ended = Some(index);
                    format!("{}{}", color::Fg(color::Reset), ch)
                } else {
                    format!("{}", ch)
                }
            }).collect();

            // Ensure that we stop highlihting things
            if ended.is_none() {
                line = format!("{}{}", line, color::Fg(color::Reset));
            }

            if i == selection {
                writeln!(&mut terminal, "{}{}{}", style::Invert, line, style::Reset).unwrap();
            } else {
                writeln!(&mut terminal, "{}", line).unwrap();
            }
        }

        // Go to the beginning again and redraw the prompt.
        // This will put the cursor at the end of it
        let prompt = format!("{:width$} > {}", choices.len(), s, width = width);
        write!(&mut terminal, "{}{}", termion::cursor::Goto(1, 1), prompt).unwrap();

        terminal.flush().unwrap();

        for c in terminal.input().keys() {
            debug!("In for loop");
            match c.unwrap() {
                Key::Backspace => {
                    let _ = query.pop();
                },
                Key::Ctrl('p') | Key::Up => {
                    selection = if selection == 0 {
                        // TODO: This should be only over the visible
                        // window
                        choices.len() - 1
                    } else {
                        selection - 1
                    };
                },
                Key::Ctrl('n') | Key::Down => {
                    // TODO: This should be only over the visible
                    // window
                    // TODO: The loop shouldn't be trigger again,
                    // we should render the terminal without
                    // doing a full search. In the next event loop
                    // iteration the selection should be reset to
                    // 0 again
                    selection = if selection == (choices.len() - 1) {
                        0
                    } else {
                        selection + 1
                    };
                },
                Key::Char('\n') | Key::Ctrl('j') | Key::Ctrl('m') => {
                    let choice = choices[selection];
                    result = choice.to_string();

                    break 'event
                },
                Key::Ctrl('u') => {
                    query.clear();
                },
                Key::Char(c) => {
                    query.push(c);
                },
                Key::Esc => break 'event,
                _ => {},
            }
        }
    };

    Ok(result)
}

pub fn main() {
    docopt::Docopt::new(USAGE)
        .and_then(|doc| {
            doc.argv(env::args())
                .version(Some(scout::version()))
                .parse()
        })
        .unwrap_or_else(|e| e.exit());;

    match magic() {
        Ok(result) => println!("{}", result),
        Err(e) => panic!(e),
    }
}
<commit_msg>Remove wrong debug! call<commit_after>// TODO: Clean code, please
// TODO: Check types used. Try to minimize the memory used
// TODO: Add more tests.
// TODO: Better UI. Colors? Num of matches?
// TODO: Try to do the fuzzy search async?
extern crate scout;
extern crate docopt;
extern crate termios;
extern crate termion;

use std::env;

use termion::event::Key;
use termion::input::TermRead;
use termion::{color, style};
use std::io::{self, Read, Write};

const USAGE: &'static str = "
Scout: Small fuzzy finder

This program expects a list of items in the
standard input, so it is better to use it
with pipes.

Usage:
  scout [options]

Options:
  -h --help      Show this screen.
  -v --version   Show version.

Example:
  $ ls | scout
";

fn magic() -> Result<String, io::Error> {
    // Collect initial input
    let mut buffer = String::new();
    try!(io::stdin().read_to_string(&mut buffer));
    let input: Vec<&str> = buffer.split("\n")
        .map(|s| s.trim())
        .filter(|s| !s.is_empty())
        .collect();
    let width = format!("{}", input.len()).len();
    let mut selection = 0; // current selected item

    let mut terminal = scout::Terminal::new();
    let mut result = String::new();
    let mut query: Vec<char> = vec![];

    'event: loop {
        let s: String = query.iter().cloned().collect();
        let query_chars: Vec<char> = query.iter().cloned().collect();
        let choices = scout::explore(&input, &query_chars);

        // Clear the terminal and put the cursor at the beginning
        writeln!(
            &mut terminal,
            "{}{}",
            termion::clear::All,
            termion::cursor::Goto(1, 1),
        ).unwrap();

        // Print all the choices
        for (i, choice) in choices.iter().take(21).cloned().enumerate() {
            // Split the string in different areas
            // to highlight the matching part
            let string = choice.to_string();
            let chars = string.char_indices();
            let mut ended = None;
            let mut line: String = chars.map(|(index, ch)| {
                if index == choice.start() && index < choice.end() {
                    format!("{}{}", color::Fg(color::LightGreen), ch)
                } else if index == choice.end() {
                    ended = Some(index);
                    format!("{}{}", color::Fg(color::Reset), ch)
                } else {
                    format!("{}", ch)
                }
            }).collect();

            // Ensure that we stop highlihting things
            if ended.is_none() {
                line = format!("{}{}", line, color::Fg(color::Reset));
            }

            if i == selection {
                writeln!(&mut terminal, "{}{}{}", style::Invert, line, style::Reset).unwrap();
            } else {
                writeln!(&mut terminal, "{}", line).unwrap();
            }
        }

        // Go to the beginning again and redraw the prompt.
        // This will put the cursor at the end of it
        let prompt = format!("{:width$} > {}", choices.len(), s, width = width);
        write!(&mut terminal, "{}{}", termion::cursor::Goto(1, 1), prompt).unwrap();

        terminal.flush().unwrap();

        for c in terminal.input().keys() {
            match c.unwrap() {
                Key::Backspace => {
                    let _ = query.pop();
                },
                Key::Ctrl('p') | Key::Up => {
                    selection = if selection == 0 {
                        // TODO: This should be only over the visible
                        // window
                        choices.len() - 1
                    } else {
                        selection - 1
                    };
                },
                Key::Ctrl('n') | Key::Down => {
                    // TODO: This should be only over the visible
                    // window
                    // TODO: The loop shouldn't be trigger again,
                    // we should render the terminal without
                    // doing a full search. In the next event loop
                    // iteration the selection should be reset to
                    // 0 again
                    selection = if selection == (choices.len() - 1) {
                        0
                    } else {
                        selection + 1
                    };
                },
                Key::Char('\n') | Key::Ctrl('j') | Key::Ctrl('m') => {
                    let choice = choices[selection];
                    result = choice.to_string();

                    break 'event
                },
                Key::Ctrl('u') => {
                    query.clear();
                },
                Key::Char(c) => {
                    query.push(c);
                },
                Key::Esc => break 'event,
                _ => {},
            }
        }
    };

    Ok(result)
}

pub fn main() {
    docopt::Docopt::new(USAGE)
        .and_then(|doc| {
            doc.argv(env::args())
                .version(Some(scout::version()))
                .parse()
        })
        .unwrap_or_else(|e| e.exit());;

    match magic() {
        Ok(result) => println!("{}", result),
        Err(e) => panic!(e),
    }
}
<|endoftext|>