<commit_before>// Copyright 2016 Mozilla Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use ::compiler::{
    Cacheable,
    Compiler,
    CompilerArguments,
    ParsedArguments,
    run_input_output,
};
use log::LogLevel::Trace;
use mock_command::{
    CommandCreator,
    CommandCreatorSync,
    RunCommand,
};
use std::collections::HashMap;
use std::fs::File;
use std::io::{
    self,
    Error,
    ErrorKind,
    Write,
};
use std::path::Path;
use std::process;
use tempdir::TempDir;

pub fn parse_arguments(arguments: &[String]) -> CompilerArguments {
    let mut output_arg = None;
    let mut input_arg = None;
    let mut common_args = vec!();
    let mut compilation = false;
    let mut debug_info = false;
    let mut pdb = None;

    //TODO: support arguments that start with / as well.
    let mut it = arguments.iter();
    loop {
        match it.next() {
            Some(arg) => {
                match arg.as_ref() {
                    //TODO: support -dep
                    "-c" => compilation = true,
                    v @ _ if v.starts_with("-Fo") => {
                        output_arg = Some(String::from(&v[3..]));
                    }
                    // Arguments that take a value.
                    "-FI" => {
                        common_args.push(arg.clone());
                        if let Some(arg_val) = it.next() {
                            common_args.push(arg_val.clone());
                        }
                    }
                    // Arguments we can't handle.
                    "-showIncludes" => return CompilerArguments::CannotCache,
                    a @ _ if a.starts_with('@') => return CompilerArguments::CannotCache,
                    // Arguments we can't handle because they output more files.
                    // TODO: support more multi-file outputs.
                    "-FA" | "-Fa" | "-Fe" | "-Fm" | "-Fp" | "-FR" | "-Fx" => return CompilerArguments::CannotCache,
                    "-Zi" => {
                        debug_info = true;
                        common_args.push(arg.clone());
                    }
                    v @ _ if v.starts_with("-Fd") => {
                        pdb = Some(String::from(&v[3..]));
                        common_args.push(arg.clone());
                    }
                    // Other options.
                    v @ _ if v.starts_with('-') && v.len() > 1 => {
                        common_args.push(arg.clone());
                    }
                    // Anything else is an input file.
                    v @ _ => {
                        if input_arg.is_some() {
                            // Can't cache compilations with multiple inputs.
                            return CompilerArguments::CannotCache;
                        }
                        input_arg = Some(v);
                    }
                }
            }
            None => break,
        }
    }
    // We only support compilation.
    if !compilation {
        return CompilerArguments::NotCompilation;
    }
    let (input, extension) = match input_arg {
        Some(i) => {
            match Path::new(i).extension().and_then(|e| e.to_str()) {
                Some(e) => (i.to_owned(), e.to_owned()),
                _ => {
                    trace!("Bad or missing source extension: {:?}", i);
                    return CompilerArguments::CannotCache;
                }
            }
        }
        // We can't cache compilation without an input.
        None => return CompilerArguments::CannotCache,
    };
    let mut outputs = HashMap::new();
    match output_arg {
        // We can't cache compilation that doesn't go to a file
        None => return CompilerArguments::CannotCache,
        Some(o) => {
            outputs.insert("obj", o.to_owned());
            // -Fd is not taken into account unless -Zi is given
            if debug_info {
                match pdb {
                    Some(p) => outputs.insert("pdb", p.to_owned()),
                    None => {
                        // -Zi without -Fd defaults to vcxxx.pdb (where xxx depends on the
                        // MSVC version), and that's used for all compilations with the same
                        // working directory. We can't cache such a pdb.
                        return CompilerArguments::CannotCache;
                    }
                };
            }
        }
    }
    CompilerArguments::Ok(ParsedArguments {
        input: input,
        extension: extension,
        outputs: outputs,
        preprocessor_args: vec!(),
        common_args: common_args,
    })
}

pub fn preprocess<T : CommandCreatorSync>(mut creator: T, compiler: &Compiler, parsed_args: &ParsedArguments, cwd: &str) -> io::Result<process::Output> {
    trace!("preprocess");
    //TODO: support depfile by way of -showIncludes
    let mut cmd = creator.new_command_sync(&compiler.executable);
    cmd.arg("-E")
        .arg(&parsed_args.input)
        .arg("-nologo")
        .args(&parsed_args.common_args)
        .current_dir(cwd);

    if log_enabled!(Trace) {
        trace!("preprocess: {:?}", cmd);
    }

    run_input_output(cmd, None)
}

pub fn compile<T : CommandCreatorSync>(mut creator: T, compiler: &Compiler, preprocessor_output: Vec<u8>, parsed_args: &ParsedArguments, cwd: &str) -> io::Result<(Cacheable, process::Output)> {
    trace!("compile");
    let output = try!(parsed_args.outputs.get("obj").ok_or(Error::new(ErrorKind::Other, "Missing object file output")));
    // See if this compilation will produce a PDB.
    let cacheable = parsed_args.outputs.get("pdb")
        .map(|pdb| {
            // If the PDB exists, we don't know if it's shared with another
            // compilation. If it is, we can't cache.
            if Path::new(cwd).join(pdb).exists() {
                Cacheable::No
            } else {
                Cacheable::Yes
            }
        })
        .unwrap_or(Cacheable::Yes);
    // MSVC doesn't read anything from stdin, so it needs a temporary file
    // as input.
    let tempdir = try!(TempDir::new("sccache"));
    let filename = try!(Path::new(&parsed_args.input).file_name().ok_or(Error::new(ErrorKind::Other, "Missing input filename")));
    let input = tempdir.path().join(filename);
    {
        try!(File::create(&input)
             .and_then(|mut f| f.write_all(&preprocessor_output)))
    }

    let mut cmd = creator.new_command_sync(&compiler.executable);
    cmd.arg("-c")
        .arg(&input)
        .arg(&format!("-Fo{}", output))
        .args(&parsed_args.common_args)
        .current_dir(cwd);

    let output = try!(run_input_output(cmd, None));
    Ok((cacheable, output))
}

#[cfg(test)]
mod test {
    use super::*;
    use ::compiler::*;

    #[test]
    fn test_parse_arguments_simple() {
        match parse_arguments(&stringvec!["-c", "foo.c", "-Fofoo.obj"]) {
            CompilerArguments::Ok(ParsedArguments { input, extension, outputs, preprocessor_args, common_args }) => {
                assert!(true, "Parsed ok");
                assert_eq!("foo.c", input);
                assert_eq!("c", extension);
                assert_map_contains!(outputs, ("obj", "foo.obj"));
                //TODO: fix assert_map_contains to assert no extra keys!
                assert_eq!(1, outputs.len());
                assert!(preprocessor_args.is_empty());
                assert!(common_args.is_empty());
            }
            o @ _ => assert!(false, format!("Got unexpected parse result: {:?}", o)),
        }
    }

    #[test]
    fn test_parse_arguments_extra() {
        match parse_arguments(&stringvec!["-c", "foo.c", "-foo", "-Fofoo.obj", "-bar"]) {
            CompilerArguments::Ok(ParsedArguments { input, extension, outputs, preprocessor_args, common_args }) => {
                assert!(true, "Parsed ok");
                assert_eq!("foo.c", input);
                assert_eq!("c", extension);
                assert_map_contains!(outputs, ("obj", "foo.obj"));
                //TODO: fix assert_map_contains to assert no extra keys!
                assert_eq!(1, outputs.len());
                assert!(preprocessor_args.is_empty());
                assert_eq!(common_args, &["-foo", "-bar"]);
            }
            o @ _ => assert!(false, format!("Got unexpected parse result: {:?}", o)),
        }
    }

    #[test]
    fn test_parse_arguments_values() {
        match parse_arguments(&stringvec!["-c", "foo.c", "-FI", "file", "-Fofoo.obj"]) {
            CompilerArguments::Ok(ParsedArguments { input, extension, outputs, preprocessor_args, common_args }) => {
                assert!(true, "Parsed ok");
                assert_eq!("foo.c", input);
                assert_eq!("c", extension);
                assert_map_contains!(outputs, ("obj", "foo.obj"));
                //TODO: fix assert_map_contains to assert no extra keys!
                assert_eq!(1, outputs.len());
                assert!(preprocessor_args.is_empty());
                assert_eq!(common_args, &["-FI", "file"]);
            }
            o @ _ => assert!(false, format!("Got unexpected parse result: {:?}", o)),
        }
    }

    #[test]
    fn test_parse_arguments_pdb() {
        match parse_arguments(&stringvec!["-c", "foo.c", "-Zi", "-Fdfoo.pdb", "-Fofoo.obj"]) {
            CompilerArguments::Ok(ParsedArguments { input, extension, outputs, preprocessor_args, common_args }) => {
                assert!(true, "Parsed ok");
                assert_eq!("foo.c", input);
                assert_eq!("c", extension);
                assert_map_contains!(outputs, ("obj", "foo.obj"), ("pdb", "foo.pdb"));
                //TODO: fix assert_map_contains to assert no extra keys!
                assert_eq!(2, outputs.len());
                assert!(preprocessor_args.is_empty());
                assert_eq!(common_args, &["-Zi", "-Fdfoo.pdb"]);
            }
            o @ _ => assert!(false, format!("Got unexpected parse result: {:?}", o)),
        }
    }

    #[test]
    fn test_parse_arguments_empty_args() {
        assert_eq!(CompilerArguments::NotCompilation,
                   parse_arguments(&vec!()));
    }

    #[test]
    fn test_parse_arguments_not_compile() {
        assert_eq!(CompilerArguments::NotCompilation,
                   parse_arguments(&stringvec!["-Fofoo", "foo.c"]));
    }

    #[test]
    fn test_parse_arguments_too_many_inputs() {
        assert_eq!(CompilerArguments::CannotCache,
                   parse_arguments(&stringvec!["-c", "foo.c", "-Fofoo.obj", "bar.c"]));
    }

    #[test]
    fn test_parse_arguments_unsupported() {
        assert_eq!(CompilerArguments::CannotCache,
                   parse_arguments(&stringvec!["-c", "foo.c", "-Fofoo.obj", "-FA"]));

        assert_eq!(CompilerArguments::CannotCache,
                   parse_arguments(&stringvec!["-Fa", "-c", "foo.c", "-Fofoo.obj"]));

        assert_eq!(CompilerArguments::CannotCache,
                   parse_arguments(&stringvec!["-c", "foo.c", "-FR", "-Fofoo.obj"]));
    }

    #[test]
    fn test_parse_arguments_response_file() {
        assert_eq!(CompilerArguments::CannotCache,
                   parse_arguments(&stringvec!["-c", "foo.c", "@foo", "-Fofoo.obj"]));
    }

    #[test]
    fn test_parse_arguments_missing_pdb() {
        assert_eq!(CompilerArguments::CannotCache,
                   parse_arguments(&stringvec!["-c", "foo.c", "-Zi", "-Fofoo.obj"]));
    }
}
<commit_msg>add some better logging<commit_after>// Copyright 2016 Mozilla Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use ::compiler::{
    Cacheable,
    Compiler,
    CompilerArguments,
    ParsedArguments,
    run_input_output,
};
use log::LogLevel::Trace;
use mock_command::{
    CommandCreator,
    CommandCreatorSync,
    RunCommand,
};
use std::collections::HashMap;
use std::fs::File;
use std::io::{
    self,
    Error,
    ErrorKind,
    Write,
};
use std::path::Path;
use std::process;
use tempdir::TempDir;

pub fn parse_arguments(arguments: &[String]) -> CompilerArguments {
    let mut output_arg = None;
    let mut input_arg = None;
    let mut common_args = vec!();
    let mut compilation = false;
    let mut debug_info = false;
    let mut pdb = None;

    //TODO: support arguments that start with / as well.
    let mut it = arguments.iter();
    loop {
        match it.next() {
            Some(arg) => {
                match arg.as_ref() {
                    //TODO: support -dep
                    "-c" => compilation = true,
                    v @ _ if v.starts_with("-Fo") => {
                        output_arg = Some(String::from(&v[3..]));
                    }
                    // Arguments that take a value.
                    "-FI" => {
                        common_args.push(arg.clone());
                        if let Some(arg_val) = it.next() {
                            common_args.push(arg_val.clone());
                        }
                    }
                    // Arguments we can't handle.
                    "-showIncludes" => return CompilerArguments::CannotCache,
                    a @ _ if a.starts_with('@') => return CompilerArguments::CannotCache,
                    // Arguments we can't handle because they output more files.
                    // TODO: support more multi-file outputs.
                    "-FA" | "-Fa" | "-Fe" | "-Fm" | "-Fp" | "-FR" | "-Fx" => return CompilerArguments::CannotCache,
                    "-Zi" => {
                        debug_info = true;
                        common_args.push(arg.clone());
                    }
                    v @ _ if v.starts_with("-Fd") => {
                        pdb = Some(String::from(&v[3..]));
                        common_args.push(arg.clone());
                    }
                    // Other options.
                    v @ _ if v.starts_with('-') && v.len() > 1 => {
                        common_args.push(arg.clone());
                    }
                    // Anything else is an input file.
                    v @ _ => {
                        if input_arg.is_some() {
                            // Can't cache compilations with multiple inputs.
                            return CompilerArguments::CannotCache;
                        }
                        input_arg = Some(v);
                    }
                }
            }
            None => break,
        }
    }
    // We only support compilation.
    if !compilation {
        return CompilerArguments::NotCompilation;
    }
    let (input, extension) = match input_arg {
        Some(i) => {
            match Path::new(i).extension().and_then(|e| e.to_str()) {
                Some(e) => (i.to_owned(), e.to_owned()),
                _ => {
                    trace!("Bad or missing source extension: {:?}", i);
                    return CompilerArguments::CannotCache;
                }
            }
        }
        // We can't cache compilation without an input.
        None => return CompilerArguments::CannotCache,
    };
    let mut outputs = HashMap::new();
    match output_arg {
        // We can't cache compilation that doesn't go to a file
        None => return CompilerArguments::CannotCache,
        Some(o) => {
            outputs.insert("obj", o.to_owned());
            // -Fd is not taken into account unless -Zi is given
            if debug_info {
                match pdb {
                    Some(p) => outputs.insert("pdb", p.to_owned()),
                    None => {
                        // -Zi without -Fd defaults to vcxxx.pdb (where xxx depends on the
                        // MSVC version), and that's used for all compilations with the same
                        // working directory. We can't cache such a pdb.
                        return CompilerArguments::CannotCache;
                    }
                };
            }
        }
    }
    CompilerArguments::Ok(ParsedArguments {
        input: input,
        extension: extension,
        outputs: outputs,
        preprocessor_args: vec!(),
        common_args: common_args,
    })
}

pub fn preprocess<T : CommandCreatorSync>(mut creator: T, compiler: &Compiler, parsed_args: &ParsedArguments, cwd: &str) -> io::Result<process::Output> {
    trace!("preprocess");
    //TODO: support depfile by way of -showIncludes
    let mut cmd = creator.new_command_sync(&compiler.executable);
    cmd.arg("-E")
        .arg(&parsed_args.input)
        .arg("-nologo")
        .args(&parsed_args.common_args)
        .current_dir(cwd);

    if log_enabled!(Trace) {
        trace!("preprocess: {:?}", cmd);
    }

    run_input_output(cmd, None)
}

pub fn compile<T : CommandCreatorSync>(mut creator: T, compiler: &Compiler, preprocessor_output: Vec<u8>, parsed_args: &ParsedArguments, cwd: &str) -> io::Result<(Cacheable, process::Output)> {
    trace!("compile");
    let output = try!(parsed_args.outputs.get("obj").ok_or(Error::new(ErrorKind::Other, "Missing object file output")));
    // See if this compilation will produce a PDB.
    let cacheable = parsed_args.outputs.get("pdb")
        .map(|pdb| {
            // If the PDB exists, we don't know if it's shared with another
            // compilation. If it is, we can't cache.
            if Path::new(cwd).join(pdb).exists() {
                Cacheable::No
            } else {
                Cacheable::Yes
            }
        })
        .unwrap_or(Cacheable::Yes);
    // MSVC doesn't read anything from stdin, so it needs a temporary file
    // as input.
    let tempdir = try!(TempDir::new("sccache"));
    let filename = try!(Path::new(&parsed_args.input).file_name().ok_or(Error::new(ErrorKind::Other, "Missing input filename")));
    let input = tempdir.path().join(filename);
    {
        try!(File::create(&input)
             .and_then(|mut f| f.write_all(&preprocessor_output)))
    }

    let mut cmd = creator.new_command_sync(&compiler.executable);
    cmd.arg("-c")
        .arg(&input)
        .arg(&format!("-Fo{}", output))
        .args(&parsed_args.common_args)
        .current_dir(cwd);

    if log_enabled!(Trace) {
        trace!("compile: {:?}", cmd);
    }

    let output = try!(run_input_output(cmd, None));
    Ok((cacheable, output))
}

#[cfg(test)]
mod test {
    use super::*;
    use ::compiler::*;

    #[test]
    fn test_parse_arguments_simple() {
        match parse_arguments(&stringvec!["-c", "foo.c", "-Fofoo.obj"]) {
            CompilerArguments::Ok(ParsedArguments { input, extension, outputs, preprocessor_args, common_args }) => {
                assert!(true, "Parsed ok");
                assert_eq!("foo.c", input);
                assert_eq!("c", extension);
                assert_map_contains!(outputs, ("obj", "foo.obj"));
                //TODO: fix assert_map_contains to assert no extra keys!
                assert_eq!(1, outputs.len());
                assert!(preprocessor_args.is_empty());
                assert!(common_args.is_empty());
            }
            o @ _ => assert!(false, format!("Got unexpected parse result: {:?}", o)),
        }
    }

    #[test]
    fn test_parse_arguments_extra() {
        match parse_arguments(&stringvec!["-c", "foo.c", "-foo", "-Fofoo.obj", "-bar"]) {
            CompilerArguments::Ok(ParsedArguments { input, extension, outputs, preprocessor_args, common_args }) => {
                assert!(true, "Parsed ok");
                assert_eq!("foo.c", input);
                assert_eq!("c", extension);
                assert_map_contains!(outputs, ("obj", "foo.obj"));
                //TODO: fix assert_map_contains to assert no extra keys!
                assert_eq!(1, outputs.len());
                assert!(preprocessor_args.is_empty());
                assert_eq!(common_args, &["-foo", "-bar"]);
            }
            o @ _ => assert!(false, format!("Got unexpected parse result: {:?}", o)),
        }
    }

    #[test]
    fn test_parse_arguments_values() {
        match parse_arguments(&stringvec!["-c", "foo.c", "-FI", "file", "-Fofoo.obj"]) {
            CompilerArguments::Ok(ParsedArguments { input, extension, outputs, preprocessor_args, common_args }) => {
                assert!(true, "Parsed ok");
                assert_eq!("foo.c", input);
                assert_eq!("c", extension);
                assert_map_contains!(outputs, ("obj", "foo.obj"));
                //TODO: fix assert_map_contains to assert no extra keys!
                assert_eq!(1, outputs.len());
                assert!(preprocessor_args.is_empty());
                assert_eq!(common_args, &["-FI", "file"]);
            }
            o @ _ => assert!(false, format!("Got unexpected parse result: {:?}", o)),
        }
    }

    #[test]
    fn test_parse_arguments_pdb() {
        match parse_arguments(&stringvec!["-c", "foo.c", "-Zi", "-Fdfoo.pdb", "-Fofoo.obj"]) {
            CompilerArguments::Ok(ParsedArguments { input, extension, outputs, preprocessor_args, common_args }) => {
                assert!(true, "Parsed ok");
                assert_eq!("foo.c", input);
                assert_eq!("c", extension);
                assert_map_contains!(outputs, ("obj", "foo.obj"), ("pdb", "foo.pdb"));
                //TODO: fix assert_map_contains to assert no extra keys!
                assert_eq!(2, outputs.len());
                assert!(preprocessor_args.is_empty());
                assert_eq!(common_args, &["-Zi", "-Fdfoo.pdb"]);
            }
            o @ _ => assert!(false, format!("Got unexpected parse result: {:?}", o)),
        }
    }

    #[test]
    fn test_parse_arguments_empty_args() {
        assert_eq!(CompilerArguments::NotCompilation,
                   parse_arguments(&vec!()));
    }

    #[test]
    fn test_parse_arguments_not_compile() {
        assert_eq!(CompilerArguments::NotCompilation,
                   parse_arguments(&stringvec!["-Fofoo", "foo.c"]));
    }

    #[test]
    fn test_parse_arguments_too_many_inputs() {
        assert_eq!(CompilerArguments::CannotCache,
                   parse_arguments(&stringvec!["-c", "foo.c", "-Fofoo.obj", "bar.c"]));
    }

    #[test]
    fn test_parse_arguments_unsupported() {
        assert_eq!(CompilerArguments::CannotCache,
                   parse_arguments(&stringvec!["-c", "foo.c", "-Fofoo.obj", "-FA"]));

        assert_eq!(CompilerArguments::CannotCache,
                   parse_arguments(&stringvec!["-Fa", "-c", "foo.c", "-Fofoo.obj"]));

        assert_eq!(CompilerArguments::CannotCache,
                   parse_arguments(&stringvec!["-c", "foo.c", "-FR", "-Fofoo.obj"]));
    }

    #[test]
    fn test_parse_arguments_response_file() {
        assert_eq!(CompilerArguments::CannotCache,
                   parse_arguments(&stringvec!["-c", "foo.c", "@foo", "-Fofoo.obj"]));
    }

    #[test]
    fn test_parse_arguments_missing_pdb() {
        assert_eq!(CompilerArguments::CannotCache,
                   parse_arguments(&stringvec!["-c", "foo.c", "-Zi", "-Fofoo.obj"]));
    }
}
<|endoftext|>


<commit_before>use std::cell::RefCell;
use std::collections::{HashMap, HashSet};
use std::rc::Rc;

use super::ast::nodes::*;
use super::ir::layout::{
    Address, Import, Instruction, InstructionBuilder, Module as IrModule, SharedFunction,
    SharedSlot, SharedValue,
};
use super::vm::prelude::is_in_prelude;

trait Scope {
    fn add_local(&mut self, name: &String) -> SharedSlot;

    fn resolve(&mut self, name: &String) -> SharedSlot;

    /// Acts the same as `resolve`, but records bindings along the way.
    fn resolve_as_binding(&mut self, name: &String) -> SharedSlot;
}

struct FunctionScope<'a> {
    parent: &'a mut dyn Scope,
    function: SharedFunction,
    // Keep track of every variable we've seen. Some of them may end up bound,
    // so we'll actually make them proper local variables or bound variables
    // when we traverse upwards.
    locals: HashMap<String, SharedSlot>,
    // Keep track of which variables need to be bound and come from parent
    // bindings.
    bindings: HashSet<String>,
    parent_bindings: HashMap<String, SharedSlot>,
}

impl<'a> FunctionScope<'a> {
    fn new(parent: &'a mut dyn Scope, function: SharedFunction) -> Self {
        Self {
            parent,
            function,
            locals: HashMap::new(),
            bindings: HashSet::new(),
            parent_bindings: HashMap::new(),
        }
    }
}

impl<'a> Scope for FunctionScope<'a> {
    fn add_local(&mut self, name: &String) -> SharedSlot {
        let slot = SharedSlot::new_local(name.clone());
        self.locals.insert(name.clone(), slot.clone());
        slot
    }

    fn resolve(&mut self, name: &String) -> SharedSlot {
        if let Some(resolved) = self.locals.get(name) {
            resolved.clone()
        } else {
            let slot = self.parent.resolve_as_binding(name);
            self.parent_bindings.insert(name.clone(), slot.clone());
            slot
        }
    }

    fn resolve_as_binding(&mut self, name: &String) -> SharedSlot {
        if let Some(resolved) = self.locals.get(name) {
            // Since this was called from a lower function scope we now know
            // we need to bind it for that lower scope.
            self.bindings.insert(name.clone());
            resolved.clone()
        } else {
            let slot = self.parent.resolve_as_binding(name);
            self.parent_bindings.insert(name.clone(), slot.clone());
            slot
        }
    }
}

struct ModuleScope {
    module: Rc<RefCell<IrModule>>,
    bindings: HashMap<String, SharedSlot>,
    // Map an import name to its source.
    imports: HashMap<String, Import>,
}

impl Scope for ModuleScope {
    fn add_local(&mut self, name: &String) -> SharedSlot {
        let slot = SharedSlot::new_static(name.clone());
        self.bindings.insert(name.clone(), slot.clone());
        slot
    }

    fn resolve(&mut self, name: &String) -> SharedSlot {
        if is_in_prelude(name) {
            self.imports.insert(
                name.to_owned(),
                Import::Named("prelude".to_owned(), name.to_owned()),
            );
            return SharedSlot::new_static(name.clone());
        }
        if let Some(slot) = self.bindings.get(name) {
            slot.clone()
        } else {
            // Automatically add it because it might be imported via
            // an `import *`.
            self.add_local(name)
        }
    }

    fn resolve_as_binding(&mut self, name: &String) -> SharedSlot {
        self.resolve(name)
    }
}

struct Compiler {
    module: Rc<RefCell<IrModule>>,
    current: SharedFunction,
}

impl Compiler {
    fn new() -> Self {
        let module = Rc::new(RefCell::new(IrModule::new()));
        let current = module.borrow().main_function();
        Self { module, current }
    }

    fn compile_module(&mut self, module: &Module) {
        let mut module_scope = ModuleScope {
            module: self.module.clone(),
            bindings: HashMap::new(),
            imports: HashMap::new(),
        };

        // We should start in the main function.
        assert_eq!(self.current, self.module.borrow().main_function());

        let mut scope = FunctionScope::new(&mut module_scope, self.current.clone());
        for node in module.nodes.iter() {
            self.compile_node(node, &mut scope);
            // We should always end up back in the main function.
            assert_eq!(self.current, self.module.borrow().main_function());
        }
        // Insert a return at the end to make sure empty modules don't crash.
        self.build_return_null();
        Compiler::finalize_function(self.current.clone(), &scope);

        // Now that we've visited the whole program we can write out the
        // imports we've found.
        self.module.borrow_mut().imports = module_scope.imports;
    }

    fn compile_node(&mut self, node: &Node, scope: &mut dyn Scope) -> SharedValue {
        match node {
            &Node::Assignment(ref assignment) => self.compile_assignment(assignment, scope),
            &Node::Block(ref block) => self.compile_anonymous_block(block, scope),
            &Node::Function(ref function) => self.compile_function(function, scope),
            &Node::Identifier(ref identifier) => self.compile_identifier(identifier, scope),
            &Node::Infix(ref infix) => self.compile_infix(infix, scope),
            &Node::Integer(ref integer) => self.compile_integer(integer, scope),
            &Node::PostfixCall(ref call) => self.compile_postfix_call(call, scope),
            &Node::Return(ref ret) => self.compile_return(ret, scope),
            &Node::Var(ref var) => self.compile_var(var, scope),
            &Node::While(ref while_) => self.compile_while(while_, scope),
            _ => panic!("Cannot compile node: {:?}", node),
        }
    }

    fn compile_assignment(
        &mut self,
        assignment: &Assignment,
        scope: &mut dyn Scope,
    ) -> SharedValue {
        let lhs = &*assignment.lhs;
        let assignee = match &lhs {
            &Node::Identifier(identifier) => {
                let local = &identifier.value;
                scope.resolve(local)
            }
            _ => unreachable!("Cannot assign to: {:?}", lhs),
        };

        let rval = self.compile_node(&assignment.rhs, scope);
        self.build_set(assignee, rval.clone());
        rval
    }

    fn compile_anonymous_block(&mut self, block: &Block, scope: &mut dyn Scope) -> SharedValue {
        // Push a new basic block and branch to it.
        self.current.borrow_mut().push_basic_block(true);

        let mut implicit_return = self.null_value();
        for node in block.nodes.iter() {
            implicit_return = self.compile_node(node, scope);
        }

        // Exit from the current block to the new block.
        self.current.borrow_mut().push_basic_block(true);

        implicit_return
    }

    fn compile_function(&mut self, function: &Function, scope: &mut dyn Scope) -> SharedValue {
        let name = function.name.to_owned();
        let enclosing_function = self.current.clone();
        let new_function = match &name {
            Some(name) => self.module.borrow_mut().new_named_function(name.clone()),
            None => self
                .module
                .borrow_mut()
                .new_anonymous_function(enclosing_function.clone()),
        };
        self.current = new_function.clone();

        let slot = name.map(|name| scope.add_local(&name));
        let mut function_scope = FunctionScope::new(scope, self.current.clone());
        let body = &*function.body;
        let lval = self.compile_node(body, &mut function_scope);
        self.build_return(lval);
        // Save the bindings so that we know how to build the closure.
        Compiler::finalize_function(new_function.clone(), &function_scope);

        self.current = enclosing_function;
        let lval = self.build_make_function(new_function);
        if let Some(slot) = slot {
            self.build_set(slot, lval.clone());
        };
        lval
    }

    fn finalize_function(function: SharedFunction, scope: &FunctionScope) {
        let mut function = function.borrow_mut();

        let mut all_locals = scope.locals.clone();
        // The bindings will have been filled in by nested functions within
        // this function. We therefore now need to remove any local that's
        // been bound and promote it to lexical.
        for binding in scope.bindings.iter() {
            let slot = all_locals
                .remove(binding)
                .expect(&format!("Missing local for binding: {}", binding));
            slot.promote_from_local_to_lexical(binding.clone());
        }
        // Remaining entries in `all_locals` will now only be unbound locals
        // (ie. ones that won't be captured in a closure).
        let unbound_locals = all_locals;

        let mut locals = vec![];
        for (index, (name, slot)) in unbound_locals.iter().enumerate() {
            locals.push(name.clone());
            slot.set_local_index(index as u8);
        }

        function.locals = locals;
        function.bindings = scope.bindings.clone();
        // We only need to capture our parent's bindings if we use non-static
        // ones (eg. lexical slots or local slots that will be promoted
        // to lexical).
        function.parent_bindings = scope.parent_bindings.values().any(|slot| !slot.is_static());
    }

    fn compile_identifier(
        &mut self,
        identifier: &Identifier,
        scope: &mut dyn Scope,
    ) -> SharedValue {
        let local = &identifier.value;
        self.build_get(scope.resolve(local))
    }

    fn compile_infix(&mut self, infix: &Infix, scope: &mut dyn Scope) -> SharedValue {
        let lhs = self.compile_node(&infix.lhs, scope);
        let rhs = self.compile_node(&infix.rhs, scope);
        match &infix.op {
            &InfixOp::Add => self.build_op_add(lhs, rhs),
            &InfixOp::LessThan => self.build_op_less_than(lhs, rhs),
            other @ _ => panic!("Cannot compile infix op: {:?}", other),
        }
    }

    fn compile_integer(&mut self, integer: &Integer, _scope: &mut dyn Scope) -> SharedValue {
        self.build_make_integer(integer.value)
    }

    fn compile_postfix_call(&mut self, call: &PostfixCall, scope: &mut dyn Scope) -> SharedValue {
        let target = self.compile_node(&call.target, scope);
        let mut arguments = vec![];
        for argument in call.arguments.iter() {
            arguments.push(self.compile_node(argument, scope));
        }
        self.build_call(target, arguments)
    }

    fn compile_return(&mut self, ret: &Return, scope: &mut dyn Scope) -> SharedValue {
        if let Some(ref rhs) = ret.rhs {
            let rval = self.compile_node(&rhs, scope);
            self.build_return(rval);
        } else {
            self.build_return_null();
        }
        self.null_value()
    }

    fn compile_var(&mut self, var: &Var, scope: &mut dyn Scope) -> SharedValue {
        let slot = scope.add_local(&var.lhs.value);
        if let Some(ref rhs) = var.rhs {
            let rval = self.compile_node(rhs, scope);
            self.build_set(slot, rval);
        }
        self.null_value()
    }

    fn compile_while(&mut self, while_: &While, scope: &mut dyn Scope) -> SharedValue {
        // Make a block for the condition and branch to it.
        let condition_block = self.current.borrow_mut().push_basic_block(true);
        // The block for the loop to run in.
        let loop_block = self.current.borrow_mut().push_basic_block(false);
        // The block after the while.
        let successor_block = self.current.borrow_mut().push_basic_block(false);

        self.current
            .borrow_mut()
            .set_current_basic_block(condition_block.clone());
        let condition_value = self.compile_node(&while_.condition, scope);
        // If it's true branch to the loop block, else branch to the after block.
        self.build_branch_if(loop_block.clone(), condition_value);
        self.build_branch(successor_block.clone());

        self.current
            .borrow_mut()
            .set_current_basic_block(loop_block.clone());
        for node in while_.block.nodes.iter() {
            self.compile_node(node, scope);
        }
        // After executing the block go back to the condition.
        self.build_branch(condition_block.clone());

        self.current
            .borrow_mut()
            .set_current_basic_block(successor_block.clone());
        self.null_value()
    }

    // Convenience proxy methods to the current function:

    fn null_value(&self) -> SharedValue {
        self.current.borrow().null_value()
    }
}

// Forward all the heavy lifting to the current function. This allows us to do
// stuff like `self.build_get_local()` above inside of `Compiler`.
impl InstructionBuilder for Compiler {
    fn new_value(&mut self) -> SharedValue {
        self.current.borrow_mut().new_value()
    }

    fn push(&mut self, instruction: Instruction) -> Address {
        self.current.borrow_mut().push(instruction)
    }

    fn track(&mut self, rval: SharedValue, address: Address) {
        self.current.borrow_mut().track(rval, address)
    }
}

pub fn compile(module: &Module) -> IrModule {
    let mut compiler = Compiler::new();
    compiler.compile_module(module);
    // The reference count should be 1 at this point.
    let module_cell = Rc::try_unwrap(compiler.module).unwrap();
    module_cell.into_inner()
}
<commit_msg>Use flags to control AST-to-IR compilation for REPL evaluation<commit_after>use std::cell::RefCell;
use std::collections::{HashMap, HashSet};
use std::rc::Rc;

use super::ast::nodes::*;
use super::ir::layout::{
    Address, Import, Instruction, InstructionBuilder, Module as IrModule, SharedFunction,
    SharedSlot, SharedValue,
};
use super::vm::prelude::is_in_prelude;

#[derive(PartialEq)]
enum ScopeFlags {
    /// Evaluate scoping rules normally.
    None,
    /// Evaluate scoping rules for use in a REPL (ie. promote all locals
    /// to statics).
    Repl,
}

impl Into<ScopeFlags> for CompilationFlags {
    fn into(self) -> ScopeFlags {
        match self {
            CompilationFlags::None => ScopeFlags::None,
            CompilationFlags::Repl => ScopeFlags::Repl,
        }
    }
}

trait Scope {
    fn add_local(&mut self, name: &String) -> SharedSlot;

    fn resolve(&mut self, name: &String) -> SharedSlot;

    /// Acts the same as `resolve`, but records bindings along the way.
    fn resolve_as_binding(&mut self, name: &String) -> SharedSlot;
}

struct FunctionScope<'a> {
    parent: &'a mut dyn Scope,
    function: SharedFunction,
    // Keep track of every variable we've seen. Some of them may end up bound,
    // so we'll actually make them proper local variables or bound variables
    // when we traverse upwards.
    locals: HashMap<String, SharedSlot>,
    // Keep track of which variables need to be bound and come from parent
    // bindings.
    bindings: HashSet<String>,
    parent_bindings: HashMap<String, SharedSlot>,
}

impl<'a> FunctionScope<'a> {
    fn new(parent: &'a mut dyn Scope, function: SharedFunction) -> Self {
        Self {
            parent,
            function,
            locals: HashMap::new(),
            bindings: HashSet::new(),
            parent_bindings: HashMap::new(),
        }
    }
}

impl<'a> Scope for FunctionScope<'a> {
    fn add_local(&mut self, name: &String) -> SharedSlot {
        let slot = SharedSlot::new_local(name.clone());
        self.locals.insert(name.clone(), slot.clone());
        slot
    }

    fn resolve(&mut self, name: &String) -> SharedSlot {
        if let Some(resolved) = self.locals.get(name) {
            resolved.clone()
        } else {
            let slot = self.parent.resolve_as_binding(name);
            self.parent_bindings.insert(name.clone(), slot.clone());
            slot
        }
    }

    fn resolve_as_binding(&mut self, name: &String) -> SharedSlot {
        if let Some(resolved) = self.locals.get(name) {
            // Since this was called from a lower function scope we now know
            // we need to bind it for that lower scope.
            self.bindings.insert(name.clone());
            resolved.clone()
        } else {
            let slot = self.parent.resolve_as_binding(name);
            self.parent_bindings.insert(name.clone(), slot.clone());
            slot
        }
    }
}

struct ModuleScope {
    module: Rc<RefCell<IrModule>>,
    bindings: HashMap<String, SharedSlot>,
    // Map an import name to its source.
    imports: HashMap<String, Import>,
}

impl Scope for ModuleScope {
    fn add_local(&mut self, name: &String) -> SharedSlot {
        let slot = SharedSlot::new_static(name.clone());
        self.bindings.insert(name.clone(), slot.clone());
        slot
    }

    fn resolve(&mut self, name: &String) -> SharedSlot {
        if is_in_prelude(name) {
            self.imports.insert(
                name.to_owned(),
                Import::Named("prelude".to_owned(), name.to_owned()),
            );
            return SharedSlot::new_static(name.clone());
        }
        if let Some(slot) = self.bindings.get(name) {
            slot.clone()
        } else {
            // Automatically add it because it might be imported via
            // an `import *`.
            self.add_local(name)
        }
    }

    fn resolve_as_binding(&mut self, name: &String) -> SharedSlot {
        self.resolve(name)
    }
}

struct Compiler {
    module: Rc<RefCell<IrModule>>,
    current: SharedFunction,
}

impl Compiler {
    fn new() -> Self {
        let module = Rc::new(RefCell::new(IrModule::new()));
        let current = module.borrow().main_function();
        Self { module, current }
    }

    fn compile_module(&mut self, module: &Module, flags: CompilationFlags) {
        let mut module_scope = ModuleScope {
            module: self.module.clone(),
            bindings: HashMap::new(),
            imports: HashMap::new(),
        };

        // We should start in the main function.
        assert_eq!(self.current, self.module.borrow().main_function());

        let mut scope = FunctionScope::new(&mut module_scope, self.current.clone());
        let mut implicit_return = self.null_value();
        for node in module.nodes.iter() {
            implicit_return = self.compile_node(node, &mut scope);
            // We should always end up back in the main function.
            assert_eq!(self.current, self.module.borrow().main_function());
        }
        match flags {
            CompilationFlags::Repl => {
                // If compiling for a REPL then we want to implicit return the
                // last statement.
                self.build_return(implicit_return);
            }
            _ => {
                // Insert a return at the end to make sure empty modules don't crash.
                self.build_return_null();
            }
        };
        Compiler::finalize_function(self.current.clone(), &scope, flags.into());

        // Now that we've visited the whole program we can write out the
        // imports we've found.
        self.module.borrow_mut().imports = module_scope.imports;
    }

    fn compile_node(&mut self, node: &Node, scope: &mut dyn Scope) -> SharedValue {
        match node {
            &Node::Assignment(ref assignment) => self.compile_assignment(assignment, scope),
            &Node::Block(ref block) => self.compile_anonymous_block(block, scope),
            &Node::Function(ref function) => self.compile_function(function, scope),
            &Node::Identifier(ref identifier) => self.compile_identifier(identifier, scope),
            &Node::Infix(ref infix) => self.compile_infix(infix, scope),
            &Node::Integer(ref integer) => self.compile_integer(integer, scope),
            &Node::PostfixCall(ref call) => self.compile_postfix_call(call, scope),
            &Node::Return(ref ret) => self.compile_return(ret, scope),
            &Node::Var(ref var) => self.compile_var(var, scope),
            &Node::While(ref while_) => self.compile_while(while_, scope),
            _ => panic!("Cannot compile node: {:?}", node),
        }
    }

    fn compile_assignment(
        &mut self,
        assignment: &Assignment,
        scope: &mut dyn Scope,
    ) -> SharedValue {
        let lhs = &*assignment.lhs;
        let assignee = match &lhs {
            &Node::Identifier(identifier) => {
                let local = &identifier.value;
                scope.resolve(local)
            }
            _ => unreachable!("Cannot assign to: {:?}", lhs),
        };

        let rval = self.compile_node(&assignment.rhs, scope);
        self.build_set(assignee, rval.clone());
        rval
    }

    fn compile_anonymous_block(&mut self, block: &Block, scope: &mut dyn Scope) -> SharedValue {
        // Push a new basic block and branch to it.
        self.current.borrow_mut().push_basic_block(true);

        let mut implicit_return = self.null_value();
        for node in block.nodes.iter() {
            implicit_return = self.compile_node(node, scope);
        }

        // Exit from the current block to the new block.
        self.current.borrow_mut().push_basic_block(true);

        implicit_return
    }

    fn compile_function(&mut self, function: &Function, scope: &mut dyn Scope) -> SharedValue {
        let name = function.name.to_owned();
        let enclosing_function = self.current.clone();
        let new_function = match &name {
            Some(name) => self.module.borrow_mut().new_named_function(name.clone()),
            None => self
                .module
                .borrow_mut()
                .new_anonymous_function(enclosing_function.clone()),
        };
        self.current = new_function.clone();

        let slot = name.map(|name| scope.add_local(&name));
        let mut function_scope = FunctionScope::new(scope, self.current.clone());
        let body = &*function.body;
        let lval = self.compile_node(body, &mut function_scope);
        self.build_return(lval);
        // Save the bindings so that we know how to build the closure.
        Compiler::finalize_function(new_function.clone(), &function_scope, ScopeFlags::None);

        self.current = enclosing_function;
        let lval = self.build_make_function(new_function);
        if let Some(slot) = slot {
            self.build_set(slot, lval.clone());
        };
        lval
    }

    fn finalize_function(function: SharedFunction, scope: &FunctionScope, flags: ScopeFlags) {
        let mut function = function.borrow_mut();

        // If we're in a REPL then promote everything to lexical and always
        // capture our parent bindings.
        if flags == ScopeFlags::Repl {
            for (name, slot) in scope.locals.iter() {
                slot.promote_from_local_to_lexical(name.clone());
            }
            function.locals = vec![];
            function.bindings = scope.bindings.clone();
            function.parent_bindings = true;
            return;
        }

        let mut all_locals = scope.locals.clone();
        // The bindings will have been filled in by nested functions within
        // this function. We therefore now need to remove any local that's
        // been bound and promote it to lexical.
        for binding in scope.bindings.iter() {
            let slot = all_locals
                .remove(binding)
                .expect(&format!("Missing local for binding: {}", binding));
            slot.promote_from_local_to_lexical(binding.clone());
        }
        // Remaining entries in `all_locals` will now only be unbound locals
        // (ie. ones that won't be captured in a closure).
        let unbound_locals = all_locals;

        let mut locals = vec![];
        for (index, (name, slot)) in unbound_locals.iter().enumerate() {
            locals.push(name.clone());
            slot.set_local_index(index as u8);
        }

        function.locals = locals;
        function.bindings = scope.bindings.clone();
        // We only need to capture our parent's bindings if we use non-static
        // ones (eg. lexical slots or local slots that will be promoted
        // to lexical).
        function.parent_bindings = scope.parent_bindings.values().any(|slot| !slot.is_static());
    }

    fn compile_identifier(
        &mut self,
        identifier: &Identifier,
        scope: &mut dyn Scope,
    ) -> SharedValue {
        let local = &identifier.value;
        self.build_get(scope.resolve(local))
    }

    fn compile_infix(&mut self, infix: &Infix, scope: &mut dyn Scope) -> SharedValue {
        let lhs = self.compile_node(&infix.lhs, scope);
        let rhs = self.compile_node(&infix.rhs, scope);
        match &infix.op {
            &InfixOp::Add => self.build_op_add(lhs, rhs),
            &InfixOp::LessThan => self.build_op_less_than(lhs, rhs),
            other @ _ => panic!("Cannot compile infix op: {:?}", other),
        }
    }

    fn compile_integer(&mut self, integer: &Integer, _scope: &mut dyn Scope) -> SharedValue {
        self.build_make_integer(integer.value)
    }

    fn compile_postfix_call(&mut self, call: &PostfixCall, scope: &mut dyn Scope) -> SharedValue {
        let target = self.compile_node(&call.target, scope);
        let mut arguments = vec![];
        for argument in call.arguments.iter() {
            arguments.push(self.compile_node(argument, scope));
        }
        self.build_call(target, arguments)
    }

    fn compile_return(&mut self, ret: &Return, scope: &mut dyn Scope) -> SharedValue {
        if let Some(ref rhs) = ret.rhs {
            let rval = self.compile_node(&rhs, scope);
            self.build_return(rval);
        } else {
            self.build_return_null();
        }
        self.null_value()
    }

    fn compile_var(&mut self, var: &Var, scope: &mut dyn Scope) -> SharedValue {
        let slot = scope.add_local(&var.lhs.value);
        if let Some(ref rhs) = var.rhs {
            let rval = self.compile_node(rhs, scope);
            self.build_set(slot, rval);
        }
        self.null_value()
    }

    fn compile_while(&mut self, while_: &While, scope: &mut dyn Scope) -> SharedValue {
        // Make a block for the condition and branch to it.
        let condition_block = self.current.borrow_mut().push_basic_block(true);
        // The block for the loop to run in.
        let loop_block = self.current.borrow_mut().push_basic_block(false);
        // The block after the while.
        let successor_block = self.current.borrow_mut().push_basic_block(false);

        self.current
            .borrow_mut()
            .set_current_basic_block(condition_block.clone());
        let condition_value = self.compile_node(&while_.condition, scope);
        // If it's true branch to the loop block, else branch to the after block.
        self.build_branch_if(loop_block.clone(), condition_value);
        self.build_branch(successor_block.clone());

        self.current
            .borrow_mut()
            .set_current_basic_block(loop_block.clone());
        for node in while_.block.nodes.iter() {
            self.compile_node(node, scope);
        }
        // After executing the block go back to the condition.
        self.build_branch(condition_block.clone());

        self.current
            .borrow_mut()
            .set_current_basic_block(successor_block.clone());
        self.null_value()
    }

    // Convenience proxy methods to the current function:

    fn null_value(&self) -> SharedValue {
        self.current.borrow().null_value()
    }
}

// Forward all the heavy lifting to the current function. This allows us to do
// stuff like `self.build_get_local()` above inside of `Compiler`.
impl InstructionBuilder for Compiler {
    fn new_value(&mut self) -> SharedValue {
        self.current.borrow_mut().new_value()
    }

    fn push(&mut self, instruction: Instruction) -> Address {
        self.current.borrow_mut().push(instruction)
    }

    fn track(&mut self, rval: SharedValue, address: Address) {
        self.current.borrow_mut().track(rval, address)
    }
}

pub enum CompilationFlags {
    None,
    Repl,
}

impl Default for CompilationFlags {
    fn default() -> Self {
        Self::None
    }
}

pub fn compile(module: &Module, flags: CompilationFlags) -> IrModule {
    let mut compiler = Compiler::new();
    compiler.compile_module(module, flags);
    // The reference count should be 1 at this point.
    let module_cell = Rc::try_unwrap(compiler.module).unwrap();
    module_cell.into_inner()
}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate log;
extern crate osmpbfreader;
extern crate rustc_serialize;
extern crate docopt;
extern crate mimir;
extern crate rs_es;

use std::collections::HashSet;
use std::collections::HashMap;
use mimirsbrunn::rubber::Rubber;
use mimirsbrunn::objects::{Polygon, MultiPolygon};

pub type AdminsVec = Vec<mimir::Admin>;
pub type StreetsVec = Vec<mimir::Street>;
pub type ParsedPbf = osmpbfreader::OsmPbfReader<std::fs::File>;

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_level: Vec<u32>,
    flag_connection_string: String,
}

static USAGE: &'static str = "
Usage:
    osm2mimir --help
    osm2mimir --input=<file> [--connection-string=<connection-string>] --level=<level>...

Options:
    -h, --help            Show this message.
    -i, --input=<file>    OSM PBF file.
    -l, --level=<level>   Admin levels to keep.
    -c, --connection-string=<connection-string>
                          Elasticsearch parameters, [default: http://localhost:9200/munin]
";

#[derive(Debug)]
struct AdminMatcher {
    admin_levels: HashSet<u32>,
}
impl AdminMatcher {
    pub fn new(levels: HashSet<u32>) -> AdminMatcher {
        AdminMatcher { admin_levels: levels }
    }

    pub fn is_admin(&self, obj: &osmpbfreader::OsmObj) -> bool {
        match *obj {
            osmpbfreader::OsmObj::Relation(ref rel) => {
                rel.tags.get("boundary").map_or(false, |v| v == "administrative") &&
                rel.tags.get("admin_level").map_or(false, |lvl| {
                    self.admin_levels.contains(&lvl.parse::<u32>().unwrap_or(0))
                })
            }
            _ => false,
        }
    }
}

struct BoundaryPart {
    nodes: Vec<osmpbfreader::Node>,
}

impl BoundaryPart {
    pub fn new(nodes: Vec<osmpbfreader::Node>) -> BoundaryPart {
        BoundaryPart { nodes: nodes }
    }
    pub fn first(&self) -> i64 {
        self.nodes.first().unwrap().id
    }
    pub fn last(&self) -> i64 {
        self.nodes.last().unwrap().id
    }
}

fn get_nodes(way: &osmpbfreader::Way,
             objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
             -> Vec<osmpbfreader::Node> {
    way.nodes
       .iter()
       .filter_map(|node_id| objects.get(&osmpbfreader::OsmId::Node(*node_id)))
       .filter_map(|node_obj| if let &osmpbfreader::OsmObj::Node(ref node) = node_obj {
           Some(node.clone())
       } else {
           None
       })
       .collect()
}

fn build_boundary(relation: &osmpbfreader::Relation,
                  objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
                  -> Option<MultiPolygon> {
    let mut boundary_parts: Vec<BoundaryPart> =
        relation.refs
                .iter()
                .filter(|rf| rf.role == "outer" || rf.role == "" || rf.role == "enclave")
                .filter_map(|refe| {
                    objects.get(&refe.member).or_else(|| {
                        warn!("missing element for relation {}", relation.id);
                        None
                    })
                })
                .filter_map(|way_obj| if let &osmpbfreader::OsmObj::Way(ref way) = way_obj {
                    Some(way)
                } else {
                    None
                })
                .map(|way| get_nodes(&way, objects))
                .filter(|nodes| nodes.len() > 1)
                .map(|nodes| BoundaryPart::new(nodes))
                .collect();
    let mut multipoly = MultiPolygon { polygons: Vec::new() };
    // we want to try build a polygon for a least each way
    while !boundary_parts.is_empty() {
        let mut current = -1;
        let mut first = 0;
        let mut nb_try = 0;

        let mut outer: Vec<osmpbfreader::Node> = Vec::new();
        // we try to close the polygon, if we can't we want to at least have tried one time per
        // way. We could improve that latter by trying to attach the way to both side of the
        // polygon
        let max_try = boundary_parts.len();
        while current != first && nb_try < max_try {
            let mut i = 0;
            while i < boundary_parts.len() {
                if outer.is_empty() {
                    // our polygon is empty, we initialise it with the current way
                    first = boundary_parts[i].first();
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    // this way has been used, we remove it from the pool
                    boundary_parts.remove(i);
                    continue;
                }
                if current == boundary_parts[i].first() {
                    // the start of current way touch the polygon, we add it and remove it from the
                    // pool
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else if current == boundary_parts[i].last() {
                    // the end of the current way touch the polygon, we reverse the way and add it
                    current = boundary_parts[i].first();
                    boundary_parts[i].nodes.reverse();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else {
                    i += 1;
                    // didnt do anything, we want to explore the next way, if we had do something we
                    // will have removed the current way and there will be no need to increment
                }
                if current == first {
                    // our polygon is closed, we create it and add it to the multipolygon
                    let polygon = Polygon::new(outer.iter()
                                                    .map(|n| {
                                                        mimir::Coord {
                                                            lat: n.lat,
                                                            lon: n.lon,
                                                        }
                                                    })
                                                    .collect());
                    multipoly.polygons.push(polygon);
                    break;
                }
            }
            nb_try += 1;
        }
    }
    debug!("polygon for relation {};{}", relation.id, multipoly.to_wkt());
    if multipoly.polygons.is_empty() {
        None
    } else {
        Some(multipoly)
    }
}

fn parse_osm_pbf(path: &String) -> ParsedPbf {
    let path = std::path::Path::new(&path);
    osmpbfreader::OsmPbfReader::new(std::fs::File::open(&path).unwrap())
}

fn administrative_regions(pbf: &mut ParsedPbf, levels: HashSet<u32>) -> AdminsVec {
    let mut administrative_regions = AdminsVec::new();
    let matcher = AdminMatcher::new(levels);
    let objects = osmpbfreader::get_objs_and_deps(pbf, |o| matcher.is_admin(o)).unwrap();
    // load administratives regions
    for (_, obj) in &objects {
        if !matcher.is_admin(&obj) {
            continue;
        }
        if let &osmpbfreader::OsmObj::Relation(ref relation) = obj {
            let level = relation.tags
                                .get("admin_level")
                                .and_then(|s| s.parse().ok());
            let level = match level {
                None => {
                    info!("invalid admin_level for relation {}: admin_level {:?}",
                          relation.id,
                          relation.tags.get("admin_level"));
                    continue;
                }
                Some(l) => l,
            };
            // administrative region with name ?
            let name = match relation.tags.get("name") {
                Some(val) => val,
                None => {
                    warn!("adminstrative region without name for relation {}:  admin_level {} \
                           ignored.",
                          relation.id,
                          level);
                    continue;
                }
            };
            // admininstrative region without coordinates
            let coord_centre = relation.refs
                                       .iter()
                                       .find(|rf| rf.role == "admin_centre")
                                       .and_then(|r| {
                                           objects.get(&r.member).and_then(|value| {
                                               match value {
                                                   &osmpbfreader::OsmObj::Node(ref node) => {
                                                       Some(mimir::Coord {
                                                           lat: node.lat,
                                                           lon: node.lon,
                                                       })
                                                   }
                                                   _ => None,
                                               }
                                           })
                                       });

            let admin_id = match relation.tags.get("ref:INSEE") {
                Some(val) => format!("admin:fr:{}", val.trim_left_matches('0')),
                None => format!("admin:osm:{}", relation.id),
            };
            let zip_code = match relation.tags.get("addr:postcode") {
                Some(val) => &val[..],
                None => "",
            };
            let boundary = build_boundary(&relation, &objects);
            let admin = mimir::Admin {
                id: admin_id,
                level: level,
                name: name.to_string(),
                zip_code: zip_code.to_string(),
                // TODO weight value ?
                weight: 1,
                coord: coord_centre,
                boundary: boundary,
            };
            administrative_regions.push(admin);
        }
    }
    return administrative_regions;
}

fn streets(pbf: &mut ParsedPbf) -> StreetsVec {

    let is_valid_street = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Way(ref way) => {
                way.tags.get("highway").map_or(false, |x: &String| !x.is_empty()) &&
                way.tags.get("name").map_or(false, |x: &String| !x.is_empty())
            }
            _ => false,
        }
    };

    let is_associated_street_relation = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Relation(ref rel) => {
                rel.tags.get("type").map_or(false, |v| v == "associatedStreet")
            }
            _ => false,
        }
    };

    let mut street_map = osmpbfreader::get_objs_and_deps(pbf, is_valid_street).unwrap();
    let relation_map = osmpbfreader::get_objs_and_deps(pbf, is_associated_street_relation).unwrap();

    // Sometimes, streets can be devided into several "way"s that still have the same street name.
    // The reason why a street is devided may be that a part of the street become a bridge/tunne/etc.
    // In this case, a "relation" tagged with (type = associatedStreet) is used to group all these "way"s.
    // In order not to have doublons in autocompleion, we should keep only one "way" in the relation
    // and remove all the rest way whose street name is the same.
    for (_, rel_obj) in &relation_map {
        if let &osmpbfreader::OsmObj::Relation(ref rel) = rel_obj {
            let mut found = false;
            for ref_obj in &rel.refs {
                if let &osmpbfreader::OsmId::Way(_) = &ref_obj.member {
                    if !found {
                        found = true;
                        continue;
                    };
                    street_map.remove(&ref_obj.member);
                };
            }
        };
    }

    street_map.iter()
              .filter_map(|(osm_id, obj)| {
                  if let &osmpbfreader::OsmObj::Way(ref way) = obj {
                      if let &osmpbfreader::OsmId::Way(ref way_id) = osm_id {
                          way.tags.get("name").and_then(|way_name| {
                              Some(mimirsbrunn::Street {
                                  id: way_id.to_string(),
                                  street_name: way_name.to_string(),
                                  name: way_name.to_string(),
                                  weight: 1,
                                  administrative_region: None,
                              })
                          })
                      } else {
                          None
                      }
                  } else {
                      None
                  }
              })
              .collect()

}

fn index_osm(es_cnx_string: &str, admins: &AdminsVec, streets: &StreetsVec) {
    let mut rubber = Rubber::new(es_cnx_string);
    rubber.create_index();
    match rubber.clean_db_by_doc_type(&["admin", "street"]) {
        Err(e) => panic!("failed to clean data by document type: {}", e),
        Ok(nb) => info!("clean data by document type : {}", nb),
    }
    info!("Add data in elasticsearch db.");
    match rubber.bulk_index(admins.iter()) {
        Err(e) => panic!("failed to index admins of osm because: {}", e),
        Ok(nb) => info!("Nb of indexed adminstrative regions: {}", nb),
    }

    // TODO: make it (de)activable from command line
    if false {
        match rubber.bulk_index(streets.iter()) {
            Err(e) => panic!("failed to index streets of osm because: {}", e),
            Ok(nb) => info!("Nb of indexed streets: {}", nb),
        }
    }
}

fn main() {
    mimir::logger_init().unwrap();
    debug!("importing adminstrative region into Mimir");
    let args: Args = docopt::Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());

    let levels = args.flag_level.iter().cloned().collect();
    let mut parsed_pbf = parse_osm_pbf(&args.flag_input);
    let admins = administrative_regions(&mut parsed_pbf, levels);
    let streets = streets(&mut parsed_pbf);
    index_osm(&args.flag_connection_string, &admins, &streets);
}
<commit_msg>import way can be selected from command line<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate log;
extern crate osmpbfreader;
extern crate rustc_serialize;
extern crate docopt;
extern crate mimir;
extern crate rs_es;

use std::collections::HashSet;
use std::collections::HashMap;
use mimirsbrunn::rubber::Rubber;
use mimirsbrunn::objects::{Polygon, MultiPolygon};

pub type AdminsVec = Vec<mimir::Admin>;
pub type StreetsVec = Vec<mimir::Street>;
pub type ParsedPbf = osmpbfreader::OsmPbfReader<std::fs::File>;

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_level: Vec<u32>,
    flag_connection_string: String,
    flag_import_way: bool
}

static USAGE: &'static str = "
Usage:
    osm2mimir --help
    osm2mimir --input=<file> [--connection-string=<connection-string>] --level=<level> [--import-way]...

Options:
    -h, --help            Show this message.
    -i, --input=<file>    OSM PBF file.
    -l, --level=<level>   Admin levels to keep.
    -w, --import-way      By default, streets are not imported
    -c, --connection-string=<connection-string>
                          Elasticsearch parameters, [default: http://localhost:9200/munin]
";

#[derive(Debug)]
struct AdminMatcher {
    admin_levels: HashSet<u32>,
}
impl AdminMatcher {
    pub fn new(levels: HashSet<u32>) -> AdminMatcher {
        AdminMatcher { admin_levels: levels }
    }

    pub fn is_admin(&self, obj: &osmpbfreader::OsmObj) -> bool {
        match *obj {
            osmpbfreader::OsmObj::Relation(ref rel) => {
                rel.tags.get("boundary").map_or(false, |v| v == "administrative") &&
                rel.tags.get("admin_level").map_or(false, |lvl| {
                    self.admin_levels.contains(&lvl.parse::<u32>().unwrap_or(0))
                })
            }
            _ => false,
        }
    }
}

struct BoundaryPart {
    nodes: Vec<osmpbfreader::Node>,
}

impl BoundaryPart {
    pub fn new(nodes: Vec<osmpbfreader::Node>) -> BoundaryPart {
        BoundaryPart { nodes: nodes }
    }
    pub fn first(&self) -> i64 {
        self.nodes.first().unwrap().id
    }
    pub fn last(&self) -> i64 {
        self.nodes.last().unwrap().id
    }
}

fn get_nodes(way: &osmpbfreader::Way,
             objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
             -> Vec<osmpbfreader::Node> {
    way.nodes
       .iter()
       .filter_map(|node_id| objects.get(&osmpbfreader::OsmId::Node(*node_id)))
       .filter_map(|node_obj| if let &osmpbfreader::OsmObj::Node(ref node) = node_obj {
           Some(node.clone())
       } else {
           None
       })
       .collect()
}

fn build_boundary(relation: &osmpbfreader::Relation,
                  objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
                  -> Option<MultiPolygon> {
    let mut boundary_parts: Vec<BoundaryPart> =
        relation.refs
                .iter()
                .filter(|rf| rf.role == "outer" || rf.role == "" || rf.role == "enclave")
                .filter_map(|refe| {
                    objects.get(&refe.member).or_else(|| {
                        warn!("missing element for relation {}", relation.id);
                        None
                    })
                })
                .filter_map(|way_obj| if let &osmpbfreader::OsmObj::Way(ref way) = way_obj {
                    Some(way)
                } else {
                    None
                })
                .map(|way| get_nodes(&way, objects))
                .filter(|nodes| nodes.len() > 1)
                .map(|nodes| BoundaryPart::new(nodes))
                .collect();
    let mut multipoly = MultiPolygon { polygons: Vec::new() };
    // we want to try build a polygon for a least each way
    while !boundary_parts.is_empty() {
        let mut current = -1;
        let mut first = 0;
        let mut nb_try = 0;

        let mut outer: Vec<osmpbfreader::Node> = Vec::new();
        // we try to close the polygon, if we can't we want to at least have tried one time per
        // way. We could improve that latter by trying to attach the way to both side of the
        // polygon
        let max_try = boundary_parts.len();
        while current != first && nb_try < max_try {
            let mut i = 0;
            while i < boundary_parts.len() {
                if outer.is_empty() {
                    // our polygon is empty, we initialise it with the current way
                    first = boundary_parts[i].first();
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    // this way has been used, we remove it from the pool
                    boundary_parts.remove(i);
                    continue;
                }
                if current == boundary_parts[i].first() {
                    // the start of current way touch the polygon, we add it and remove it from the
                    // pool
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else if current == boundary_parts[i].last() {
                    // the end of the current way touch the polygon, we reverse the way and add it
                    current = boundary_parts[i].first();
                    boundary_parts[i].nodes.reverse();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else {
                    i += 1;
                    // didnt do anything, we want to explore the next way, if we had do something we
                    // will have removed the current way and there will be no need to increment
                }
                if current == first {
                    // our polygon is closed, we create it and add it to the multipolygon
                    let polygon = Polygon::new(outer.iter()
                                                    .map(|n| {
                                                        mimir::Coord {
                                                            lat: n.lat,
                                                            lon: n.lon,
                                                        }
                                                    })
                                                    .collect());
                    multipoly.polygons.push(polygon);
                    break;
                }
            }
            nb_try += 1;
        }
    }
    debug!("polygon for relation {};{}", relation.id, multipoly.to_wkt());
    if multipoly.polygons.is_empty() {
        None
    } else {
        Some(multipoly)
    }
}

fn parse_osm_pbf(path: &String) -> ParsedPbf {
    let path = std::path::Path::new(&path);
    osmpbfreader::OsmPbfReader::new(std::fs::File::open(&path).unwrap())
}

fn administrative_regions(pbf: &mut ParsedPbf, levels: HashSet<u32>) -> AdminsVec {
    let mut administrative_regions = AdminsVec::new();
    let matcher = AdminMatcher::new(levels);
    let objects = osmpbfreader::get_objs_and_deps(pbf, |o| matcher.is_admin(o)).unwrap();
    // load administratives regions
    for (_, obj) in &objects {
        if !matcher.is_admin(&obj) {
            continue;
        }
        if let &osmpbfreader::OsmObj::Relation(ref relation) = obj {
            let level = relation.tags
                                .get("admin_level")
                                .and_then(|s| s.parse().ok());
            let level = match level {
                None => {
                    info!("invalid admin_level for relation {}: admin_level {:?}",
                          relation.id,
                          relation.tags.get("admin_level"));
                    continue;
                }
                Some(l) => l,
            };
            // administrative region with name ?
            let name = match relation.tags.get("name") {
                Some(val) => val,
                None => {
                    warn!("adminstrative region without name for relation {}:  admin_level {} \
                           ignored.",
                          relation.id,
                          level);
                    continue;
                }
            };
            // admininstrative region without coordinates
            let coord_centre = relation.refs
                                       .iter()
                                       .find(|rf| rf.role == "admin_centre")
                                       .and_then(|r| {
                                           objects.get(&r.member).and_then(|value| {
                                               match value {
                                                   &osmpbfreader::OsmObj::Node(ref node) => {
                                                       Some(mimir::Coord {
                                                           lat: node.lat,
                                                           lon: node.lon,
                                                       })
                                                   }
                                                   _ => None,
                                               }
                                           })
                                       });

            let admin_id = match relation.tags.get("ref:INSEE") {
                Some(val) => format!("admin:fr:{}", val.trim_left_matches('0')),
                None => format!("admin:osm:{}", relation.id),
            };
            let zip_code = match relation.tags.get("addr:postcode") {
                Some(val) => &val[..],
                None => "",
            };
            let boundary = build_boundary(&relation, &objects);
            let admin = mimir::Admin {
                id: admin_id,
                level: level,
                name: name.to_string(),
                zip_code: zip_code.to_string(),
                // TODO weight value ?
                weight: 1,
                coord: coord_centre,
                boundary: boundary,
            };
            administrative_regions.push(admin);
        }
    }
    return administrative_regions;
}

fn streets(pbf: &mut ParsedPbf) -> StreetsVec {

    let is_valid_street = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Way(ref way) => {
                way.tags.get("highway").map_or(false, |x: &String| !x.is_empty()) &&
                way.tags.get("name").map_or(false, |x: &String| !x.is_empty())
            }
            _ => false,
        }
    };

    let is_associated_street_relation = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Relation(ref rel) => {
                rel.tags.get("type").map_or(false, |v| v == "associatedStreet")
            }
            _ => false,
        }
    };

    let mut street_map = osmpbfreader::get_objs_and_deps(pbf, is_valid_street).unwrap();
    let relation_map = osmpbfreader::get_objs_and_deps(pbf, is_associated_street_relation).unwrap();

    // Sometimes, streets can be devided into several "way"s that still have the same street name.
    // The reason why a street is devided may be that a part of the street become a bridge/tunne/etc.
    // In this case, a "relation" tagged with (type = associatedStreet) is used to group all these "way"s.
    // In order not to have doublons in autocompleion, we should keep only one "way" in the relation
    // and remove all the rest way whose street name is the same.
    for (_, rel_obj) in &relation_map {
        if let &osmpbfreader::OsmObj::Relation(ref rel) = rel_obj {
            let mut found = false;
            for ref_obj in &rel.refs {
                if let &osmpbfreader::OsmId::Way(_) = &ref_obj.member {
                    if !found {
                        found = true;
                        continue;
                    };
                    street_map.remove(&ref_obj.member);
                };
            }
        };
    }

    street_map.iter()
              .filter_map(|(osm_id, obj)| {
                  if let &osmpbfreader::OsmObj::Way(ref way) = obj {
                      if let &osmpbfreader::OsmId::Way(ref way_id) = osm_id {
                          way.tags.get("name").and_then(|way_name| {
                              Some(mimirsbrunn::Street {
                                  id: way_id.to_string(),
                                  street_name: way_name.to_string(),
                                  name: way_name.to_string(),
                                  weight: 1,
                                  administrative_region: None,
                              })
                          })
                      } else {
                          None
                      }
                  } else {
                      None
                  }
              })
              .collect()

}
// TODO: template these two functions
fn index_admin(es_cnx_string: &str, admins: AdminsVec) {
    let mut rubber = Rubber::new(es_cnx_string);
    match rubber.bulk_index(admins.iter()) {
        Err(e) => panic!("failed to index admins of osm because: {}", e),
        Ok(nb) => info!("Nb of indexed adminstrative regions: {}", nb),
    }
}

fn index_ways(es_cnx_string: &str, streets: StreetsVec) {
    let mut rubber = Rubber::new(es_cnx_string);
    match rubber.bulk_index(streets.iter()) {
        Err(e) => panic!("failed to index streets of osm because: {}", e),
        Ok(nb) => info!("Nb of indexed street: {}", nb),
    }
}

fn index_settings(es_cnx_string: &str) {
    let mut rubber = Rubber::new(es_cnx_string);
    rubber.create_index();
}

fn main() {
    mimir::logger_init().unwrap();
    debug!("importing adminstrative region into Mimir");
    let args: Args = docopt::Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());

    let levels = args.flag_level.iter().cloned().collect();
    let mut parsed_pbf = parse_osm_pbf(&args.flag_input);

    let admins = administrative_regions(&mut parsed_pbf, levels);

    index_settings(&args.flag_connection_string);
    index_admin(&args.flag_connection_string, admins);

    let mut optinoal_streets: Option<StreetsVec> = None;
    if args.flag_import_way  {
        optinoal_streets = Some(streets(&mut parsed_pbf));
    }

    if let Some(streets) = optinoal_streets {
	index_ways(&args.flag_connection_string, streets);
    }
}
<|endoftext|>


<commit_before>//! Processor state stored in the RFLAGS register.
//!
//! In 64-bit mode, EFLAGS is extended to 64 bits and called RFLAGS.
//! The upper 32 bits of RFLAGS register is reserved.
//! The lower 32 bits of RFLAGS is the same as EFLAGS.

use Ring;

/// The RFLAGS register.
/// This is duplicated code from bits32 eflags.rs.
bitflags! {
    pub struct RFlags: u64 {
        /// ID Flag (ID)
        const FLAGS_ID = 1 << 21;
        /// Virtual Interrupt Pending (VIP)
        const FLAGS_VIP = 1 << 20;
        /// Virtual Interrupt Flag (VIF)
        const FLAGS_VIF = 1 << 19;
        /// Alignment Check (AC)
        const FLAGS_AC = 1 << 18;
        /// Virtual-8086 Mode (VM)
        const FLAGS_VM = 1 << 17;
        /// Resume Flag (RF)
        const FLAGS_RF = 1 << 16;
        /// Nested Task (NT)
        const FLAGS_NT = 1 << 14;
        /// I/O Privilege Level (IOPL) 0
        const FLAGS_IOPL0 = 0b00 << 12;
        /// I/O Privilege Level (IOPL) 1
        const FLAGS_IOPL1 = 0b01 << 12;
        /// I/O Privilege Level (IOPL) 2
        const FLAGS_IOPL2 = 0b10 << 12;
        /// I/O Privilege Level (IOPL) 3
        const FLAGS_IOPL3 = 0b11 << 12;
        /// Overflow Flag (OF)
        const FLAGS_OF = 1 << 11;
        /// Direction Flag (DF)
        const FLAGS_DF = 1 << 10;
        /// Interrupt Enable Flag (IF)
        const FLAGS_IF = 1 << 9;
        /// Trap Flag (TF)
        const FLAGS_TF = 1 << 8;
        /// Sign Flag (SF)
        const FLAGS_SF = 1 << 7;
        /// Zero Flag (ZF)
        const FLAGS_ZF = 1 << 6;
        /// Auxiliary Carry Flag (AF)
        const FLAGS_AF = 1 << 4;
        /// Parity Flag (PF)
        const FLAGS_PF = 1 << 2;
        /// Bit 1 is always 1.
        const FLAGS_A1 = 1 << 1;
        /// Carry Flag (CF)
        const FLAGS_CF = 1 << 0;
    }
}

impl RFlags {
    /// Creates a new Flags entry. Ensures bit 1 is set.
    pub const fn new() -> RFlags {
        RFlags::FLAGS_A1
    }

    /// Creates a new Flags with the given I/O privilege level.
    pub const fn from_priv(iopl: Ring) -> RFlags {
        RFlags {
            bits: (iopl as u64) << 12,
        }
    }

    pub const fn from_raw(bits: u64) -> RFlags {
        RFlags { bits: bits }
    }
}

#[cfg(target_arch = "x86-64")]
pub unsafe fn read() -> RFlags {
    let r: u64;
    asm!("pushfq; popq $0" : "=r"(r) :: "memory");
    RFlags::from_bits_truncate(r)
}

#[cfg(target_arch = "x86-64")]
pub unsafe fn set(val: RFlags) {
    asm!("pushq $0; popfq" :: "r"(val.bits()) : "memory" "flags");
}
<commit_msg>use x86_64 instead x86-64 as architecture target<commit_after>//! Processor state stored in the RFLAGS register.
//!
//! In 64-bit mode, EFLAGS is extended to 64 bits and called RFLAGS.
//! The upper 32 bits of RFLAGS register is reserved.
//! The lower 32 bits of RFLAGS is the same as EFLAGS.

use Ring;

/// The RFLAGS register.
/// This is duplicated code from bits32 eflags.rs.
bitflags! {
    pub struct RFlags: u64 {
        /// ID Flag (ID)
        const FLAGS_ID = 1 << 21;
        /// Virtual Interrupt Pending (VIP)
        const FLAGS_VIP = 1 << 20;
        /// Virtual Interrupt Flag (VIF)
        const FLAGS_VIF = 1 << 19;
        /// Alignment Check (AC)
        const FLAGS_AC = 1 << 18;
        /// Virtual-8086 Mode (VM)
        const FLAGS_VM = 1 << 17;
        /// Resume Flag (RF)
        const FLAGS_RF = 1 << 16;
        /// Nested Task (NT)
        const FLAGS_NT = 1 << 14;
        /// I/O Privilege Level (IOPL) 0
        const FLAGS_IOPL0 = 0b00 << 12;
        /// I/O Privilege Level (IOPL) 1
        const FLAGS_IOPL1 = 0b01 << 12;
        /// I/O Privilege Level (IOPL) 2
        const FLAGS_IOPL2 = 0b10 << 12;
        /// I/O Privilege Level (IOPL) 3
        const FLAGS_IOPL3 = 0b11 << 12;
        /// Overflow Flag (OF)
        const FLAGS_OF = 1 << 11;
        /// Direction Flag (DF)
        const FLAGS_DF = 1 << 10;
        /// Interrupt Enable Flag (IF)
        const FLAGS_IF = 1 << 9;
        /// Trap Flag (TF)
        const FLAGS_TF = 1 << 8;
        /// Sign Flag (SF)
        const FLAGS_SF = 1 << 7;
        /// Zero Flag (ZF)
        const FLAGS_ZF = 1 << 6;
        /// Auxiliary Carry Flag (AF)
        const FLAGS_AF = 1 << 4;
        /// Parity Flag (PF)
        const FLAGS_PF = 1 << 2;
        /// Bit 1 is always 1.
        const FLAGS_A1 = 1 << 1;
        /// Carry Flag (CF)
        const FLAGS_CF = 1 << 0;
    }
}

impl RFlags {
    /// Creates a new Flags entry. Ensures bit 1 is set.
    pub const fn new() -> RFlags {
        RFlags::FLAGS_A1
    }

    /// Creates a new Flags with the given I/O privilege level.
    pub const fn from_priv(iopl: Ring) -> RFlags {
        RFlags {
            bits: (iopl as u64) << 12,
        }
    }

    pub const fn from_raw(bits: u64) -> RFlags {
        RFlags { bits: bits }
    }
}

#[cfg(target_arch = "x86_64")]
pub unsafe fn read() -> RFlags {
    let r: u64;
    asm!("pushfq; popq $0" : "=r"(r) :: "memory");
    RFlags::from_bits_truncate(r)
}

#[cfg(target_arch = "x86_64")]
pub unsafe fn set(val: RFlags) {
    asm!("pushq $0; popfq" :: "r"(val.bits()) : "memory" "flags");
}
<|endoftext|>


<commit_before>use std::old_io::{File, Open, ReadWrite};
use std::old_io::IoResult;
use super::GapBuffer;
use super::gap_buffer;
use super::Position;
use super::Range;

pub struct Buffer {
    data: GapBuffer,
    file: Option<File>,
}

impl Buffer {
    /// Returns the contents of the buffer as a string.
    ///
    /// # Examples
    ///
    /// ```
    /// let buffer = scribe::buffer::from_file(&Path::new("tests/sample/file")).unwrap();
    /// assert_eq!(buffer.data(), "it works!\n");
    /// ```
    pub fn data(&self) -> String {
        self.data.to_string()
    }
}

/// Creates a new empty buffer.
///
/// # Examples
///
/// ```
/// let buffer = scribe::buffer::new();
/// ```
pub fn new() -> Buffer {
    Buffer{ data: gap_buffer::new(String::new()), file: None }
}

/// Creates a new buffer by reading the UTF-8 interpreted file contents of the specified path.
///
/// # Examples
///
/// ```
/// let buffer = scribe::buffer::from_file(&Path::new("tests/sample/file")).unwrap();
/// assert_eq!(buffer.data(), "it works!\n");
/// ```
pub fn from_file(path: &Path) -> IoResult<Buffer> {
    // Try to open and read the file, returning any errors encountered.
    let mut file = match File::open_mode(path, Open, ReadWrite) {
        Ok(f) => f,
        Err(error) => return Err(error),
    };
    let mut data = match file.read_to_string() {
        Ok(d) => d,
        Err(error) => return Err(error),
    };

    let data = gap_buffer::new(data);

    // Create a new buffer using the loaded data, file, and other defaults.
    Ok(Buffer{ data: data, file: Some(file) })
}
<commit_msg>Add a cursor field to the buffer struct. Initialize it in constructors.<commit_after>use std::old_io::{File, Open, ReadWrite};
use std::old_io::IoResult;
use super::GapBuffer;
use super::gap_buffer;
use super::Position;
use super::Range;

pub struct Buffer {
    data: GapBuffer,
    file: Option<File>,
    pub cursor: Position,
}

impl Buffer {
    /// Returns the contents of the buffer as a string.
    ///
    /// # Examples
    ///
    /// ```
    /// let buffer = scribe::buffer::from_file(&Path::new("tests/sample/file")).unwrap();
    /// assert_eq!(buffer.data(), "it works!\n");
    /// ```
    pub fn data(&self) -> String {
        self.data.to_string()
    }
}

/// Creates a new empty buffer. The buffer's cursor is set to the beginning of the buffer.
///
/// # Examples
///
/// ```
/// let buffer = scribe::buffer::new();
/// # assert_eq!(buffer.cursor.line, 0);
/// # assert_eq!(buffer.cursor.offset, 0);
/// ```
pub fn new() -> Buffer {
    let data = gap_buffer::new(String::new());
    let cursor = Position{ line: 0, offset: 0 };

    Buffer{ data: data, file: None, cursor: cursor }
}

/// Creates a new buffer by reading the UTF-8 interpreted file contents of the specified path.
/// The buffer's cursor is set to the beginning of the buffer.
///
/// # Examples
///
/// ```
/// let buffer = scribe::buffer::from_file(&Path::new("tests/sample/file")).unwrap();
/// assert_eq!(buffer.data(), "it works!\n");
/// # assert_eq!(buffer.cursor.line, 0);
/// # assert_eq!(buffer.cursor.offset, 0);
/// ```
pub fn from_file(path: &Path) -> IoResult<Buffer> {
    // Try to open and read the file, returning any errors encountered.
    let mut file = match File::open_mode(path, Open, ReadWrite) {
        Ok(f) => f,
        Err(error) => return Err(error),
    };
    let mut data = match file.read_to_string() {
        Ok(d) => d,
        Err(error) => return Err(error),
    };

    let data = gap_buffer::new(data);
    let cursor = Position{ line: 0, offset: 0 };

    // Create a new buffer using the loaded data, file, and other defaults.
    Ok(Buffer{ data: data, file: Some(file), cursor: cursor })
}
<|endoftext|>


<commit_before>use std::io::Error;
use std::io::Write;
use std::path::Path;
use std::fs::File;
use std::fmt::{Arguments};
use std::time::{Instant, Duration};

const NANO_FACTOR: f64 = 0.000000001;

fn duration_as_f64(d: Duration) -> f64 {
    (d.as_secs() as f64) + (NANO_FACTOR * d.subsec_nanos() as f64)
}

pub struct TimeLog {
    file: File,
    start: Instant,
    prev: Instant,
}

impl TimeLog {
    pub fn start(path: &Path) -> Result<TimeLog, Error> {
        let now = Instant::now();
        let mut res = TimeLog {
            file: try!(File::create(path)),
            start: now,
            prev: now,
        };
        try!(res.mark(format_args!("Start {:?}", now)));
        Ok(res)
    }
    pub fn mark(&mut self, args: Arguments) -> Result<(), Error> {
        let now = Instant::now();
        let d_start = duration_as_f64(now.duration_since(self.start));
        let d_prev = duration_as_f64(now.duration_since(self.prev));
        write!(&mut self.file, "{:7.3} {:7.3}   ", d_start, d_prev)
        .and_then(|()| self.file.write_fmt(args))
        .and_then(|()| writeln!(&mut self.file, ""))
        .map(|()| self.prev = now)
    }
}
<commit_msg>Use SystemTime for to print the start time<commit_after>use std::io::Error;
use std::io::Write;
use std::path::Path;
use std::fs::File;
use std::fmt::{Arguments};
use std::time::{Instant, Duration, SystemTime, UNIX_EPOCH};

const NANO_FACTOR: f64 = 0.000000001;

fn duration_as_f64(d: Duration) -> f64 {
    (d.as_secs() as f64) + (NANO_FACTOR * d.subsec_nanos() as f64)
}

pub struct TimeLog {
    file: File,
    start: Instant,
    prev: Instant,
}

impl TimeLog {
    pub fn start(path: &Path) -> Result<TimeLog, Error> {
        let now = SystemTime::now();
        let now_instant = Instant::now();
        let mut res = TimeLog {
            file: try!(File::create(path)),
            start: now_instant,
            prev: now_instant,
        };
        let duration = now.duration_since(UNIX_EPOCH)
                          .expect("FATAL: now was created after the epoch");
        try!(res.mark(format_args!("Start {:?}", duration_as_f64(duration))));
        Ok(res)
    }
    pub fn mark(&mut self, args: Arguments) -> Result<(), Error> {
        let now = Instant::now();
        let d_start = duration_as_f64(now.duration_since(self.start));
        let d_prev = duration_as_f64(now.duration_since(self.prev));
        write!(&mut self.file, "{:7.3} {:7.3}   ", d_start, d_prev)
        .and_then(|()| self.file.write_fmt(args))
        .and_then(|()| writeln!(&mut self.file, ""))
        .map(|()| self.prev = now)
    }
}
<|endoftext|>