<commit_before>use action::Action;
use config;
use config::action::message::InjectMode;
use context::base::BaseContext;
use dispatcher::Response;
use dispatcher::response::ResponseSender;
use message::{Message, MessageBuilder};

use handlebars::Context;
use std::borrow::Borrow;
use std::collections::BTreeMap;
use state::State;
use self::error::Error;
use self::renderer_context::RendererContext;

mod error;
mod renderer_context;
#[cfg(test)]
mod test;

pub const CONTEXT_UUID: &'static str = "context_uuid";
pub const CONTEXT_NAME: &'static str = "context_name";
pub const CONTEXT_LEN: &'static str = "context_len";
pub const MESSAGES: &'static str = "messages";
const MESSAGE: &'static str = "MESSAGE";

pub struct MessageAction {
    sender: Box<ResponseSender>,
    action: config::action::MessageAction,
}

impl MessageAction {
    pub fn new(sender: Box<ResponseSender>,
               mut action: config::action::MessageAction)
               -> MessageAction {
        let message = action.message.clone();
        action.values.register_template(MESSAGE, message);

        MessageAction {
            sender: sender,
            action: action,
        }
    }

    fn render_value(&self, key: &String, template_context: &Context) -> Result<String, Error> {
        let mut writer = Vec::new();
        {
            try!(self.action.values.renderw(key, &template_context, &mut writer));
        }
        let string = try!(String::from_utf8(writer));
        Ok(string)
    }

    fn render_values(&self, template_context: &Context) -> Result<BTreeMap<String, String>, Error> {
        let mut rendered_values = BTreeMap::new();
        for (key, _) in self.action.values.get_templates() {
            let rendered_value = try!(self.render_value(key, &template_context));
            rendered_values.insert(key.to_string(), rendered_value);
        }
        Ok(rendered_values)
    }

    fn render_message(&self, state: &State, context: &BaseContext) -> Result<Message, Error> {
        let template_context = {
            use handlebars::Context;
            let context = RendererContext::new(state, context);
            Context::wraps(&context)
        };

        let mut rendered_values = try!(self.render_values(&template_context));
        MessageAction::extend_with_context_information(&mut rendered_values, state, context);
        let message = rendered_values.remove(MESSAGE)
                                     .expect(&format!("There is no '{}' key in the renderer \
                                                       key-value pairs",
                                                      MESSAGE));
        let name = self.action.name.as_ref().map(|name| name.borrow());
        let message = MessageBuilder::new(&self.action.uuid, message)
                          .name(name)
                          .values(rendered_values)
                          .build();
        Ok(message)
    }

    fn extend_with_context_information(values: &mut BTreeMap<String, String>,
                                       state: &State,
                                       context: &BaseContext) {
        values.insert(CONTEXT_UUID.to_string(),
                      context.uuid().to_hyphenated_string());
        values.insert(CONTEXT_LEN.to_string(), state.messages().len().to_string());
        if let Some(name) = context.name() {
            values.insert(CONTEXT_NAME.to_string(), name.to_string());
        }
    }

    fn execute(&self, _state: &State, _context: &BaseContext, responder: &mut ResponseSender) {
        match self.render_message(_state, _context) {
            Ok(message) => {
                let response = Alert {
                    message: message,
                    inject_mode: self.action.inject_mode.clone(),
                };
                self.sender.send_response(Response::Alert(response));
            }
            Err(error) => {
                error!("{}", error);
            }
        }
    }
}

#[derive(Debug, Clone)]
pub struct Alert {
    message: Message,
    inject_mode: InjectMode,
}

impl Alert {
    pub fn message(&self) -> &Message {
        &self.message
    }
}

impl Action for MessageAction {
    fn on_opened(&self, _state: &State, _context: &BaseContext, responder: &mut ResponseSender) {
        if self.action.when.on_opened {
            trace!("MessageAction: on_opened()");
            self.execute(_state, _context, responder);
        }
    }

    fn on_closed(&self, _state: &State, _context: &BaseContext, responder: &mut ResponseSender) {
        if self.action.when.on_closed {
            trace!("MessageAction: on_closed()");
            self.execute(_state, _context, responder);
        }
    }
}
<commit_msg>MessageAction: use responder instead of owned response sender<commit_after>use action::Action;
use config;
use config::action::message::InjectMode;
use context::base::BaseContext;
use dispatcher::Response;
use dispatcher::response::ResponseSender;
use message::{Message, MessageBuilder};

use handlebars::Context;
use std::borrow::Borrow;
use std::collections::BTreeMap;
use state::State;
use self::error::Error;
use self::renderer_context::RendererContext;

mod error;
mod renderer_context;
#[cfg(test)]
mod test;

pub const CONTEXT_UUID: &'static str = "context_uuid";
pub const CONTEXT_NAME: &'static str = "context_name";
pub const CONTEXT_LEN: &'static str = "context_len";
pub const MESSAGES: &'static str = "messages";
const MESSAGE: &'static str = "MESSAGE";

pub struct MessageAction {
    sender: Box<ResponseSender>,
    action: config::action::MessageAction,
}

impl MessageAction {
    pub fn new(sender: Box<ResponseSender>,
               mut action: config::action::MessageAction)
               -> MessageAction {
        let message = action.message.clone();
        action.values.register_template(MESSAGE, message);

        MessageAction {
            sender: sender,
            action: action,
        }
    }

    fn render_value(&self, key: &String, template_context: &Context) -> Result<String, Error> {
        let mut writer = Vec::new();
        {
            try!(self.action.values.renderw(key, &template_context, &mut writer));
        }
        let string = try!(String::from_utf8(writer));
        Ok(string)
    }

    fn render_values(&self, template_context: &Context) -> Result<BTreeMap<String, String>, Error> {
        let mut rendered_values = BTreeMap::new();
        for (key, _) in self.action.values.get_templates() {
            let rendered_value = try!(self.render_value(key, &template_context));
            rendered_values.insert(key.to_string(), rendered_value);
        }
        Ok(rendered_values)
    }

    fn render_message(&self, state: &State, context: &BaseContext) -> Result<Message, Error> {
        let template_context = {
            use handlebars::Context;
            let context = RendererContext::new(state, context);
            Context::wraps(&context)
        };

        let mut rendered_values = try!(self.render_values(&template_context));
        MessageAction::extend_with_context_information(&mut rendered_values, state, context);
        let message = rendered_values.remove(MESSAGE)
                                     .expect(&format!("There is no '{}' key in the renderer \
                                                       key-value pairs",
                                                      MESSAGE));
        let name = self.action.name.as_ref().map(|name| name.borrow());
        let message = MessageBuilder::new(&self.action.uuid, message)
                          .name(name)
                          .values(rendered_values)
                          .build();
        Ok(message)
    }

    fn extend_with_context_information(values: &mut BTreeMap<String, String>,
                                       state: &State,
                                       context: &BaseContext) {
        values.insert(CONTEXT_UUID.to_string(),
                      context.uuid().to_hyphenated_string());
        values.insert(CONTEXT_LEN.to_string(), state.messages().len().to_string());
        if let Some(name) = context.name() {
            values.insert(CONTEXT_NAME.to_string(), name.to_string());
        }
    }

    fn execute(&self, _state: &State, _context: &BaseContext, responder: &mut ResponseSender) {
        match self.render_message(_state, _context) {
            Ok(message) => {
                let response = Alert {
                    message: message,
                    inject_mode: self.action.inject_mode.clone(),
                };
                responder.send_response(Response::Alert(response));
            }
            Err(error) => {
                error!("{}", error);
            }
        }
    }
}

#[derive(Debug, Clone)]
pub struct Alert {
    message: Message,
    inject_mode: InjectMode,
}

impl Alert {
    pub fn message(&self) -> &Message {
        &self.message
    }
}

impl Action for MessageAction {
    fn on_opened(&self, _state: &State, _context: &BaseContext, responder: &mut ResponseSender) {
        if self.action.when.on_opened {
            trace!("MessageAction: on_opened()");
            self.execute(_state, _context, responder);
        }
    }

    fn on_closed(&self, _state: &State, _context: &BaseContext, responder: &mut ResponseSender) {
        if self.action.when.on_closed {
            trace!("MessageAction: on_closed()");
            self.execute(_state, _context, responder);
        }
    }
}
<|endoftext|>


<commit_before>use ffi;
use libc;

use AsLua;
use AsMutLua;
use LuaContext;
use LuaRead;
use Push;
use PushGuard;

use std::marker::PhantomData;
use std::fmt::Debug;
use std::mem;
use std::ptr;

///
pub fn function<F, P>(f: F) -> Function<F, P, <F as FnMut<P>>::Output> where F: FnMut<P> {
    Function {
        function: f,
        marker: PhantomData,
    }
}

pub struct Function<F, P, R> {
    function: F,
    marker: PhantomData<(P, R)>,
}

// this function is the main entry point when Lua wants to call one of our functions
extern fn wrapper1(lua: *mut ffi::lua_State) -> ::libc::c_int {
    // we load the pointer to the wrapper2 function from an upvalue (an upvalue is a value that was pushed alongside our function)
    let wrapper2raw = unsafe { ffi::lua_touserdata(lua, ffi::lua_upvalueindex(2)) };
    let wrapper2: fn(*mut ffi::lua_State)->::libc::c_int = unsafe { ::std::mem::transmute(wrapper2raw) };

    wrapper2(lua)
}

// this function is called when Lua wants to call one of our functions
fn wrapper2<T, P, R>(lua: *mut ffi::lua_State) -> libc::c_int
                     where T: FnMut(P) -> R,
                           P: for<'p> LuaRead<&'p mut InsideCallback>,
                           R: for<'p> Push<&'p mut InsideCallback>
{
    // loading the object that we want to call from the Lua context
    let data_raw = unsafe { ffi::lua_touserdata(lua, ffi::lua_upvalueindex(1)) };
    let data: &mut T = unsafe { mem::transmute(data_raw) };

    // creating a temporary Lua context in order to pass it to push & read functions
    let mut tmp_lua = InsideCallback { lua: LuaContext(lua) };

    // trying to read the arguments
    let arguments_count = unsafe { ffi::lua_gettop(lua) } as i32;
    let args = match LuaRead::lua_read_at_position(&mut tmp_lua, -arguments_count as libc::c_int) {      // TODO: what if the user has the wrong params?
        None => {
            let err_msg = format!("wrong parameter types for callback function");
            err_msg.push_to_lua(&mut tmp_lua);
            unsafe { ffi::lua_error(lua); }
            unreachable!()
        },
        Some(a) => a
    };

    let ret_value = data(args);

    // pushing back the result of the function on the stack
    let nb = {
        let guard = ret_value.push_to_lua(&mut tmp_lua);
        let nb = guard.size;
        unsafe { mem::forget(guard) };
        nb
    };

    nb as libc::c_int
}

// this trait should be implemented on objects that are pushed to be callbacks
trait AnyCallable {
    fn load_args_and_call(&mut self, lua: *mut ffi::lua_State) -> ::libc::c_int;
}

// lua context used inside callbacks
#[doc(hidden)]
pub struct InsideCallback {
    lua: LuaContext,
}

unsafe impl<'a> AsLua for &'a InsideCallback {
    fn as_lua(&self) -> LuaContext {
        self.lua
    }
}

unsafe impl<'a> AsLua for &'a mut InsideCallback {
    fn as_lua(&self) -> LuaContext {
        self.lua
    }
}

unsafe impl<'a> AsMutLua for &'a mut InsideCallback {
    fn as_mut_lua(&mut self) -> LuaContext {
        self.lua
    }
}

impl<L, F, P, R> Push<L> for Function<F, P, R>
        where L: AsMutLua, P: 'static,
              P: for<'p> LuaRead<&'p mut InsideCallback>,
              R: for<'a> Push<&'a mut InsideCallback> + 'static
{
    fn push_to_lua(self, mut lua: L) -> PushGuard<L> {
        // pushing the function pointer as a userdata
        let lua_data_raw = unsafe {
            ffi::lua_newuserdata(lua.as_mut_lua().0, mem::size_of::<F>() as libc::size_t)
        };

        let lua_data: *mut F = unsafe { mem::transmute(lua_data_raw) };
        unsafe { ptr::write(lua_data, self.function) };

        // pushing wrapper2 as a lightuserdata
        let wrapper2: fn(*mut ffi::lua_State) -> libc::c_int = wrapper2::<fn(P) -> R, _, _>;
        unsafe { ffi::lua_pushlightuserdata(lua.as_mut_lua().0, mem::transmute(wrapper2)) };

        // pushing wrapper1 as a closure
        unsafe { ffi::lua_pushcclosure(lua.as_mut_lua().0, mem::transmute(wrapper1), 2) };

        PushGuard { lua: lua, size: 1 }
    }
}

impl<'a, T, E> Push<&'a mut InsideCallback> for Result<T, E>
                where T: Push<&'a mut InsideCallback> + for<'b> Push<&'b mut &'a mut InsideCallback>, E: Debug
{
    fn push_to_lua(self, mut lua: &'a mut InsideCallback) -> PushGuard<&'a mut InsideCallback> {
        match self {
            Ok(val) => val.push_to_lua(lua),
            Err(val) => {
                let msg = format!("{:?}", val);
                msg.push_to_lua(&mut lua);
                unsafe { ffi::lua_error(lua.as_mut_lua().0); }
                unreachable!()
            }
        }
    }
}
<commit_msg>Fix function calls crashing<commit_after>use ffi;
use libc;

use AsLua;
use AsMutLua;
use LuaContext;
use LuaRead;
use Push;
use PushGuard;

use std::marker::PhantomData;
use std::fmt::Debug;
use std::mem;
use std::ptr;

///
pub fn function<F, P>(f: F) -> Function<F, P, <F as FnMut<P>>::Output> where F: FnMut<P> {
    Function {
        function: f,
        marker: PhantomData,
    }
}

pub struct Function<F, P, R> {
    function: F,
    marker: PhantomData<(P, R)>,
}

// this function is the main entry point when Lua wants to call one of our functions
extern fn wrapper1(lua: *mut ffi::lua_State) -> ::libc::c_int {
    // we load the pointer to the wrapper2 function from an upvalue (an upvalue is a value that was pushed alongside our function)
    let wrapper2raw = unsafe { ffi::lua_touserdata(lua, ffi::lua_upvalueindex(2)) };
    let wrapper2: fn(*mut ffi::lua_State)->::libc::c_int = unsafe { ::std::mem::transmute(wrapper2raw) };

    wrapper2(lua)
}

// this function is called when Lua wants to call one of our functions
fn wrapper2<T, P, R>(lua: *mut ffi::lua_State) -> libc::c_int
                     where T: FnMut<P, Output=R>,
                           P: for<'p> LuaRead<&'p mut InsideCallback>,
                           R: for<'p> Push<&'p mut InsideCallback>
{
    // loading the object that we want to call from the Lua context
    let data_raw = unsafe { ffi::lua_touserdata(lua, ffi::lua_upvalueindex(1)) };
    let data: &mut T = unsafe { mem::transmute(data_raw) };

    // creating a temporary Lua context in order to pass it to push & read functions
    let mut tmp_lua = InsideCallback { lua: LuaContext(lua) };

    // trying to read the arguments
    let arguments_count = unsafe { ffi::lua_gettop(lua) } as i32;
    let args = match LuaRead::lua_read_at_position(&mut tmp_lua, -arguments_count as libc::c_int) {      // TODO: what if the user has the wrong params?
        None => {
            let err_msg = format!("wrong parameter types for callback function");
            err_msg.push_to_lua(&mut tmp_lua);
            unsafe { ffi::lua_error(lua); }
            unreachable!()
        },
        Some(a) => a
    };

    let ret_value = data.call_mut(args);

    // pushing back the result of the function on the stack
    let nb = {
        let guard = ret_value.push_to_lua(&mut tmp_lua);
        let nb = guard.size;
        unsafe { mem::forget(guard) };
        nb
    };

    nb as libc::c_int
}

// this trait should be implemented on objects that are pushed to be callbacks
trait AnyCallable {
    fn load_args_and_call(&mut self, lua: *mut ffi::lua_State) -> ::libc::c_int;
}

// lua context used inside callbacks
#[doc(hidden)]
pub struct InsideCallback {
    lua: LuaContext,
}

unsafe impl<'a> AsLua for &'a InsideCallback {
    fn as_lua(&self) -> LuaContext {
        self.lua
    }
}

unsafe impl<'a> AsLua for &'a mut InsideCallback {
    fn as_lua(&self) -> LuaContext {
        self.lua
    }
}

unsafe impl<'a> AsMutLua for &'a mut InsideCallback {
    fn as_mut_lua(&mut self) -> LuaContext {
        self.lua
    }
}

impl<L, F, P, R> Push<L> for Function<F, P, R>
        where L: AsMutLua, P: 'static,
              F: FnMut<P, Output=R>,
              P: for<'p> LuaRead<&'p mut InsideCallback>,
              R: for<'a> Push<&'a mut InsideCallback> + 'static
{
    fn push_to_lua(self, mut lua: L) -> PushGuard<L> {
        // pushing the function pointer as a userdata
        let lua_data_raw = unsafe {
            ffi::lua_newuserdata(lua.as_mut_lua().0, mem::size_of::<F>() as libc::size_t)
        };

        let lua_data: *mut F = unsafe { mem::transmute(lua_data_raw) };
        unsafe { ptr::write(lua_data, self.function) };

        // pushing wrapper2 as a lightuserdata
        let wrapper2: fn(*mut ffi::lua_State) -> libc::c_int = wrapper2::<F, P, R>;
        unsafe { ffi::lua_pushlightuserdata(lua.as_mut_lua().0, mem::transmute(wrapper2)) };

        // pushing wrapper1 as a closure
        unsafe { ffi::lua_pushcclosure(lua.as_mut_lua().0, mem::transmute(wrapper1), 2) };

        PushGuard { lua: lua, size: 1 }
    }
}

impl<'a, T, E> Push<&'a mut InsideCallback> for Result<T, E>
                where T: Push<&'a mut InsideCallback> + for<'b> Push<&'b mut &'a mut InsideCallback>, E: Debug
{
    fn push_to_lua(self, mut lua: &'a mut InsideCallback) -> PushGuard<&'a mut InsideCallback> {
        match self {
            Ok(val) => val.push_to_lua(lua),
            Err(val) => {
                let msg = format!("{:?}", val);
                msg.push_to_lua(&mut lua);
                unsafe { ffi::lua_error(lua.as_mut_lua().0); }
                unreachable!()
            }
        }
    }
}
<|endoftext|>


<commit_before>use std::process;
use std::process::Command;

fn main () {
    #[cfg(target_os = "macos")]
    fn inner() -> &'static str {
        "open"
    }
    #[cfg(target_os = "windows")]
    fn inner() -> &'static str {
        "start"
    }
    #[cfg(target_os = "unix")]
    fn inner() -> &'static str {
        "xdg-open"
    }

    let command = inner();
    let arg = "http://groonga.org/docs/";
    let err = match Command::new(command)
        .args(&[arg])
        // .stdout(Stdio::inherit())
        // .stderr(Stdio::inherit())
        .spawn() {
        Ok(_) => return (),
        Err(e) => e,
        };
    println!("Failed to execute docopen subcommand. reason: {:?}", err);
    process::exit(1);

}
<commit_msg>rustfmt<commit_after>use std::process;
use std::process::Command;

fn main() {
    #[cfg(target_os = "macos")]
    fn inner() -> &'static str {
        "open"
    }
    #[cfg(target_os = "windows")]
    fn inner() -> &'static str {
        "start"
    }
    #[cfg(target_os = "unix")]
    fn inner() -> &'static str {
        "xdg-open"
    }

    let command = inner();
    let arg = "http://groonga.org/docs/";
    let err = match Command::new(command)
        .args(&[arg])
        // .stdout(Stdio::inherit())
        // .stderr(Stdio::inherit())
        .spawn() {
        Ok(_) => return (),
        Err(e) => e,
    };
    println!("Failed to execute docopen subcommand. reason: {:?}", err);
    process::exit(1);

}
<|endoftext|>


<commit_before>extern crate rand;
extern crate evo;
extern crate petgraph;
extern crate graph_annealing;
extern crate clap;
extern crate crossbeam;
extern crate simple_parallel;
extern crate num_cpus;
extern crate pcg;
extern crate graph_sgf;
extern crate triadic_census;
extern crate time;

use std::f32;
use std::fmt::Debug;
use std::str::FromStr;
use clap::{App, Arg};
use rand::{Rng, SeedableRng};
use rand::distributions::{IndependentSample, Range};
use rand::os::OsRng;
use pcg::PcgRng;

use evo::Probability;
use evo::nsga2::{self, FitnessEval, MultiObjective3};
use graph_annealing::repr::edge_ops_genome::{EdgeOpsGenome, Toolbox};
use graph_annealing::helper::draw_graph;
use graph_annealing::goal::Goal;
use simple_parallel::Pool;

use petgraph::{Directed, Graph};
use graph_sgf::{PetgraphReader, Unweighted};
use triadic_census::OptDenseDigraph;

use std::io::BufReader;
use std::fs::File;

#[derive(Debug, Copy, Clone)]
enum FitnessFunction {
    Null,
    ConnectedComponents,
    StronglyConnectedComponents,
    NeighborMatching,
    TriadicDistance,
}

fn apply_fitness_function<N: Clone + Default, E: Clone + Default>(fitfun: FitnessFunction,
                                                                  goal: &Goal<N, E>,
                                                                  g: &OptDenseDigraph<(), ()>)
                                                                  -> f32 {
    match fitfun {
        FitnessFunction::Null => {
            0.0
        }
        FitnessFunction::ConnectedComponents => {
            goal.connected_components_distance(g) as f32
        }
        FitnessFunction::StronglyConnectedComponents => {
            goal.strongly_connected_components_distance(g) as f32
        }
        FitnessFunction::NeighborMatching => {
            goal.neighbor_matching_score(g) as f32
        }
        FitnessFunction::TriadicDistance => {
            goal.triadic_distance(g) as f32
        }
    }
}

struct MyEval<N, E> {
    goal: Goal<N, E>,
    pool: Pool,
    fitness_functions: (FitnessFunction, FitnessFunction, FitnessFunction),
}

#[inline]
fn fitness<N: Clone + Default, E: Clone + Default>(fitness_functions: (FitnessFunction,
                                                                       FitnessFunction,
                                                                       FitnessFunction),
                                                   goal: &Goal<N, E>,
                                                   ind: &EdgeOpsGenome)
                                                   -> MultiObjective3<f32> {
    let g = ind.to_graph();
    MultiObjective3::from((apply_fitness_function(fitness_functions.0, goal, &g),
                           apply_fitness_function(fitness_functions.1, goal, &g),
                           apply_fitness_function(fitness_functions.2, goal, &g)))

}

impl<N:Clone+Sync+Default,E:Clone+Sync+Default> FitnessEval<EdgeOpsGenome, MultiObjective3<f32>> for MyEval<N,E> {
    fn fitness(&mut self, pop: &[EdgeOpsGenome]) -> Vec<MultiObjective3<f32>> {
        let pool = &mut self.pool;
        let goal = &self.goal;

        let fitness_functions = self.fitness_functions;

        crossbeam::scope(|scope| {
            pool.map(scope, pop, |ind| fitness(fitness_functions, goal, ind))
                .collect()
        })
    }
}

#[derive(Debug)]
struct Stat<T: Debug> {
    min: T,
    max: T,
    avg: T,
}

impl Stat<f32> {
    fn for_objectives(fit: &[MultiObjective3<f32>], i: usize) -> Stat<f32> {
        let min = fit.iter().fold(f32::INFINITY, |acc, f| {
            let x = f.objectives[i];
            if x < acc {
                x
            } else {
                acc
            }
        });
        let max = fit.iter().fold(-f32::INFINITY, |acc, f| {
            let x = f.objectives[i];
            if x > acc {
                x
            } else {
                acc
            }
        });
        let sum = fit.iter()
                     .fold(0.0, |acc, f| acc + f.objectives[i]);
        Stat {
            min: min,
            max: max,
            avg: if fit.is_empty() {
                0.0
            } else {
                sum / fit.len() as f32
            },
        }
    }
}

#[allow(non_snake_case)]
fn main() {
    let ncpus = num_cpus::get();
    println!("Using {} CPUs", ncpus);
    let matches = App::new("nsga2_edge")
                      .arg(Arg::with_name("NGEN")
                               .long("ngen")
                               .help("Number of generations to run")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("OPS")
                               .long("ops")
                               .help("Edge operation and weight specification, e.g. \
                                      Dup:1,Split:3,Parent:2")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("GRAPH")
                               .long("graph")
                               .help("SGF graph file")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("MU")
                               .long("mu")
                               .help("Population size")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("LAMBDA")
                               .long("lambda")
                               .help("Size of offspring population")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("K")
                               .long("k")
                               .help("Tournament selection (default: 2)")
                               .takes_value(true))
                      .arg(Arg::with_name("SEED")
                               .long("seed")
                               .help("Seed value for Rng")
                               .takes_value(true))
                      .arg(Arg::with_name("PMUT")
                               .long("pmut")
                               .help("Probability for element mutation")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("MUTOPS")
                               .long("mutops")
                               .help("Mutation operation and weight specification, e.g. \
                                      Insert:1,Remove:1,Replace:2,ModifyOp:0,ModifyParam:2,Copy:\
                                      0")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("ILEN")
                               .long("ilen")
                               .help("Initial genome length (random range)")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("OBJECTIVES")
                               .long("objectives")
                               .help("Specify 3 objective functions comma separated (null, cc, \
                                      scc, nm, td), e.g. cc,nm,td")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("THRESHOLD")
                               .long("threshold")
                               .help("Abort if for all fitness[i] <= value[i] (default: 0,0,0)")
                               .takes_value(true)
                               .required(false))
                      .get_matches();

    // number of generations
    let NGEN: usize = FromStr::from_str(matches.value_of("NGEN").unwrap()).unwrap();
    println!("NGEN: {}", NGEN);

    // size of population
    let MU: usize = FromStr::from_str(matches.value_of("MU").unwrap()).unwrap();
    println!("MU: {}", MU);

    // size of offspring population
    let LAMBDA: usize = FromStr::from_str(matches.value_of("LAMBDA").unwrap()).unwrap();
    println!("LAMBDA: {}", LAMBDA);

    // tournament selection
    let K: usize = FromStr::from_str(matches.value_of("K").unwrap_or("2")).unwrap();
    assert!(K > 0);
    println!("K: {}", K);

    let seed: Vec<u64>;
    if let Some(seed_str) = matches.value_of("SEED") {
        seed = seed_str.split(",").map(|s| FromStr::from_str(s).unwrap()).collect();
    } else {
        println!("Use OsRng to generate seed..");
        let mut rng = OsRng::new().unwrap();
        seed = (0..2).map(|_| rng.next_u64()).collect();
    }
    println!("SEED: {:?}", seed);

    let PMUT: f32 = FromStr::from_str(matches.value_of("PMUT").unwrap()).unwrap();
    println!("PMUT: {:?}", PMUT);

    // Parse weighted operation choice from command line
    let mutops = Toolbox::parse_weighted_mutops(matches.value_of("MUTOPS").unwrap()).unwrap();
    println!("mut ops: {:?}", mutops);

    let ilen_str = matches.value_of("ILEN").unwrap();
    let ilen: Vec<usize> = ilen_str.split(",").map(|s| FromStr::from_str(s).unwrap()).collect();
    assert!(ilen.len() == 1 || ilen.len() == 2);

    let ilen_from = ilen[0];
    let ilen_to = if ilen.len() == 1 {
        ilen[0]
    } else {
        ilen[1]
    };
    assert!(ilen_from <= ilen_to);

    // read objective functions
    let mut objectives_arr: Vec<FitnessFunction> = Vec::new();
    for s in matches.value_of("OBJECTIVES").unwrap().split(",") {
        objectives_arr.push(match s {
            "null" => {
                FitnessFunction::Null
            }
            "cc" => {
                FitnessFunction::ConnectedComponents
            }
            "scc" => {
                FitnessFunction::StronglyConnectedComponents
            }
            "nm" => {
                FitnessFunction::NeighborMatching
            }
            "td" => {
                FitnessFunction::TriadicDistance
            }
            _ => {
                panic!("Invalid objective function");
            }
        });
    }

    while objectives_arr.len() < 3 {
        objectives_arr.push(FitnessFunction::Null);
    }

    if objectives_arr.len() > 3 {
        panic!("Max 3 objectives allowed");
    }

    println!("objectives={:?}", objectives_arr);

    // read objective functions
    let mut threshold_arr: Vec<f32> = Vec::new();
    for s in matches.value_of("THRESHOLD").unwrap_or("").split(",") {
        let value: f32 = FromStr::from_str(s).unwrap();
        threshold_arr.push(value);
    }

    while threshold_arr.len() < 3 {
        threshold_arr.push(0.0);
    }

    if threshold_arr.len() > 3 {
        panic!("Max 3 threshold values allowed");
    }

    // read graph
    let graph_file = matches.value_of("GRAPH").unwrap();
    println!("Using graph file: {}", graph_file);
    let graph = {
        let mut f = BufReader::new(File::open(graph_file).unwrap());
        let graph: Graph<Unweighted, Unweighted, Directed> = PetgraphReader::from_sgf(&mut f);
        graph
    };

    // Parse weighted operation choice from command line
    let ops = Toolbox::parse_weighted_op_choice_list(matches.value_of("OPS").unwrap()).unwrap();
    println!("edge ops: {:?}", ops);

    let mut toolbox = Toolbox::new(&ops[..], Probability::new(PMUT), &mutops[..]);

    let mut evaluator = MyEval {
        goal: Goal::new(OptDenseDigraph::from(graph)),
        pool: Pool::new(ncpus),
        fitness_functions: (objectives_arr[0], objectives_arr[1], objectives_arr[2]),
    };

    assert!(seed.len() == 2);
    let mut rng: PcgRng = SeedableRng::from_seed([seed[0], seed[1]]);

    // create initial random population
    let initial_population: Vec<EdgeOpsGenome> = (0..MU)
                                                     .map(|_| {
                                                         let len = if ilen_from == ilen_to {
                                                             ilen_from
                                                         } else {
                                                             Range::new(ilen_from, ilen_to)
                                                                 .ind_sample(&mut rng)
                                                         };
                                                         toolbox.random_genome(&mut rng, len)
                                                     })
                                                     .collect();

    // evaluate fitness
    let fitness: Vec<_> = evaluator.fitness(&initial_population[..]);
    assert!(fitness.len() == initial_population.len());

    let mut pop = initial_population;
    let mut fit = fitness;

    for iteration in 0..NGEN {
        print!("# {:>6}", iteration);
        let before = time::precise_time_ns();
        let (new_pop, new_fit) = nsga2::iterate(&mut rng,
                                                pop,
                                                fit,
                                                &mut evaluator,
                                                MU,
                                                LAMBDA,
                                                K,
                                                &mut toolbox);
        let duration = time::precise_time_ns() - before;
        pop = new_pop;
        fit = new_fit;

        // calculate a min/max/avg value for each objective.
        let stat0 = Stat::<f32>::for_objectives(&fit[..], 0);
        let stat1 = Stat::<f32>::for_objectives(&fit[..], 1);
        let stat2 = Stat::<f32>::for_objectives(&fit[..], 2);

        let duration_ms = (duration as f32) / 1_000_000.0;

        let mut num_optima = 0;
        for f in fit.iter() {
            if f.objectives[0] <= threshold_arr[0] && f.objectives[1] <= threshold_arr[1] &&
               f.objectives[2] <= threshold_arr[2] {
                num_optima += 1;
            }
        }

        print!(" | ");
        print!("{:>8.1}", stat0.min);
        print!("{:>9.1}", stat0.avg);
        print!("{:>10.1}", stat0.max);
        print!(" | ");
        print!("{:>8.1}", stat1.min);
        print!("{:>9.1}", stat1.avg);
        print!("{:>10.1}", stat1.max);
        print!(" | ");
        print!("{:>8.1}", stat2.min);
        print!("{:>9.1}", stat2.avg);
        print!("{:>10.1}", stat2.max);

        print!(" | {:>5} | {:>8.0} ms", num_optima, duration_ms);

        println!("");

        if num_optima > 0 {
            println!("Found premature optimum in Iteration {}", iteration);
            break;
        }
    }
    println!("===========================================================");

    // finally evaluate rank and crowding distance (using select()).
    let rank_dist = nsga2::select(&fit[..], MU);
    assert!(rank_dist.len() == MU);

    let mut j = 0;

    for rd in rank_dist.iter() {
        if rd.rank == 0 {
            println!("-------------------------------------------");
            println!("rd: {:?}", rd);
            println!("fitness: {:?}", fit[rd.idx]);
            println!("genome: {:?}", pop[rd.idx]);

            // if fit[rd.idx].objectives[0] < 1.0 {
            draw_graph((&pop[rd.idx].to_graph()).ref_graph(),
                       // XXX: name
                       &format!("nsga2edgeops_g{}_f{}_i{}.svg",
                                NGEN,
                                fit[rd.idx].objectives[1] as usize,
                                j));
            j += 1;
            // }
        }
    }
}
<commit_msg>Rename --pmut to --mutp<commit_after>extern crate rand;
extern crate evo;
extern crate petgraph;
extern crate graph_annealing;
extern crate clap;
extern crate crossbeam;
extern crate simple_parallel;
extern crate num_cpus;
extern crate pcg;
extern crate graph_sgf;
extern crate triadic_census;
extern crate time;

use std::f32;
use std::fmt::Debug;
use std::str::FromStr;
use clap::{App, Arg};
use rand::{Rng, SeedableRng};
use rand::distributions::{IndependentSample, Range};
use rand::os::OsRng;
use pcg::PcgRng;

use evo::Probability;
use evo::nsga2::{self, FitnessEval, MultiObjective3};
use graph_annealing::repr::edge_ops_genome::{EdgeOpsGenome, Toolbox};
use graph_annealing::helper::draw_graph;
use graph_annealing::goal::Goal;
use simple_parallel::Pool;

use petgraph::{Directed, Graph};
use graph_sgf::{PetgraphReader, Unweighted};
use triadic_census::OptDenseDigraph;

use std::io::BufReader;
use std::fs::File;

#[derive(Debug, Copy, Clone)]
enum FitnessFunction {
    Null,
    ConnectedComponents,
    StronglyConnectedComponents,
    NeighborMatching,
    TriadicDistance,
}

fn apply_fitness_function<N: Clone + Default, E: Clone + Default>(fitfun: FitnessFunction,
                                                                  goal: &Goal<N, E>,
                                                                  g: &OptDenseDigraph<(), ()>)
                                                                  -> f32 {
    match fitfun {
        FitnessFunction::Null => {
            0.0
        }
        FitnessFunction::ConnectedComponents => {
            goal.connected_components_distance(g) as f32
        }
        FitnessFunction::StronglyConnectedComponents => {
            goal.strongly_connected_components_distance(g) as f32
        }
        FitnessFunction::NeighborMatching => {
            goal.neighbor_matching_score(g) as f32
        }
        FitnessFunction::TriadicDistance => {
            goal.triadic_distance(g) as f32
        }
    }
}

struct MyEval<N, E> {
    goal: Goal<N, E>,
    pool: Pool,
    fitness_functions: (FitnessFunction, FitnessFunction, FitnessFunction),
}

#[inline]
fn fitness<N: Clone + Default, E: Clone + Default>(fitness_functions: (FitnessFunction,
                                                                       FitnessFunction,
                                                                       FitnessFunction),
                                                   goal: &Goal<N, E>,
                                                   ind: &EdgeOpsGenome)
                                                   -> MultiObjective3<f32> {
    let g = ind.to_graph();
    MultiObjective3::from((apply_fitness_function(fitness_functions.0, goal, &g),
                           apply_fitness_function(fitness_functions.1, goal, &g),
                           apply_fitness_function(fitness_functions.2, goal, &g)))

}

impl<N:Clone+Sync+Default,E:Clone+Sync+Default> FitnessEval<EdgeOpsGenome, MultiObjective3<f32>> for MyEval<N,E> {
    fn fitness(&mut self, pop: &[EdgeOpsGenome]) -> Vec<MultiObjective3<f32>> {
        let pool = &mut self.pool;
        let goal = &self.goal;

        let fitness_functions = self.fitness_functions;

        crossbeam::scope(|scope| {
            pool.map(scope, pop, |ind| fitness(fitness_functions, goal, ind))
                .collect()
        })
    }
}

#[derive(Debug)]
struct Stat<T: Debug> {
    min: T,
    max: T,
    avg: T,
}

impl Stat<f32> {
    fn for_objectives(fit: &[MultiObjective3<f32>], i: usize) -> Stat<f32> {
        let min = fit.iter().fold(f32::INFINITY, |acc, f| {
            let x = f.objectives[i];
            if x < acc {
                x
            } else {
                acc
            }
        });
        let max = fit.iter().fold(-f32::INFINITY, |acc, f| {
            let x = f.objectives[i];
            if x > acc {
                x
            } else {
                acc
            }
        });
        let sum = fit.iter()
                     .fold(0.0, |acc, f| acc + f.objectives[i]);
        Stat {
            min: min,
            max: max,
            avg: if fit.is_empty() {
                0.0
            } else {
                sum / fit.len() as f32
            },
        }
    }
}

#[allow(non_snake_case)]
fn main() {
    let ncpus = num_cpus::get();
    println!("Using {} CPUs", ncpus);
    let matches = App::new("nsga2_edge")
                      .arg(Arg::with_name("NGEN")
                               .long("ngen")
                               .help("Number of generations to run")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("OPS")
                               .long("ops")
                               .help("Edge operation and weight specification, e.g. \
                                      Dup:1,Split:3,Parent:2")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("GRAPH")
                               .long("graph")
                               .help("SGF graph file")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("MU")
                               .long("mu")
                               .help("Population size")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("LAMBDA")
                               .long("lambda")
                               .help("Size of offspring population")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("K")
                               .long("k")
                               .help("Tournament selection (default: 2)")
                               .takes_value(true))
                      .arg(Arg::with_name("SEED")
                               .long("seed")
                               .help("Seed value for Rng")
                               .takes_value(true))
                      .arg(Arg::with_name("MUTP")
                               .long("mutp")
                               .help("Probability for element mutation")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("MUTOPS")
                               .long("mutops")
                               .help("Mutation operation and weight specification, e.g. \
                                      Insert:1,Remove:1,Replace:2,ModifyOp:0,ModifyParam:2,Copy:\
                                      0")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("ILEN")
                               .long("ilen")
                               .help("Initial genome length (random range)")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("OBJECTIVES")
                               .long("objectives")
                               .help("Specify 3 objective functions comma separated (null, cc, \
                                      scc, nm, td), e.g. cc,nm,td")
                               .takes_value(true)
                               .required(true))
                      .arg(Arg::with_name("THRESHOLD")
                               .long("threshold")
                               .help("Abort if for all fitness[i] <= value[i] (default: 0,0,0)")
                               .takes_value(true)
                               .required(false))
                      .get_matches();

    // number of generations
    let NGEN: usize = FromStr::from_str(matches.value_of("NGEN").unwrap()).unwrap();
    println!("NGEN: {}", NGEN);

    // size of population
    let MU: usize = FromStr::from_str(matches.value_of("MU").unwrap()).unwrap();
    println!("MU: {}", MU);

    // size of offspring population
    let LAMBDA: usize = FromStr::from_str(matches.value_of("LAMBDA").unwrap()).unwrap();
    println!("LAMBDA: {}", LAMBDA);

    // tournament selection
    let K: usize = FromStr::from_str(matches.value_of("K").unwrap_or("2")).unwrap();
    assert!(K > 0);
    println!("K: {}", K);

    let seed: Vec<u64>;
    if let Some(seed_str) = matches.value_of("SEED") {
        seed = seed_str.split(",").map(|s| FromStr::from_str(s).unwrap()).collect();
    } else {
        println!("Use OsRng to generate seed..");
        let mut rng = OsRng::new().unwrap();
        seed = (0..2).map(|_| rng.next_u64()).collect();
    }
    println!("SEED: {:?}", seed);

    let MUTP: f32 = FromStr::from_str(matches.value_of("MUTP").unwrap()).unwrap();
    println!("MUTP: {:?}", MUTP);

    // Parse weighted operation choice from command line
    let mutops = Toolbox::parse_weighted_mutops(matches.value_of("MUTOPS").unwrap()).unwrap();
    println!("mut ops: {:?}", mutops);

    let ilen_str = matches.value_of("ILEN").unwrap();
    let ilen: Vec<usize> = ilen_str.split(",").map(|s| FromStr::from_str(s).unwrap()).collect();
    assert!(ilen.len() == 1 || ilen.len() == 2);

    let ilen_from = ilen[0];
    let ilen_to = if ilen.len() == 1 {
        ilen[0]
    } else {
        ilen[1]
    };
    assert!(ilen_from <= ilen_to);

    // read objective functions
    let mut objectives_arr: Vec<FitnessFunction> = Vec::new();
    for s in matches.value_of("OBJECTIVES").unwrap().split(",") {
        objectives_arr.push(match s {
            "null" => {
                FitnessFunction::Null
            }
            "cc" => {
                FitnessFunction::ConnectedComponents
            }
            "scc" => {
                FitnessFunction::StronglyConnectedComponents
            }
            "nm" => {
                FitnessFunction::NeighborMatching
            }
            "td" => {
                FitnessFunction::TriadicDistance
            }
            _ => {
                panic!("Invalid objective function");
            }
        });
    }

    while objectives_arr.len() < 3 {
        objectives_arr.push(FitnessFunction::Null);
    }

    if objectives_arr.len() > 3 {
        panic!("Max 3 objectives allowed");
    }

    println!("objectives={:?}", objectives_arr);

    // read objective functions
    let mut threshold_arr: Vec<f32> = Vec::new();
    for s in matches.value_of("THRESHOLD").unwrap_or("").split(",") {
        let value: f32 = FromStr::from_str(s).unwrap();
        threshold_arr.push(value);
    }

    while threshold_arr.len() < 3 {
        threshold_arr.push(0.0);
    }

    if threshold_arr.len() > 3 {
        panic!("Max 3 threshold values allowed");
    }

    // read graph
    let graph_file = matches.value_of("GRAPH").unwrap();
    println!("Using graph file: {}", graph_file);
    let graph = {
        let mut f = BufReader::new(File::open(graph_file).unwrap());
        let graph: Graph<Unweighted, Unweighted, Directed> = PetgraphReader::from_sgf(&mut f);
        graph
    };

    // Parse weighted operation choice from command line
    let ops = Toolbox::parse_weighted_op_choice_list(matches.value_of("OPS").unwrap()).unwrap();
    println!("edge ops: {:?}", ops);

    let mut toolbox = Toolbox::new(&ops[..], Probability::new(MUTP), &mutops[..]);

    let mut evaluator = MyEval {
        goal: Goal::new(OptDenseDigraph::from(graph)),
        pool: Pool::new(ncpus),
        fitness_functions: (objectives_arr[0], objectives_arr[1], objectives_arr[2]),
    };

    assert!(seed.len() == 2);
    let mut rng: PcgRng = SeedableRng::from_seed([seed[0], seed[1]]);

    // create initial random population
    let initial_population: Vec<EdgeOpsGenome> = (0..MU)
                                                     .map(|_| {
                                                         let len = if ilen_from == ilen_to {
                                                             ilen_from
                                                         } else {
                                                             Range::new(ilen_from, ilen_to)
                                                                 .ind_sample(&mut rng)
                                                         };
                                                         toolbox.random_genome(&mut rng, len)
                                                     })
                                                     .collect();

    // evaluate fitness
    let fitness: Vec<_> = evaluator.fitness(&initial_population[..]);
    assert!(fitness.len() == initial_population.len());

    let mut pop = initial_population;
    let mut fit = fitness;

    for iteration in 0..NGEN {
        print!("# {:>6}", iteration);
        let before = time::precise_time_ns();
        let (new_pop, new_fit) = nsga2::iterate(&mut rng,
                                                pop,
                                                fit,
                                                &mut evaluator,
                                                MU,
                                                LAMBDA,
                                                K,
                                                &mut toolbox);
        let duration = time::precise_time_ns() - before;
        pop = new_pop;
        fit = new_fit;

        // calculate a min/max/avg value for each objective.
        let stat0 = Stat::<f32>::for_objectives(&fit[..], 0);
        let stat1 = Stat::<f32>::for_objectives(&fit[..], 1);
        let stat2 = Stat::<f32>::for_objectives(&fit[..], 2);

        let duration_ms = (duration as f32) / 1_000_000.0;

        let mut num_optima = 0;
        for f in fit.iter() {
            if f.objectives[0] <= threshold_arr[0] && f.objectives[1] <= threshold_arr[1] &&
               f.objectives[2] <= threshold_arr[2] {
                num_optima += 1;
            }
        }

        print!(" | ");
        print!("{:>8.1}", stat0.min);
        print!("{:>9.1}", stat0.avg);
        print!("{:>10.1}", stat0.max);
        print!(" | ");
        print!("{:>8.1}", stat1.min);
        print!("{:>9.1}", stat1.avg);
        print!("{:>10.1}", stat1.max);
        print!(" | ");
        print!("{:>8.1}", stat2.min);
        print!("{:>9.1}", stat2.avg);
        print!("{:>10.1}", stat2.max);

        print!(" | {:>5} | {:>8.0} ms", num_optima, duration_ms);

        println!("");

        if num_optima > 0 {
            println!("Found premature optimum in Iteration {}", iteration);
            break;
        }
    }
    println!("===========================================================");

    // finally evaluate rank and crowding distance (using select()).
    let rank_dist = nsga2::select(&fit[..], MU);
    assert!(rank_dist.len() == MU);

    let mut j = 0;

    for rd in rank_dist.iter() {
        if rd.rank == 0 {
            println!("-------------------------------------------");
            println!("rd: {:?}", rd);
            println!("fitness: {:?}", fit[rd.idx]);
            println!("genome: {:?}", pop[rd.idx]);

            // if fit[rd.idx].objectives[0] < 1.0 {
            draw_graph((&pop[rd.idx].to_graph()).ref_graph(),
                       // XXX: name
                       &format!("nsga2edgeops_g{}_f{}_i{}.svg",
                                NGEN,
                                fit[rd.idx].objectives[1] as usize,
                                j));
            j += 1;
            // }
        }
    }
}
<|endoftext|>


<commit_before>use std::ops::BitXor;
use std::cmp;
use std::iter;
use num_rational::Ratio;

pub fn block_xor<T>(block1: &[T], block2: &[T]) -> Vec<T::Output>
    where T: BitXor + Clone + Copy {

    let block_size = cmp::min(block1.len(), block2.len());

    let mut new_block = Vec::with_capacity(block_size);

    for i in 0..block_size {
        new_block.push(block1[i] ^ block2[i]);
    }

    new_block
}

pub fn exact_block_xor<T>(block1: &[T], block2: &[T]) -> Option<Vec<T::Output>>
    where T: BitXor + Clone + Copy {

    if block1.len() != block2.len() {
        return None;
    }

    Some(block_xor(block1, block2))
}

fn repeat_char(character: u8, n: usize) -> Vec<u8> {
    iter::repeat(character).take(n).collect()
}

fn op_with_char<F>(op: F, character: u8, string: &[u8]) -> Vec<u8>
    where F: Fn(&[u8], &[u8]) -> Vec<u8> {

    let other_string = repeat_char(character, string.len());

    op(string, &other_string)
}

pub fn xor_with_char(character: u8, string: &[u8]) -> Vec<u8> {
    op_with_char(&block_xor, character, string)
}

#[allow(unused_variables)]
fn with_key(key: &[u8], string: &[u8]) -> Vec<u8> {
    let rep_count = string.len() / key.len();
    let remainder = string.len() % key.len();
    let mut vec   = Vec::new();

    assert!(string.len() == rep_count*key.len() + remainder);

    for i in 0..rep_count {
        for ch in key {
            vec.push(*ch);
        }
    }

    for item in key.iter().take(remainder) {
        vec.push(*item);
    }

    vec
}

fn op_with_key<F>(op: F, key: &[u8], string: &[u8]) -> Vec<u8>
    where F: Fn(&[u8], &[u8]) -> Vec<u8> {

    let other_string = with_key(key, string);

    op(string, other_string.as_ref())
}

pub fn xor_with_key(key: &[u8], string: &[u8]) -> Vec<u8> {
    op_with_key(&block_xor, key, string)
}

pub fn edit_distance(str1: &[u8], str2: &[u8]) -> Option<usize> {
    let hmetric = |x, y| {
        let mut val  = x ^ y;
        let mut dist = 0;

        while val != 0 {
            dist += 1;
            val  &= val - 1;
        }

        dist
    };

    if str1.len() != str2.len() {
        return None;
    }

    let mut dist = 0;
    for (byte1, byte2) in str1.into_iter().zip(str2) {
        dist += hmetric(byte1, byte2);
    }

    Some(dist)
}

pub fn normalized_edit_distance(str1: &[u8], str2: &[u8]) -> Option<Ratio<usize>> {
    let dist = edit_distance(str1, str2);
    let normalized_edit_distance = match dist {
        None       => None,
        Some(edit_dist) => Some(Ratio::new_raw(edit_dist, str1.len())),
    };

    normalized_edit_distance
}

fn total_edit_distance(seq1: &[&[u8]], seq2: &[&[u8]]) -> Option<usize> {
    if seq1.is_empty() || seq2.is_empty() {
        return None;
    }

    if seq1.len() != seq2.len() {
        return None;
    }

    for (str1, str2) in seq1.into_iter().zip(seq2) {
        if str1.len() != str2.len() {
            return None;
        }
    }

    let edit_dist = seq1.into_iter().zip(seq2).fold(0, |dist, (str1, str2)| {
        dist + edit_distance(str1, str2).unwrap()
    });

    Some(edit_dist)
}

pub fn mean_edit_distance(strings: &[&[u8]]) -> Option<Ratio<usize>> {
    let length = |strings: &[&[u8]]| {
        strings.into_iter()
               .fold(0, |acc, st| { acc + st.len() })
    };

    let seq1: Vec<&[u8]> = strings.into_iter()
                                  .take(strings.len()-1)
                                  .map(|slice| { *slice })
                                  .collect();

    let seq2: Vec<&[u8]> = strings.into_iter()
                                  .skip(1)
                                  .map(|slice| { *slice })
                                  .collect();

    let dist  = total_edit_distance(seq1.as_ref(), seq2.as_ref());
    let total = length(strings);

    let mean = match dist {
        None      => None,
        Some(val) => Some(Ratio::new_raw(val, total)),
    };

    mean
}
<commit_msg>Adjust some match blocks to return directly from the function.<commit_after>use std::ops::BitXor;
use std::cmp;
use std::iter;
use num_rational::Ratio;

pub fn block_xor<T>(block1: &[T], block2: &[T]) -> Vec<T::Output>
    where T: BitXor + Clone + Copy {

    let block_size = cmp::min(block1.len(), block2.len());

    let mut new_block = Vec::with_capacity(block_size);

    for i in 0..block_size {
        new_block.push(block1[i] ^ block2[i]);
    }

    new_block
}

pub fn exact_block_xor<T>(block1: &[T], block2: &[T]) -> Option<Vec<T::Output>>
    where T: BitXor + Clone + Copy {

    if block1.len() != block2.len() {
        return None;
    }

    Some(block_xor(block1, block2))
}

fn repeat_char(character: u8, n: usize) -> Vec<u8> {
    iter::repeat(character).take(n).collect()
}

fn op_with_char<F>(op: F, character: u8, string: &[u8]) -> Vec<u8>
    where F: Fn(&[u8], &[u8]) -> Vec<u8> {

    let other_string = repeat_char(character, string.len());

    op(string, &other_string)
}

pub fn xor_with_char(character: u8, string: &[u8]) -> Vec<u8> {
    op_with_char(&block_xor, character, string)
}

#[allow(unused_variables)]
fn with_key(key: &[u8], string: &[u8]) -> Vec<u8> {
    let rep_count = string.len() / key.len();
    let remainder = string.len() % key.len();
    let mut vec   = Vec::new();

    assert!(string.len() == rep_count*key.len() + remainder);

    for i in 0..rep_count {
        for ch in key {
            vec.push(*ch);
        }
    }

    for item in key.iter().take(remainder) {
        vec.push(*item);
    }

    vec
}

fn op_with_key<F>(op: F, key: &[u8], string: &[u8]) -> Vec<u8>
    where F: Fn(&[u8], &[u8]) -> Vec<u8> {

    let other_string = with_key(key, string);

    op(string, other_string.as_ref())
}

pub fn xor_with_key(key: &[u8], string: &[u8]) -> Vec<u8> {
    op_with_key(&block_xor, key, string)
}

pub fn edit_distance(str1: &[u8], str2: &[u8]) -> Option<usize> {
    let hmetric = |x, y| {
        let mut val  = x ^ y;
        let mut dist = 0;

        while val != 0 {
            dist += 1;
            val  &= val - 1;
        }

        dist
    };

    if str1.len() != str2.len() {
        return None;
    }

    let mut dist = 0;
    for (byte1, byte2) in str1.into_iter().zip(str2) {
        dist += hmetric(byte1, byte2);
    }

    Some(dist)
}

pub fn normalized_edit_distance(str1: &[u8], str2: &[u8]) -> Option<Ratio<usize>> {
    let dist = edit_distance(str1, str2);

    match dist {
        None            => None,
        Some(edit_dist) => Some(Ratio::new_raw(edit_dist, str1.len())),
    }
}

fn total_edit_distance(seq1: &[&[u8]], seq2: &[&[u8]]) -> Option<usize> {
    if seq1.is_empty() || seq2.is_empty() {
        return None;
    }

    if seq1.len() != seq2.len() {
        return None;
    }

    for (str1, str2) in seq1.into_iter().zip(seq2) {
        if str1.len() != str2.len() {
            return None;
        }
    }

    let edit_dist = seq1.into_iter().zip(seq2).fold(0, |dist, (str1, str2)| {
        dist + edit_distance(str1, str2).unwrap()
    });

    Some(edit_dist)
}

pub fn mean_edit_distance(strings: &[&[u8]]) -> Option<Ratio<usize>> {
    let length = |strings: &[&[u8]]| {
        strings.into_iter()
               .fold(0, |acc, st| { acc + st.len() })
    };

    let seq1: Vec<&[u8]> = strings.into_iter()
                                  .take(strings.len()-1)
                                  .map(|slice| { *slice })
                                  .collect();

    let seq2: Vec<&[u8]> = strings.into_iter()
                                  .skip(1)
                                  .map(|slice| { *slice })
                                  .collect();

    let dist  = total_edit_distance(seq1.as_ref(), seq2.as_ref());
    let total = length(strings);

    match dist {
        None      => None,
        Some(val) => Some(Ratio::new_raw(val, total)),
    }
}
<|endoftext|>


<commit_before>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#![feature(globs, phase, macro_rules)]

extern crate syntax;
extern crate rustc;

#[phase(link)]
extern crate regex;

#[phase(link, plugin)]
extern crate log;

#[phase(plugin)] extern crate regex_macros;

use std::collections::HashMap;
use std::io::File;

use syntax::parse;
use syntax::parse::lexer;
use rustc::driver::{session, config};

use syntax::ast;
use syntax::ast::Name;
use syntax::parse::token;
use syntax::parse::lexer::TokenAndSpan;

fn parse_token_list(file: &str) -> HashMap<String, Token> {
    fn id() -> Token {
        token::Ident(ast::Ident { name: Name(0), ctxt: 0, }, token::Plain)
    }

    let mut res = HashMap::new();

    res.insert("-1".to_string(), EOF);

    for line in file.split('\n') {
        let eq = match line.trim().rfind('=') {
            Some(val) => val,
            None => continue
        };

        let val = line.slice_to(eq);
        let num = line.slice_from(eq + 1);

        let tok = match val {
            "SHR"               => token::BinOp(token::Shr),
            "DOLLAR"            => token::Dollar,
            "LT"                => token::Lt,
            "STAR"              => token::BinOp(token::Star),
            "FLOAT_SUFFIX"      => id(),
            "INT_SUFFIX"        => id(),
            "SHL"               => token::BinOp(token::Shl),
            "LBRACE"            => token::OpenDelim(token::Brace),
            "RARROW"            => token::Rarrow,
            "LIT_STR"           => token::Literal(token::Str_(Name(0))),
            "DOTDOT"            => token::DotDot,
            "MOD_SEP"           => token::ModSep,
            "DOTDOTDOT"         => token::DotDotDot,
            "NOT"               => token::Not,
            "AND"               => token::BinOp(token::And),
            "LPAREN"            => token::OpenDelim(token::Paren),
            "ANDAND"            => token::AndAnd,
            "AT"                => token::At,
            "LBRACKET"          => token::OpenDelim(token::Bracket),
            "LIT_STR_RAW"       => token::Literal(token::StrRaw(Name(0), 0)),
            "RPAREN"            => token::CloseDelim(token::Paren),
            "SLASH"             => token::BinOp(token::Slash),
            "COMMA"             => token::Comma,
            "LIFETIME"          => token::Lifetime(ast::Ident { name: Name(0), ctxt: 0 }),
            "CARET"             => token::BinOp(token::Caret),
            "TILDE"             => token::Tilde,
            "IDENT"             => token::Id(),
            "PLUS"              => token::BinOp(token::Plus),
            "LIT_CHAR"          => token::Literal(token::Char(Name(0))),
            "LIT_BYTE"          => token::Literal(token::Byte(Name(0))),
            "EQ"                => token::Eq,
            "RBRACKET"          => token::CloseDelim(token::Bracket),
            "COMMENT"           => token::Comment,
            "DOC_COMMENT"       => token::DocComment(Name(0)),
            "DOT"               => token::Dot,
            "EQEQ"              => token::EqEq,
            "NE"                => token::Ne,
            "GE"                => token::Ge,
            "PERCENT"           => token::BinOp(token::Percent),
            "RBRACE"            => token::CloseDelim(token::Brace),
            "BINOP"             => token::BinOp(token::Plus),
            "POUND"             => token::Pound,
            "OROR"              => token::OrOr,
            "LIT_INTEGER"       => token::Literal(token::Integer(Name(0))),
            "BINOPEQ"           => token::BinOpEq(token::Plus),
            "LIT_FLOAT"         => token::Literal(token::Float(Name(0))),
            "WHITESPACE"        => token::Whitespace,
            "UNDERSCORE"        => token::Underscore,
            "MINUS"             => token::BinOp(token::Minus),
            "SEMI"              => token::Semi,
            "COLON"             => token::Colon,
            "FAT_ARROW"         => token::FatArrow,
            "OR"                => token::BinOp(token::Or),
            "GT"                => token::Gt,
            "LE"                => token::Le,
            "LIT_BINARY"        => token::Literal(token::Binary(Name(0))),
            "LIT_BINARY_RAW"    => token::Literal(token::BinaryRaw(Name(0), 0)),
            _                   => continue,
        };

        res.insert(num.to_string(), tok);
    }

    debug!("Token map: {}", res);
    res
}

fn str_to_binop(s: &str) -> BinOpToken {
    match s {
        "+"     => token::Plus,
        "/"     => token::Slash,
        "-"     => token::Minus,
        "*"     => token::Star,
        "%"     => token::Percent,
        "^"     => token::Caret,
        "&"     => token::And,
        "|"     => token::Or,
        "<<"    => token::Shl,
        ">>"    => token::Shr,
        _       => panic!("Bad binop str `{}`", s),
    }
}

/// Assuming a string/binary literal, strip out the leading/trailing
/// hashes and surrounding quotes/raw/binary prefix.
fn fix(mut lit: &str) -> ast::Name {
    if lit.char_at(0) == 'r' {
        if lit.char_at(1) == 'b' {
            lit = lit.slice_from(2)
        } else {
            lit = lit.slice_from(1);
        }
    } else if lit.char_at(0) == 'b' {
        lit = lit.slice_from(1);
    }

    let leading_hashes = count(lit);

    // +1/-1 to adjust for single quotes
    parse::token::intern(lit.slice(leading_hashes + 1, lit.len() - leading_hashes - 1))
}

/// Assuming a char/byte literal, strip the 'b' prefix and the single quotes.
fn fixchar(mut lit: &str) -> ast::Name {
    if lit.char_at(0) == 'b' {
        lit = lit.slice_from(1);
    }

    parse::token::intern(lit.slice(1, lit.len() - 1))
}

fn count(lit: &str) -> uint {
    lit.chars().take_while(|c| *c == '#').count()
}

fn parse_antlr_token(s: &str, tokens: &HashMap<String, Token>) -> TokenAndSpan {
    let re = regex!(
      r"\[@(?P<seq>\d+),(?P<start>\d+):(?P<end>\d+)='(?P<content>.+?)',<(?P<toknum>-?\d+)>,\d+:\d+]"
    );

    let m = re.captures(s).expect(format!("The regex didn't match {}", s).as_slice());
    let start = m.name("start");
    let end = m.name("end");
    let toknum = m.name("toknum");
    let content = m.name("content");

    let proto_tok = tokens.get(&toknum).expect(format!("didn't find token {} in the map",
                                                              toknum).as_slice());

    let nm = parse::token::intern(content);

    debug!("What we got: content (`{}`), proto: {}", content, proto_tok);

    let real_tok = match *proto_tok {
        token::BinOp(..)           => token::BinOp(str_to_binop(content)),
        token::BinOpEq(..)         => token::BinOpEq(str_to_binop(content.slice_to(
                                                                    content.len() - 1))),
        token::Literal(token::Str_(..))      => token::Literal(token::Str_(fix(content))),
        token::Literal(token::StrRaw(..))    => token::Literal(token::StrRaw(fix(content),
                                                                             count(content))),
        token::Literal(token::Char(..))      => token::Literal(token::Char(fixchar(content))),
        token::Literal(token::Byte(..))      => token::Literal(token::Byte(fixchar(content))),
        token::DocComment(..)      => token::DocComment(nm),
        token::Literal(token::Integer(..))   => token::Literal(token::Integer(nm)),
        token::Literal(token::Float(..))     => token::Literal(token::Float(nm)),
        token::Literal(token::Binary(..))    => token::Literal(token::Binary(nm)),
        token::Literal(token::BinaryRaw(..)) => token::Literal(token::BinaryRaw(fix(content),
                                                                                count(content))),
        token::Ident(..)           => token::Ident(ast::Ident { name: nm, ctxt: 0 },
                                                   token::ModName),
        token::Lifetime(..)        => token::Lifetime(ast::Ident { name: nm, ctxt: 0 }),
        ref t => t.clone()
    };

    let offset = if real_tok == EOF {
        1
    } else {
        0
    };

    let sp = syntax::codemap::Span {
        lo: syntax::codemap::BytePos(from_str::<u32>(start).unwrap() - offset),
        hi: syntax::codemap::BytePos(from_str::<u32>(end).unwrap() + 1),
        expn_id: syntax::codemap::NO_EXPANSION
    };

    TokenAndSpan {
        tok: real_tok,
        sp: sp
    }
}

fn tok_cmp(a: &Token, b: &Token) -> bool {
    match a {
        &token::Ident(id, _) => match b {
                &token::Ident(id2, _) => id == id2,
                _ => false
        },
        _ => a == b
    }
}

fn main() {
    fn next(r: &mut lexer::StringReader) -> TokenAndSpan {
        use syntax::parse::lexer::Reader;
        r.next_token()
    }

    let args = std::os::args();

    let mut token_file = File::open(&Path::new(args.get(2).as_slice()));
    let token_map = parse_token_list(token_file.read_to_string().unwrap().as_slice());

    let mut stdin = std::io::stdin();
    let mut antlr_tokens = stdin.lines().map(|l| parse_antlr_token(l.unwrap().as_slice().trim(),
                                                                   &token_map));

    let code = File::open(&Path::new(args.get(1).as_slice())).unwrap().read_to_string().unwrap();
    let options = config::basic_options();
    let session = session::build_session(options, None,
                                         syntax::diagnostics::registry::Registry::new([]));
    let filemap = parse::string_to_filemap(&session.parse_sess,
                                           code,
                                           String::from_str("<n/a>"));
    let mut lexer = lexer::StringReader::new(session.diagnostic(), filemap);

    for antlr_tok in antlr_tokens {
        let rustc_tok = next(&mut lexer);
        if rustc_tok.tok == EOF && antlr_tok.tok == EOF {
            continue
        }

        assert!(rustc_tok.sp == antlr_tok.sp, "{} and {} have different spans", rustc_tok,
                antlr_tok);

        macro_rules! matches (
            ( $($x:pat),+ ) => (
                match rustc_tok.tok {
                    $($x => match antlr_tok.tok {
                        $x => {
                            if !tok_cmp(&rustc_tok.tok, &antlr_tok.tok) {
                                // FIXME #15677: needs more robust escaping in
                                // antlr
                                warn!("Different names for {} and {}", rustc_tok, antlr_tok);
                            }
                        }
                        _ => panic!("{} is not {}", antlr_tok, rustc_tok)
                    },)*
                    ref c => assert!(c == &antlr_tok.tok, "{} is not {}", rustc_tok, antlr_tok)
                }
            )
        )

        matches!(
            token::Literal(token::Byte(..)),
            token::Literal(token::Char(..)),
            token::Literal(token::Integer(..)),
            token::Literal(token::Float(..)),
            token::Literal(token::Str_(..)),
            token::Literal(token::StrRaw(..)),
            token::Literal(token::Binary(..)),
            token::Literal(token::BinaryRaw(..)),
            Ident(..),
            Lifetime(..),
            Interpolated(..),
            DocComment(..),
            Shebang(..)
        );
    }
}
<commit_msg>Update src/grammar for language changes.<commit_after>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#![feature(globs, phase, macro_rules)]

extern crate syntax;
extern crate rustc;

#[phase(link)]
extern crate regex;

#[phase(link, plugin)]
extern crate log;

#[phase(plugin)] extern crate regex_macros;

use std::collections::HashMap;
use std::io::File;

use syntax::parse;
use syntax::parse::lexer;
use rustc::session::{mod, config};

use syntax::ast;
use syntax::ast::Name;
use syntax::parse::token;
use syntax::parse::lexer::TokenAndSpan;

fn parse_token_list(file: &str) -> HashMap<String, token::Token> {
    fn id() -> token::Token {
        token::Ident(ast::Ident { name: Name(0), ctxt: 0, }, token::Plain)
    }

    let mut res = HashMap::new();

    res.insert("-1".to_string(), token::Eof);

    for line in file.split('\n') {
        let eq = match line.trim().rfind('=') {
            Some(val) => val,
            None => continue
        };

        let val = line.slice_to(eq);
        let num = line.slice_from(eq + 1);

        let tok = match val {
            "SHR"               => token::BinOp(token::Shr),
            "DOLLAR"            => token::Dollar,
            "LT"                => token::Lt,
            "STAR"              => token::BinOp(token::Star),
            "FLOAT_SUFFIX"      => id(),
            "INT_SUFFIX"        => id(),
            "SHL"               => token::BinOp(token::Shl),
            "LBRACE"            => token::OpenDelim(token::Brace),
            "RARROW"            => token::RArrow,
            "LIT_STR"           => token::Literal(token::Str_(Name(0))),
            "DOTDOT"            => token::DotDot,
            "MOD_SEP"           => token::ModSep,
            "DOTDOTDOT"         => token::DotDotDot,
            "NOT"               => token::Not,
            "AND"               => token::BinOp(token::And),
            "LPAREN"            => token::OpenDelim(token::Paren),
            "ANDAND"            => token::AndAnd,
            "AT"                => token::At,
            "LBRACKET"          => token::OpenDelim(token::Bracket),
            "LIT_STR_RAW"       => token::Literal(token::StrRaw(Name(0), 0)),
            "RPAREN"            => token::CloseDelim(token::Paren),
            "SLASH"             => token::BinOp(token::Slash),
            "COMMA"             => token::Comma,
            "LIFETIME"          => token::Lifetime(ast::Ident { name: Name(0), ctxt: 0 }),
            "CARET"             => token::BinOp(token::Caret),
            "TILDE"             => token::Tilde,
            "IDENT"             => id(),
            "PLUS"              => token::BinOp(token::Plus),
            "LIT_CHAR"          => token::Literal(token::Char(Name(0))),
            "LIT_BYTE"          => token::Literal(token::Byte(Name(0))),
            "EQ"                => token::Eq,
            "RBRACKET"          => token::CloseDelim(token::Bracket),
            "COMMENT"           => token::Comment,
            "DOC_COMMENT"       => token::DocComment(Name(0)),
            "DOT"               => token::Dot,
            "EQEQ"              => token::EqEq,
            "NE"                => token::Ne,
            "GE"                => token::Ge,
            "PERCENT"           => token::BinOp(token::Percent),
            "RBRACE"            => token::CloseDelim(token::Brace),
            "BINOP"             => token::BinOp(token::Plus),
            "POUND"             => token::Pound,
            "OROR"              => token::OrOr,
            "LIT_INTEGER"       => token::Literal(token::Integer(Name(0))),
            "BINOPEQ"           => token::BinOpEq(token::Plus),
            "LIT_FLOAT"         => token::Literal(token::Float(Name(0))),
            "WHITESPACE"        => token::Whitespace,
            "UNDERSCORE"        => token::Underscore,
            "MINUS"             => token::BinOp(token::Minus),
            "SEMI"              => token::Semi,
            "COLON"             => token::Colon,
            "FAT_ARROW"         => token::FatArrow,
            "OR"                => token::BinOp(token::Or),
            "GT"                => token::Gt,
            "LE"                => token::Le,
            "LIT_BINARY"        => token::Literal(token::Binary(Name(0))),
            "LIT_BINARY_RAW"    => token::Literal(token::BinaryRaw(Name(0), 0)),
            _                   => continue,
        };

        res.insert(num.to_string(), tok);
    }

    debug!("Token map: {}", res);
    res
}

fn str_to_binop(s: &str) -> token::BinOpToken {
    match s {
        "+"     => token::Plus,
        "/"     => token::Slash,
        "-"     => token::Minus,
        "*"     => token::Star,
        "%"     => token::Percent,
        "^"     => token::Caret,
        "&"     => token::And,
        "|"     => token::Or,
        "<<"    => token::Shl,
        ">>"    => token::Shr,
        _       => panic!("Bad binop str `{}`", s),
    }
}

/// Assuming a string/binary literal, strip out the leading/trailing
/// hashes and surrounding quotes/raw/binary prefix.
fn fix(mut lit: &str) -> ast::Name {
    if lit.char_at(0) == 'r' {
        if lit.char_at(1) == 'b' {
            lit = lit.slice_from(2)
        } else {
            lit = lit.slice_from(1);
        }
    } else if lit.char_at(0) == 'b' {
        lit = lit.slice_from(1);
    }

    let leading_hashes = count(lit);

    // +1/-1 to adjust for single quotes
    parse::token::intern(lit.slice(leading_hashes + 1, lit.len() - leading_hashes - 1))
}

/// Assuming a char/byte literal, strip the 'b' prefix and the single quotes.
fn fixchar(mut lit: &str) -> ast::Name {
    if lit.char_at(0) == 'b' {
        lit = lit.slice_from(1);
    }

    parse::token::intern(lit.slice(1, lit.len() - 1))
}

fn count(lit: &str) -> uint {
    lit.chars().take_while(|c| *c == '#').count()
}

fn parse_antlr_token(s: &str, tokens: &HashMap<String, token::Token>) -> TokenAndSpan {
    let re = regex!(
      r"\[@(?P<seq>\d+),(?P<start>\d+):(?P<end>\d+)='(?P<content>.+?)',<(?P<toknum>-?\d+)>,\d+:\d+]"
    );

    let m = re.captures(s).expect(format!("The regex didn't match {}", s).as_slice());
    let start = m.name("start");
    let end = m.name("end");
    let toknum = m.name("toknum");
    let content = m.name("content");

    let proto_tok = tokens.get(toknum).expect(format!("didn't find token {} in the map",
                                                              toknum).as_slice());

    let nm = parse::token::intern(content);

    debug!("What we got: content (`{}`), proto: {}", content, proto_tok);

    let real_tok = match *proto_tok {
        token::BinOp(..)           => token::BinOp(str_to_binop(content)),
        token::BinOpEq(..)         => token::BinOpEq(str_to_binop(content.slice_to(
                                                                    content.len() - 1))),
        token::Literal(token::Str_(..))      => token::Literal(token::Str_(fix(content))),
        token::Literal(token::StrRaw(..))    => token::Literal(token::StrRaw(fix(content),
                                                                             count(content))),
        token::Literal(token::Char(..))      => token::Literal(token::Char(fixchar(content))),
        token::Literal(token::Byte(..))      => token::Literal(token::Byte(fixchar(content))),
        token::DocComment(..)      => token::DocComment(nm),
        token::Literal(token::Integer(..))   => token::Literal(token::Integer(nm)),
        token::Literal(token::Float(..))     => token::Literal(token::Float(nm)),
        token::Literal(token::Binary(..))    => token::Literal(token::Binary(nm)),
        token::Literal(token::BinaryRaw(..)) => token::Literal(token::BinaryRaw(fix(content),
                                                                                count(content))),
        token::Ident(..)           => token::Ident(ast::Ident { name: nm, ctxt: 0 },
                                                   token::ModName),
        token::Lifetime(..)        => token::Lifetime(ast::Ident { name: nm, ctxt: 0 }),
        ref t => t.clone()
    };

    let offset = if real_tok == token::Eof
 {
        1
    } else {
        0
    };

    let sp = syntax::codemap::Span {
        lo: syntax::codemap::BytePos(from_str::<u32>(start).unwrap() - offset),
        hi: syntax::codemap::BytePos(from_str::<u32>(end).unwrap() + 1),
        expn_id: syntax::codemap::NO_EXPANSION
    };

    TokenAndSpan {
        tok: real_tok,
        sp: sp
    }
}

fn tok_cmp(a: &token::Token, b: &token::Token) -> bool {
    match a {
        &token::Ident(id, _) => match b {
                &token::Ident(id2, _) => id == id2,
                _ => false
        },
        _ => a == b
    }
}

fn main() {
    fn next(r: &mut lexer::StringReader) -> TokenAndSpan {
        use syntax::parse::lexer::Reader;
        r.next_token()
    }

    let args = std::os::args();

    let mut token_file = File::open(&Path::new(args[2].as_slice()));
    let token_map = parse_token_list(token_file.read_to_string().unwrap().as_slice());

    let mut stdin = std::io::stdin();
    let mut antlr_tokens = stdin.lines().map(|l| parse_antlr_token(l.unwrap().as_slice().trim(),
                                                                   &token_map));

    let code = File::open(&Path::new(args[1].as_slice())).unwrap().read_to_string().unwrap();
    let options = config::basic_options();
    let session = session::build_session(options, None,
                                         syntax::diagnostics::registry::Registry::new(&[]));
    let filemap = parse::string_to_filemap(&session.parse_sess,
                                           code,
                                           String::from_str("<n/a>"));
    let mut lexer = lexer::StringReader::new(session.diagnostic(), filemap);

    for antlr_tok in antlr_tokens {
        let rustc_tok = next(&mut lexer);
        if rustc_tok.tok == token::Eof && antlr_tok.tok == token::Eof {
            continue
        }

        assert!(rustc_tok.sp == antlr_tok.sp, "{} and {} have different spans", rustc_tok,
                antlr_tok);

        macro_rules! matches (
            ( $($x:pat),+ ) => (
                match rustc_tok.tok {
                    $($x => match antlr_tok.tok {
                        $x => {
                            if !tok_cmp(&rustc_tok.tok, &antlr_tok.tok) {
                                // FIXME #15677: needs more robust escaping in
                                // antlr
                                warn!("Different names for {} and {}", rustc_tok, antlr_tok);
                            }
                        }
                        _ => panic!("{} is not {}", antlr_tok, rustc_tok)
                    },)*
                    ref c => assert!(c == &antlr_tok.tok, "{} is not {}", rustc_tok, antlr_tok)
                }
            )
        )

        matches!(
            token::Literal(token::Byte(..)),
            token::Literal(token::Char(..)),
            token::Literal(token::Integer(..)),
            token::Literal(token::Float(..)),
            token::Literal(token::Str_(..)),
            token::Literal(token::StrRaw(..)),
            token::Literal(token::Binary(..)),
            token::Literal(token::BinaryRaw(..)),
            token::Ident(..),
            token::Lifetime(..),
            token::Interpolated(..),
            token::DocComment(..),
            token::Shebang(..)
        );
    }
}
<|endoftext|>


<commit_before>use crate::utils::printing;

pub fn next() {
    printing::subheading("Read a document so that I increase what I know.");
    printing::list(vec![
        "[X] http://linuxcommand.org/tlcl.php",
        "[X] https://pragprog.com/book/dnvim2/practical-vim-second-edition",
        "[ ] https://doc.rust-lang.org/book/",
        "[ ] https://doc.rust-lang.org/stable/rust-by-example/",
        "[ ] https://doc.rust-lang.org/std/index.html",
        "[ ] https://rust-lang-nursery.github.io/cli-wg/",
        "[ ] https://rustwasm.github.io/docs/book/",
        "[ ] https://archive.org/details/MythicalManMonth",
        "[ ] http://www.charlespetzold.com/code/",
        "[ ] https://pragprog.com/book/tpp/the-pragmatic-programmer",
        "[ ] http://codersatwork.com",
        "[ ] https://mitpress.mit.edu/books/introduction-algorithms"
    ]);
    printing::pause();
}
<commit_msg>Format<commit_after>use crate::utils::printing;

pub fn next() {
    printing::subheading("Read a document so that I increase what I know.");
    printing::list(vec![
        "[X] http://linuxcommand.org/tlcl.php",
        "[X] https://pragprog.com/book/dnvim2/practical-vim-second-edition",
        "[ ] https://doc.rust-lang.org/book/",
        "[ ] https://doc.rust-lang.org/stable/rust-by-example/",
        "[ ] https://doc.rust-lang.org/std/index.html",
        "[ ] https://rust-lang-nursery.github.io/cli-wg/",
        "[ ] https://rustwasm.github.io/docs/book/",
        "[ ] https://archive.org/details/MythicalManMonth",
        "[ ] http://www.charlespetzold.com/code/",
        "[ ] https://pragprog.com/book/tpp/the-pragmatic-programmer",
        "[ ] http://codersatwork.com",
        "[ ] https://mitpress.mit.edu/books/introduction-algorithms",
    ]);
    printing::pause();
}
<|endoftext|>


<commit_before>use super::ast::{Span, SourceMap, ModuleId};
use super::core::Name;

use std::cell::RefCell;
use std::collections::{HashSet, HashMap};
use std::env;
use std::path::{PathBuf, Path};
use std::io;
use std::rc::Rc;
use std::io::prelude::*;

use term::{self, Terminal, color, stdout, StdoutTerminal};

/// A type that contains a session either directly or
/// transitively.
pub trait HasSession {
    #[inline]
    fn session(&self) -> &Session;

    #[inline]
    fn report<E: Reportable>(&self, error: E) -> io::Result<()> {
        error.report(self.session())
    }
}

/// A trait for errors that constructs the appropriate
/// messages
pub trait Reportable {
    fn report(self, session: &Session) -> io::Result<()>;
}

/// The session is a global configuration object which
/// stores information that needs to be known across
/// elaboration, type checking and eventually
/// code generation.
pub struct Session {
    /// The fields that are mutable and must be shared across
    /// all copies of the session object.
    data: Rc<RefCell<SessionData>>,

    /// This might be the wrong set up.
    pub ty: SessionType,
}

impl Clone for Session {
    fn clone(&self) -> Session {
        Session {
            data: self.data.clone(),
            ty: self.ty.clone(),
        }
    }
}

pub struct SessionData {
    /// The terminal that is used for errors reporting.
    pub terminal: Box<StdoutTerminal>,
    /// A global counter used to track how many module ids
    /// we have handed out.
    module_id_counter: usize,
    /// The set of things that have been imported.
    imports: HashSet<Name>,
    /// An index from module id to source map.
    source_maps: HashMap<ModuleId, SourceMap>,
}

#[derive(Clone)]
pub enum SessionType {
    Compiler {
        /// The file that is the module root.
        root_file: PathBuf,
    },
    Repl {
        /// The file passed on the command line.
        loaded_file: Option<PathBuf>,
    }
}

impl Session   {
    pub fn empty() -> Session {
        Session {
            data: Rc::new(RefCell::new(SessionData {
                terminal: term::stdout().unwrap(), // Not sure about this, we can revisit it later.
                module_id_counter: 0,
                imports: HashSet::new(),
                source_maps: HashMap::new(),
            })),
            ty: SessionType::Repl { loaded_file: None },
        }
    }

    pub fn from_root(path: &Path) -> Session {
        Session {
            data: Rc::new(RefCell::new(SessionData {
                terminal: term::stdout().unwrap(), // Not sure about this, we can revisit it later.
                module_id_counter: 0,
                imports: HashSet::new(),
                source_maps: HashMap::new(),
            })),
            ty: SessionType::Compiler { root_file: path.to_owned() }
        }
    }

    /// Return the root file if this is a compiler session.
    pub fn root_file(&self) -> PathBuf {
        match self.ty {
            SessionType::Compiler { ref root_file, .. } =>
                root_file.to_owned(),
            SessionType::Repl { ref loaded_file, .. } =>
                loaded_file.as_ref()
                           .map(|x| x.to_owned())
                           .unwrap_or(env::current_dir().unwrap()),
        }
    }

    pub fn next_module_id(&self) -> ModuleId {
        let id = self.data.borrow().module_id_counter;
        self.data.borrow_mut().module_id_counter += 1;
        ModuleId(id)
    }

    pub fn add_source_map_for(&self, id: ModuleId, source_map: SourceMap) {
        let mut data = self.data.borrow_mut();
        data.source_maps.insert(id, source_map);
    }

    /// Reports a message at a given location. Underlines the Span.
    pub fn span_error(&self,
                      span: Span,
                      message: String) -> io::Result<()> {

        let mut session_data = self.data.borrow_mut();
        let &mut SessionData {
            ref mut terminal,
            ref mut source_maps,
            .. } = &mut *session_data;

        let module_id = span.module_id;
        let source_map = source_maps.get(&module_id).unwrap();

        // TODO: We need to know if we wrap around to more then one line.
        let (line_no, col_no) = source_map.position(span)
                                          .unwrap_or((0,0));

        let (line_with_padding, marker) = source_map
                                              .underline_span(span)
                                              .unwrap_or((format!("??"),format!("??")));

        let filename_str = format!("{}:{}:{}: {}:{} ",
            source_map.file_name,
            line_no,
            col_no,
            line_no, // this should be the line where we end, not the same line
            col_no + (span.hi - span.lo)); // this should be the column we end at

        try!(write!(terminal, "{}", filename_str));

        try!(terminal.fg(color::RED));
        try!(write!(terminal, "error: "));
        try!(terminal.reset());
        try!(writeln!(terminal, "{}", message));

        let file_str_simple =
            format!("{}:{}: ",
                source_map.file_name,
                line_no);

        try!(write!(terminal, "{} {}", file_str_simple, line_with_padding));

        let mut marker_padding = "".to_string();

        for _ in 0..file_str_simple.len() {
            marker_padding.push(' ');
        }

        try!(write!(terminal, "{}", marker_padding));
        try!(terminal.fg(color::RED));
        try!(writeln!(terminal, "{}", marker));
        try!(terminal.reset());
        try!(terminal.flush());

        Ok(())
    }

    pub fn error(&self, message: String) -> io::Result<()> {
        let mut session_data = self.data.borrow_mut();
        let &mut SessionData {
            ref mut terminal,
            ref mut source_maps,
            .. } = &mut *session_data;

        try!(terminal.fg(color::RED));
        try!(writeln!(terminal, "{}", message));
        try!(terminal.reset());
        try!(terminal.flush());
        Ok(())
    }
}

impl HasSession for Session {
    fn session(&self) -> &Session {
        self
    }
}
<commit_msg>Fix panic when sourcemap does not exist in repl<commit_after>use super::ast::{Span, SourceMap, ModuleId};
use super::core::Name;

use std::cell::RefCell;
use std::collections::{HashSet, HashMap};
use std::env;
use std::path::{PathBuf, Path};
use std::io;
use std::rc::Rc;
use std::io::prelude::*;

use term::{self, Terminal, color, stdout, StdoutTerminal};

/// A type that contains a session either directly or
/// transitively.
pub trait HasSession {
    #[inline]
    fn session(&self) -> &Session;

    #[inline]
    fn report<E: Reportable>(&self, error: E) -> io::Result<()> {
        error.report(self.session())
    }
}

/// A trait for errors that constructs the appropriate
/// messages
pub trait Reportable {
    fn report(self, session: &Session) -> io::Result<()>;
}

/// The session is a global configuration object which
/// stores information that needs to be known across
/// elaboration, type checking and eventually
/// code generation.
pub struct Session {
    /// The fields that are mutable and must be shared across
    /// all copies of the session object.
    data: Rc<RefCell<SessionData>>,

    /// This might be the wrong set up.
    pub ty: SessionType,
}

impl Clone for Session {
    fn clone(&self) -> Session {
        Session {
            data: self.data.clone(),
            ty: self.ty.clone(),
        }
    }
}

pub struct SessionData {
    /// The terminal that is used for errors reporting.
    pub terminal: Box<StdoutTerminal>,
    /// A global counter used to track how many module ids
    /// we have handed out.
    module_id_counter: usize,
    /// The set of things that have been imported.
    imports: HashSet<Name>,
    /// An index from module id to source map.
    source_maps: HashMap<ModuleId, SourceMap>,
}

#[derive(Clone)]
pub enum SessionType {
    Compiler {
        /// The file that is the module root.
        root_file: PathBuf,
    },
    Repl {
        /// The file passed on the command line.
        loaded_file: Option<PathBuf>,
    }
}

impl Session   {
    pub fn empty() -> Session {
        Session {
            data: Rc::new(RefCell::new(SessionData {
                terminal: term::stdout().unwrap(), // Not sure about this, we can revisit it later.
                module_id_counter: 0,
                imports: HashSet::new(),
                source_maps: HashMap::new(),
            })),
            ty: SessionType::Repl { loaded_file: None },
        }
    }

    pub fn from_root(path: &Path) -> Session {
        Session {
            data: Rc::new(RefCell::new(SessionData {
                terminal: term::stdout().unwrap(), // Not sure about this, we can revisit it later.
                module_id_counter: 0,
                imports: HashSet::new(),
                source_maps: HashMap::new(),
            })),
            ty: SessionType::Compiler { root_file: path.to_owned() }
        }
    }

    /// Return the root file if this is a compiler session.
    pub fn root_file(&self) -> PathBuf {
        match self.ty {
            SessionType::Compiler { ref root_file, .. } =>
                root_file.to_owned(),
            SessionType::Repl { ref loaded_file, .. } =>
                loaded_file.as_ref()
                           .map(|x| x.to_owned())
                           .unwrap_or(env::current_dir().unwrap()),
        }
    }

    pub fn next_module_id(&self) -> ModuleId {
        let id = self.data.borrow().module_id_counter;
        self.data.borrow_mut().module_id_counter += 1;
        ModuleId(id)
    }

    pub fn add_source_map_for(&self, id: ModuleId, source_map: SourceMap) {
        let mut data = self.data.borrow_mut();
        data.source_maps.insert(id, source_map);
    }

    /// Reports a message at a given location. Underlines the Span.
    pub fn span_error(&self,
                      span: Span,
                      message: String) -> io::Result<()> {

        let mut session_data = self.data.borrow_mut();
        let &mut SessionData {
            ref mut terminal,
            ref mut source_maps,
            .. } = &mut *session_data;

        let module_id = span.module_id;
        let emp = SourceMap::empty();
        let source_map = source_maps.get(&module_id).unwrap_or(&emp);

        // TODO: We need to know if we wrap around to more then one line.
        let (line_no, col_no) = source_map.position(span)
                                          .unwrap_or((0,0));

        let (line_with_padding, marker) = source_map
                                              .underline_span(span)
                                              .unwrap_or((format!("??"),format!("??")));

        let filename_str = format!("{}:{}:{}: {}:{} ",
            source_map.file_name,
            line_no,
            col_no,
            line_no, // this should be the line where we end, not the same line
            col_no + (span.hi - span.lo)); // this should be the column we end at

        try!(write!(terminal, "{}", filename_str));

        try!(terminal.fg(color::RED));
        try!(write!(terminal, "error: "));
        try!(terminal.reset());
        try!(writeln!(terminal, "{}", message));

        let file_str_simple =
            format!("{}:{}: ",
                source_map.file_name,
                line_no);

        try!(write!(terminal, "{} {}", file_str_simple, line_with_padding));

        let mut marker_padding = "".to_string();

        for _ in 0..file_str_simple.len() {
            marker_padding.push(' ');
        }

        try!(write!(terminal, "{}", marker_padding));
        try!(terminal.fg(color::RED));
        try!(writeln!(terminal, "{}", marker));
        try!(terminal.reset());
        try!(terminal.flush());

        Ok(())
    }

    pub fn error(&self, message: String) -> io::Result<()> {
        let mut session_data = self.data.borrow_mut();
        let &mut SessionData {
            ref mut terminal,
            ref mut source_maps,
            .. } = &mut *session_data;

        try!(terminal.fg(color::RED));
        try!(writeln!(terminal, "{}", message));
        try!(terminal.reset());
        try!(terminal.flush());
        Ok(())
    }
}

impl HasSession for Session {
    fn session(&self) -> &Session {
        self
    }
}
<|endoftext|>