<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

extern crate rustc_serialize;
extern crate docopt;
extern crate csv;
extern crate mimir;
extern crate itertools;
extern crate mimirsbrunn;
#[macro_use]
extern crate log;

use std::rc::Rc;
use mimir::rubber::Rubber;
use docopt::Docopt;
use mimirsbrunn::utils::{format_label, get_zip_codes_from_admins};
use mimirsbrunn::admin_geofinder::AdminGeoFinder;
use std::collections::HashMap;

const USAGE: &'static str = "
Usage:
    stops2mimir --help
    stops2mimir --input=<file> \
     [--connection-string=<connection-string>] [--dataset=<dataset>] [--city-level=<level>]

Options:
    -h, --help                Show this message.
    -i, --input=<file>        NTFS stops.txt file.
    -c, --connection-string=<connection-string>   \
                              Elasticsearch parameters [default: http://localhost:9200/munin].
    -d, --dataset=<dataset>   Name of the dataset [default: fr].
    -C, --city-level=<level>  City level to calculate weight [default: 8]
";

#[derive(Debug, RustcDecodable)]
struct Args {
    flag_input: String,
    flag_dataset: String,
    flag_connection_string: String,
    flag_city_level: u32,
}

struct StopPointIter<'a, R: std::io::Read + 'a> {
    iter: csv::StringRecords<'a, R>,
    stop_id_pos: usize,
    stop_lat_pos: usize,
    stop_lon_pos: usize,
    stop_name_pos: usize,
    location_type_pos: Option<usize>,
    stop_visible_pos: Option<usize>,
    parent_station_pos: Option<usize>,
    nb_stop_points: &'a mut HashMap<String, u32>,
}

impl<'a, R: std::io::Read + 'a> StopPointIter<'a, R> {
    fn new(r: &'a mut csv::Reader<R>,
           nb_stop_points: &'a mut HashMap<String, u32>)
           -> csv::Result<Self> {
        let headers = try!(r.headers());
        let get_optional_pos = |name| headers.iter().position(|s| s == name);

        let get_pos = |field| {
            get_optional_pos(field).ok_or_else(|| {
                csv::Error::Decode(format!("Invalid file, cannot find column '{}'", field))
            })
        };

        Ok(StopPointIter {
            iter: r.records(),
            stop_id_pos: try!(get_pos("stop_id")),
            stop_lat_pos: try!(get_pos("stop_lat")),
            stop_lon_pos: try!(get_pos("stop_lon")),
            stop_name_pos: try!(get_pos("stop_name")),
            location_type_pos: get_optional_pos("location_type"),
            stop_visible_pos: get_optional_pos("visible"),
            parent_station_pos: get_optional_pos("parent_station"),
            nb_stop_points: nb_stop_points,
        })
    }
    fn get_location_type(&self, record: &[String]) -> Option<u8> {
        self.location_type_pos.and_then(|pos| record.get(pos).and_then(|s| s.parse().ok()))
    }
    fn get_visible(&self, record: &[String]) -> Option<u8> {
        self.stop_visible_pos.and_then(|pos| record.get(pos).and_then(|s| s.parse().ok()))
    }

    fn get_parent_station<'b>(&self, record: &'b [String]) -> Option<&'b String> {
        self.parent_station_pos.and_then(|pos| record.get(pos))
    }
}

impl<'a, R: std::io::Read + 'a> Iterator for StopPointIter<'a, R> {
    type Item = csv::Result<mimir::Stop>;
    fn next(&mut self) -> Option<Self::Item> {
        fn get(record: &[String], pos: usize) -> csv::Result<&str> {
            record.get(pos)
                .map(|s| s.as_str())
                .ok_or_else(|| csv::Error::Decode(format!("Failed accessing record '{}'.", pos)))
        }
        fn parse_f64(s: &str) -> csv::Result<f64> {
            s.parse()
                .map_err(|_| csv::Error::Decode(format!("Failed converting '{}' from str.", s)))
        }

        fn is_valid_stop_area(location_type: &Option<u8>, visible: &Option<u8>) -> csv::Result<()> {
            if *location_type != Some(1) {
                Err(csv::Error::Decode("not a stop_area.".to_string()))
            } else if *visible == Some(0) {
                Err(csv::Error::Decode("stop_area invisible.".to_string()))
            } else {
                Ok(())
            }
        }

        self.iter.next().map(|r| {
            r.and_then(|r| {
                let location_type = self.get_location_type(&r);
                let visible = self.get_visible(&r);
                let parent_station = self.get_parent_station(&r);
                //if it's a stop, we update its stop_area counter
                if let (Some(0), Some(id)) = (location_type, parent_station) {
                    if !id.is_empty() {
                        *self.nb_stop_points.entry(format!("stop_area:{}", id)).or_insert(0) += 1;
                    }
                }
                try!(is_valid_stop_area(&location_type, &visible));
                let stop_id = try!(get(&r, self.stop_id_pos));
                let stop_lat = try!(get(&r, self.stop_lat_pos));
                let stop_lat = try!(parse_f64(stop_lat));
                let stop_lon = try!(get(&r, self.stop_lon_pos));
                let stop_lon = try!(parse_f64(stop_lon));
                let stop_name = try!(get(&r, self.stop_name_pos));
                Ok(mimir::Stop {
                    id: format!("stop_area:{}", stop_id), // prefix to match navitia's id
                    coord: mimir::Coord::new(stop_lat, stop_lon),
                    label: stop_name.to_string(),
                    weight: 1,
                    zip_codes: vec![],
                    administrative_regions: vec![],
                    name: stop_name.to_string(),
                })
            })
        })
    }
}

fn attach_stop(stop: &mut mimir::Stop, admins: Vec<Rc<mimir::Admin>>, city_level: u32) {
    stop.administrative_regions = admins;
    stop.label = format_label(&stop.administrative_regions, city_level, &stop.name);
    stop.zip_codes = get_zip_codes_from_admins(&stop.administrative_regions);
}

/// Attach the stops to administrative regions
///
/// The admins are loaded from Elasticsearch and stored in a quadtree
/// We attach a stop with all the admins that have a boundary containing
/// the coordinate of the stop
fn attach_stops_to_admins<'a, It: Iterator<Item = &'a mut mimir::Stop>>(stops: It,
                                                                        rubber: &mut Rubber,
                                                                        city_level: u32) {
    let admins = rubber.get_all_admins().unwrap_or_else(|_| {
        info!("Administratives regions not found in elasticsearch db");
        vec![]
    });

    info!("{} administrative regions loaded from mimir", admins.len());

    let admins_geofinder = admins.into_iter().collect::<AdminGeoFinder>();

    let mut nb_unmatched = 0u32;
    let mut nb_matched = 0u32;
    for mut stop in stops {
        let admins = admins_geofinder.get(&stop.coord);

        if admins.is_empty() {
            nb_unmatched += 1;
        } else {
            nb_matched += 1;
        }

        attach_stop(&mut stop, admins, city_level);
    }

    info!("there is {}/{} stops without any admin",
          nb_unmatched,
          nb_matched + nb_unmatched);
}

// Update weight value for each stop_area from HashMap.
fn finalize_stop_area_weight<'a, It: Iterator<Item = &'a mut mimir::Stop>>(stops: It,
    nb_stop_points: &HashMap<String, u32>) {
    for stop in stops {
        if let Some(weight) = nb_stop_points.get(&stop.id) {
            stop.weight = *weight;
        }
    }
}

fn main() {
    mimir::logger_init().unwrap();
    info!("Launching stops2mimir...");

    let args: Args = Docopt::new(USAGE).and_then(|dopt| dopt.decode()).unwrap_or_else(|e| e.exit());

    info!("creation of indexes");
    let mut rubber = Rubber::new(&args.flag_connection_string);
    let mut rdr = csv::Reader::from_file(args.flag_input).unwrap().double_quote(true);

    let mut nb_stop_points = HashMap::new();

    let mut stops: Vec<mimir::Stop> = StopPointIter::new(&mut rdr, &mut nb_stop_points)
        .unwrap()
        .filter_map(|rc| {
            rc.map_err(|e| debug!("skip csv line because: {}", e))
                .ok()
        })
        .collect();

    attach_stops_to_admins(stops.iter_mut(), &mut rubber, args.flag_city_level);

    finalize_stop_area_weight(stops.iter_mut(), &nb_stop_points);

    info!("Importing stops into Mimir");
    let nb_stops = rubber.index(&args.flag_dataset, stops.iter()).unwrap();

    info!("Nb of indexed stops: {}", nb_stops);

}

#[test]
fn test_load_stops() {
    use itertools::Itertools;
    // stops.txt:
    // SP:main_station : StopPoint object
    // SA:main_station: StopArea valid
    // SA:second_station: StopArea valid with visible is empty
    // SA:invisible_station: invisible StopArea
    // SA:without_lat: StopArea object without lattitude coord
    // SA:witout_lon: StopArea object without longitude coord
    // SA:station_no_city: StopArea far away, we won't be able to attach it to a city
    let mut rdr = csv::Reader::from_file("./tests/fixtures/stops.txt".to_string())
        .unwrap()
        .double_quote(true);

    let mut nb_stop_points = HashMap::new();
    let stops: Vec<mimir::Stop> = StopPointIter::new(&mut rdr, &mut nb_stop_points)
        .unwrap()
        .filter_map(|rc| {
            rc.map_err(|e| println!("error at csv line decoding : {}", e))
                .ok()
        })
        .collect();
    assert_eq!(stops.len(), 5);
    let ids: Vec<_> = stops.iter().map(|s| s.id.clone()).sorted();
    assert_eq!(ids,
               vec!["stop_area:SA:main_station",
                    "stop_area:SA:second_station",
                    "stop_area:SA:station_no_city",
                    "stop_area:SA:weight_1_station",
                    "stop_area:SA:weight_3_station"]);
}
<commit_msg>mimir: comment modified<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

extern crate rustc_serialize;
extern crate docopt;
extern crate csv;
extern crate mimir;
extern crate itertools;
extern crate mimirsbrunn;
#[macro_use]
extern crate log;

use std::rc::Rc;
use mimir::rubber::Rubber;
use docopt::Docopt;
use mimirsbrunn::utils::{format_label, get_zip_codes_from_admins};
use mimirsbrunn::admin_geofinder::AdminGeoFinder;
use std::collections::HashMap;

const USAGE: &'static str = "
Usage:
    stops2mimir --help
    stops2mimir --input=<file> \
     [--connection-string=<connection-string>] [--dataset=<dataset>] [--city-level=<level>]

Options:
    -h, --help                Show this message.
    -i, --input=<file>        NTFS stops.txt file.
    -c, --connection-string=<connection-string>   \
                              Elasticsearch parameters [default: http://localhost:9200/munin].
    -d, --dataset=<dataset>   Name of the dataset [default: fr].
    -C, --city-level=<level>  City level to calculate weight [default: 8]
";

#[derive(Debug, RustcDecodable)]
struct Args {
    flag_input: String,
    flag_dataset: String,
    flag_connection_string: String,
    flag_city_level: u32,
}

struct StopPointIter<'a, R: std::io::Read + 'a> {
    iter: csv::StringRecords<'a, R>,
    stop_id_pos: usize,
    stop_lat_pos: usize,
    stop_lon_pos: usize,
    stop_name_pos: usize,
    location_type_pos: Option<usize>,
    stop_visible_pos: Option<usize>,
    parent_station_pos: Option<usize>,
    nb_stop_points: &'a mut HashMap<String, u32>,
}

impl<'a, R: std::io::Read + 'a> StopPointIter<'a, R> {
    fn new(r: &'a mut csv::Reader<R>,
           nb_stop_points: &'a mut HashMap<String, u32>)
           -> csv::Result<Self> {
        let headers = try!(r.headers());
        let get_optional_pos = |name| headers.iter().position(|s| s == name);

        let get_pos = |field| {
            get_optional_pos(field).ok_or_else(|| {
                csv::Error::Decode(format!("Invalid file, cannot find column '{}'", field))
            })
        };

        Ok(StopPointIter {
            iter: r.records(),
            stop_id_pos: try!(get_pos("stop_id")),
            stop_lat_pos: try!(get_pos("stop_lat")),
            stop_lon_pos: try!(get_pos("stop_lon")),
            stop_name_pos: try!(get_pos("stop_name")),
            location_type_pos: get_optional_pos("location_type"),
            stop_visible_pos: get_optional_pos("visible"),
            parent_station_pos: get_optional_pos("parent_station"),
            nb_stop_points: nb_stop_points,
        })
    }
    fn get_location_type(&self, record: &[String]) -> Option<u8> {
        self.location_type_pos.and_then(|pos| record.get(pos).and_then(|s| s.parse().ok()))
    }
    fn get_visible(&self, record: &[String]) -> Option<u8> {
        self.stop_visible_pos.and_then(|pos| record.get(pos).and_then(|s| s.parse().ok()))
    }

    fn get_parent_station<'b>(&self, record: &'b [String]) -> Option<&'b String> {
        self.parent_station_pos.and_then(|pos| record.get(pos))
    }
}

impl<'a, R: std::io::Read + 'a> Iterator for StopPointIter<'a, R> {
    type Item = csv::Result<mimir::Stop>;
    fn next(&mut self) -> Option<Self::Item> {
        fn get(record: &[String], pos: usize) -> csv::Result<&str> {
            record.get(pos)
                .map(|s| s.as_str())
                .ok_or_else(|| csv::Error::Decode(format!("Failed accessing record '{}'.", pos)))
        }
        fn parse_f64(s: &str) -> csv::Result<f64> {
            s.parse()
                .map_err(|_| csv::Error::Decode(format!("Failed converting '{}' from str.", s)))
        }

        fn is_valid_stop_area(location_type: &Option<u8>, visible: &Option<u8>) -> csv::Result<()> {
            if *location_type != Some(1) {
                Err(csv::Error::Decode("not a stop_area.".to_string()))
            } else if *visible == Some(0) {
                Err(csv::Error::Decode("stop_area invisible.".to_string()))
            } else {
                Ok(())
            }
        }

        self.iter.next().map(|r| {
            r.and_then(|r| {
                let location_type = self.get_location_type(&r);
                let visible = self.get_visible(&r);
                let parent_station = self.get_parent_station(&r);
                //if it's a stop point, we update its stop_area counter
                if let (Some(0), Some(id)) = (location_type, parent_station) {
                    if !id.is_empty() {
                        *self.nb_stop_points.entry(format!("stop_area:{}", id)).or_insert(0) += 1;
                    }
                }
                try!(is_valid_stop_area(&location_type, &visible));
                let stop_id = try!(get(&r, self.stop_id_pos));
                let stop_lat = try!(get(&r, self.stop_lat_pos));
                let stop_lat = try!(parse_f64(stop_lat));
                let stop_lon = try!(get(&r, self.stop_lon_pos));
                let stop_lon = try!(parse_f64(stop_lon));
                let stop_name = try!(get(&r, self.stop_name_pos));
                Ok(mimir::Stop {
                    id: format!("stop_area:{}", stop_id), // prefix to match navitia's id
                    coord: mimir::Coord::new(stop_lat, stop_lon),
                    label: stop_name.to_string(),
                    weight: 1,
                    zip_codes: vec![],
                    administrative_regions: vec![],
                    name: stop_name.to_string(),
                })
            })
        })
    }
}

fn attach_stop(stop: &mut mimir::Stop, admins: Vec<Rc<mimir::Admin>>, city_level: u32) {
    stop.administrative_regions = admins;
    stop.label = format_label(&stop.administrative_regions, city_level, &stop.name);
    stop.zip_codes = get_zip_codes_from_admins(&stop.administrative_regions);
}

/// Attach the stops to administrative regions
///
/// The admins are loaded from Elasticsearch and stored in a quadtree
/// We attach a stop with all the admins that have a boundary containing
/// the coordinate of the stop
fn attach_stops_to_admins<'a, It: Iterator<Item = &'a mut mimir::Stop>>(stops: It,
                                                                        rubber: &mut Rubber,
                                                                        city_level: u32) {
    let admins = rubber.get_all_admins().unwrap_or_else(|_| {
        info!("Administratives regions not found in elasticsearch db");
        vec![]
    });

    info!("{} administrative regions loaded from mimir", admins.len());

    let admins_geofinder = admins.into_iter().collect::<AdminGeoFinder>();

    let mut nb_unmatched = 0u32;
    let mut nb_matched = 0u32;
    for mut stop in stops {
        let admins = admins_geofinder.get(&stop.coord);

        if admins.is_empty() {
            nb_unmatched += 1;
        } else {
            nb_matched += 1;
        }

        attach_stop(&mut stop, admins, city_level);
    }

    info!("there is {}/{} stops without any admin",
          nb_unmatched,
          nb_matched + nb_unmatched);
}

// Update weight value for each stop_area from HashMap.
fn finalize_stop_area_weight<'a, It: Iterator<Item = &'a mut mimir::Stop>>(stops: It,
    nb_stop_points: &HashMap<String, u32>) {
    for stop in stops {
        if let Some(weight) = nb_stop_points.get(&stop.id) {
            stop.weight = *weight;
        }
    }
}

fn main() {
    mimir::logger_init().unwrap();
    info!("Launching stops2mimir...");

    let args: Args = Docopt::new(USAGE).and_then(|dopt| dopt.decode()).unwrap_or_else(|e| e.exit());

    info!("creation of indexes");
    let mut rubber = Rubber::new(&args.flag_connection_string);
    let mut rdr = csv::Reader::from_file(args.flag_input).unwrap().double_quote(true);

    let mut nb_stop_points = HashMap::new();

    let mut stops: Vec<mimir::Stop> = StopPointIter::new(&mut rdr, &mut nb_stop_points)
        .unwrap()
        .filter_map(|rc| {
            rc.map_err(|e| debug!("skip csv line because: {}", e))
                .ok()
        })
        .collect();

    attach_stops_to_admins(stops.iter_mut(), &mut rubber, args.flag_city_level);

    finalize_stop_area_weight(stops.iter_mut(), &nb_stop_points);

    info!("Importing stops into Mimir");
    let nb_stops = rubber.index(&args.flag_dataset, stops.iter()).unwrap();

    info!("Nb of indexed stops: {}", nb_stops);

}

#[test]
fn test_load_stops() {
    use itertools::Itertools;
    // stops.txt:
    // SP:main_station : StopPoint object
    // SA:main_station: StopArea valid
    // SA:second_station: StopArea valid with visible is empty
    // SA:invisible_station: invisible StopArea
    // SA:without_lat: StopArea object without lattitude coord
    // SA:witout_lon: StopArea object without longitude coord
    // SA:station_no_city: StopArea far away, we won't be able to attach it to a city
    let mut rdr = csv::Reader::from_file("./tests/fixtures/stops.txt".to_string())
        .unwrap()
        .double_quote(true);

    let mut nb_stop_points = HashMap::new();
    let stops: Vec<mimir::Stop> = StopPointIter::new(&mut rdr, &mut nb_stop_points)
        .unwrap()
        .filter_map(|rc| {
            rc.map_err(|e| println!("error at csv line decoding : {}", e))
                .ok()
        })
        .collect();
    assert_eq!(stops.len(), 5);
    let ids: Vec<_> = stops.iter().map(|s| s.id.clone()).sorted();
    assert_eq!(ids,
               vec!["stop_area:SA:main_station",
                    "stop_area:SA:second_station",
                    "stop_area:SA:station_no_city",
                    "stop_area:SA:weight_1_station",
                    "stop_area:SA:weight_3_station"]);
}
<|endoftext|>


<commit_before>use std::fmt;
use std::ops::{Deref, DerefMut};
use std::ptr::{copy_nonoverlapping};
use std::mem::{uninitialized, transmute, forget, size_of, align_of};
use std::slice::{from_raw_parts, from_raw_parts_mut};
use std::marker;
use ::{ArrayVec, Vector};

#[derive(Clone)]
pub struct SmallDST<T: ?Sized, A: Vector<Item=usize> = ArrayVec<[usize; 9]>> {
    data: A,
    _phantom: marker::PhantomData<T>,
}

impl<T: ?Sized, A: Vector<Item=usize>> SmallDST<T, A> {
    fn ptr_len() -> usize {
        assert_eq!(size_of::<*const T>() % size_of::<usize>(), 0);
        size_of::<*const T>() / size_of::<usize>()
    }

    fn data_len() -> usize {
        Self::ptr_len() - 1
    }

    fn ref_data<'a>(v: &'a *const T) -> &'a [usize] {
        unsafe { from_raw_parts(v as *const *const T as *const usize, Self::ptr_len()) }
    }

    fn mut_data<'a>(v: &'a mut *const T) -> &'a mut [usize] {
        unsafe { from_raw_parts_mut(v as *mut *const T as *mut usize, Self::ptr_len()) }
    }

    #[cfg(feature = "unstable")]
    pub fn new<U: marker::Unsize<T>>(v: U) -> Self {
        let ret = unsafe { Self::new_ref(&v, &v) };
        forget(v);
        ret
    }

    #[inline]
    pub unsafe fn new_ref<U>(v: &T, u: &U) -> Self {
        let t = v as *const T;
        let data = Self::ref_data(&t);
        assert_eq!(data[0], u as *const U as usize);

        Self::from_ref(u, &data[1..])
    }

    #[inline]
    pub unsafe fn data(&self) -> &A {
        &self.data
    }

    unsafe fn from_value<U>(v: U, data: &[usize]) -> Self {
        let ret = Self::from_ref(&v, data);
        forget(v);
        ret
    }

    unsafe fn from_ref<U>(v: &U, data: &[usize]) -> Self {
        assert!(align_of::<U>() <= align_of::<A::Item>()); // TODO: support arbitrary alignments
        assert_eq!(data.len(), Self::data_len());

        let size = (size_of::<U>() + size_of::<usize>() - 1) / size_of::<usize>();

        let mut a = A::with_capacity(Self::data_len() + size);
        a.set_len(Self::data_len() + size);
        copy_nonoverlapping(data.as_ptr(), a.as_mut_ptr(), Self::data_len());
        copy_nonoverlapping(v as *const U as *const u8, a.as_mut_ptr().offset(Self::data_len() as isize) as *mut u8, size_of::<U>());

        SmallDST {
            data: a,
            _phantom: marker::PhantomData,
        }
    }
}

impl<T: ?Sized, A: Vector<Item=usize>> Drop for SmallDST<T, A> {
    #[cfg(feature = "unstable")]
    #[inline]
    fn drop(&mut self) {
        use std::intrinsics;

        unsafe { intrinsics::drop_in_place(&mut **self); }
    }

    #[cfg(not(feature = "unstable"))]
    fn drop(&mut self) {
        use ::SmallVec;
        use std::rc::Rc;

        struct RcBox<T: ?Sized> {
            _strong: usize,
            _weak: usize,
            _value: T,
        }

        unsafe {
            let mut rc_box: SmallDST<T, SmallVec<[usize; 0x10]>> = SmallDST::from_ref(&RcBox {
                _strong: 1,
                _weak: 2,
                _value: (),
            }, &Self::ref_data(&(&**self as *const T))[1..]);
            rc_box.data.reserve_exact(self.data.len() - Self::data_len());
            rc_box.data.extend(self.data.as_slice()[Self::data_len()..].iter().cloned());
            let rc: Rc<T> = transmute(&mut *rc_box);
            drop(rc); // TODO: DropGuard
            forget(rc_box);
        }
    }
}

impl<T: Sized, A: Vector<Item=usize>> From<T> for SmallDST<T, A> {
    #[inline]
    fn from(t: T) -> Self {
        unsafe { Self::from_value(t, &[]) }
    }
}

impl<T: ?Sized, A: Vector<Item=usize>> Deref for SmallDST<T, A> {
    type Target = T;

    #[inline]
    fn deref(&self) -> &Self::Target {
        unsafe {
            let mut v: *const T = uninitialized();
            {
                let v = Self::mut_data(&mut v);
                v[0] = self.data.as_ptr().offset(Self::data_len() as isize) as usize;
                copy_nonoverlapping(self.data.as_ptr(), v[1..].as_mut_ptr(), Self::data_len());
            }
            transmute(v)
        }
    }
}

impl<T: ?Sized, A: Vector<Item=usize>> DerefMut for SmallDST<T, A> {
    #[inline]
    fn deref_mut(&mut self) -> &mut Self::Target {
        unsafe {
            let mut v: *const T = uninitialized();
            {
                let v = Self::mut_data(&mut v);
                v[0] = self.data.as_ptr().offset(Self::data_len() as isize) as usize;
                copy_nonoverlapping(self.data.as_ptr(), v[1..].as_mut_ptr(), Self::data_len());
            }
            transmute(v as *mut T)
        }
    }
}

impl<T: ?Sized + fmt::Debug, A: Vector<Item=usize>> fmt::Debug for SmallDST<T, A> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt::Debug::fmt(&**self, fmt)
    }
}

#[macro_export]
macro_rules! small_dst {
    ($x:expr) => {
        {
            let v = $x;
            let dst = unsafe { $crate::SmallDST::new_ref(&v as &_, &v) };
            ::std::mem::forget(v);
            dst
        }
    };
}
<commit_msg>SmallDST clone implementation<commit_after>use std::fmt;
use std::ops::{Deref, DerefMut};
use std::ptr::{copy_nonoverlapping};
use std::mem::{uninitialized, transmute, forget, size_of, align_of};
use std::slice::{from_raw_parts, from_raw_parts_mut};
use std::marker;
use ::{ArrayVec, Vector};

#[macro_export]
macro_rules! small_dst {
    ($x:expr) => {
        {
            let v = $x;
            let dst = unsafe { $crate::SmallDST::new_ref(&v as &_, &v) };
            ::std::mem::forget(v);
            dst
        }
    };
}

pub struct SmallDST<T: ?Sized, A: Vector<Item=usize> = ArrayVec<[usize; 9]>> {
    data: A,
    _phantom: marker::PhantomData<T>,
}

impl<T: ?Sized + Clone, A: Vector<Item=usize>> Clone for SmallDST<T, A> {
    fn clone(&self) -> Self {
        small_dst!((&**self).clone())
    }
}

impl<T: ?Sized, A: Vector<Item=usize>> SmallDST<T, A> {
    fn ptr_len() -> usize {
        assert_eq!(size_of::<*const T>() % size_of::<usize>(), 0);
        size_of::<*const T>() / size_of::<usize>()
    }

    fn data_len() -> usize {
        Self::ptr_len() - 1
    }

    fn ref_data<'a>(v: &'a *const T) -> &'a [usize] {
        unsafe { from_raw_parts(v as *const *const T as *const usize, Self::ptr_len()) }
    }

    fn mut_data<'a>(v: &'a mut *const T) -> &'a mut [usize] {
        unsafe { from_raw_parts_mut(v as *mut *const T as *mut usize, Self::ptr_len()) }
    }

    #[cfg(feature = "unstable")]
    pub fn new<U: marker::Unsize<T>>(v: U) -> Self {
        let ret = unsafe { Self::new_ref(&v, &v) };
        forget(v);
        ret
    }

    #[inline]
    pub unsafe fn new_ref<U>(v: &T, u: &U) -> Self {
        let t = v as *const T;
        let data = Self::ref_data(&t);
        assert_eq!(data[0], u as *const U as usize);

        Self::from_ref(u, &data[1..])
    }

    #[inline]
    pub unsafe fn data(&self) -> &A {
        &self.data
    }

    unsafe fn from_value<U>(v: U, data: &[usize]) -> Self {
        let ret = Self::from_ref(&v, data);
        forget(v);
        ret
    }

    unsafe fn from_ref<U>(v: &U, data: &[usize]) -> Self {
        assert!(align_of::<U>() <= align_of::<A::Item>()); // TODO: support arbitrary alignments
        assert_eq!(data.len(), Self::data_len());

        let size = (size_of::<U>() + size_of::<usize>() - 1) / size_of::<usize>();

        let mut a = A::with_capacity(Self::data_len() + size);
        a.set_len(Self::data_len() + size);
        copy_nonoverlapping(data.as_ptr(), a.as_mut_ptr(), Self::data_len());
        copy_nonoverlapping(v as *const U as *const u8, a.as_mut_ptr().offset(Self::data_len() as isize) as *mut u8, size_of::<U>());

        SmallDST {
            data: a,
            _phantom: marker::PhantomData,
        }
    }
}

impl<T: ?Sized, A: Vector<Item=usize>> Drop for SmallDST<T, A> {
    #[cfg(feature = "unstable")]
    #[inline]
    fn drop(&mut self) {
        use std::intrinsics;

        unsafe { intrinsics::drop_in_place(&mut **self); }
    }

    #[cfg(not(feature = "unstable"))]
    fn drop(&mut self) {
        use ::SmallVec;
        use std::rc::Rc;

        struct RcBox<T: ?Sized> {
            _strong: usize,
            _weak: usize,
            _value: T,
        }

        unsafe {
            let mut rc_box: SmallDST<T, SmallVec<[usize; 0x10]>> = SmallDST::from_ref(&RcBox {
                _strong: 1,
                _weak: 2,
                _value: (),
            }, &Self::ref_data(&(&**self as *const T))[1..]);
            rc_box.data.reserve_exact(self.data.len() - Self::data_len());
            rc_box.data.extend(self.data.as_slice()[Self::data_len()..].iter().cloned());
            let rc: Rc<T> = transmute(&mut *rc_box);
            drop(rc); // TODO: DropGuard
            forget(rc_box);
        }
    }
}

impl<T: Sized, A: Vector<Item=usize>> From<T> for SmallDST<T, A> {
    #[inline]
    fn from(t: T) -> Self {
        unsafe { Self::from_value(t, &[]) }
    }
}

impl<T: ?Sized, A: Vector<Item=usize>> Deref for SmallDST<T, A> {
    type Target = T;

    #[inline]
    fn deref(&self) -> &Self::Target {
        unsafe {
            let mut v: *const T = uninitialized();
            {
                let v = Self::mut_data(&mut v);
                v[0] = self.data.as_ptr().offset(Self::data_len() as isize) as usize;
                copy_nonoverlapping(self.data.as_ptr(), v[1..].as_mut_ptr(), Self::data_len());
            }
            transmute(v)
        }
    }
}

impl<T: ?Sized, A: Vector<Item=usize>> DerefMut for SmallDST<T, A> {
    #[inline]
    fn deref_mut(&mut self) -> &mut Self::Target {
        unsafe {
            let mut v: *const T = uninitialized();
            {
                let v = Self::mut_data(&mut v);
                v[0] = self.data.as_ptr().offset(Self::data_len() as isize) as usize;
                copy_nonoverlapping(self.data.as_ptr(), v[1..].as_mut_ptr(), Self::data_len());
            }
            transmute(v as *mut T)
        }
    }
}

impl<T: ?Sized + fmt::Debug, A: Vector<Item=usize>> fmt::Debug for SmallDST<T, A> {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt::Debug::fmt(&**self, fmt)
    }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use flow::Flow;
use flow_ref::{self, FlowRef};
use std::collections::{LinkedList, linked_list};

// This needs to be reworked now that we have dynamically-sized types in Rust.
// Until then, it's just a wrapper around LinkedList.

pub struct FlowList {
    flows: LinkedList<FlowRef>,
}

pub struct FlowListIterator<'a> {
    it: linked_list::Iter<'a, FlowRef>,
}

pub struct MutFlowListIterator<'a> {
    it: linked_list::IterMut<'a, FlowRef>,
}

impl FlowList {
    /// Add an element last in the list
    ///
    /// O(1)
    pub fn push_back(&mut self, new_tail: FlowRef) {
        self.flows.push_back(new_tail);
    }

    pub fn back(&self) -> Option<&Flow> {
        self.flows.back().map(|x| &**x)
    }

    /// Add an element first in the list
    ///
    /// O(1)
    pub fn push_front(&mut self, new_head: FlowRef) {
        self.flows.push_front(new_head);
    }

    pub fn pop_front(&mut self) -> Option<FlowRef> {
        self.flows.pop_front()
    }

    pub fn front(&self) -> Option<&Flow> {
        self.flows.front().map(|x| &**x)
    }

    /// Create an empty list
    #[inline]
    pub fn new() -> FlowList {
        FlowList {
            flows: LinkedList::new(),
        }
    }

    /// Provide a forward iterator
    #[inline]
    pub fn iter(&self) -> FlowListIterator {
        FlowListIterator {
            it: self.flows.iter(),
        }
    }

    /// Provide a forward iterator with mutable references
    #[inline]
    pub fn iter_mut(&mut self) -> MutFlowListIterator {
        MutFlowListIterator {
            it: self.flows.iter_mut(),
        }
    }

    /// Provide a forward iterator with FlowRef items
    #[inline]
    pub fn iter_flow_ref_mut<'a>(&'a mut self) -> linked_list::IterMut<'a, FlowRef> {
        self.flows.iter_mut()
    }

    /// O(1)
    #[inline]
    pub fn is_empty(&self) -> bool {
        self.flows.is_empty()
    }

    /// O(1)
    #[inline]
    pub fn len(&self) -> usize {
        self.flows.len()
    }

    #[inline]
    pub fn split_off(&mut self, i: usize) -> Self {
        FlowList {
            flows: self.flows.split_off(i)
        }
    }
}

impl<'a> Iterator for FlowListIterator<'a> {
    type Item = &'a Flow;
    #[inline]
    fn next(&mut self) -> Option<&'a Flow> {
        self.it.next().map(|x| &**x)
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        self.it.size_hint()
    }
}

impl<'a> DoubleEndedIterator for FlowListIterator<'a> {
    fn next_back(&mut self) -> Option<&'a Flow> {
        self.it.next_back().map(|x| &**x)
    }
}

impl<'a> Iterator for MutFlowListIterator<'a> {
    type Item = &'a mut Flow;
    #[inline]
    fn next(&mut self) -> Option<&'a mut Flow> {
        self.it.next().map(flow_ref::deref_mut)
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        self.it.size_hint()
    }
}
<commit_msg>Auto merge of #11778 - avadacatavra:mutflowlistdoubleiter, r=Ms2ger<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use flow::Flow;
use flow_ref::{self, FlowRef};
use std::collections::{LinkedList, linked_list};

// This needs to be reworked now that we have dynamically-sized types in Rust.
// Until then, it's just a wrapper around LinkedList.

pub struct FlowList {
    flows: LinkedList<FlowRef>,
}

pub struct FlowListIterator<'a> {
    it: linked_list::Iter<'a, FlowRef>,
}

pub struct MutFlowListIterator<'a> {
    it: linked_list::IterMut<'a, FlowRef>,
}

impl FlowList {
    /// Add an element last in the list
    ///
    /// O(1)
    pub fn push_back(&mut self, new_tail: FlowRef) {
        self.flows.push_back(new_tail);
    }

    pub fn back(&self) -> Option<&Flow> {
        self.flows.back().map(|x| &**x)
    }

    /// Add an element first in the list
    ///
    /// O(1)
    pub fn push_front(&mut self, new_head: FlowRef) {
        self.flows.push_front(new_head);
    }

    pub fn pop_front(&mut self) -> Option<FlowRef> {
        self.flows.pop_front()
    }

    pub fn front(&self) -> Option<&Flow> {
        self.flows.front().map(|x| &**x)
    }

    /// Create an empty list
    #[inline]
    pub fn new() -> FlowList {
        FlowList {
            flows: LinkedList::new(),
        }
    }

    /// Provide a forward iterator
    #[inline]
    pub fn iter(&self) -> FlowListIterator {
        FlowListIterator {
            it: self.flows.iter(),
        }
    }

    /// Provide a forward iterator with mutable references
    #[inline]
    pub fn iter_mut(&mut self) -> MutFlowListIterator {
        MutFlowListIterator {
            it: self.flows.iter_mut(),
        }
    }

    /// Provide a forward iterator with FlowRef items
    #[inline]
    pub fn iter_flow_ref_mut<'a>(&'a mut self) -> linked_list::IterMut<'a, FlowRef> {
        self.flows.iter_mut()
    }

    /// O(1)
    #[inline]
    pub fn is_empty(&self) -> bool {
        self.flows.is_empty()
    }

    /// O(1)
    #[inline]
    pub fn len(&self) -> usize {
        self.flows.len()
    }

    #[inline]
    pub fn split_off(&mut self, i: usize) -> Self {
        FlowList {
            flows: self.flows.split_off(i)
        }
    }
}

impl<'a> Iterator for FlowListIterator<'a> {
    type Item = &'a Flow;
    #[inline]
    fn next(&mut self) -> Option<&'a Flow> {
        self.it.next().map(|x| &**x)
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        self.it.size_hint()
    }
}

impl<'a> DoubleEndedIterator for FlowListIterator<'a> {
    fn next_back(&mut self) -> Option<&'a Flow> {
        self.it.next_back().map(|x| &**x)
    }
}

impl<'a> DoubleEndedIterator for MutFlowListIterator<'a> {
    fn next_back(&mut self) -> Option<&'a mut Flow> {
        self.it.next_back().map(flow_ref::deref_mut)
    }
}

impl<'a> Iterator for MutFlowListIterator<'a> {
    type Item = &'a mut Flow;
    #[inline]
    fn next(&mut self) -> Option<&'a mut Flow> {
        self.it.next().map(flow_ref::deref_mut)
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        self.it.size_hint()
    }
}
<|endoftext|>


<commit_before>//!Neural Network Layers

use linalg::{Matrix, MatrixSlice, BaseMatrix, BaseMatrixMut};
use rulinalg::utils;

use learning::LearningResult;
use learning::error::{Error, ErrorKind};
use learning::toolkit::activ_fn::ActivationFunc;

use rand::thread_rng;
use rand::distributions::Sample;
use rand::distributions::normal::Normal;

use std::fmt::Debug;

/// Trait for neural net layers
pub trait NetLayer : Debug {
    /// The result of propogating data forward through this layer
    fn forward(&self, input: &Matrix<f64>, params: MatrixSlice<f64>) -> LearningResult<Matrix<f64>>;

    /// The gradient of the output of this layer with respect to its input
    fn back_input(&self, out_grad: &Matrix<f64>, input: &Matrix<f64>, params: MatrixSlice<f64>) -> Matrix<f64>;

    /// The gradient of the output of this layer with respect to its parameters
    fn back_params(&self, out_grad: &Matrix<f64>, input: &Matrix<f64>, params: MatrixSlice<f64>) -> Matrix<f64>;

    /// The default value of the parameters of this layer before training
    fn default_params(&self) -> Vec<f64>;

    /// The shape of the parameters used by this layer
    fn param_shape(&self) -> (usize, usize);

    /// The number of parameters used by this layer
    fn num_params(&self) -> usize {
        let shape = self.param_shape();
        shape.0 * shape.1
    }
}

/// Linear network layer
///
/// Represents a fully connected layer with optional bias term
///
/// The parameters are a matrix of weights of size I x O
/// where O is the dimensionality of the output and I the dimensionality of the input
#[derive(Debug, Clone, Copy)]
pub struct Linear {
    /// The number of dimensions of the input
    input_size: usize,
    /// The number of dimensions of the output
    output_size: usize,
    /// Whether or not to include a bias term
    has_bias: bool,
}

impl Linear {
    /// Construct a new Linear layer
    pub fn new(input_size: usize, output_size: usize) -> Linear {
        Linear {
            input_size: input_size + 1,
            output_size: output_size,
            has_bias: true
        }
    }

    /// Construct a Linear layer with a bias term
    pub fn without_bias(input_size: usize, output_size: usize) -> Linear {
        Linear {
            input_size: input_size,
            output_size: output_size,
            has_bias: false
        }
    }
}

impl NetLayer for Linear {
    /// Computes a matrix product
    ///
    /// input should have dimensions N x I
    /// where N is the number of samples and I is the dimensionality of the input
    fn forward(&self, input: &Matrix<f64>, params: MatrixSlice<f64>) -> LearningResult<Matrix<f64>> {
        if self.has_bias {
            if input.cols()+1 != params.rows() {
                Err(Error::new(ErrorKind::InvalidData, "The input had the wrong number of columns"))
            } else {
                Ok(&input.hcat(&Matrix::ones(input.rows(), 1)) * &params)
            }
        } else {
            if input.cols() != params.rows() {
                Err(Error::new(ErrorKind::InvalidData, "The input had the wrong number of columns"))
            } else {
                Ok(input * &params)
            }
        }
    }

    fn back_input(&self, out_grad: &Matrix<f64>, _: &Matrix<f64>, params: MatrixSlice<f64>) -> Matrix<f64> {
        debug_assert_eq!(out_grad.cols(), params.cols());
        let gradient = out_grad * &params.transpose();
        if self.has_bias {
            let rows = gradient.rows();
            let cols = gradient.cols() - 1;
            gradient.sub_slice([0, 0], rows, cols).into()
        } else {
            gradient
        }
    }

    fn back_params(&self, out_grad: &Matrix<f64>, input: &Matrix<f64>, _: MatrixSlice<f64>) -> Matrix<f64> {
        debug_assert_eq!(input.rows(), out_grad.rows());
        if self.has_bias {
            &input.hcat(&Matrix::ones(input.rows(), 1)).transpose() * out_grad
        } else {
            &input.transpose() * out_grad
        }
    }

    /// Initializes weights using Xavier initialization
    ///
    /// weights drawn from gaussian distribution with 0 mean and variance 2/(input_size+output_size)
    fn default_params(&self) -> Vec<f64> {
        let mut distro = Normal::new(0.0, (2.0/(self.input_size+self.output_size) as f64).sqrt());
        let mut rng = thread_rng();

        (0..self.input_size*self.output_size).map(|_| distro.sample(&mut rng))
                                             .collect()
    }

    fn param_shape(&self) -> (usize, usize) {
        (self.input_size, self.output_size)
    }
}

impl<T: ActivationFunc + Debug> NetLayer for T {
    /// Applys the activation function to each element of the input
    fn forward(&self, input: &Matrix<f64>, _: MatrixSlice<f64>) -> LearningResult<Matrix<f64>> {
        Ok(input.clone().apply(&T::func))
    }

    fn back_input(&self, out_grad: &Matrix<f64>, input: &Matrix<f64>, _: MatrixSlice<f64>) -> Matrix<f64> {
        let mut in_grad = input.clone();
        utils::in_place_vec_bin_op(in_grad.mut_data(), out_grad.data(), |x, &y| {
            *x = T::func_grad(*x) * y
        });
        in_grad
    }

    fn back_params(&self, _: &Matrix<f64>, _: &Matrix<f64>, _: MatrixSlice<f64>) -> Matrix<f64> {
        Matrix::new(0, 0, Vec::new())
    }

    fn default_params(&self) -> Vec<f64> {
        Vec::new()
    }

    fn param_shape(&self) -> (usize, usize) {
        (0, 0)
    }
}
<commit_msg>bias appended to beginning of inputs<commit_after>//!Neural Network Layers

use linalg::{Matrix, MatrixSlice, BaseMatrix, BaseMatrixMut};
use rulinalg::utils;

use learning::LearningResult;
use learning::error::{Error, ErrorKind};
use learning::toolkit::activ_fn::ActivationFunc;

use rand::thread_rng;
use rand::distributions::Sample;
use rand::distributions::normal::Normal;

use std::fmt::Debug;

/// Trait for neural net layers
pub trait NetLayer : Debug {
    /// The result of propogating data forward through this layer
    fn forward(&self, input: &Matrix<f64>, params: MatrixSlice<f64>) -> LearningResult<Matrix<f64>>;

    /// The gradient of the output of this layer with respect to its input
    fn back_input(&self, out_grad: &Matrix<f64>, input: &Matrix<f64>, params: MatrixSlice<f64>) -> Matrix<f64>;

    /// The gradient of the output of this layer with respect to its parameters
    fn back_params(&self, out_grad: &Matrix<f64>, input: &Matrix<f64>, params: MatrixSlice<f64>) -> Matrix<f64>;

    /// The default value of the parameters of this layer before training
    fn default_params(&self) -> Vec<f64>;

    /// The shape of the parameters used by this layer
    fn param_shape(&self) -> (usize, usize);

    /// The number of parameters used by this layer
    fn num_params(&self) -> usize {
        let shape = self.param_shape();
        shape.0 * shape.1
    }
}

/// Linear network layer
///
/// Represents a fully connected layer with optional bias term
///
/// The parameters are a matrix of weights of size I x O
/// where O is the dimensionality of the output and I the dimensionality of the input
#[derive(Debug, Clone, Copy)]
pub struct Linear {
    /// The number of dimensions of the input
    input_size: usize,
    /// The number of dimensions of the output
    output_size: usize,
    /// Whether or not to include a bias term
    has_bias: bool,
}

impl Linear {
    /// Construct a new Linear layer
    pub fn new(input_size: usize, output_size: usize) -> Linear {
        Linear {
            input_size: input_size + 1,
            output_size: output_size,
            has_bias: true
        }
    }

    /// Construct a Linear layer with a bias term
    pub fn without_bias(input_size: usize, output_size: usize) -> Linear {
        Linear {
            input_size: input_size,
            output_size: output_size,
            has_bias: false
        }
    }
}

impl NetLayer for Linear {
    /// Computes a matrix product
    ///
    /// input should have dimensions N x I
    /// where N is the number of samples and I is the dimensionality of the input
    fn forward(&self, input: &Matrix<f64>, params: MatrixSlice<f64>) -> LearningResult<Matrix<f64>> {
        if self.has_bias {
            if input.cols()+1 != params.rows() {
                Err(Error::new(ErrorKind::InvalidData, "The input had the wrong number of columns"))
            } else {
                Ok(&Matrix::ones(input.rows(), 1).hcat(input) * &params)
            }
        } else {
            if input.cols() != params.rows() {
                Err(Error::new(ErrorKind::InvalidData, "The input had the wrong number of columns"))
            } else {
                Ok(input * &params)
            }
        }
    }

    fn back_input(&self, out_grad: &Matrix<f64>, _: &Matrix<f64>, params: MatrixSlice<f64>) -> Matrix<f64> {
        debug_assert_eq!(out_grad.cols(), params.cols());
        let gradient = out_grad * &params.transpose();
        if self.has_bias {
            let rows = gradient.rows();
            let cols = gradient.cols() - 1;
            gradient.sub_slice([0, 1], rows, cols).into()
        } else {
            gradient
        }
    }

    fn back_params(&self, out_grad: &Matrix<f64>, input: &Matrix<f64>, _: MatrixSlice<f64>) -> Matrix<f64> {
        debug_assert_eq!(input.rows(), out_grad.rows());
        if self.has_bias {
            &Matrix::ones(input.rows(), 1).hcat(input).transpose() * out_grad
        } else {
            &input.transpose() * out_grad
        }
    }

    /// Initializes weights using Xavier initialization
    ///
    /// weights drawn from gaussian distribution with 0 mean and variance 2/(input_size+output_size)
    fn default_params(&self) -> Vec<f64> {
        let mut distro = Normal::new(0.0, (2.0/(self.input_size+self.output_size) as f64).sqrt());
        let mut rng = thread_rng();

        (0..self.input_size*self.output_size).map(|_| distro.sample(&mut rng))
                                             .collect()
    }

    fn param_shape(&self) -> (usize, usize) {
        (self.input_size, self.output_size)
    }
}

impl<T: ActivationFunc + Debug> NetLayer for T {
    /// Applys the activation function to each element of the input
    fn forward(&self, input: &Matrix<f64>, _: MatrixSlice<f64>) -> LearningResult<Matrix<f64>> {
        Ok(input.clone().apply(&T::func))
    }

    fn back_input(&self, out_grad: &Matrix<f64>, input: &Matrix<f64>, _: MatrixSlice<f64>) -> Matrix<f64> {
        let mut in_grad = input.clone();
        utils::in_place_vec_bin_op(in_grad.mut_data(), out_grad.data(), |x, &y| {
            *x = T::func_grad(*x) * y
        });
        in_grad
    }

    fn back_params(&self, _: &Matrix<f64>, _: &Matrix<f64>, _: MatrixSlice<f64>) -> Matrix<f64> {
        Matrix::new(0, 0, Vec::new())
    }

    fn default_params(&self) -> Vec<f64> {
        Vec::new()
    }

    fn param_shape(&self) -> (usize, usize) {
        (0, 0)
    }
}
<|endoftext|>


<commit_before>use std::io::{Result, Write};

use analysis;
use analysis::symbols;
use analysis::namespaces::MAIN;
use env::Env;
use file_saver::save_to_file;
use library::*;
use library::Type as LType;
use nameutil;
use regex::{Captures, Regex};
use self::format::reformat_doc;
use stripper_lib::Type as SType;
use stripper_lib::{TypeStruct, write_file_name, write_item_doc};
use traits::*;

mod format;

trait FunctionTraitType {
    fn functions(&self) -> &Vec<Function>;
    fn name(&self) -> &str;
}

macro_rules! impl_function_trait_type {
    ($ty:ident) => {
        impl FunctionTraitType for $ty {
            fn functions(&self) -> &Vec<Function> {
                &self.functions
            }

            fn name(&self) -> &str {
                &self.name
            }
        }
    }
}

impl_function_trait_type!(Class);
impl_function_trait_type!(Interface);

trait ToStripperType {
    fn convert(&self) -> TypeStruct;
    fn doc(&self) -> Option<&str>;
    fn doc_deprecated(&self) -> Option<&str>;
}

macro_rules! impl_to_stripper_type {
    ($ty:ident, $enum_var:ident) => {
        impl ToStripperType for $ty {
            fn convert(&self) -> TypeStruct {
                TypeStruct::new(SType::$enum_var, &self.name)
            }

            fn doc(&self) -> Option<&str> {
                self.doc.as_ref().map(|s| &s[..])
            }

            fn doc_deprecated(&self) -> Option<&str> {
                self.doc_deprecated.as_ref().map(|s| &s[..])
            }
        }
    }
}

impl_to_stripper_type!(Alias, Struct);
impl_to_stripper_type!(Constant, Const);
impl_to_stripper_type!(Member, Variant);
impl_to_stripper_type!(Enumeration, Enum);
impl_to_stripper_type!(Bitfield, Type);
impl_to_stripper_type!(Record, Struct);
impl_to_stripper_type!(Field, Variant);
impl_to_stripper_type!(Union, Struct);
impl_to_stripper_type!(Function, Fn);
impl_to_stripper_type!(Interface, Trait);
impl_to_stripper_type!(Class, Struct);
impl_to_stripper_type!(Namespace, Mod);
impl_to_stripper_type!(Parameter, Variant);

pub fn generate(env: &Env) {
    let path =  env.config.target_path.join("docs.md");
    println!("Generating documentation {:?}", path);
    save_to_file(&path, env.config.make_backup,
        |w| generate_doc(w, &env));
}

fn generate_doc(mut w: &mut Write, env: &Env) -> Result<()> {
    try!(write_file_name(w, None));

    let namespace = env.library.namespace(MAIN);
    for obj in env.config.objects.values() {
        if obj.status.ignored() {
            continue;
        }

        let info = analysis::object::class(env, obj)
            .or_else(|| analysis::object::interface(env, obj));
        if let Some(info) = info {
            if info.type_id.ns_id == MAIN {
                try!(create_object_doc(w, env, &info));
            }
        }

        if let Some(info) = analysis::record::new(env, obj) {
            if info.type_id.ns_id == MAIN {
                println!("documenting struct {}", info.name);
                let record = env.library.type_(info.type_id).to_ref_as::<Record>();
                let symbols = env.symbols.borrow();
                try!(create_record_doc(w, record, &symbols));
            }
        }
    }

    let symbols = env.symbols.borrow();
    for ty in namespace.types.iter().filter_map(|t| t.as_ref()) {
        try!(handle_type(&mut w, &ty, &symbols));
    }
    Ok(())
}

fn handle_type(w: &mut Write, ty: &LType, symbols: &symbols::Info) -> Result<()> {
    match *ty {
        LType::Alias(ref a) => create_sub_doc(w, a, symbols),
        LType::Enumeration(ref e) => create_enum_doc(w, &e, symbols),
        LType::Function(ref f) => create_fn_doc(w, &f, None, symbols),
        _ => Ok(()),
    }
}

fn create_object_doc(w: &mut Write, env: &Env, info: &analysis::object::Info) -> Result<()> {
    let symbols = env.symbols.borrow();
    let ty = TypeStruct::new(SType::Struct, &info.name);
    let ty_ext = TypeStruct::new(SType::Trait, &format!("{}Ext", info.name));
    let has_trait = info.has_children;
    let class: &ToStripperType;
    let functions: &[Function];

    match *env.library.type_(info.type_id) {
        Type::Class(ref cl) => {
            class = cl;
            functions = &cl.functions;
        }
        Type::Interface(ref iface) => {
            class = iface;
            functions = &iface.functions;
        }
        _ => unreachable!(),
    }

    if class.doc().is_some() || class.doc_deprecated().is_some() {
        try!(write_item_doc(w, &ty, |w| {
            if let Some(doc) = class.doc() {
                try!(writeln!(w, "{}", reformat_doc(doc, &symbols)));
            }

            try!(writeln!(w, "\n# Implements\n"));
            let impl_self = if has_trait { Some(info.type_id) } else { None };
            let implements = impl_self.iter()
                .chain(env.class_hierarchy.supertypes(info.type_id))
                .filter(|&tid| !env.type_status(&tid.full_name(&env.library)).ignored())
                .map(|&tid| format!("[`{name}Ext`](trait.{name}Ext.html)",
                                    name = env.library.type_(tid).get_name()))
                .collect::<Vec<_>>();
            try!(writeln!(w, "{}", &implements.join(", ")));

            if let Some(doc) = class.doc_deprecated() {
                try!(writeln!(w, "\n# Deprecated\n"));
                try!(writeln!(w, "{}", reformat_doc(doc, &symbols)));
            }
            Ok(())
        }));
    }

    if has_trait {
        try!(write_item_doc(w, &ty_ext, |w| {
            writeln!(w, "Trait containing all `{}` methods.", ty.name)
        }));
    }

    let ty = TypeStruct { ty: SType::Impl, ..ty };

    for function in functions {
        let ty = if has_trait && function.parameters.iter().any(|p| p.instance_parameter) {
            ty_ext.clone()
        }
        else {
            ty.clone()
        };
        try!(create_fn_doc(w, &function, Some(Box::new(ty)), &symbols));
    }
    Ok(())
}

fn create_record_doc(w: &mut Write, record: &Record, symbols: &symbols::Info) -> Result<()> {
    let ty = record.convert();

    if record.doc().is_some() || record.doc_deprecated().is_some() {
        try!(write_item_doc(w, &ty, |w| {
            if let Some(doc) = record.doc() {
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }

            if let Some(doc) = record.doc_deprecated() {
                try!(writeln!(w, "\n# Deprecated\n"));
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }
            Ok(())
        }));
    }

    let ty = TypeStruct { ty: SType::Impl, ..ty };
    for function in &record.functions {
        try!(create_fn_doc(w, &function, Some(Box::new(ty.clone())), &symbols));
    }
    Ok(())
}

fn create_enum_doc(w: &mut Write, enum_: &Enumeration, symbols: &symbols::Info) -> Result<()> {
    let ty = enum_.convert();

    try!(write_item_doc(w, &ty, |w| {
        if let Some(doc) = enum_.doc() {
            try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
        }
        if let Some(doc) = enum_.doc_deprecated() {
            try!(writeln!(w, "\n# Deprecated\n"));
            try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
        }
        Ok(())
    }));

    for member in enum_.members.iter() {
        let mut sub_ty : TypeStruct = member.convert();

        if member.doc().is_some() || member.doc_deprecated().is_some() {
            sub_ty.parent = Some(Box::new(ty.clone()));
            try!(write_item_doc(w, &sub_ty, |w| {
                if let Some(doc) = member.doc() {
                    try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
                }
                if let Some(doc) = member.doc_deprecated() {
                    try!(writeln!(w, "\n# Deprecated\n"));
                    try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
                }
                Ok(())
            }));
        }
    }
    Ok(())
}

lazy_static! {
    static ref PARAM_NAME: Regex = Regex::new(r"@(\w+)\b").unwrap();
}

fn fix_param_names(doc: &str, self_name: &Option<String>) -> String {
    PARAM_NAME.replace_all(doc, |caps: &Captures| {
        if let Some(ref self_name) = *self_name {
            if &caps[1] == self_name {
                return "@self".into()
            }
        }
        format!("@{}", nameutil::mangle_keywords(&caps[1]))
    })
}

fn create_fn_doc(w: &mut Write, fn_: &Function, parent: Option<Box<TypeStruct>>,
                 symbols: &symbols::Info) -> Result<()> {
    if fn_.doc().is_none() && fn_.doc_deprecated().is_none() && fn_.ret.doc().is_none() {
        if fn_.parameters.iter().all(|x| {
            x.doc().is_none()
        }) {
            return Ok(());
        }
    }
    let mut sub_ty = fn_.convert();
    sub_ty.parent = parent;

    let self_name: Option<String> = fn_.parameters.iter()
        .find(|p| p.instance_parameter)
        .map(|p| p.name.clone());

    write_item_doc(w, &sub_ty, |w| {
        if let Some(doc) = fn_.doc() {
            try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
        };
        if let Some(doc) = fn_.doc_deprecated() {
            try!(writeln!(w, "\n# Deprecated\n"));
            try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
        };

        for parameter in fn_.parameters.iter() {
            if parameter.instance_parameter || parameter.name.is_empty() {
                continue
            }
            if let Some(doc) = parameter.doc() {
                try!(writeln!(w, "## `{}`", nameutil::mangle_keywords(&parameter.name[..])));
                try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
            }
        }

        if let Some(doc) = fn_.ret.doc() {
            try!(writeln!(w, "\n# Returns\n"));
            try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
        }
        Ok(())
    })
}

fn create_sub_doc<T: ToStripperType>(w: &mut Write, ty: &T, symbols: &symbols::Info) -> Result<()> {
    let sub_ty = ty.convert();

    if ty.doc().is_some() || ty.doc_deprecated().is_some() {
        try!(write_item_doc(w, &sub_ty, |w| {
            if let Some(doc) = ty.doc() {
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }
            if let Some(doc) = ty.doc_deprecated() {
                try!(writeln!(w, "\n# Deprecated\n"));
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }
            Ok(())
        }));
    }
    Ok(())
}
<commit_msg>List implementors in trait documentation<commit_after>use std::io::{Result, Write};

use analysis;
use analysis::symbols;
use analysis::namespaces::MAIN;
use env::Env;
use file_saver::save_to_file;
use library::*;
use library::Type as LType;
use nameutil;
use regex::{Captures, Regex};
use self::format::reformat_doc;
use stripper_lib::Type as SType;
use stripper_lib::{TypeStruct, write_file_name, write_item_doc};
use traits::*;

mod format;

trait FunctionTraitType {
    fn functions(&self) -> &Vec<Function>;
    fn name(&self) -> &str;
}

macro_rules! impl_function_trait_type {
    ($ty:ident) => {
        impl FunctionTraitType for $ty {
            fn functions(&self) -> &Vec<Function> {
                &self.functions
            }

            fn name(&self) -> &str {
                &self.name
            }
        }
    }
}

impl_function_trait_type!(Class);
impl_function_trait_type!(Interface);

trait ToStripperType {
    fn convert(&self) -> TypeStruct;
    fn doc(&self) -> Option<&str>;
    fn doc_deprecated(&self) -> Option<&str>;
}

macro_rules! impl_to_stripper_type {
    ($ty:ident, $enum_var:ident) => {
        impl ToStripperType for $ty {
            fn convert(&self) -> TypeStruct {
                TypeStruct::new(SType::$enum_var, &self.name)
            }

            fn doc(&self) -> Option<&str> {
                self.doc.as_ref().map(|s| &s[..])
            }

            fn doc_deprecated(&self) -> Option<&str> {
                self.doc_deprecated.as_ref().map(|s| &s[..])
            }
        }
    }
}

impl_to_stripper_type!(Alias, Struct);
impl_to_stripper_type!(Constant, Const);
impl_to_stripper_type!(Member, Variant);
impl_to_stripper_type!(Enumeration, Enum);
impl_to_stripper_type!(Bitfield, Type);
impl_to_stripper_type!(Record, Struct);
impl_to_stripper_type!(Field, Variant);
impl_to_stripper_type!(Union, Struct);
impl_to_stripper_type!(Function, Fn);
impl_to_stripper_type!(Interface, Trait);
impl_to_stripper_type!(Class, Struct);
impl_to_stripper_type!(Namespace, Mod);
impl_to_stripper_type!(Parameter, Variant);

pub fn generate(env: &Env) {
    let path =  env.config.target_path.join("docs.md");
    println!("Generating documentation {:?}", path);
    save_to_file(&path, env.config.make_backup,
        |w| generate_doc(w, &env));
}

fn generate_doc(mut w: &mut Write, env: &Env) -> Result<()> {
    try!(write_file_name(w, None));

    let namespace = env.library.namespace(MAIN);
    for obj in env.config.objects.values() {
        if obj.status.ignored() {
            continue;
        }

        let info = analysis::object::class(env, obj)
            .or_else(|| analysis::object::interface(env, obj));
        if let Some(info) = info {
            if info.type_id.ns_id == MAIN {
                try!(create_object_doc(w, env, &info));
            }
        }

        if let Some(info) = analysis::record::new(env, obj) {
            if info.type_id.ns_id == MAIN {
                println!("documenting struct {}", info.name);
                let record = env.library.type_(info.type_id).to_ref_as::<Record>();
                let symbols = env.symbols.borrow();
                try!(create_record_doc(w, record, &symbols));
            }
        }
    }

    let symbols = env.symbols.borrow();
    for ty in namespace.types.iter().filter_map(|t| t.as_ref()) {
        try!(handle_type(&mut w, &ty, &symbols));
    }
    Ok(())
}

fn handle_type(w: &mut Write, ty: &LType, symbols: &symbols::Info) -> Result<()> {
    match *ty {
        LType::Alias(ref a) => create_sub_doc(w, a, symbols),
        LType::Enumeration(ref e) => create_enum_doc(w, &e, symbols),
        LType::Function(ref f) => create_fn_doc(w, &f, None, symbols),
        _ => Ok(()),
    }
}

fn create_object_doc(w: &mut Write, env: &Env, info: &analysis::object::Info) -> Result<()> {
    let symbols = env.symbols.borrow();
    let ty = TypeStruct::new(SType::Struct, &info.name);
    let ty_ext = TypeStruct::new(SType::Trait, &format!("{}Ext", info.name));
    let has_trait = info.has_children;
    let class: &ToStripperType;
    let functions: &[Function];

    match *env.library.type_(info.type_id) {
        Type::Class(ref cl) => {
            class = cl;
            functions = &cl.functions;
        }
        Type::Interface(ref iface) => {
            class = iface;
            functions = &iface.functions;
        }
        _ => unreachable!(),
    }

    if class.doc().is_some() || class.doc_deprecated().is_some() {
        try!(write_item_doc(w, &ty, |w| {
            if let Some(doc) = class.doc() {
                try!(writeln!(w, "{}", reformat_doc(doc, &symbols)));
            }

            try!(writeln!(w, "\n# Implements\n"));
            let impl_self = if has_trait { Some(info.type_id) } else { None };
            let implements = impl_self.iter()
                .chain(env.class_hierarchy.supertypes(info.type_id))
                .filter(|&tid| !env.type_status(&tid.full_name(&env.library)).ignored())
                .map(|&tid| format!("[`{name}Ext`](trait.{name}Ext.html)",
                                    name = env.library.type_(tid).get_name()))
                .collect::<Vec<_>>();
            try!(writeln!(w, "{}", &implements.join(", ")));

            if let Some(doc) = class.doc_deprecated() {
                try!(writeln!(w, "\n# Deprecated\n"));
                try!(writeln!(w, "{}", reformat_doc(doc, &symbols)));
            }
            Ok(())
        }));
    }

    if has_trait {
        try!(write_item_doc(w, &ty_ext, |w| {
            try!(writeln!(w, "Trait containing all `{}` methods.", ty.name));
            let mut implementors = Some(info.type_id).into_iter()
                .chain(env.class_hierarchy.subtypes(info.type_id))
                .filter(|&tid| !env.type_status(&tid.full_name(&env.library)).ignored())
                .map(|tid| format!("[`{name}`](struct.{name}.html)",
                                    name = env.library.type_(tid).get_name()))
                .collect::<Vec<_>>();
            implementors.sort();

            try!(writeln!(w, "\n# Implementors\n"));
            try!(writeln!(w, "{}", implementors.join(", ")));
            Ok(())
        }));
    }

    let ty = TypeStruct { ty: SType::Impl, ..ty };

    for function in functions {
        let ty = if has_trait && function.parameters.iter().any(|p| p.instance_parameter) {
            ty_ext.clone()
        }
        else {
            ty.clone()
        };
        try!(create_fn_doc(w, &function, Some(Box::new(ty)), &symbols));
    }
    Ok(())
}

fn create_record_doc(w: &mut Write, record: &Record, symbols: &symbols::Info) -> Result<()> {
    let ty = record.convert();

    if record.doc().is_some() || record.doc_deprecated().is_some() {
        try!(write_item_doc(w, &ty, |w| {
            if let Some(doc) = record.doc() {
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }

            if let Some(doc) = record.doc_deprecated() {
                try!(writeln!(w, "\n# Deprecated\n"));
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }
            Ok(())
        }));
    }

    let ty = TypeStruct { ty: SType::Impl, ..ty };
    for function in &record.functions {
        try!(create_fn_doc(w, &function, Some(Box::new(ty.clone())), &symbols));
    }
    Ok(())
}

fn create_enum_doc(w: &mut Write, enum_: &Enumeration, symbols: &symbols::Info) -> Result<()> {
    let ty = enum_.convert();

    try!(write_item_doc(w, &ty, |w| {
        if let Some(doc) = enum_.doc() {
            try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
        }
        if let Some(doc) = enum_.doc_deprecated() {
            try!(writeln!(w, "\n# Deprecated\n"));
            try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
        }
        Ok(())
    }));

    for member in enum_.members.iter() {
        let mut sub_ty : TypeStruct = member.convert();

        if member.doc().is_some() || member.doc_deprecated().is_some() {
            sub_ty.parent = Some(Box::new(ty.clone()));
            try!(write_item_doc(w, &sub_ty, |w| {
                if let Some(doc) = member.doc() {
                    try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
                }
                if let Some(doc) = member.doc_deprecated() {
                    try!(writeln!(w, "\n# Deprecated\n"));
                    try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
                }
                Ok(())
            }));
        }
    }
    Ok(())
}

lazy_static! {
    static ref PARAM_NAME: Regex = Regex::new(r"@(\w+)\b").unwrap();
}

fn fix_param_names(doc: &str, self_name: &Option<String>) -> String {
    PARAM_NAME.replace_all(doc, |caps: &Captures| {
        if let Some(ref self_name) = *self_name {
            if &caps[1] == self_name {
                return "@self".into()
            }
        }
        format!("@{}", nameutil::mangle_keywords(&caps[1]))
    })
}

fn create_fn_doc(w: &mut Write, fn_: &Function, parent: Option<Box<TypeStruct>>,
                 symbols: &symbols::Info) -> Result<()> {
    if fn_.doc().is_none() && fn_.doc_deprecated().is_none() && fn_.ret.doc().is_none() {
        if fn_.parameters.iter().all(|x| {
            x.doc().is_none()
        }) {
            return Ok(());
        }
    }
    let mut sub_ty = fn_.convert();
    sub_ty.parent = parent;

    let self_name: Option<String> = fn_.parameters.iter()
        .find(|p| p.instance_parameter)
        .map(|p| p.name.clone());

    write_item_doc(w, &sub_ty, |w| {
        if let Some(doc) = fn_.doc() {
            try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
        };
        if let Some(doc) = fn_.doc_deprecated() {
            try!(writeln!(w, "\n# Deprecated\n"));
            try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
        };

        for parameter in fn_.parameters.iter() {
            if parameter.instance_parameter || parameter.name.is_empty() {
                continue
            }
            if let Some(doc) = parameter.doc() {
                try!(writeln!(w, "## `{}`", nameutil::mangle_keywords(&parameter.name[..])));
                try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
            }
        }

        if let Some(doc) = fn_.ret.doc() {
            try!(writeln!(w, "\n# Returns\n"));
            try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
        }
        Ok(())
    })
}

fn create_sub_doc<T: ToStripperType>(w: &mut Write, ty: &T, symbols: &symbols::Info) -> Result<()> {
    let sub_ty = ty.convert();

    if ty.doc().is_some() || ty.doc_deprecated().is_some() {
        try!(write_item_doc(w, &sub_ty, |w| {
            if let Some(doc) = ty.doc() {
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }
            if let Some(doc) = ty.doc_deprecated() {
                try!(writeln!(w, "\n# Deprecated\n"));
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }
            Ok(())
        }));
    }
    Ok(())
}
<|endoftext|>


<commit_before>use LalafellBot;
use commands::*;
use lodestone::Lodestone;

use discord::builders::EmbedBuilder;
use discord::model::Channel;

use xivdb::error::*;

use std::sync::Arc;

pub struct VerifyCommand {
  bot: Arc<LalafellBot>
}

impl VerifyCommand {
  pub fn new(bot: Arc<LalafellBot>) -> VerifyCommand {
    VerifyCommand {
      bot: bot
    }
  }
}

impl<'a> Command<'a> for VerifyCommand {
  fn run(&self, message: &Message, _: &[&str]) -> CommandResult<'a> {
    let channel = self.bot.discord.get_channel(message.channel_id).chain_err(|| "could not get channel for message")?;
    let server_id = match channel {
      Channel::Public(c) => c.server_id,
      _ => {
        let err: error::Error = "channel was not public".into();
        return Err(err.into());
      }
    };
    let mut database = self.bot.database.lock().unwrap();
    let user = database.autotags.users.iter_mut().find(|u| u.user_id == message.author.id.0 && u.server_id == server_id.0);
    let mut user = match user {
      Some(u) => u,
      None => return Err(ExternalCommandFailure::default()
          .message(|e: EmbedBuilder| e
            .title("Not tagged.")
            .description("Please tag yourself with an account before verifying it."))
          .wrap())
    };
    let verification_string = match user.verification.verification_string {
      Some(ref v) => v,
      None => {
        let verification_string = user.verification.create_verification_string();
        let chan = self.bot.discord.create_private_channel(message.author.id).chain_err(|| "could not create private channel")?;
        self.bot.discord.send_embed(chan.id, "", |e| e
          .title("Verification instructions")
          .description(&format!("Edit your Lodestone profile to contain `{}`.\nRerun the `!verify` command afterward.", verification_string))
          .url("http://na.finalfantasyxiv.com/lodestone/my/setting/profile/")).ok();
        return Ok(CommandSuccess::default());
      }
    };
    let profile = Lodestone::new().character_profile(user.character_id)?;
    if profile.contains(verification_string) {
      let mut state_option = self.bot.state.lock().unwrap();
      let state = state_option.as_mut().unwrap();
      let server = match state.servers().iter().find(|x| x.id == server_id) {
        Some(s) => s,
        None => {
          let err: error::Error = "could not find server for channel".into();
          return Err(err.into());
        }
      };

      user.verification.verified = true;
      if let Some(r) = server.roles.iter().find(|x| x.name.to_lowercase() == "verified") {
        let member = self.bot.discord.get_member(server_id, message.author.id).chain_err(|| "could not get member for tagging")?;

        if !member.roles.contains(&r.id) {
          self.bot.discord.edit_member_roles(server_id, message.author.id, &[r.id]).chain_err(|| "could not add roles")?;
        }
      }
      let char_name = user.character.clone();
      let serv_name = user.server.clone();
      Ok(CommandSuccess::default()
        .message(move |e: EmbedBuilder| e
          .title("Verified!")
          .description(&format!("You have successfully verified yourself as {} on {}.", char_name, serv_name))))
    } else {
      Err(ExternalCommandFailure::default()
        .wrap())
    }
  }
}
<commit_msg>fix: make things work<commit_after>use LalafellBot;
use commands::*;
use lodestone::Lodestone;

use discord::builders::EmbedBuilder;
use discord::model::Channel;

use xivdb::error::*;

use std::sync::Arc;

pub struct VerifyCommand {
  bot: Arc<LalafellBot>
}

impl VerifyCommand {
  pub fn new(bot: Arc<LalafellBot>) -> VerifyCommand {
    VerifyCommand {
      bot: bot
    }
  }
}

impl<'a> Command<'a> for VerifyCommand {
  fn run(&self, message: &Message, _: &[&str]) -> CommandResult<'a> {
    let channel = self.bot.discord.get_channel(message.channel_id).chain_err(|| "could not get channel for message")?;
    let server_id = match channel {
      Channel::Public(c) => c.server_id,
      _ => {
        let err: error::Error = "channel was not public".into();
        return Err(err.into());
      }
    };
    let mut database = self.bot.database.lock().unwrap();
    let user = database.autotags.users.iter_mut().find(|u| u.user_id == message.author.id.0 && u.server_id == server_id.0);
    let mut user = match user {
      Some(u) => u,
      None => return Err(ExternalCommandFailure::default()
          .message(|e: EmbedBuilder| e
            .title("Not tagged.")
            .description("Please tag yourself with an account before verifying it."))
          .wrap())
    };
    if user.verification.verified {
      return Err(ExternalCommandFailure::default()
        .message(|e: EmbedBuilder| e.description("You are already verified."))
        .wrap());
    }
    let verification_string = match user.verification.verification_string {
      Some(ref v) => v,
      None => {
        let verification_string = user.verification.create_verification_string();
        let chan = self.bot.discord.create_private_channel(message.author.id).chain_err(|| "could not create private channel")?;
        self.bot.discord.send_embed(chan.id, "", |e| e
          .title("Verification instructions")
          .description(&format!("Edit your Lodestone profile to contain `{}`.\nRerun the `!verify` command afterward.", verification_string))
          .url("http://na.finalfantasyxiv.com/lodestone/my/setting/profile/")).ok();
        return Ok(CommandSuccess::default());
      }
    };
    let profile = Lodestone::new().character_profile(user.character_id)?;
    if profile.contains(verification_string) {
      let mut state_option = self.bot.state.lock().unwrap();
      let state = state_option.as_mut().unwrap();
      let server = match state.servers().iter().find(|x| x.id == server_id) {
        Some(s) => s,
        None => {
          let err: error::Error = "could not find server for channel".into();
          return Err(err.into());
        }
      };

      user.verification.verified = true;
      if let Some(r) = server.roles.iter().find(|x| x.name.to_lowercase() == "verified") {
        let mut member = self.bot.discord.get_member(server_id, message.author.id).chain_err(|| "could not get member for tagging")?;

        if !member.roles.contains(&r.id) {
          member.roles.push(r.id);
          self.bot.discord.edit_member_roles(server_id, message.author.id, &member.roles).chain_err(|| "could not add roles")?;
        }
      }
      let char_name = user.character.clone();
      let serv_name = user.server.clone();
      Ok(CommandSuccess::default()
        .message(move |e: EmbedBuilder| e
          .title("Verified!")
          .description(&format!("You have successfully verified yourself as {} on {}.", char_name, serv_name))))
    } else {
      Err(ExternalCommandFailure::default()
        .wrap())
    }
  }
}
<|endoftext|>


<commit_before>/// The tokenizer for the query interpreter

use error::*;
use std::rc::Rc;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Rc<Token>>
    },

    Index {
        idx: i64,
        next: Option<Rc<Token>>
    }
}

impl Token {

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { next: ref mut next, .. } => *next = Some(Rc::new(token)),
            &mut Token::Index { next: ref mut next, .. }      => *next = Some(Rc::new(token)),
        }
    }

    #[cfg(test)]
    pub fn identifier(&self) -> &String {
        match self {
            &Token::Identifier { ident: ref ident, .. } => &ident,
            _ => unreachable!(),
        }
    }

    #[cfg(test)]
    pub fn idx(&self) -> i64 {
        match self {
            &Token::Index { idx: i, .. } => i,
            _ => unreachable!(),
        }
    }
}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    /// Creates a Token object from a string
    ///
    /// # Panics
    ///
    /// * If the internal regex does not compile (should never happen)
    /// * If the token is non-valid (that is, a array index with a non-i64)
    /// * If the regex does not find anything
    /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
    ///
    /// # Incorrect behaviour
    ///
    /// * If the regex finds multiple captures
    ///
    /// # Returns
    ///
    /// The `Token` object with the correct identifier/index for this token and no next token.
    ///
    fn mk_token_object(s: &str) -> Result<Token> {
        use regex::Regex;
        use std::str::FromStr;

        lazy_static! {
            static ref RE: Regex = Regex::new(r"^\[-?\d+\]$").unwrap();
        }

        if s.as_bytes()[0] == b'[' && s.as_bytes()[s.len() - 1] == b']' {
            match RE.captures(s) {
                None => return Err(Error::from(ErrorKind::ArrayAccessWithoutIndex)),
                Some(captures) => {
                    match captures.get(0) {
                        None => Ok(Token::Identifier { ident: String::from(s), next: None }),
                        Some(mtch) => {
                            let mtch = mtch.as_str().replace("[","").replace("]","");
                            let i : i64 = FromStr::from_str(&mtch).unwrap(); // save because regex
                            Ok(Token::Index {
                                idx: i,
                                next: None,
                            })
                        }
                    }
                }
            }
        } else {
            Ok(Token::Identifier { ident: String::from(s), next: None })
        }

    }

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) -> Result<()> {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                let mut token = try!(mk_token_object(token));
                try!(build_token_tree(split, &mut token));
                last.set_next(token);
            }
        }
        Ok(())
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            if token.len() == 0 {
                return Err(Error::from(ErrorKind::EmptyIdentifier));
            }
            let mut tok = try!(mk_token_object(token));
            let _       = try!(build_token_tree(&mut tokens, &mut tok));
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::*;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_seperator_only() {
        let tokens = tokenize_with_seperator(&String::from("."), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    #[test]
    fn test_tokenize_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_with_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_in_array_brackets() {
        let tokens = tokenize_with_seperator(&String::from("[a]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier {
                ident: ident,
                next: None
            } => {
                assert_eq!(String::from("example"), ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                assert_eq!("b", next.deref().identifier());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        use std::ops::Deref;

        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Rc::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Rc::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Rc::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    quickcheck! {
        fn test_array_index(i: i64) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index {
                    idx: i,
                    next: None,
                }) => true,
                _ => false,
            }
        }
    }

}
<commit_msg>Refactor: Complex if condition as fn<commit_after>/// The tokenizer for the query interpreter

use error::*;
use std::rc::Rc;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Rc<Token>>
    },

    Index {
        idx: i64,
        next: Option<Rc<Token>>
    }
}

impl Token {

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { next: ref mut next, .. } => *next = Some(Rc::new(token)),
            &mut Token::Index { next: ref mut next, .. }      => *next = Some(Rc::new(token)),
        }
    }

    #[cfg(test)]
    pub fn identifier(&self) -> &String {
        match self {
            &Token::Identifier { ident: ref ident, .. } => &ident,
            _ => unreachable!(),
        }
    }

    #[cfg(test)]
    pub fn idx(&self) -> i64 {
        match self {
            &Token::Index { idx: i, .. } => i,
            _ => unreachable!(),
        }
    }
}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    /// Creates a Token object from a string
    ///
    /// # Panics
    ///
    /// * If the internal regex does not compile (should never happen)
    /// * If the token is non-valid (that is, a array index with a non-i64)
    /// * If the regex does not find anything
    /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
    ///
    /// # Incorrect behaviour
    ///
    /// * If the regex finds multiple captures
    ///
    /// # Returns
    ///
    /// The `Token` object with the correct identifier/index for this token and no next token.
    ///
    fn mk_token_object(s: &str) -> Result<Token> {
        use regex::Regex;
        use std::str::FromStr;

        lazy_static! {
            static ref RE: Regex = Regex::new(r"^\[-?\d+\]$").unwrap();
        }

        if has_array_brackets(s) {
            match RE.captures(s) {
                None => return Err(Error::from(ErrorKind::ArrayAccessWithoutIndex)),
                Some(captures) => {
                    match captures.get(0) {
                        None => Ok(Token::Identifier { ident: String::from(s), next: None }),
                        Some(mtch) => {
                            let mtch = mtch.as_str().replace("[","").replace("]","");
                            let i : i64 = FromStr::from_str(&mtch).unwrap(); // save because regex
                            Ok(Token::Index {
                                idx: i,
                                next: None,
                            })
                        }
                    }
                }
            }
        } else {
            Ok(Token::Identifier { ident: String::from(s), next: None })
        }

    }

    /// Check whether a str begins with '[' and ends with ']'
    fn has_array_brackets(s: &str) -> bool {
        s.as_bytes()[0] == b'[' && s.as_bytes()[s.len() - 1] == b']'
    }

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) -> Result<()> {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                let mut token = try!(mk_token_object(token));
                try!(build_token_tree(split, &mut token));
                last.set_next(token);
            }
        }
        Ok(())
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            if token.len() == 0 {
                return Err(Error::from(ErrorKind::EmptyIdentifier));
            }
            let mut tok = try!(mk_token_object(token));
            let _       = try!(build_token_tree(&mut tokens, &mut tok));
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::*;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_seperator_only() {
        let tokens = tokenize_with_seperator(&String::from("."), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    #[test]
    fn test_tokenize_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_with_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_in_array_brackets() {
        let tokens = tokenize_with_seperator(&String::from("[a]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier {
                ident: ident,
                next: None
            } => {
                assert_eq!(String::from("example"), ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                assert_eq!("b", next.deref().identifier());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        use std::ops::Deref;

        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Rc::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Rc::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Rc::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    quickcheck! {
        fn test_array_index(i: i64) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index {
                    idx: i,
                    next: None,
                }) => true,
                _ => false,
            }
        }
    }

}
<|endoftext|>