<commit_before>use std::io::{Read, BufRead, BufReader, BufWriter, Write, Result};
use std::fs::File;
use std::slice;
use std::mem;

fn main() {

    let parse_mode = match std::env::args().skip(1).next().as_ref().map(|x| &x[..]) {
        Some("parse") => true,
        Some("print") => false,
        _ => panic!("parse_mode not one of print or parse"),
    };

    let stdin = std::io::stdin();
    let mut reader = stdin.lock();
    let mut writer = std::io::stdout();

    if parse_mode {

        let mut pairs = vec![0u32; 2];
        for readline in reader.lines() {
            let line = readline.ok().expect("read error");
            if !line.starts_with('#') {
                let elts: Vec<&str> = line[..].split("\t").collect();
                pairs[0] = elts[0].parse().ok().expect("malformed src");
                pairs[1] = elts[1].parse().ok().expect("malformed dst");
                writer.write_all(unsafe { typed_as_byte_slices(&pairs[..]) }).unwrap();
            }
        }

    }
    else {

        let mut bytes = vec![0u8; 1024];
        while let Ok(edges) = read_as_typed::<_,(u32,u32)>(&mut reader, &mut bytes) {
            for &(src,dst) in edges {
                println!("{} {}", src,dst);
            }
        }

    }
}

unsafe fn typed_as_byte_slices<T>(slice: &[T]) -> &[u8] {
    slice::from_raw_parts(slice.as_ptr() as *const u8, slice.len() * mem::size_of::<T>())
}

// naughty method using unsafe transmute to read a filled binary buffer as a typed buffer
fn read_as_typed<'a, R: Read, T: Copy>(reader: &mut R, buffer: &'a mut[u8]) -> Result<&'a[T]> {
    if mem::size_of::<T>() * (buffer.len() / mem::size_of::<T>()) < buffer.len() {
        panic!("buffer size must be a multiple of mem::size_of::<T>() = {:?}", mem::size_of::<T>());
    }

    let mut read = try!(reader.read(buffer));
    while mem::size_of::<T>() * (read / mem::size_of::<T>()) < read {
        read += try!(reader.read(&mut buffer[read..]));
    }

    Ok(unsafe { ::std::slice::from_raw_parts_mut(buffer.as_mut_ptr() as *mut T, read / mem::size_of::<T>()) } )
}
<commit_msg>added printer<commit_after>use std::io::{Read, BufRead, BufReader, BufWriter, Write, Result};
use std::fs::File;
use std::slice;
use std::mem;

fn main() {

    let parse_mode = match std::env::args().skip(1).next().as_ref().map(|x| &x[..]) {
        Some("parse") => true,
        Some("print") => false,
        _ => panic!("parse_mode not one of print or parse"),
    };

    let stdin = std::io::stdin();
    let mut reader = stdin.lock();
    let mut writer = std::io::stdout();

    if parse_mode {

        let mut pairs = vec![0u32; 2];
        for readline in reader.lines() {
            let line = readline.ok().expect("read error");
            if !line.starts_with('#') {
                let elts: Vec<&str> = line[..].split("\t").collect();
                pairs[0] = elts[0].parse().ok().expect("malformed src");
                pairs[1] = elts[1].parse().ok().expect("malformed dst");
                writer.write_all(unsafe { typed_as_byte_slices(&pairs[..]) }).unwrap();
            }
        }

    }
    else {

        let mut bytes = vec![0u8; 1024];
        while let Ok(edges) = read_as_typed::<_,(u32,u32)>(&mut reader, &mut bytes) {
            if edges.len() == 0 { break; }
            for &(src,dst) in edges {
                println!("{} {}", src,dst);
            }
        }

    }
}

unsafe fn typed_as_byte_slices<T>(slice: &[T]) -> &[u8] {
    slice::from_raw_parts(slice.as_ptr() as *const u8, slice.len() * mem::size_of::<T>())
}

// naughty method using unsafe transmute to read a filled binary buffer as a typed buffer
fn read_as_typed<'a, R: Read, T: Copy>(reader: &mut R, buffer: &'a mut[u8]) -> Result<&'a[T]> {
    if mem::size_of::<T>() * (buffer.len() / mem::size_of::<T>()) < buffer.len() {
        panic!("buffer size must be a multiple of mem::size_of::<T>() = {:?}", mem::size_of::<T>());
    }

    let mut read = try!(reader.read(buffer));
    while mem::size_of::<T>() * (read / mem::size_of::<T>()) < read {
        read += try!(reader.read(&mut buffer[read..]));
    }

    Ok(unsafe { ::std::slice::from_raw_parts_mut(buffer.as_mut_ptr() as *mut T, read / mem::size_of::<T>()) } )
}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate structopt_derive;
extern crate structopt;
extern crate csv;
extern crate mimir;
extern crate itertools;
extern crate mimirsbrunn;
#[macro_use]
extern crate log;

use std::rc::Rc;
use mimir::rubber::Rubber;
use mimirsbrunn::utils::{format_label, get_zip_codes_from_admins};
use mimir::MimirObject;
use mimirsbrunn::admin_geofinder::AdminGeoFinder;
use std::collections::HashMap;
use structopt::StructOpt;

const GLOBAL_STOP_INDEX_NAME: &'static str = "munin_global_stops";

#[derive(Debug, StructOpt)]
struct Args {
    /// NTFS stops.txt file.
    #[structopt(short = "i", long = "input")]
    input: String,
    /// Name of the dataset.
    #[structopt(short = "d", long = "dataset", default_value = "fr")]
    dataset: String,
    /// Elasticsearch parameters.
    #[structopt(short = "c", long = "connection-string",
                default_value = "http://localhost:9200/munin")]
    connection_string: String,
    /// City level to calculate weight.
    #[structopt(short = "C", long = "city-level", default_value = "8")]
    city_level: u32,
}

#[derive(Debug, Deserialize)]
struct GtfsStop {
    stop_id: String,
    stop_lat: f64,
    stop_lon: f64,
    stop_name: String,
    location_type: Option<i32>,
    visible: Option<i32>,
    parent_station: Option<String>,
}

impl GtfsStop {
    fn incr_stop_point(&self, nb_stop_points: &mut HashMap<String, u32>) {
        match (self.location_type, &self.parent_station) {
            (Some(0), &Some(ref id)) |
            (None, &Some(ref id)) if !id.is_empty() => {
                *nb_stop_points
                    .entry(format!("stop_area:{}", id))
                    .or_insert(0) += 1
            }
            _ => (),
        }
    }
    // to be moved when TryInto is stablilized
    fn try_into(self) -> Option<mimir::Stop> {
        if self.location_type == Some(1) && self.visible != Some(0) {
            Some(mimir::Stop {
                id: format!("stop_area:{}", self.stop_id), // prefix to match navitia's id
                coord: mimir::Coord::new(self.stop_lat, self.stop_lon),
                label: self.stop_name.clone(),
                weight: 0.,
                zip_codes: vec![],
                administrative_regions: vec![],
                name: self.stop_name,
                coverages: vec![],
            })
        } else {
            None
        }
    }
}

fn attach_stop(stop: &mut mimir::Stop, admins: Vec<Rc<mimir::Admin>>, city_level: u32) {
    stop.administrative_regions = admins;
    stop.label = format_label(&stop.administrative_regions, city_level, &stop.name);
    stop.zip_codes = get_zip_codes_from_admins(&stop.administrative_regions);
}

/// Attach the stops to administrative regions
///
/// The admins are loaded from Elasticsearch and stored in a quadtree
/// We attach a stop with all the admins that have a boundary containing
/// the coordinate of the stop
fn attach_stops_to_admins<'a, It: Iterator<Item = &'a mut mimir::Stop>>(
    stops: It,
    rubber: &mut Rubber,
    city_level: u32,
) {
    let admins = rubber.get_all_admins().unwrap_or_else(|_| {
        info!("Administratives regions not found in elasticsearch db");
        vec![]
    });

    info!("{} administrative regions loaded from mimir", admins.len());

    let admins_geofinder = admins.into_iter().collect::<AdminGeoFinder>();

    let mut nb_unmatched = 0u32;
    let mut nb_matched = 0u32;
    for mut stop in stops {
        let admins = admins_geofinder.get(&stop.coord);

        if admins.is_empty() {
            nb_unmatched += 1;
        } else {
            nb_matched += 1;
        }

        attach_stop(&mut stop, admins, city_level);
    }

    info!(
        "there is {}/{} stops without any admin",
        nb_unmatched,
        nb_matched + nb_unmatched
    );
}

// Update weight value for each stop_area from HashMap and stop's coverage
fn finalize_stop_area<'a, It: Iterator<Item = &'a mut mimir::Stop>>(
    stops: It,
    nb_stop_points: &HashMap<String, u32>,
    dataset: &str,
) {
    let max = *nb_stop_points.values().max().unwrap_or(&1) as f64;
    for stop in stops {
        if let Some(weight) = nb_stop_points.get(&stop.id) {
            stop.weight = *weight as f64 / max;
        }
        stop.coverages.push(dataset.to_string());
    }
}

/// merge the stops from all the different indexes
/// for the moment the merge is very simple and uses only the ID
/// (and we take the data from the first stop inserted)
fn merge_stops<It: IntoIterator<Item = mimir::Stop>>(
    stops: It,
) -> Box<Iterator<Item = mimir::Stop>> {
    let mut stops_by_id = HashMap::<String, mimir::Stop>::new();
    for mut stop in stops.into_iter() {
        let cov = std::mem::replace(&mut stop.coverages, vec![]);
        stops_by_id
            .entry(stop.id.clone())
            .or_insert(stop)
            .coverages
            .extend(cov.into_iter());
    }
    Box::new(stops_by_id.into_iter().map(|(_, v)| v))
}

fn get_all_stops(rubber: &mut Rubber, index: String) -> Result<Vec<mimir::Stop>, String> {
    rubber.get_all_objects_from_index(&index).map_err(
        |e| e.to_string(),
    )
}

fn update_global_stop_index<'a, It: Iterator<Item = &'a mimir::Stop>>(
    rubber: &mut Rubber,
    stops: It,
    dataset: &str,
) -> Result<String, String> {
    let dataset_index =
        mimir::rubber::get_main_type_and_dataset_index(mimir::Stop::doc_type(), dataset);
    let stops_indexes = rubber
        .get_all_aliased_index(&mimir::rubber::get_main_type_index(mimir::Stop::doc_type()))?
        .into_iter()
        .filter(|&(_, ref aliases)| !aliases.contains(&dataset_index))
        .map(|(index, _)| index);

    let all_other_es_stops: Vec<_> = stops_indexes
        .map(|index| get_all_stops(rubber, index).unwrap()) // TODO no unwrap
        .flat_map(|stops| stops.into_iter())
        .collect();

    let all_es_stops = all_other_es_stops.into_iter().chain(
        stops.into_iter().cloned(),
    );

    let all_merged_stops = merge_stops(all_es_stops);
    let es_index_name = mimir::rubber::get_date_index_name(GLOBAL_STOP_INDEX_NAME);

    let nb_stops_added = rubber
        .bulk_index(&es_index_name, all_merged_stops)
        .map_err(|e| e.to_string())?;
    info!("{} stops added in the global index", nb_stops_added);
    // create global index
    // fill structure for each stop indexes
    Ok(es_index_name)
}

// publish the global stop index
// alias the new index to the global stop alias, and remove the old index
fn publish_global_index(rubber: &mut Rubber, new_global_index: &str) -> Result<(), String> {
    let last_global_indexes = rubber
        .get_all_aliased_index(GLOBAL_STOP_INDEX_NAME)?
        .into_iter()
        .map(|(k, _)| k)
        .filter(|k| k != new_global_index)
        .collect();
    rubber.alias(
        GLOBAL_STOP_INDEX_NAME,
        &vec![new_global_index.to_string()],
        &last_global_indexes,
    )?;

    for index in last_global_indexes {
        rubber.delete_index(&index)?;
    }
    Ok(())
}

fn main() {
    mimir::logger_init().unwrap();
    info!("Launching stops2mimir...");

    let args = Args::from_args();

    info!("creation of indexes");
    let mut rubber = Rubber::new(&args.connection_string);
    let mut rdr = csv::Reader::from_path(args.input).unwrap();

    let mut nb_stop_points = HashMap::new();

    let mut stops: Vec<mimir::Stop> = rdr.deserialize()
        .filter_map(|rc| {
            rc.map_err(|e| warn!("skip csv line because: {}", e)).ok()
        })
        .filter_map(|stop: GtfsStop| {
            stop.incr_stop_point(&mut nb_stop_points);
            stop.try_into()
        })
        .collect();

    attach_stops_to_admins(stops.iter_mut(), &mut rubber, args.city_level);

    finalize_stop_area(stops.iter_mut(), &nb_stop_points, &args.dataset);

    let global_index = update_global_stop_index(&mut rubber, stops.iter(), &args.dataset).unwrap();

    info!("Importing {} stops into Mimir", stops.len());
    let nb_stops = rubber.index(&args.dataset, stops.iter()).unwrap();
    info!("Nb of indexed stops: {}", nb_stops);

    publish_global_index(&mut rubber, &global_index).unwrap();
}

#[test]
fn test_load_stops() {
    use itertools::Itertools;
    let mut rdr = csv::Reader::from_path("./tests/fixtures/stops.txt".to_string()).unwrap();

    let mut nb_stop_points = HashMap::new();
    let stops: Vec<mimir::Stop> = rdr.deserialize()
        .filter_map(Result::ok)
        .filter_map(|stop: GtfsStop| {
            stop.incr_stop_point(&mut nb_stop_points);
            stop.try_into()
        })
        .collect();
    let ids: Vec<_> = stops.iter().map(|s| s.id.clone()).sorted();
    assert_eq!(
        ids,
        vec![
            "stop_area:SA:known_by_all_dataset",
            "stop_area:SA:main_station",
            "stop_area:SA:second_station",
            "stop_area:SA:station_no_city",
            "stop_area:SA:weight_1_station",
            "stop_area:SA:weight_3_station",
        ]
    );
    let weights: Vec<_> = ids.iter().map(|id| nb_stop_points.get(id)).collect();
    assert_eq!(
        weights,
        vec![None, Some(&1), Some(&1), None, Some(&1), Some(&3)]
    );
}
<commit_msg>small change<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate structopt_derive;
extern crate structopt;
extern crate csv;
extern crate mimir;
extern crate itertools;
extern crate mimirsbrunn;
#[macro_use]
extern crate log;

use std::rc::Rc;
use mimir::rubber::Rubber;
use mimirsbrunn::utils::{format_label, get_zip_codes_from_admins};
use mimir::MimirObject;
use mimirsbrunn::admin_geofinder::AdminGeoFinder;
use std::collections::HashMap;
use structopt::StructOpt;

const GLOBAL_STOP_INDEX_NAME: &'static str = "munin_global_stops";

#[derive(Debug, StructOpt)]
struct Args {
    /// NTFS stops.txt file.
    #[structopt(short = "i", long = "input")]
    input: String,
    /// Name of the dataset.
    #[structopt(short = "d", long = "dataset", default_value = "fr")]
    dataset: String,
    /// Elasticsearch parameters.
    #[structopt(short = "c", long = "connection-string",
                default_value = "http://localhost:9200/munin")]
    connection_string: String,
    /// City level to calculate weight.
    #[structopt(short = "C", long = "city-level", default_value = "8")]
    city_level: u32,
}

#[derive(Debug, Deserialize)]
struct GtfsStop {
    stop_id: String,
    stop_lat: f64,
    stop_lon: f64,
    stop_name: String,
    location_type: Option<i32>,
    visible: Option<i32>,
    parent_station: Option<String>,
}

impl GtfsStop {
    fn incr_stop_point(&self, nb_stop_points: &mut HashMap<String, u32>) {
        match (self.location_type, &self.parent_station) {
            (Some(0), &Some(ref id)) |
            (None, &Some(ref id)) if !id.is_empty() => {
                *nb_stop_points
                    .entry(format!("stop_area:{}", id))
                    .or_insert(0) += 1
            }
            _ => (),
        }
    }
    // to be moved when TryInto is stablilized
    fn try_into(self) -> Option<mimir::Stop> {
        if self.location_type == Some(1) && self.visible != Some(0) {
            Some(mimir::Stop {
                id: format!("stop_area:{}", self.stop_id), // prefix to match navitia's id
                coord: mimir::Coord::new(self.stop_lat, self.stop_lon),
                label: self.stop_name.clone(),
                weight: 0.,
                zip_codes: vec![],
                administrative_regions: vec![],
                name: self.stop_name,
                coverages: vec![],
            })
        } else {
            None
        }
    }
}

fn attach_stop(stop: &mut mimir::Stop, admins: Vec<Rc<mimir::Admin>>, city_level: u32) {
    stop.administrative_regions = admins;
    stop.label = format_label(&stop.administrative_regions, city_level, &stop.name);
    stop.zip_codes = get_zip_codes_from_admins(&stop.administrative_regions);
}

/// Attach the stops to administrative regions
///
/// The admins are loaded from Elasticsearch and stored in a quadtree
/// We attach a stop with all the admins that have a boundary containing
/// the coordinate of the stop
fn attach_stops_to_admins<'a, It: Iterator<Item = &'a mut mimir::Stop>>(
    stops: It,
    rubber: &mut Rubber,
    city_level: u32,
) {
    let admins = rubber.get_all_admins().unwrap_or_else(|_| {
        info!("Administratives regions not found in elasticsearch db");
        vec![]
    });

    info!("{} administrative regions loaded from mimir", admins.len());

    let admins_geofinder = admins.into_iter().collect::<AdminGeoFinder>();

    let mut nb_unmatched = 0u32;
    let mut nb_matched = 0u32;
    for mut stop in stops {
        let admins = admins_geofinder.get(&stop.coord);

        if admins.is_empty() {
            nb_unmatched += 1;
        } else {
            nb_matched += 1;
        }

        attach_stop(&mut stop, admins, city_level);
    }

    info!(
        "there is {}/{} stops without any admin",
        nb_unmatched,
        nb_matched + nb_unmatched
    );
}

// Update weight value for each stop_area from HashMap and stop's coverage
fn finalize_stop_area<'a, It: Iterator<Item = &'a mut mimir::Stop>>(
    stops: It,
    nb_stop_points: &HashMap<String, u32>,
    dataset: &str,
) {
    let max = *nb_stop_points.values().max().unwrap_or(&1) as f64;
    for stop in stops {
        if let Some(weight) = nb_stop_points.get(&stop.id) {
            stop.weight = *weight as f64 / max;
        }
        stop.coverages.push(dataset.to_string());
    }
}

/// merge the stops from all the different indexes
/// for the moment the merge is very simple and uses only the ID
/// (and we take the data from the first stop inserted)
fn merge_stops<It: IntoIterator<Item = mimir::Stop>>(
    stops: It,
) -> Box<Iterator<Item = mimir::Stop>> {
    let mut stops_by_id = HashMap::<String, mimir::Stop>::new();
    for mut stop in stops.into_iter() {
        let cov = std::mem::replace(&mut stop.coverages, vec![]);
        stops_by_id
            .entry(stop.id.clone())
            .or_insert(stop)
            .coverages
            .extend(cov.into_iter());
    }
    Box::new(stops_by_id.into_iter().map(|(_, v)| v))
}

fn get_all_stops(rubber: &mut Rubber, index: String) -> Result<Vec<mimir::Stop>, String> {
    rubber.get_all_objects_from_index(&index).map_err(
        |e| e.to_string(),
    )
}

fn update_global_stop_index<'a, It: Iterator<Item = &'a mimir::Stop>>(
    rubber: &mut Rubber,
    stops: It,
    dataset: &str,
) -> Result<String, String> {
    let dataset_index =
        mimir::rubber::get_main_type_and_dataset_index(mimir::Stop::doc_type(), dataset);
    let stops_indexes = rubber
        .get_all_aliased_index(&mimir::rubber::get_main_type_index(mimir::Stop::doc_type()))?
        .into_iter()
        .filter(|&(_, ref aliases)| !aliases.contains(&dataset_index))
        .map(|(index, _)| index);

    let all_other_es_stops: Vec<_> = stops_indexes
        .map(|index| get_all_stops(rubber, index).unwrap())
        .flat_map(|stops| stops.into_iter())
        .collect();

    let all_es_stops = all_other_es_stops.into_iter().chain(
        stops.into_iter().cloned(),
    );

    let all_merged_stops = merge_stops(all_es_stops);
    let es_index_name = mimir::rubber::get_date_index_name(GLOBAL_STOP_INDEX_NAME);

    let nb_stops_added = rubber
        .bulk_index(&es_index_name, all_merged_stops)
        .map_err(|e| e.to_string())?;
    info!("{} stops added in the global index", nb_stops_added);
    // create global index
    // fill structure for each stop indexes
    Ok(es_index_name)
}

// publish the global stop index
// alias the new index to the global stop alias, and remove the old index
fn publish_global_index(rubber: &mut Rubber, new_global_index: &str) -> Result<(), String> {
    let last_global_indexes = rubber
        .get_all_aliased_index(GLOBAL_STOP_INDEX_NAME)?
        .into_iter()
        .map(|(k, _)| k)
        .filter(|k| k != new_global_index)
        .collect();
    rubber.alias(
        GLOBAL_STOP_INDEX_NAME,
        &vec![new_global_index.to_string()],
        &last_global_indexes,
    )?;

    for index in last_global_indexes {
        rubber.delete_index(&index)?;
    }
    Ok(())
}

fn main() {
    mimir::logger_init().unwrap();
    info!("Launching stops2mimir...");

    let args = Args::from_args();

    info!("creation of indexes");
    let mut rubber = Rubber::new(&args.connection_string);
    let mut rdr = csv::Reader::from_path(args.input).unwrap();

    let mut nb_stop_points = HashMap::new();

    let mut stops: Vec<mimir::Stop> = rdr.deserialize()
        .filter_map(|rc| {
            rc.map_err(|e| warn!("skip csv line because: {}", e)).ok()
        })
        .filter_map(|stop: GtfsStop| {
            stop.incr_stop_point(&mut nb_stop_points);
            stop.try_into()
        })
        .collect();

    attach_stops_to_admins(stops.iter_mut(), &mut rubber, args.city_level);

    finalize_stop_area(stops.iter_mut(), &nb_stop_points, &args.dataset);

    let global_index = update_global_stop_index(&mut rubber, stops.iter(), &args.dataset).unwrap();

    info!("Importing {} stops into Mimir", stops.len());
    let nb_stops = rubber.index(&args.dataset, stops.iter()).unwrap();
    info!("Nb of indexed stops: {}", nb_stops);

    publish_global_index(&mut rubber, &global_index).unwrap();
}

#[test]
fn test_load_stops() {
    use itertools::Itertools;
    let mut rdr = csv::Reader::from_path("./tests/fixtures/stops.txt".to_string()).unwrap();

    let mut nb_stop_points = HashMap::new();
    let stops: Vec<mimir::Stop> = rdr.deserialize()
        .filter_map(Result::ok)
        .filter_map(|stop: GtfsStop| {
            stop.incr_stop_point(&mut nb_stop_points);
            stop.try_into()
        })
        .collect();
    let ids: Vec<_> = stops.iter().map(|s| s.id.clone()).sorted();
    assert_eq!(
        ids,
        vec![
            "stop_area:SA:known_by_all_dataset",
            "stop_area:SA:main_station",
            "stop_area:SA:second_station",
            "stop_area:SA:station_no_city",
            "stop_area:SA:weight_1_station",
            "stop_area:SA:weight_3_station",
        ]
    );
    let weights: Vec<_> = ids.iter().map(|id| nb_stop_points.get(id)).collect();
    assert_eq!(
        weights,
        vec![None, Some(&1), Some(&1), None, Some(&1), Some(&3)]
    );
}
<|endoftext|>


<commit_before>//! Binary encoding for lambda expressions

use lambda_calculus::term::*;
use lambda_calculus::term::Term::*;
use self::Error::*;

#[derive(Debug, PartialEq)]
pub enum Error {
    NotATerm
}

fn _from_binary(input: &[u8]) -> Option<(Term, &[u8])> {
    if input.len() == 0 { return None }

    if [9, 10, 13, 32].contains(&input[0]) {
        _from_binary(&input[1..])
    } else {
        match &input[0..2] {
            b"00" => {
                if let Some((term, rest)) = _from_binary(&input[2..]) {
                    Some((abs(term), rest))
                } else {
                    None
                }
            },
            b"01" => {
                if let Some((term1, rest1)) = _from_binary(&input[2..]) {
                    if let Some((term2, rest2)) = _from_binary(&rest1) {
                        Some((app(term1, term2), &rest2))
                    } else {
                        None
                    }
                } else {
                    None
                }
            },
            b"10" | b"11" => {
                let i = input.iter().take_while(|&b| *b == b'1').count();
                if input[2..].len() == 0 {
                    Some((Var(i), &*b""))
                } else {
                    Some((Var(i), &input[i+1..]))
                }
            },
            _ => None
        }
    }
}

/// Parse a binary-encoded lambda `Term`.
///
/// # Example
/// ```
/// use blc::binary_encoding::{from_binary, to_binary};
///
/// let k = from_binary(b"0000110");
///
/// assert!(k.is_ok());
/// assert_eq!(to_binary(&k.unwrap()), "0000110".to_owned());
/// ```
pub fn from_binary(input: &[u8]) -> Result<Term, Error> {
    if let Some((result, _)) = _from_binary(input) {
        Ok(result)
    } else {
        Err(NotATerm)
    }
}

/// Represent a lambda `Term` in binary.
///
/// # Example
/// ```
/// use blc::binary_encoding::{from_binary, to_binary};
///
/// let k = from_binary(b"0000110");
///
/// assert!(k.is_ok());
/// assert_eq!(to_binary(&k.unwrap()), "0000110".to_owned());
/// ```

fn _to_binary(term: &Term, output: &mut Vec<u8>) {
    match *term {
        Var(i) => {
            for _ in 0..i { output.push(b'1') }
            output.push(b'0');
        }
        Abs(ref t) => {
            output.extend_from_slice(b"00");
            output.append(&mut to_binary(t));
        }
        App(ref t1, ref t2) => {
            output.extend_from_slice(b"01");
            output.append(&mut to_binary(t1));
            output.append(&mut to_binary(t2));
        }
    }
}

pub fn to_binary(term: &Term) -> Vec<u8> {
    let mut output = Vec::new();
    _to_binary(term, &mut output);

    output
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn variables() {
        assert_eq!(from_binary(b"10"),   Ok(Var(1)));
        assert_eq!(from_binary(b"110"),  Ok(Var(2)));
        assert_eq!(from_binary(b"1110"), Ok(Var(3)));
    }

    #[test]
    fn abstractions() {
        assert_eq!(from_binary(b"0010"),     Ok(abs(Var(1))));
        assert_eq!(from_binary(b"000010"),   Ok(abs(abs(Var(1)))));
        assert_eq!(from_binary(b"00000010"), Ok(abs(abs(abs((Var(1)))))));
    }

    #[test]
    fn applications() {
        assert_eq!(from_binary(b"011010"),  Ok(app(Var(1), Var(1))));
        assert_eq!(from_binary(b"0110110"), Ok(app(Var(1), Var(2))));
        assert_eq!(from_binary(b"0111010"), Ok(app(Var(2), Var(1))));
    }

    #[test]
    fn ignoring_whitespaces() {
        assert_eq!(from_binary(b"00 00\t00\n10\r\n"), Ok(abs(abs(abs((Var(1)))))));
    }

    #[test]
    fn from_binary_and_back() {
        let k =    b"0000110";
        let v15 =  b"1111111111111110";
        let s =    b"00000001011110100111010";
        let succ = b"000000011100101111011010";

        let quine = Vec::from(&
            b"000101100100011010000000000001011011110010111100111111011111011010"[..]);
        let primes = Vec::from(&
            b"000100011001100101000110100000000101100000100100010101111101111010010001101\
              000011100110100000000001011011100111001111111011110000000011111001101110000\
              00101100000110110"[..]);
        let blc = Vec::from(&
            b"010100011010000000010101100000000001111000010111111001111000010111001111000\
              000111100001011011011100111110000111110000101111010011101001011001110000110\
              110000101111100001111100001110011011110111110011110111011000011001000110100\
              0011010"[..]);

        assert_eq!(to_binary(&from_binary(&*k).unwrap()),      k);
        assert_eq!(to_binary(&from_binary(&*v15).unwrap()),    v15);
        assert_eq!(to_binary(&from_binary(&*s).unwrap()),      s);
        assert_eq!(to_binary(&from_binary(&*succ).unwrap()),   succ);
        assert_eq!(to_binary(&from_binary(&*quine).unwrap()),  quine);
        assert_eq!(to_binary(&from_binary(&*primes).unwrap()), primes);
        assert_eq!(to_binary(&from_binary(&*blc).unwrap()),    blc);
    }
}
<commit_msg>Fix doc placement<commit_after>//! Binary encoding for lambda expressions

use lambda_calculus::term::*;
use lambda_calculus::term::Term::*;
use self::Error::*;

#[derive(Debug, PartialEq)]
pub enum Error {
    NotATerm
}

fn _from_binary(input: &[u8]) -> Option<(Term, &[u8])> {
    if input.len() == 0 { return None }

    if [9, 10, 13, 32].contains(&input[0]) {
        _from_binary(&input[1..])
    } else {
        match &input[0..2] {
            b"00" => {
                if let Some((term, rest)) = _from_binary(&input[2..]) {
                    Some((abs(term), rest))
                } else {
                    None
                }
            },
            b"01" => {
                if let Some((term1, rest1)) = _from_binary(&input[2..]) {
                    if let Some((term2, rest2)) = _from_binary(&rest1) {
                        Some((app(term1, term2), &rest2))
                    } else {
                        None
                    }
                } else {
                    None
                }
            },
            b"10" | b"11" => {
                let i = input.iter().take_while(|&b| *b == b'1').count();
                if input[2..].len() == 0 {
                    Some((Var(i), &*b""))
                } else {
                    Some((Var(i), &input[i+1..]))
                }
            },
            _ => None
        }
    }
}

/// Parse a binary-encoded lambda `Term`.
///
/// # Example
/// ```
/// use blc::binary_encoding::{from_binary, to_binary};
///
/// let k = from_binary(b"0000110");
///
/// assert!(k.is_ok());
/// assert_eq!(to_binary(&k.unwrap()), "0000110".to_owned());
/// ```
pub fn from_binary(input: &[u8]) -> Result<Term, Error> {
    if let Some((result, _)) = _from_binary(input) {
        Ok(result)
    } else {
        Err(NotATerm)
    }
}

/// Represent a lambda `Term` in binary.
///
/// # Example
/// ```
/// use blc::binary_encoding::{from_binary, to_binary};
///
/// let k = from_binary(b"0000110");
///
/// assert!(k.is_ok());
/// assert_eq!(to_binary(&k.unwrap()), "0000110".to_owned());
/// ```
pub fn to_binary(term: &Term) -> Vec<u8> {
    let mut output = Vec::new();
    _to_binary(term, &mut output);

    output
}

fn _to_binary(term: &Term, output: &mut Vec<u8>) {
    match *term {
        Var(i) => {
            for _ in 0..i { output.push(b'1') }
            output.push(b'0');
        }
        Abs(ref t) => {
            output.extend_from_slice(b"00");
            output.append(&mut to_binary(t));
        }
        App(ref t1, ref t2) => {
            output.extend_from_slice(b"01");
            output.append(&mut to_binary(t1));
            output.append(&mut to_binary(t2));
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn variables() {
        assert_eq!(from_binary(b"10"),   Ok(Var(1)));
        assert_eq!(from_binary(b"110"),  Ok(Var(2)));
        assert_eq!(from_binary(b"1110"), Ok(Var(3)));
    }

    #[test]
    fn abstractions() {
        assert_eq!(from_binary(b"0010"),     Ok(abs(Var(1))));
        assert_eq!(from_binary(b"000010"),   Ok(abs(abs(Var(1)))));
        assert_eq!(from_binary(b"00000010"), Ok(abs(abs(abs((Var(1)))))));
    }

    #[test]
    fn applications() {
        assert_eq!(from_binary(b"011010"),  Ok(app(Var(1), Var(1))));
        assert_eq!(from_binary(b"0110110"), Ok(app(Var(1), Var(2))));
        assert_eq!(from_binary(b"0111010"), Ok(app(Var(2), Var(1))));
    }

    #[test]
    fn ignoring_whitespaces() {
        assert_eq!(from_binary(b"00 00\t00\n10\r\n"), Ok(abs(abs(abs((Var(1)))))));
    }

    #[test]
    fn from_binary_and_back() {
        let k =    b"0000110";
        let v15 =  b"1111111111111110";
        let s =    b"00000001011110100111010";
        let succ = b"000000011100101111011010";

        let quine = Vec::from(&
            b"000101100100011010000000000001011011110010111100111111011111011010"[..]);
        let primes = Vec::from(&
            b"000100011001100101000110100000000101100000100100010101111101111010010001101\
              000011100110100000000001011011100111001111111011110000000011111001101110000\
              00101100000110110"[..]);
        let blc = Vec::from(&
            b"010100011010000000010101100000000001111000010111111001111000010111001111000\
              000111100001011011011100111110000111110000101111010011101001011001110000110\
              110000101111100001111100001110011011110111110011110111011000011001000110100\
              0011010"[..]);

        assert_eq!(to_binary(&from_binary(&*k).unwrap()),      k);
        assert_eq!(to_binary(&from_binary(&*v15).unwrap()),    v15);
        assert_eq!(to_binary(&from_binary(&*s).unwrap()),      s);
        assert_eq!(to_binary(&from_binary(&*succ).unwrap()),   succ);
        assert_eq!(to_binary(&from_binary(&*quine).unwrap()),  quine);
        assert_eq!(to_binary(&from_binary(&*primes).unwrap()), primes);
        assert_eq!(to_binary(&from_binary(&*blc).unwrap()),    blc);
    }
}
<|endoftext|>


<commit_before>extern crate zim;
extern crate clap;
extern crate stopwatch;
extern crate crossbeam;

use std::fs::File;
use std::io::{Write, BufWriter};
use std::error::Error;
use std::collections::HashMap;
use std::path::Path;

use clap::{Arg, App};
use stopwatch::Stopwatch;

use zim::{Zim, Target};

fn main() {
    let matches = App::new("zimextractor")
        .version("0.1")
        .about("Extract zim files")
        .arg(Arg::with_name("out")
                 .short("o")
                 .long("out")
                 .help("Output directory")
                 .takes_value(true))
        .arg(Arg::with_name("skip-link")
                 .long("skip-link")
                 .help("Skip genrating hard links")
                 .takes_value(false))
        .arg(Arg::with_name("INPUT")
                 .help("Set the zim file to extract")
                 .required(true)
                 .index(1))
        .get_matches();

    let skip_link = matches.is_present("skip-link");
    let out = matches.value_of("out").unwrap_or("out");
    let root_output = Path::new(out);

    let input = matches.value_of("INPUT").unwrap();

    println!("Extracting file: {} to {}\n", input, out);

    let sw = Stopwatch::start_new();

    let zim = Zim::new(input).ok().unwrap();

    // map between cluster and directory entry
    let mut cluster_map = HashMap::new();

    println!("  Creating cluster map");
    for i in zim.iterate_by_urls() {
        if let Some(Target::Cluster(cid, _)) = i.target {
            cluster_map.entry(cid).or_insert(Vec::new()).push(i);
        }
    }

    println!("  Extracting entries: {}", zim.header.cluster_count);

    crossbeam::scope(|scope| for (cid, entries) in cluster_map {
                         let mut cluster = zim.get_cluster(cid).unwrap();

                         scope.spawn(move || {
            cluster.decompress();

            for entry in entries {
                if let Some(Target::Cluster(_cid, bid)) = entry.target {
                    assert_eq!(cid, _cid);
                    let mut s = String::new();
                    s.push(entry.namespace);
                    let out_path = root_output.join(&s).join(&entry.url);
                    safe_write(&out_path, cluster.get_blob(bid), 1);
                }
            }
        });
                     });

    println!("  Extraction done in {}ms", sw.elapsed_ms());

    if !skip_link {
        println!("  Linking redirects");

        // link all redirects
        for entry in zim.iterate_by_urls() {
            // get redirect entry
            if let Some(Target::Redirect(redir)) = entry.target {
                let redir = zim.get_by_url_index(redir).unwrap();

                let mut s = String::new();
                s.push(redir.namespace);
                let src = root_output.join(&s).join(&redir.url);

                let mut d = String::new();
                d.push(entry.namespace);
                let dst = root_output.join(&s).join(&entry.url);

                if !dst.exists() {
                    // println!("{:?} -> {:?}", src, dst);
                    std::fs::hard_link(src, dst).unwrap();
                }
            }
        }

        println!("  Linking done in {}ms", sw.elapsed_ms());
    } else {
        println!("  Skipping linking...");
    }

    if let Some(main_page_idx) = zim.header.main_page {
        let page = zim.get_by_url_index(main_page_idx).unwrap();
        println!("  Main page is {}", page.url);
    }
}

fn safe_write(path: &Path, data: &[u8], count: usize) {
    let display = path.display();
    let contain_path = path.parent().unwrap();

    ensure_dir(contain_path);

    match File::create(&path) {
        Err(why) => {
            println!("couldn't create {}: {}", display, why.description());

            if count < 3 {
                safe_write(path, data, count + 1);
            } else {
                panic!("failed retry: couldn't create {}: {}",
                       display,
                       why.description());
            }
        }
        Ok(file) => {
            let mut writer = BufWriter::new(&file);

            if let Err(why) = writer.write_all(data) {
                println!("couldn't write to {}: {}", display, why.description());
            }
        }
    }
}

fn ensure_dir(path: &Path) {
    match std::fs::create_dir_all(path) {
        Err(e) => {
            use std::io::ErrorKind::*;

            match e.kind() {
                // do not panic if it already exists, that's fine, we just want to make
                // sure we have it before moving on
                AlreadyExists => {}
                _ => {
                    panic!("failed to create {}: {}", path.display(), e.description());
                }
            }
        }
        _ => {}
    }
}
<commit_msg>Fix non-relative paths<commit_after>extern crate zim;
extern crate clap;
extern crate stopwatch;
extern crate crossbeam;

use std::fs::File;
use std::io::{Write, BufWriter};
use std::error::Error;
use std::collections::HashMap;
use std::path::Path;

use clap::{Arg, App};
use stopwatch::Stopwatch;

use zim::{Zim, Target};

fn main() {
    let matches = App::new("zimextractor")
        .version("0.1")
        .about("Extract zim files")
        .arg(Arg::with_name("out")
                 .short("o")
                 .long("out")
                 .help("Output directory")
                 .takes_value(true))
        .arg(Arg::with_name("skip-link")
                 .long("skip-link")
                 .help("Skip genrating hard links")
                 .takes_value(false))
        .arg(Arg::with_name("INPUT")
                 .help("Set the zim file to extract")
                 .required(true)
                 .index(1))
        .get_matches();

    let skip_link = matches.is_present("skip-link");
    let out = matches.value_of("out").unwrap_or("out");
    let root_output = Path::new(out);

    let input = matches.value_of("INPUT").unwrap();

    println!("Extracting file: {} to {}\n", input, out);

    let sw = Stopwatch::start_new();

    let zim = Zim::new(input).ok().unwrap();

    // map between cluster and directory entry
    let mut cluster_map = HashMap::new();

    println!("  Creating cluster map");
    for i in zim.iterate_by_urls() {
        if let Some(Target::Cluster(cid, _)) = i.target {
            cluster_map.entry(cid).or_insert(Vec::new()).push(i);
        }
    }

    println!("  Extracting entries: {}", zim.header.cluster_count);

    crossbeam::scope(|scope| for (cid, entries) in cluster_map {
                         let mut cluster = zim.get_cluster(cid).unwrap();

                         scope.spawn(move || {
            cluster.decompress();

            for entry in entries {
                if let Some(Target::Cluster(_cid, bid)) = entry.target {
                    assert_eq!(cid, _cid);
                    let mut s = String::new();
                    s.push(entry.namespace);
                    let out_path = root_output.join(&s).join(&entry.url);
                    if out_path.starts_with(&out) {
                      safe_write(&out_path, cluster.get_blob(bid), 1);
                    } else {
                      let new_out=&root_output.join(Path::new(&out_path.to_str().unwrap().replacen("/","./",1)));
                      safe_write(&new_out, cluster.get_blob(bid), 1);
                    }
                }
            }
        });
                     });

    println!("  Extraction done in {}ms", sw.elapsed_ms());

    if !skip_link {
        println!("  Linking redirects");

        // link all redirects
        for entry in zim.iterate_by_urls() {
            // get redirect entry
            if let Some(Target::Redirect(redir)) = entry.target {
                let redir = zim.get_by_url_index(redir).unwrap();

                let mut s = String::new();
                s.push(redir.namespace);
                let src = root_output.join(&s).join(&redir.url);

                let mut d = String::new();
                d.push(entry.namespace);
                let dst = root_output.join(&s).join(&entry.url);

                if !dst.exists() {
                    // println!("{:?} -> {:?}", src, dst);
                    std::fs::hard_link(src, dst).unwrap();
                }
            }
        }

        println!("  Linking done in {}ms", sw.elapsed_ms());
    } else {
        println!("  Skipping linking...");
    }

    if let Some(main_page_idx) = zim.header.main_page {
        let page = zim.get_by_url_index(main_page_idx).unwrap();
        println!("  Main page is {}", page.url);
    }
}

fn safe_write(path: &Path, data: &[u8], count: usize) {
    let display = path.display();
    let contain_path = path.parent().unwrap();

    ensure_dir(contain_path);

    match File::create(&path) {
        Err(why) => {
            println!("couldn't create {}: {}", display, why.description());

            if count < 3 {
                safe_write(path, data, count + 1);
            } else {
                panic!("failed retry: couldn't create {}: {}",
                       display,
                       why.description());
            }
        }
        Ok(file) => {
            let mut writer = BufWriter::new(&file);

            if let Err(why) = writer.write_all(data) {
                println!("couldn't write to {}: {}", display, why.description());
            }
        }
    }
}

fn ensure_dir(path: &Path) {
    match std::fs::create_dir_all(path) {
        Err(e) => {
            use std::io::ErrorKind::*;

            match e.kind() {
                // do not panic if it already exists, that's fine, we just want to make
                // sure we have it before moving on
                AlreadyExists => {}
                _ => {
                    panic!("failed to create {}: {}", path.display(), e.description());
                }
            }
        }
        _ => {}
    }
}
<|endoftext|>


<commit_before>//! Implementation of compiling the compiler and standard library, in "check"-based modes.

use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
use crate::cache::Interned;
use crate::compile::{add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo};
use crate::config::TargetSelection;
use crate::tool::{prepare_tool_cargo, SourceType};
use crate::INTERNER;
use crate::{Compiler, Mode, Subcommand};
use std::path::PathBuf;

#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct Std {
    pub target: TargetSelection,
}

/// Returns args for the subcommand itself (not for cargo)
fn args(builder: &Builder<'_>) -> Vec<String> {
    fn strings<'a>(arr: &'a [&str]) -> impl Iterator<Item = String> + 'a {
        arr.iter().copied().map(String::from)
    }

    if let Subcommand::Clippy { fix, .. } = builder.config.cmd {
        let mut args = vec![];
        if fix {
            #[rustfmt::skip]
            args.extend(strings(&[
                "--fix", "-Zunstable-options",
                // FIXME: currently, `--fix` gives an error while checking tests for libtest,
                // possibly because libtest is not yet built in the sysroot.
                // As a workaround, avoid checking tests and benches when passed --fix.
                "--lib", "--bins", "--examples",
            ]));
        }
        args.extend(strings(&["--", "--cap-lints", "warn"]));
        args
    } else {
        vec![]
    }
}

fn cargo_subcommand(kind: Kind) -> &'static str {
    match kind {
        Kind::Check => "check",
        Kind::Clippy => "clippy",
        Kind::Fix => "fix",
        _ => unreachable!(),
    }
}

impl Step for Std {
    type Output = ();
    const DEFAULT: bool = true;

    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
        run.all_krates("test")
    }

    fn make_run(run: RunConfig<'_>) {
        run.builder.ensure(Std { target: run.target });
    }

    fn run(self, builder: &Builder<'_>) {
        let target = self.target;
        let compiler = builder.compiler(0, builder.config.build);

        let mut cargo = builder.cargo(
            compiler,
            Mode::Std,
            SourceType::InTree,
            target,
            cargo_subcommand(builder.kind),
        );
        std_cargo(builder, target, compiler.stage, &mut cargo);

        builder.info(&format!("Checking std artifacts ({} -> {})", &compiler.host, target));
        run_cargo(
            builder,
            cargo,
            args(builder),
            &libstd_stamp(builder, compiler, target),
            vec![],
            true,
        );

        let libdir = builder.sysroot_libdir(compiler, target);
        let hostdir = builder.sysroot_libdir(compiler, compiler.host);
        add_to_sysroot(&builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target));

        // Then run cargo again, once we've put the rmeta files for the library
        // crates into the sysroot. This is needed because e.g., core's tests
        // depend on `libtest` -- Cargo presumes it will exist, but it doesn't
        // since we initialize with an empty sysroot.
        //
        // Currently only the "libtest" tree of crates does this.

        if let Subcommand::Check { all_targets: true, .. } = builder.config.cmd {
            let mut cargo = builder.cargo(
                compiler,
                Mode::Std,
                SourceType::InTree,
                target,
                cargo_subcommand(builder.kind),
            );
            std_cargo(builder, target, compiler.stage, &mut cargo);
            cargo.arg("--all-targets");

            // Explicitly pass -p for all dependencies krates -- this will force cargo
            // to also check the tests/benches/examples for these crates, rather
            // than just the leaf crate.
            for krate in builder.in_tree_crates("test", Some(target)) {
                cargo.arg("-p").arg(krate.name);
            }

            builder.info(&format!(
                "Checking std test/bench/example targets ({} -> {})",
                &compiler.host, target
            ));
            run_cargo(
                builder,
                cargo,
                args(builder),
                &libstd_test_stamp(builder, compiler, target),
                vec![],
                true,
            );
        }
    }
}

#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct Rustc {
    pub target: TargetSelection,
}

impl Step for Rustc {
    type Output = ();
    const ONLY_HOSTS: bool = true;
    const DEFAULT: bool = true;

    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
        run.all_krates("rustc-main")
    }

    fn make_run(run: RunConfig<'_>) {
        run.builder.ensure(Rustc { target: run.target });
    }

    /// Builds the compiler.
    ///
    /// This will build the compiler for a particular stage of the build using
    /// the `compiler` targeting the `target` architecture. The artifacts
    /// created will also be linked into the sysroot directory.
    fn run(self, builder: &Builder<'_>) {
        let compiler = builder.compiler(0, builder.config.build);
        let target = self.target;

        builder.ensure(Std { target });

        let mut cargo = builder.cargo(
            compiler,
            Mode::Rustc,
            SourceType::InTree,
            target,
            cargo_subcommand(builder.kind),
        );
        rustc_cargo(builder, &mut cargo, target);
        if let Subcommand::Check { all_targets: true, .. } = builder.config.cmd {
            cargo.arg("--all-targets");
        }

        // Explicitly pass -p for all compiler krates -- this will force cargo
        // to also check the tests/benches/examples for these crates, rather
        // than just the leaf crate.
        for krate in builder.in_tree_crates("rustc-main", Some(target)) {
            cargo.arg("-p").arg(krate.name);
        }

        builder.info(&format!("Checking compiler artifacts ({} -> {})", &compiler.host, target));
        run_cargo(
            builder,
            cargo,
            args(builder),
            &librustc_stamp(builder, compiler, target),
            vec![],
            true,
        );

        let libdir = builder.sysroot_libdir(compiler, target);
        let hostdir = builder.sysroot_libdir(compiler, compiler.host);
        add_to_sysroot(&builder, &libdir, &hostdir, &librustc_stamp(builder, compiler, target));
    }
}

#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct CodegenBackend {
    pub target: TargetSelection,
    pub backend: Interned<String>,
}

impl Step for CodegenBackend {
    type Output = ();
    const ONLY_HOSTS: bool = true;
    const DEFAULT: bool = true;

    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
        run.paths(&["compiler/rustc_codegen_cranelift", "rustc_codegen_cranelift"])
    }

    fn make_run(run: RunConfig<'_>) {
        for &backend in &[INTERNER.intern_str("cranelift")] {
            run.builder.ensure(CodegenBackend { target: run.target, backend });
        }
    }

    fn run(self, builder: &Builder<'_>) {
        let compiler = builder.compiler(0, builder.config.build);
        let target = self.target;
        let backend = self.backend;

        builder.ensure(Rustc { target });

        let mut cargo = builder.cargo(
            compiler,
            Mode::Codegen,
            SourceType::Submodule,
            target,
            cargo_subcommand(builder.kind),
        );
        cargo
            .arg("--manifest-path")
            .arg(builder.src.join(format!("compiler/rustc_codegen_{}/Cargo.toml", backend)));
        rustc_cargo_env(builder, &mut cargo, target);

        run_cargo(
            builder,
            cargo,
            args(builder),
            &codegen_backend_stamp(builder, compiler, target, backend),
            vec![],
            true,
        );
    }
}

macro_rules! tool_check_step {
    ($name:ident, $path:expr, $source_type:expr) => {
        #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
        pub struct $name {
            pub target: TargetSelection,
        }

        impl Step for $name {
            type Output = ();
            const ONLY_HOSTS: bool = true;
            const DEFAULT: bool = true;

            fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
                run.path($path)
            }

            fn make_run(run: RunConfig<'_>) {
                run.builder.ensure($name { target: run.target });
            }

            fn run(self, builder: &Builder<'_>) {
                let compiler = builder.compiler(0, builder.config.build);
                let target = self.target;

                builder.ensure(Rustc { target });

                let mut cargo = prepare_tool_cargo(
                    builder,
                    compiler,
                    Mode::ToolRustc,
                    target,
                    cargo_subcommand(builder.kind),
                    $path,
                    $source_type,
                    &[],
                );

                if let Subcommand::Check { all_targets: true, .. } = builder.config.cmd {
                    cargo.arg("--all-targets");
                }

                builder.info(&format!(
                    "Checking {} artifacts ({} -> {})",
                    stringify!($name).to_lowercase(),
                    &compiler.host.triple,
                    target.triple
                ));
                run_cargo(
                    builder,
                    cargo,
                    args(builder),
                    &stamp(builder, compiler, target),
                    vec![],
                    true,
                );

                let libdir = builder.sysroot_libdir(compiler, target);
                let hostdir = builder.sysroot_libdir(compiler, compiler.host);
                add_to_sysroot(&builder, &libdir, &hostdir, &stamp(builder, compiler, target));

                /// Cargo's output path in a given stage, compiled by a particular
                /// compiler for the specified target.
                fn stamp(
                    builder: &Builder<'_>,
                    compiler: Compiler,
                    target: TargetSelection,
                ) -> PathBuf {
                    builder
                        .cargo_out(compiler, Mode::ToolRustc, target)
                        .join(format!(".{}-check.stamp", stringify!($name).to_lowercase()))
                }
            }
        }
    };
}

tool_check_step!(Rustdoc, "src/tools/rustdoc", SourceType::InTree);
// Clippy is a hybrid. It is an external tool, but uses a git subtree instead
// of a submodule. Since the SourceType only drives the deny-warnings
// behavior, treat it as in-tree so that any new warnings in clippy will be
// rejected.
tool_check_step!(Clippy, "src/tools/clippy", SourceType::InTree);

tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree);

/// Cargo's output path for the standard library in a given stage, compiled
/// by a particular compiler for the specified target.
fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf {
    builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check.stamp")
}

/// Cargo's output path for the standard library in a given stage, compiled
/// by a particular compiler for the specified target.
fn libstd_test_stamp(
    builder: &Builder<'_>,
    compiler: Compiler,
    target: TargetSelection,
) -> PathBuf {
    builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check-test.stamp")
}

/// Cargo's output path for librustc in a given stage, compiled by a particular
/// compiler for the specified target.
fn librustc_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf {
    builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc-check.stamp")
}

/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular
/// compiler for the specified target and backend.
fn codegen_backend_stamp(
    builder: &Builder<'_>,
    compiler: Compiler,
    target: TargetSelection,
    backend: Interned<String>,
) -> PathBuf {
    builder
        .cargo_out(compiler, Mode::Codegen, target)
        .join(format!(".librustc_codegen_{}-check.stamp", backend))
}
<commit_msg>Rollup merge of #79059 - jyn514:cranelift, r=Mark-Simulacrum<commit_after>//! Implementation of compiling the compiler and standard library, in "check"-based modes.

use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
use crate::cache::Interned;
use crate::compile::{add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo};
use crate::config::TargetSelection;
use crate::tool::{prepare_tool_cargo, SourceType};
use crate::INTERNER;
use crate::{Compiler, Mode, Subcommand};
use std::path::PathBuf;

#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct Std {
    pub target: TargetSelection,
}

/// Returns args for the subcommand itself (not for cargo)
fn args(builder: &Builder<'_>) -> Vec<String> {
    fn strings<'a>(arr: &'a [&str]) -> impl Iterator<Item = String> + 'a {
        arr.iter().copied().map(String::from)
    }

    if let Subcommand::Clippy { fix, .. } = builder.config.cmd {
        let mut args = vec![];
        if fix {
            #[rustfmt::skip]
            args.extend(strings(&[
                "--fix", "-Zunstable-options",
                // FIXME: currently, `--fix` gives an error while checking tests for libtest,
                // possibly because libtest is not yet built in the sysroot.
                // As a workaround, avoid checking tests and benches when passed --fix.
                "--lib", "--bins", "--examples",
            ]));
        }
        args.extend(strings(&["--", "--cap-lints", "warn"]));
        args
    } else {
        vec![]
    }
}

fn cargo_subcommand(kind: Kind) -> &'static str {
    match kind {
        Kind::Check => "check",
        Kind::Clippy => "clippy",
        Kind::Fix => "fix",
        _ => unreachable!(),
    }
}

impl Step for Std {
    type Output = ();
    const DEFAULT: bool = true;

    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
        run.all_krates("test")
    }

    fn make_run(run: RunConfig<'_>) {
        run.builder.ensure(Std { target: run.target });
    }

    fn run(self, builder: &Builder<'_>) {
        let target = self.target;
        let compiler = builder.compiler(0, builder.config.build);

        let mut cargo = builder.cargo(
            compiler,
            Mode::Std,
            SourceType::InTree,
            target,
            cargo_subcommand(builder.kind),
        );
        std_cargo(builder, target, compiler.stage, &mut cargo);

        builder.info(&format!("Checking std artifacts ({} -> {})", &compiler.host, target));
        run_cargo(
            builder,
            cargo,
            args(builder),
            &libstd_stamp(builder, compiler, target),
            vec![],
            true,
        );

        let libdir = builder.sysroot_libdir(compiler, target);
        let hostdir = builder.sysroot_libdir(compiler, compiler.host);
        add_to_sysroot(&builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target));

        // Then run cargo again, once we've put the rmeta files for the library
        // crates into the sysroot. This is needed because e.g., core's tests
        // depend on `libtest` -- Cargo presumes it will exist, but it doesn't
        // since we initialize with an empty sysroot.
        //
        // Currently only the "libtest" tree of crates does this.

        if let Subcommand::Check { all_targets: true, .. } = builder.config.cmd {
            let mut cargo = builder.cargo(
                compiler,
                Mode::Std,
                SourceType::InTree,
                target,
                cargo_subcommand(builder.kind),
            );
            std_cargo(builder, target, compiler.stage, &mut cargo);
            cargo.arg("--all-targets");

            // Explicitly pass -p for all dependencies krates -- this will force cargo
            // to also check the tests/benches/examples for these crates, rather
            // than just the leaf crate.
            for krate in builder.in_tree_crates("test", Some(target)) {
                cargo.arg("-p").arg(krate.name);
            }

            builder.info(&format!(
                "Checking std test/bench/example targets ({} -> {})",
                &compiler.host, target
            ));
            run_cargo(
                builder,
                cargo,
                args(builder),
                &libstd_test_stamp(builder, compiler, target),
                vec![],
                true,
            );
        }
    }
}

#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct Rustc {
    pub target: TargetSelection,
}

impl Step for Rustc {
    type Output = ();
    const ONLY_HOSTS: bool = true;
    const DEFAULT: bool = true;

    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
        run.all_krates("rustc-main")
    }

    fn make_run(run: RunConfig<'_>) {
        run.builder.ensure(Rustc { target: run.target });
    }

    /// Builds the compiler.
    ///
    /// This will build the compiler for a particular stage of the build using
    /// the `compiler` targeting the `target` architecture. The artifacts
    /// created will also be linked into the sysroot directory.
    fn run(self, builder: &Builder<'_>) {
        let compiler = builder.compiler(0, builder.config.build);
        let target = self.target;

        builder.ensure(Std { target });

        let mut cargo = builder.cargo(
            compiler,
            Mode::Rustc,
            SourceType::InTree,
            target,
            cargo_subcommand(builder.kind),
        );
        rustc_cargo(builder, &mut cargo, target);
        if let Subcommand::Check { all_targets: true, .. } = builder.config.cmd {
            cargo.arg("--all-targets");
        }

        // Explicitly pass -p for all compiler krates -- this will force cargo
        // to also check the tests/benches/examples for these crates, rather
        // than just the leaf crate.
        for krate in builder.in_tree_crates("rustc-main", Some(target)) {
            cargo.arg("-p").arg(krate.name);
        }

        builder.info(&format!("Checking compiler artifacts ({} -> {})", &compiler.host, target));
        run_cargo(
            builder,
            cargo,
            args(builder),
            &librustc_stamp(builder, compiler, target),
            vec![],
            true,
        );

        let libdir = builder.sysroot_libdir(compiler, target);
        let hostdir = builder.sysroot_libdir(compiler, compiler.host);
        add_to_sysroot(&builder, &libdir, &hostdir, &librustc_stamp(builder, compiler, target));
    }
}

#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct CodegenBackend {
    pub target: TargetSelection,
    pub backend: Interned<String>,
}

impl Step for CodegenBackend {
    type Output = ();
    const ONLY_HOSTS: bool = true;
    const DEFAULT: bool = true;

    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
        run.paths(&["compiler/rustc_codegen_cranelift", "rustc_codegen_cranelift"])
    }

    fn make_run(run: RunConfig<'_>) {
        for &backend in &[INTERNER.intern_str("cranelift")] {
            run.builder.ensure(CodegenBackend { target: run.target, backend });
        }
    }

    fn run(self, builder: &Builder<'_>) {
        let compiler = builder.compiler(0, builder.config.build);
        let target = self.target;
        let backend = self.backend;

        builder.ensure(Rustc { target });

        let mut cargo = builder.cargo(
            compiler,
            Mode::Codegen,
            SourceType::Submodule,
            target,
            cargo_subcommand(builder.kind),
        );
        cargo
            .arg("--manifest-path")
            .arg(builder.src.join(format!("compiler/rustc_codegen_{}/Cargo.toml", backend)));
        rustc_cargo_env(builder, &mut cargo, target);

        builder.info(&format!(
            "Checking {} artifacts ({} -> {})",
            backend, &compiler.host.triple, target.triple
        ));

        run_cargo(
            builder,
            cargo,
            args(builder),
            &codegen_backend_stamp(builder, compiler, target, backend),
            vec![],
            true,
        );
    }
}

macro_rules! tool_check_step {
    ($name:ident, $path:expr, $source_type:expr) => {
        #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
        pub struct $name {
            pub target: TargetSelection,
        }

        impl Step for $name {
            type Output = ();
            const ONLY_HOSTS: bool = true;
            const DEFAULT: bool = true;

            fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
                run.path($path)
            }

            fn make_run(run: RunConfig<'_>) {
                run.builder.ensure($name { target: run.target });
            }

            fn run(self, builder: &Builder<'_>) {
                let compiler = builder.compiler(0, builder.config.build);
                let target = self.target;

                builder.ensure(Rustc { target });

                let mut cargo = prepare_tool_cargo(
                    builder,
                    compiler,
                    Mode::ToolRustc,
                    target,
                    cargo_subcommand(builder.kind),
                    $path,
                    $source_type,
                    &[],
                );

                if let Subcommand::Check { all_targets: true, .. } = builder.config.cmd {
                    cargo.arg("--all-targets");
                }

                builder.info(&format!(
                    "Checking {} artifacts ({} -> {})",
                    stringify!($name).to_lowercase(),
                    &compiler.host.triple,
                    target.triple
                ));
                run_cargo(
                    builder,
                    cargo,
                    args(builder),
                    &stamp(builder, compiler, target),
                    vec![],
                    true,
                );

                let libdir = builder.sysroot_libdir(compiler, target);
                let hostdir = builder.sysroot_libdir(compiler, compiler.host);
                add_to_sysroot(&builder, &libdir, &hostdir, &stamp(builder, compiler, target));

                /// Cargo's output path in a given stage, compiled by a particular
                /// compiler for the specified target.
                fn stamp(
                    builder: &Builder<'_>,
                    compiler: Compiler,
                    target: TargetSelection,
                ) -> PathBuf {
                    builder
                        .cargo_out(compiler, Mode::ToolRustc, target)
                        .join(format!(".{}-check.stamp", stringify!($name).to_lowercase()))
                }
            }
        }
    };
}

tool_check_step!(Rustdoc, "src/tools/rustdoc", SourceType::InTree);
// Clippy is a hybrid. It is an external tool, but uses a git subtree instead
// of a submodule. Since the SourceType only drives the deny-warnings
// behavior, treat it as in-tree so that any new warnings in clippy will be
// rejected.
tool_check_step!(Clippy, "src/tools/clippy", SourceType::InTree);

tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree);

/// Cargo's output path for the standard library in a given stage, compiled
/// by a particular compiler for the specified target.
fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf {
    builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check.stamp")
}

/// Cargo's output path for the standard library in a given stage, compiled
/// by a particular compiler for the specified target.
fn libstd_test_stamp(
    builder: &Builder<'_>,
    compiler: Compiler,
    target: TargetSelection,
) -> PathBuf {
    builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check-test.stamp")
}

/// Cargo's output path for librustc in a given stage, compiled by a particular
/// compiler for the specified target.
fn librustc_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf {
    builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc-check.stamp")
}

/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular
/// compiler for the specified target and backend.
fn codegen_backend_stamp(
    builder: &Builder<'_>,
    compiler: Compiler,
    target: TargetSelection,
    backend: Interned<String>,
) -> PathBuf {
    builder
        .cargo_out(compiler, Mode::Codegen, target)
        .join(format!(".librustc_codegen_{}-check.stamp", backend))
}
<|endoftext|>


<commit_before>use helpers::*;
use core::cell::Cell;
use core::cmp;

use pm;
use hil::spi_master;
use hil::spi_master::SpiCallback;
use hil::spi_master::ClockPolarity;
use hil::spi_master::ClockPhase;
use hil::spi_master::DataOrder;

// Driver for the SPI hardware (seperate from the USARTS, described in chapter 26 of the
// datasheet)

/// The registers used to interface with the hardware
#[repr(C, packed)]
struct SpiRegisters {
    cr: u32, // 0x0
    mr: u32, // 0x4
    rdr: u32, // 0x8
    tdr: u32, // 0xC
    sr: u32, // 0x10
    ier: u32, // 0x14
    idr: u32, // 0x18
    imr: u32, // 0x1C
    reserved0: [u32; 4], // 0x20, 0x24, 0x28, 0x2C
    csr0: u32, // 0x30
    csr1: u32, // 0x34
    csr2: u32, // 0x38
    csr3: u32, // 0x3C
    reserved1: [u32; 41], // 0x40 - 0xE0
    wpcr: u32, // 0xE4
    wpsr: u32, // 0xE8
    reserved2: [u32; 3], // 0xEC - 0xF4
    features: u32, // 0xF8
    version: u32, // 0xFC
}

const SPI_BASE: u32 = 0x40008000;

/// Values for selected peripherals
#[derive(Copy,Clone)]
pub enum Peripheral {
    Peripheral0,
    Peripheral1,
    Peripheral2,
    Peripheral3,
}

///
/// SPI implementation using the SPI hardware
/// Supports four peripherals. Each peripheral can have different settings.
///
/// The init, read, and write methods act on the currently selected peripheral.
/// The init method can be safely called more than once to configure different peripherals:
///
///     spi.set_active_peripheral(Peripheral::Peripheral0);
///     spi.init(/* Parameters for peripheral 0 */);
///     spi.set_active_peripheral(Peripheral::Peripheral1);
///     spi.init(/* Parameters for peripheral 1 */);
///
pub struct Spi {
    /// Registers
    regs: *mut SpiRegisters,
    /// Client
    callback: Cell<Option<&'static SpiCallback>>,
}

impl Spi {
    /// Creates a new SPI object, with peripheral 0 selected
   pub const fn new() -> Spi {
        Spi {
            regs: SPI_BASE as *mut SpiRegisters,
            callback: Cell::new(None)
        }
    }

    pub fn enable(&self) {
        unsafe { volatile_store(&mut (*self.regs).cr, 0b1); }
    }

    pub fn disable(&self) {
        unsafe { volatile_store(&mut (*self.regs).cr, 0b10); }
    }

    /// Sets the approximate baud rate for the active peripheral
    ///
    /// Since the only supported baud rates are 48 MHz / n where n is an integer from 1 to 255,
    /// the exact baud rate may not be available. In that case, the next lower baud rate will be
    /// selected.
    ///
    /// The lowest available baud rate is 188235 baud. If the requested rate is lower,
    /// 188235 baud will be selected.
    pub fn set_baud_rate(&self, rate: u32) -> u32 {
        // Main clock frequency
        let mut real_rate = rate;
        let clock = 48000000;

        if real_rate < 188235 {
            real_rate = 188235;
        }
        if real_rate > clock {
            real_rate = clock;
        }

        // Divide and truncate, resulting in a n value that might be too low
        let mut scbr = clock / real_rate;
        // If the division was not exact, increase the n to get a slower baud rate
        if clock % rate != 0 {
            scbr += 1;
        }
        let mut csr = self.read_active_csr();
        let csr_mask: u32 = 0xFFFF00FF;
        // Clear, then write CSR bits
        csr &= csr_mask;
        csr |= (scbr & 0xFF) << 8;
        self.write_active_csr(csr);
        clock / scbr
    }

    pub fn set_active_peripheral(&self, peripheral: Peripheral) {
        let peripheral_number: u32 = match peripheral {
            Peripheral::Peripheral0 => 0b0000,
            Peripheral::Peripheral1 => 0b0001,
            Peripheral::Peripheral2 => 0b0011,
            Peripheral::Peripheral3 => 0b0111
        };
        let mut mr = unsafe { volatile_load(& (*self.regs).mr) };
        let pcs_mask: u32 = 0xFFF0FFFF;
        mr &= pcs_mask;
        mr |= peripheral_number << 16;
        unsafe { volatile_store(&mut (*self.regs).mr, mr); }
    }

    /// Returns the currently active peripheral
    pub fn get_active_peripheral(&self) -> Peripheral {
        let mr = unsafe {volatile_load(&(*self.regs).mr)};
        let pcs = (mr >> 16) & 0xF;
        // Split into bits for matching
        let pcs_bits = ((pcs >> 3) & 1, (pcs >> 2) & 1, (pcs >> 1) & 1, pcs & 1);
        match pcs_bits {
            (_, _, _, 0) => Peripheral::Peripheral0,
            (_, _, 0, 1) => Peripheral::Peripheral1,
            (_, 0, 1, 1) => Peripheral::Peripheral2,
            (0, 1, 1, 1) => Peripheral::Peripheral3,
            _ => {
                // Invalid configuration
                // ???
                Peripheral::Peripheral0
            }
        }
    }

    /// Returns the value of CSR0, CSR1, CSR2, or CSR3, whichever corresponds to the active
    /// peripheral
    fn read_active_csr(&self) -> u32 {
        match self.get_active_peripheral() {
            Peripheral::Peripheral0 => unsafe {volatile_load(&(*self.regs).csr0)},
            Peripheral::Peripheral1 => unsafe {volatile_load(&(*self.regs).csr1)},
            Peripheral::Peripheral2 => unsafe {volatile_load(&(*self.regs).csr2)},
            Peripheral::Peripheral3 => unsafe {volatile_load(&(*self.regs).csr3)},
        }
    }
    /// Sets the value of CSR0, CSR1, CSR2, or CSR3, whichever corresponds to the active
    /// peripheral
    fn write_active_csr(&self, value: u32) {
        match self.get_active_peripheral() {
            Peripheral::Peripheral0 => unsafe {volatile_store(&mut (*self.regs).csr0, value)},
            Peripheral::Peripheral1 => unsafe {volatile_store(&mut (*self.regs).csr1, value)},
            Peripheral::Peripheral2 => unsafe {volatile_store(&mut (*self.regs).csr2, value)},
            Peripheral::Peripheral3 => unsafe {volatile_store(&mut (*self.regs).csr3, value)},
        };
    }
}

impl spi_master::SpiMaster for Spi {
    fn init(&self, callback: &'static SpiCallback) {
        // Enable clock
        // unsafe { pm::enable_clock(pm::Clock::PBA(pm::PBAClock::SPI)); }
        // Configure GPIO pins
        // PA21, PA27, PC04 -> MISO
        // PA22, PA28, PC05 -> MOSI
        // PA23, PA29, PC06 -> SCK
        // PA24, PA30, PC03 -> CS0
        // PA31 , PC02-> CS1
        // PC00 -> CS2
        // PC01 -> CR3
        self.callback.set(Some(callback));
        self.set_rate(40000); // Set initial baud rate to 8MHz
        self.set_clock(ClockPolarity::IdleLow);
        self.set_phase(ClockPhase::SampleLeading);

        // Keep slave select active until a last transfer bit is set
        let mut csr = self.read_active_csr();
        csr |= 1 << 3;
        self.write_active_csr(csr);

        // Indicate the last transfer to disable slave select
        unsafe {volatile_store(&mut (*self.regs).cr, 1 << 24)};

        let mut mode = unsafe {volatile_load(&(*self.regs).mr)};
        mode |= 1; // Enable master mode
        mode |= 1 << 4; // Disable mode fault detection (open drain outputs not supported)
        mode != 1 << 7; // Enable looopback
        unsafe {volatile_store(&mut (*self.regs).mr, mode)};
    }

    fn read_write_byte(&'static self, val: u8) -> u8 {
        self.write_byte(val);
        // Wait for receive data register full
        while (unsafe {volatile_load(&(*self.regs).sr)} & 1) != 1 {}
        // Return read value
        unsafe {volatile_load(&(*self.regs).rdr) as u8}
    }

    fn write_byte(&'static self, out_byte: u8) {
        let tdr = out_byte as u32;
        // Wait for data to leave TDR and enter serializer, so TDR is free
        // for this next byte
        while (unsafe {volatile_load(& (*self.regs).sr)} & 1 << 1) == 0 {}
        unsafe {volatile_store(&mut (*self.regs).tdr, tdr)};
    }

    fn read_byte(&'static self) -> u8 {
        self.read_write_byte(0)
    }

    /// The write buffer has to be mutable because it's passed back to
    /// the caller, and the caller may want to be able write into it.
    fn read_write_bytes(&'static self,
                        mut read_buffer:  Option<&'static mut [u8]>,
                        write_buffer: Option<&'static mut [u8]>) -> bool {
        // If both are Some, read/write minimum of lengths
        // If only read is Some, read length and write zeroes
        // If only write is Some, write length and discard reads
        // If both are None, return false
        // TODO: Asynchronous
        if read_buffer.is_none() && write_buffer.is_none() {
            return false
        }
        let reading = read_buffer.is_some();
        let writing = write_buffer.is_some();
        let read_len = match read_buffer {
            Some(ref buf) => {buf.len()},
            None          => 0
        };
        let write_len = match write_buffer {
            Some(ref buf) => {buf.len()},
            None          => 0
        };
        let count = if reading && writing {cmp::min(read_len, write_len)}
                    else                  {cmp::max(read_len, write_len)};
        for i in 0..count {
            let mut txbyte: u8 = 0;
            match write_buffer {
                Some(ref buf) => {txbyte = buf[i];}
                None          => {}
            }
            // Write the value
            let rxbyte = self.read_write_byte(txbyte);
            match read_buffer.take() {
                Some(ref mut buf) => {buf[i] = rxbyte;}
                None          => {}
            }
        }
        self.callback.get().map(|cb| {
            cb.read_write_done(read_buffer, write_buffer);
        });
        true
    }

#[allow(unused_variables)]
    fn set_rate(&self, rate: u32) -> u32 {
      self.set_baud_rate(rate)
    }

#[allow(unused_variables)]
    fn set_clock(&self, polarity: ClockPolarity) {
        let mut csr = self.read_active_csr();
        match polarity {
            ClockPolarity::IdleHigh => csr |= 1,
            ClockPolarity::IdleLow => csr &= 0xFFFFFFFE,
        };
        self.write_active_csr(csr);
    }

    fn get_clock(&self) -> ClockPolarity { ClockPolarity::IdleLow }

#[allow(unused_variables)]
    fn set_phase(&self, phase: ClockPhase) {
        let mut csr = self.read_active_csr();
        match phase {
            ClockPhase::SampleLeading => csr |= 1 << 1,
            ClockPhase::SampleTrailing => csr &= 0xFFFFFFFD,
        };
        self.write_active_csr(csr);
    }

    /// Sets the active peripheral
    fn set_chip_select(&self, cs: u8) {
        let peripheral_number: u32 = match cs {
            0 => 0b0000,
            1 => 0b0001,
            2 => 0b0011,
            3 => 0b0111,
            _ => 0b0000,
        };

        let mut mr = unsafe {volatile_load(&(*self.regs).mr)};
        // Clear and set MR.PCS
        let pcs_mask: u32 = 0xFFF0FFFF;
        mr &= pcs_mask;
        mr |= peripheral_number << 16;
        unsafe {volatile_store(&mut (*self.regs).mr, mr);}
    }

    fn clear_chip_select(&self) {
       unsafe {volatile_store(&mut (*self.regs).cr, 1 << 24)};
    }
}
<commit_msg>Reading from MISO works; something wrong with loopback setting in SPI mode register, so tested with a simple wiring. Behavior of the driver test (written in platform/storm/lib.rs) is it enters an infinite loop that writes 5 bytes to the SPI and pauses. The 5 bytes are 0x55, a counter, the byte read on the first write, the byte read on the second write, and 0. So if everything is working correctly it should be<commit_after>use helpers::*;
use core::cell::Cell;
use core::cmp;

use pm;
use hil::spi_master;
use hil::spi_master::SpiCallback;
use hil::spi_master::ClockPolarity;
use hil::spi_master::ClockPhase;
use hil::spi_master::DataOrder;

// Driver for the SPI hardware (seperate from the USARTS, described in chapter 26 of the
// datasheet)

/// The registers used to interface with the hardware
#[repr(C, packed)]
struct SpiRegisters {
    cr: u32, // 0x0
    mr: u32, // 0x4
    rdr: u32, // 0x8
    tdr: u32, // 0xC
    sr: u32, // 0x10
    ier: u32, // 0x14
    idr: u32, // 0x18
    imr: u32, // 0x1C
    reserved0: [u32; 4], // 0x20, 0x24, 0x28, 0x2C
    csr0: u32, // 0x30
    csr1: u32, // 0x34
    csr2: u32, // 0x38
    csr3: u32, // 0x3C
    reserved1: [u32; 41], // 0x40 - 0xE0
    wpcr: u32, // 0xE4
    wpsr: u32, // 0xE8
    reserved2: [u32; 3], // 0xEC - 0xF4
    features: u32, // 0xF8
    version: u32, // 0xFC
}

const SPI_BASE: u32 = 0x40008000;

/// Values for selected peripherals
#[derive(Copy,Clone)]
pub enum Peripheral {
    Peripheral0,
    Peripheral1,
    Peripheral2,
    Peripheral3,
}

///
/// SPI implementation using the SPI hardware
/// Supports four peripherals. Each peripheral can have different settings.
///
/// The init, read, and write methods act on the currently selected peripheral.
/// The init method can be safely called more than once to configure different peripherals:
///
///     spi.set_active_peripheral(Peripheral::Peripheral0);
///     spi.init(/* Parameters for peripheral 0 */);
///     spi.set_active_peripheral(Peripheral::Peripheral1);
///     spi.init(/* Parameters for peripheral 1 */);
///
pub struct Spi {
    /// Registers
    regs: *mut SpiRegisters,
    /// Client
    callback: Cell<Option<&'static SpiCallback>>,
}

impl Spi {
    /// Creates a new SPI object, with peripheral 0 selected
   pub const fn new() -> Spi {
        Spi {
            regs: SPI_BASE as *mut SpiRegisters,
            callback: Cell::new(None)
        }
    }

    pub fn enable(&self) {
        unsafe { volatile_store(&mut (*self.regs).cr, 0b1); }
    }

    pub fn disable(&self) {
        unsafe { volatile_store(&mut (*self.regs).cr, 0b10); }
    }

    /// Sets the approximate baud rate for the active peripheral
    ///
    /// Since the only supported baud rates are 48 MHz / n where n is an integer from 1 to 255,
    /// the exact baud rate may not be available. In that case, the next lower baud rate will be
    /// selected.
    ///
    /// The lowest available baud rate is 188235 baud. If the requested rate is lower,
    /// 188235 baud will be selected.
    pub fn set_baud_rate(&self, rate: u32) -> u32 {
        // Main clock frequency
        let mut real_rate = rate;
        let clock = 48000000;

        if real_rate < 188235 {
            real_rate = 188235;
        }
        if real_rate > clock {
            real_rate = clock;
        }

        // Divide and truncate, resulting in a n value that might be too low
        let mut scbr = clock / real_rate;
        // If the division was not exact, increase the n to get a slower baud rate
        if clock % rate != 0 {
            scbr += 1;
        }
        let mut csr = self.read_active_csr();
        let csr_mask: u32 = 0xFFFF00FF;
        // Clear, then write CSR bits
        csr &= csr_mask;
        csr |= (scbr & 0xFF) << 8;
        self.write_active_csr(csr);
        clock / scbr
    }

    pub fn set_active_peripheral(&self, peripheral: Peripheral) {
        let peripheral_number: u32 = match peripheral {
            Peripheral::Peripheral0 => 0b0000,
            Peripheral::Peripheral1 => 0b0001,
            Peripheral::Peripheral2 => 0b0011,
            Peripheral::Peripheral3 => 0b0111
        };
        let mut mr = unsafe { volatile_load(& (*self.regs).mr) };
        let pcs_mask: u32 = 0xFFF0FFFF;
        mr &= pcs_mask;
        mr |= peripheral_number << 16;
        unsafe { volatile_store(&mut (*self.regs).mr, mr); }
    }

    /// Returns the currently active peripheral
    pub fn get_active_peripheral(&self) -> Peripheral {
        let mr = unsafe {volatile_load(&(*self.regs).mr)};
        let pcs = (mr >> 16) & 0xF;
        // Split into bits for matching
        let pcs_bits = ((pcs >> 3) & 1, (pcs >> 2) & 1, (pcs >> 1) & 1, pcs & 1);
        match pcs_bits {
            (_, _, _, 0) => Peripheral::Peripheral0,
            (_, _, 0, 1) => Peripheral::Peripheral1,
            (_, 0, 1, 1) => Peripheral::Peripheral2,
            (0, 1, 1, 1) => Peripheral::Peripheral3,
            _ => {
                // Invalid configuration
                // ???
                Peripheral::Peripheral0
            }
        }
    }

    /// Returns the value of CSR0, CSR1, CSR2, or CSR3, whichever corresponds to the active
    /// peripheral
    fn read_active_csr(&self) -> u32 {
        match self.get_active_peripheral() {
            Peripheral::Peripheral0 => unsafe {volatile_load(&(*self.regs).csr0)},
            Peripheral::Peripheral1 => unsafe {volatile_load(&(*self.regs).csr1)},
            Peripheral::Peripheral2 => unsafe {volatile_load(&(*self.regs).csr2)},
            Peripheral::Peripheral3 => unsafe {volatile_load(&(*self.regs).csr3)},
        }
    }
    /// Sets the value of CSR0, CSR1, CSR2, or CSR3, whichever corresponds to the active
    /// peripheral
    fn write_active_csr(&self, value: u32) {
        match self.get_active_peripheral() {
            Peripheral::Peripheral0 => unsafe {volatile_store(&mut (*self.regs).csr0, value)},
            Peripheral::Peripheral1 => unsafe {volatile_store(&mut (*self.regs).csr1, value)},
            Peripheral::Peripheral2 => unsafe {volatile_store(&mut (*self.regs).csr2, value)},
            Peripheral::Peripheral3 => unsafe {volatile_store(&mut (*self.regs).csr3, value)},
        };
    }
}

impl spi_master::SpiMaster for Spi {
    fn init(&self, callback: &'static SpiCallback) {
        // Enable clock
        // unsafe { pm::enable_clock(pm::Clock::PBA(pm::PBAClock::SPI)); }
        // Configure GPIO pins
        // PA21, PA27, PC04 -> MISO
        // PA22, PA28, PC05 -> MOSI
        // PA23, PA29, PC06 -> SCK
        // PA24, PA30, PC03 -> CS0
        // PA31 , PC02-> CS1
        // PC00 -> CS2
        // PC01 -> CR3
        self.callback.set(Some(callback));
        self.set_rate(40000); // Set initial baud rate to 40kbps
        self.set_clock(ClockPolarity::IdleLow);
        self.set_phase(ClockPhase::SampleLeading);

        // Keep slave select active until a last transfer bit is set
        let mut csr = self.read_active_csr();
        csr |= 1 << 3;
        self.write_active_csr(csr);

        // Indicate the last transfer to disable slave select
        unsafe {volatile_store(&mut (*self.regs).cr, 1 << 24)};

        let mut mode = unsafe {volatile_load(&(*self.regs).mr)};
        mode |= 1; // Enable master mode
        mode |= 1 << 4; // Disable mode fault detection (open drain outputs not supported)
        unsafe {volatile_store(&mut (*self.regs).mr, mode)};
    }

    fn read_write_byte(&'static self, val: u8) -> u8 {
        self.write_byte(val);
        // Wait for receive data register full
        while (unsafe {volatile_load(&(*self.regs).sr)} & 1) != 1 {}
        // Return read value
        unsafe {volatile_load(&(*self.regs).rdr) as u8}
    }

    fn write_byte(&'static self, out_byte: u8) {
        let tdr = out_byte as u32;
        // Wait for data to leave TDR and enter serializer, so TDR is free
        // for this next byte
        while (unsafe {volatile_load(& (*self.regs).sr)} & 1 << 1) == 0 {}
        unsafe {volatile_store(&mut (*self.regs).tdr, tdr)};
    }

    fn read_byte(&'static self) -> u8 {
        self.read_write_byte(0)
    }

    /// The write buffer has to be mutable because it's passed back to
    /// the caller, and the caller may want to be able write into it.
    fn read_write_bytes(&'static self,
                        mut read_buffer:  Option<&'static mut [u8]>,
                        write_buffer: Option<&'static mut [u8]>) -> bool {
        // If both are Some, read/write minimum of lengths
        // If only read is Some, read length and write zeroes
        // If only write is Some, write length and discard reads
        // If both are None, return false
        // TODO: Asynchronous
        if read_buffer.is_none() && write_buffer.is_none() {
            return false
        }
        let reading = read_buffer.is_some();
        let writing = write_buffer.is_some();
        let read_len = match read_buffer {
            Some(ref buf) => {buf.len()},
            None          => 0
        };
        let write_len = match write_buffer {
            Some(ref buf) => {buf.len()},
            None          => 0
        };
        let count = if reading && writing {cmp::min(read_len, write_len)}
                    else                  {cmp::max(read_len, write_len)};
        for i in 0..count {
            let mut txbyte: u8 = 0;
            match write_buffer {
                Some(ref buf) => {txbyte = buf[i];}
                None          => {}
            }
            // Write the value
            let rxbyte = self.read_write_byte(txbyte);
            match read_buffer.take() {
                Some(ref mut buf) => {buf[i] = rxbyte;}
                None          => {}
            }
        }
        self.callback.get().map(|cb| {
            cb.read_write_done(read_buffer, write_buffer);
        });
        true
    }

#[allow(unused_variables)]
    fn set_rate(&self, rate: u32) -> u32 {
      self.set_baud_rate(rate)
    }

#[allow(unused_variables)]
    fn set_clock(&self, polarity: ClockPolarity) {
        let mut csr = self.read_active_csr();
        match polarity {
            ClockPolarity::IdleHigh => csr |= 1,
            ClockPolarity::IdleLow => csr &= 0xFFFFFFFE,
        };
        self.write_active_csr(csr);
    }

    fn get_clock(&self) -> ClockPolarity { ClockPolarity::IdleLow }

#[allow(unused_variables)]
    fn set_phase(&self, phase: ClockPhase) {
        let mut csr = self.read_active_csr();
        match phase {
            ClockPhase::SampleLeading => csr |= 1 << 1,
            ClockPhase::SampleTrailing => csr &= 0xFFFFFFFD,
        };
        self.write_active_csr(csr);
    }

    /// Sets the active peripheral
    fn set_chip_select(&self, cs: u8) {
        let peripheral_number: u32 = match cs {
            0 => 0b0000,
            1 => 0b0001,
            2 => 0b0011,
            3 => 0b0111,
            _ => 0b0000,
        };

        let mut mr = unsafe {volatile_load(&(*self.regs).mr)};
        // Clear and set MR.PCS
        let pcs_mask: u32 = 0xFFF0FFFF;
        mr &= pcs_mask;
        mr |= peripheral_number << 16;
        unsafe {volatile_store(&mut (*self.regs).mr, mr);}
    }

    fn clear_chip_select(&self) {
       unsafe {volatile_store(&mut (*self.regs).cr, 1 << 24)};
    }
}
<|endoftext|>