<commit_before>use std::fs;
use std::io::prelude::*;
use std::io;
use std::path::PathBuf;

pub fn create_profile_source(shim_dir: &PathBuf,
                             groonga_dir: &String,
                             install_dir: &PathBuf)
                             -> Result<(), io::Error> {
    if shim_dir.join("source-groonga.ps1").exists() {
        fs::remove_file(shim_dir.join("source-groonga.ps1")).unwrap();
    }
    let mut f = fs::File::create(shim_dir.join("source-groonga.ps1").to_str().unwrap())
        .expect("Could not create a powershell setting file.");
    let installed_groonga = install_dir.join(groonga_dir)
        .join("bin");
    let contents = format!("$Env:Path = \"{};\" + $Env:Path",
                           installed_groonga.display());
    match f.write_all(&contents.as_bytes()) {
        Ok(_) => return Ok(()),
        Err(e) => println!("{}", e),
    }
    f.sync_data()
}
<commit_msg>Check Groonga installation<commit_after>use std::fs;
use std::io::prelude::*;
use std::io;
use std::path::PathBuf;
use std::process;

pub fn create_profile_source(shim_dir: &PathBuf,
                             groonga_dir: &String,
                             install_dir: &PathBuf)
                             -> Result<(), io::Error> {
    if shim_dir.join("source-groonga.ps1").exists() {
        fs::remove_file(shim_dir.join("source-groonga.ps1")).unwrap();
    }
    let mut f = fs::File::create(shim_dir.join("source-groonga.ps1").to_str().unwrap())
        .expect("Could not create a powershell setting file.");
    let installed_groonga = install_dir.join(groonga_dir)
        .join("bin");
    if !installed_groonga.exists() {
        println!("Specified version is not installed.");
        process::exit(1);
    }
    let contents = format!("$Env:Path = \"{};\" + $Env:Path",
                           installed_groonga.display());
    match f.write_all(&contents.as_bytes()) {
        Ok(_) => return Ok(()),
        Err(e) => println!("{}", e),
    }
    f.sync_data()
}
<|endoftext|>


<commit_before>use std::io::{BufRead, BufReader, BufWriter, Write};
use std::fs::File;
use std::slice;
use std::mem;

fn main() {
    println!("usage: parse-pairs <source> <target>");
    println!("will overwrite <target>.pairs");
    let source = std::env::args().skip(1).next().unwrap();
    let target = std::env::args().skip(2).next().unwrap();

    let file = BufReader::new(File::open(source).unwrap());
    let mut writer = BufWriter::new(File::create(format!("{}.pairs", target)).unwrap());

    let mut pairs = vec![0u32; 2];
    for readline in file.lines() {
        let line = readline.ok().expect("read error");
        if !line.starts_with('#') {
            let elts: Vec<&str> = line[..].split("\t").collect();
            pairs[0] = elts[0].parse().ok().expect("malformed src");
            pairs[1] = elts[1].parse().ok().expect("malformed dst");
            writer.write_all(unsafe { _typed_as_byte_slice(&pairs[..]) }).unwrap();
        }
    }
}

unsafe fn _typed_as_byte_slice<T>(slice: &[T]) -> &[u8] {
    slice::from_raw_parts(slice.as_ptr() as *const u8, slice.len() * mem::size_of::<T>())
}
<commit_msg>stdin/stdout<commit_after>use std::io::{BufRead, BufReader, BufWriter, Write};
use std::fs::File;
use std::slice;
use std::mem;

fn main() {

    let stdin = std::io::stdin();
    let reader = stdin.lock();
    let mut writer = std::io::stdout();

    let mut pairs = vec![0u32; 2];
    for readline in reader.lines() {
        let line = readline.ok().expect("read error");
        if !line.starts_with('#') {
            let elts: Vec<&str> = line[..].split("\t").collect();
            pairs[0] = elts[0].parse().ok().expect("malformed src");
            pairs[1] = elts[1].parse().ok().expect("malformed dst");
            writer.write_all(unsafe { _typed_as_byte_slice(&pairs[..]) }).unwrap();
        }
    }
}

unsafe fn _typed_as_byte_slice<T>(slice: &[T]) -> &[u8] {
    slice::from_raw_parts(slice.as_ptr() as *const u8, slice.len() * mem::size_of::<T>())
}
<|endoftext|>


<commit_before>use combine::{Parser, Stream, ParseError, ParseResult};
use combine::{between, many, many1, one_of, optional, parser, satisfy, satisfy_map, try};
use combine::char::{digit, char, spaces};

use token::{Literal, Token};
use unicode_xid::UnicodeXID;

pub fn lex<I>(input: I) -> Result<(Vec<Token>, I), ParseError<I>>
where
    I: Stream<Item = char>,
{
    between(spaces(),
            spaces(),
            many(spaces().with(parser(token))))
        .parse(input)
}

fn token<I>(input: I) -> ParseResult<Token, I>
where
    I: Stream<Item = char>,
{
    parser(symbol)
        .or(parser(literal))
        .or(parser(punctuation))
        .parse_stream(input)
}

fn literal<I>(input: I) -> ParseResult<Token, I>
where
    I: Stream<Item = char>,
{
    let sign = optional(char('-'));
    let digits = many1::<String, _>(digit());
    let integer =
    (
        sign.clone(),
        digits.clone().and_then(|s| s.parse::<i64>())
    )
        .map(|(sign, num)| if sign.is_some() { -num } else { num })
        .map(Literal::from);

    let float = sign.clone()
        .and(
            (digits.clone(), char('.'), digits.clone())
                .map(|(prefix, _, suffix)| format!("{}.{}", prefix, suffix))
                .and_then(|num| num.parse::<f64>()),
        )
        .map(|(sign, num)| if sign.is_some() { -num } else { num })
        .map(Literal::from);

    let num = try(float).or(try(integer));

    let boolean = char('#')
        .with(satisfy_map(|c| match c {
            't' => Some(true),
            'f' => Some(false),
            _ => None,
        }))
        .map(Literal::from);

    let escaped = char('\\').with(satisfy_map(|c| match c {
        '\"' => Some('\"'),
        '\\' => Some('\\'),
        'n' => Some('\n'),
        'r' => Some('\r'),
        't' => Some('\t'),
        _ => None,
    }));

    let non_quote = try(escaped).or(satisfy(|c| c != '"'));

    let string = between(char('"'), char('"'), many::<String, _>(non_quote)).map(Literal::from);

    boolean
        .or(num)
        .or(string)
        .map(Token::from)
        .parse_stream(input)
}

fn symbol<I>(input: I) -> ParseResult<Token, I>
where
    I: Stream<Item = char>,
{
    let punctuation = one_of("_+-*/=<>!".chars());
    let start = satisfy(UnicodeXID::is_xid_start).or(punctuation.clone());
    let body = satisfy(UnicodeXID::is_xid_continue).or(punctuation.clone());
    let rest = many::<String, _>(body);
    start
        .and(rest)
        .map(|(f, mut r)| {
            r.insert(0, f);
            r
        })
        .map(Token::Symbol)
        .parse_stream(input)
}

fn punctuation<I>(input: I) -> ParseResult<Token, I>
where
    I: Stream<Item = char>,
{
    satisfy_map(|c| match c {
        '(' => Some(Token::LParen),
        ')' => Some(Token::RParen),
        '[' => Some(Token::LBracket),
        ']' => Some(Token::RBracket),
        '\'' => Some(Token::Quote),
        _ => None,
    }).parse_stream(input)
}

#[cfg(test)]
mod test {
    use super::*;
    use float_cmp::ApproxEqUlps;

    #[test]
    fn parse_bool_literal() {
        assert_eq!(Ok((Token::from(true), "")), parser(literal).parse("#t"));
        assert_eq!(Ok((Token::from(false), "")), parser(literal).parse("#f"));
    }

    #[test]
    fn parse_zero_literal() {
        // Integer case
        assert_eq!(Ok((Token::from(0i64), "")), parser(literal).parse("0"));

        // Float case
        match parser(literal).parse("0.0") {
            Ok((Token::Literal(Literal::Flt(flt)), _)) => assert!(flt.approx_eq_ulps(&0.0f64, 4)),
            Ok(x) => assert!(false, format!("0.0 parsed as {}", x.0)),
            Err(e) => assert!(false, format!("0.0 parsed as {}", e)),
        };
    }

    #[test]
    fn parse_escape_chars() {
        assert_eq!(
            Ok((Token::from("\""), "")),
            parser(literal).parse(r#""\"""#)
        );
        assert_eq!(
            Ok((Token::from("\\"), "")),
            parser(literal).parse(r#""\\""#)
        );
        assert_eq!(
            Ok((Token::from("\n"), "")),
            parser(literal).parse(r#""\n""#)
        );
        assert_eq!(
            Ok((Token::from("\r"), "")),
            parser(literal).parse(r#""\r""#)
        );
        assert_eq!(
            Ok((Token::from("\t"), "")),
            parser(literal).parse(r#""\t""#)
        );
    }

    #[test]
    fn parse_empty() {
        assert_eq!(Ok((Vec::new(), "")), lex(""));
    }

    quickcheck!{
        fn parse_int_literal(x: i64) -> bool {
            Ok((Token::from(x), "")) == parser(literal).parse(&*x.to_string())
        }

        fn parse_float_literal(x: f64) -> bool {
            let mut string = x.to_string();
            if x.trunc() == x {
                string.push_str(".0");
            }

            if let Ok((Token::Literal(Literal::Flt(y)), _)) = parser(literal).parse(&*string) {
                x.approx_eq_ulps(&y, 4)
            } else {
                false
            }
        }

        fn parse_str_literal(x: String) -> bool {
            let string = format!("\"{}\"", x
                .replace("\\", r"\\")
                .replace("\n", r"\n")
                .replace("\r", r"\r")
                .replace("\t", r"\t")
                .replace("\"", r#"\""#));
            Ok((Token::from(x.clone()), "")) == parser(literal).parse(&*string)
        }
    }
}
<commit_msg>Add lexer tests<commit_after>use combine::{Parser, Stream, ParseError, ParseResult};
use combine::{between, many, many1, one_of, optional, parser, satisfy, satisfy_map, try};
use combine::char::{digit, char, spaces};

use token::{Literal, Token};
use unicode_xid::UnicodeXID;

pub fn lex<I>(input: I) -> Result<(Vec<Token>, I), ParseError<I>>
where
    I: Stream<Item = char>,
{
    between(spaces(),
            spaces(),
            many(spaces().with(parser(token))))
        .parse(input)
}

fn token<I>(input: I) -> ParseResult<Token, I>
where
    I: Stream<Item = char>,
{
    parser(symbol)
        .or(parser(literal))
        .or(parser(punctuation))
        .parse_stream(input)
}

fn literal<I>(input: I) -> ParseResult<Token, I>
where
    I: Stream<Item = char>,
{
    let sign = optional(char('-'));
    let digits = many1::<String, _>(digit());
    let integer =
    (
        sign.clone(),
        digits.clone().and_then(|s| s.parse::<i64>())
    )
        .map(|(sign, num)| if sign.is_some() { -num } else { num })
        .map(Literal::from);

    let float = sign.clone()
        .and(
            (digits.clone(), char('.'), digits.clone())
                .map(|(prefix, _, suffix)| format!("{}.{}", prefix, suffix))
                .and_then(|num| num.parse::<f64>()),
        )
        .map(|(sign, num)| if sign.is_some() { -num } else { num })
        .map(Literal::from);

    let num = try(float).or(try(integer));

    let boolean = char('#')
        .with(satisfy_map(|c| match c {
            't' => Some(true),
            'f' => Some(false),
            _ => None,
        }))
        .map(Literal::from);

    let escaped = char('\\').with(satisfy_map(|c| match c {
        '\"' => Some('\"'),
        '\\' => Some('\\'),
        'n' => Some('\n'),
        'r' => Some('\r'),
        't' => Some('\t'),
        _ => None,
    }));

    let non_quote = try(escaped).or(satisfy(|c| c != '"'));

    let string = between(char('"'), char('"'), many::<String, _>(non_quote)).map(Literal::from);

    boolean
        .or(num)
        .or(string)
        .map(Token::from)
        .parse_stream(input)
}

fn symbol<I>(input: I) -> ParseResult<Token, I>
where
    I: Stream<Item = char>,
{
    let punctuation = one_of("_+-*/=<>!".chars());
    let start = satisfy(UnicodeXID::is_xid_start).or(punctuation.clone());
    let body = satisfy(UnicodeXID::is_xid_continue).or(punctuation.clone());
    let rest = many::<String, _>(body);
    start
        .and(rest)
        .map(|(f, mut r)| {
            r.insert(0, f);
            r
        })
        .map(Token::Symbol)
        .parse_stream(input)
}

fn punctuation<I>(input: I) -> ParseResult<Token, I>
where
    I: Stream<Item = char>,
{
    satisfy_map(|c| match c {
        '(' => Some(Token::LParen),
        ')' => Some(Token::RParen),
        '[' => Some(Token::LBracket),
        ']' => Some(Token::RBracket),
        '\'' => Some(Token::Quote),
        _ => None,
    }).parse_stream(input)
}

#[cfg(test)]
mod test {
    use super::*;
    use float_cmp::ApproxEqUlps;

    #[test]
    fn empty() {
        assert_eq!(Ok((vec![], "")), lex(""));
    }

    #[test]
    fn bool_literal() {
        assert_eq!(Ok((Token::from(true), "")), parser(literal).parse("#t"));
        assert_eq!(Ok((Token::from(false), "")), parser(literal).parse("#f"));
    }

    #[test]
    fn zero_literal() {
        // Integer case
        assert_eq!(Ok((Token::from(0i64), "")), parser(literal).parse("0"));

        // Float case
        match parser(literal).parse("0.0") {
            Ok((Token::Literal(Literal::Flt(flt)), _)) => assert!(flt.approx_eq_ulps(&0.0f64, 4)),
            Ok(x) => assert!(false, format!("0.0 parsed as {}", x.0)),
            Err(e) => assert!(false, format!("0.0 parsed as {}", e)),
        };
    }

    #[test]
    fn escape_chars() {
        assert_eq!(
            Ok((Token::from("\""), "")),
            parser(literal).parse(r#""\"""#)
        );
        assert_eq!(
            Ok((Token::from("\\"), "")),
            parser(literal).parse(r#""\\""#)
        );
        assert_eq!(
            Ok((Token::from("\n"), "")),
            parser(literal).parse(r#""\n""#)
        );
        assert_eq!(
            Ok((Token::from("\r"), "")),
            parser(literal).parse(r#""\r""#)
        );
        assert_eq!(
            Ok((Token::from("\t"), "")),
            parser(literal).parse(r#""\t""#)
        );
    }

    #[test]
    fn delimiters() {
        // Single-character tests
        assert_eq!(
            Ok((Token::LParen, "")),
            parser(punctuation).parse("(")
        );

        assert_eq!(
            Ok((Token::RParen, "")),
            parser(punctuation).parse(")")
        );

        assert_eq!(
            Ok((Token::LBracket, "")),
            parser(punctuation).parse("[")
        );

        assert_eq!(
            Ok((Token::RBracket, "")),
            parser(punctuation).parse("]")
        );
    }

    #[test]
    fn nested_lists() {
        assert_eq!(
            Ok((vec![Token::LParen, Token::LParen, Token::RParen, Token::RParen],"")),
            lex("(())")
        );
    }

    #[test]
    fn deep_nested_lists() {
        use std::iter;
        let num_layers = 1000;
        let input = ["(".repeat(num_layers), ")".repeat(num_layers)].concat();
        let left = iter::repeat(Token::LParen).take(num_layers);
        let right = iter::repeat(Token::RParen).take(num_layers);
        let output = left.chain(right).collect::<Vec<_>>();
        assert_eq!(
            Ok((output, "")),
            lex(&*input)
        );
    }

    quickcheck!{
        fn int_literal(x: i64) -> bool {
            Ok((Token::from(x), "")) == parser(literal).parse(&*x.to_string())
        }

        fn float_literal(x: f64) -> bool {
            let mut string = x.to_string();
            if x.trunc() == x {
                string.push_str(".0");
            }

            if let Ok((Token::Literal(Literal::Flt(y)), _)) = parser(literal).parse(&*string) {
                x.approx_eq_ulps(&y, 4)
            } else {
                false
            }
        }

        fn str_literal(x: String) -> bool {
            let string = format!("\"{}\"", x
                .replace("\\", r"\\")
                .replace("\n", r"\n")
                .replace("\r", r"\r")
                .replace("\t", r"\t")
                .replace("\"", r#"\""#));
            Ok((Token::from(x.clone()), "")) == parser(literal).parse(&*string)
        }
    }
}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate structopt_derive;
extern crate structopt;
extern crate csv;
extern crate mimir;
extern crate itertools;
extern crate mimirsbrunn;
#[macro_use]
extern crate log;

use std::rc::Rc;
use mimir::rubber::Rubber;
use mimirsbrunn::utils::{format_label, get_zip_codes_from_admins};
use mimir::MimirObject;
use mimirsbrunn::admin_geofinder::AdminGeoFinder;
use std::collections::HashMap;
use structopt::StructOpt;

const GLOBAL_STOP_INDEX_NAME: &'static str = "munin_global_stops";

#[derive(Debug, StructOpt)]
struct Args {
    /// NTFS stops.txt file.
    #[structopt(short = "i", long = "input")]
    input: String,
    /// Name of the dataset.
    #[structopt(short = "d", long = "dataset", default_value = "fr")]
    dataset: String,
    /// Elasticsearch parameters.
    #[structopt(short = "c", long = "connection-string",
                default_value = "http://localhost:9200/munin")]
    connection_string: String,
    /// City level to calculate weight.
    #[structopt(short = "C", long = "city-level", default_value = "8")]
    city_level: u32,
}

#[derive(Debug, Deserialize)]
struct GtfsStop {
    stop_id: String,
    stop_lat: f64,
    stop_lon: f64,
    stop_name: String,
    location_type: Option<i32>,
    visible: Option<i32>,
    parent_station: Option<String>,
}

impl GtfsStop {
    fn incr_stop_point(&self, nb_stop_points: &mut HashMap<String, u32>) {
        match (self.location_type, &self.parent_station) {
            (Some(0), &Some(ref id)) |
            (None, &Some(ref id)) if !id.is_empty() => {
                *nb_stop_points
                    .entry(format!("stop_area:{}", id))
                    .or_insert(0) += 1
            }
            _ => (),
        }
    }
    // to be moved when TryInto is stablilized
    fn try_into(self) -> Option<mimir::Stop> {
        if self.location_type == Some(1) && self.visible != Some(0) {
            Some(mimir::Stop {
                id: format!("stop_area:{}", self.stop_id), // prefix to match navitia's id
                coord: mimir::Coord::new(self.stop_lat, self.stop_lon),
                label: self.stop_name.clone(),
                weight: 0.,
                zip_codes: vec![],
                administrative_regions: vec![],
                name: self.stop_name,
                coverages: vec![],
            })
        } else {
            None
        }
    }
}

fn attach_stop(stop: &mut mimir::Stop, admins: Vec<Rc<mimir::Admin>>, city_level: u32) {
    stop.administrative_regions = admins;
    stop.label = format_label(&stop.administrative_regions, city_level, &stop.name);
    stop.zip_codes = get_zip_codes_from_admins(&stop.administrative_regions);
}

/// Attach the stops to administrative regions
///
/// The admins are loaded from Elasticsearch and stored in a quadtree
/// We attach a stop with all the admins that have a boundary containing
/// the coordinate of the stop
fn attach_stops_to_admins<'a, It: Iterator<Item = &'a mut mimir::Stop>>(
    stops: It,
    rubber: &mut Rubber,
    city_level: u32,
) {
    let admins = rubber.get_all_admins().unwrap_or_else(|_| {
        info!("Administratives regions not found in elasticsearch db");
        vec![]
    });

    info!("{} administrative regions loaded from mimir", admins.len());

    let admins_geofinder = admins.into_iter().collect::<AdminGeoFinder>();

    let mut nb_unmatched = 0u32;
    let mut nb_matched = 0u32;
    for mut stop in stops {
        let admins = admins_geofinder.get(&stop.coord);

        if admins.is_empty() {
            nb_unmatched += 1;
        } else {
            nb_matched += 1;
        }

        attach_stop(&mut stop, admins, city_level);
    }

    info!(
        "there is {}/{} stops without any admin",
        nb_unmatched,
        nb_matched + nb_unmatched
    );
}

// Update weight value for each stop_area from HashMap and stop's coverage
fn finalize_stop_area<'a, It: Iterator<Item = &'a mut mimir::Stop>>(
    stops: It,
    nb_stop_points: &HashMap<String, u32>,
    dataset: &str,
) {
    let max = *nb_stop_points.values().max().unwrap_or(&1) as f64;
    for stop in stops {
        if let Some(weight) = nb_stop_points.get(&stop.id) {
            stop.weight = *weight as f64 / max;
        }
        stop.coverages.push(dataset.to_string());
    }
}

/// merge the stops from all the different indexes
/// for the moment the merge is very simple and uses only the ID
/// (and we take the data from the first stop inserted)
fn merge_stops<It: IntoIterator<Item = mimir::Stop>>(
    stops: It,
) -> Box<Iterator<Item = mimir::Stop>> {
    let mut stops_by_id = HashMap::<String, mimir::Stop>::new();
    for mut stop in stops.into_iter() {
        let cov = std::mem::replace(&mut stop.coverages, vec![]);
        stops_by_id
            .entry(stop.id.clone())
            .or_insert(stop)
            .coverages
            .extend(cov.into_iter());
    }
    Box::new(stops_by_id.into_iter().map(|(_, v)| v))
}

fn get_all_stops(rubber: &mut Rubber, index: String) -> Result<Vec<mimir::Stop>, String> {
    rubber.get_all_objects_from_index(&index).map_err(
        |e| e.to_string(),
    )
}

fn update_global_stop_index<'a, It: Iterator<Item = &'a mimir::Stop>>(
    rubber: &mut Rubber,
    stops: It,
    dataset: &str,
) -> Result<String, String> {
    let dataset_index =
        mimir::rubber::get_main_type_and_dataset_index(mimir::Stop::doc_type(), dataset);
    let stops_indexes = rubber
        .get_all_aliased_index(&mimir::rubber::get_main_type_index(mimir::Stop::doc_type()))?
        .into_iter()
        .filter(|&(_, ref aliases)| !aliases.contains(&dataset_index))
        .map(|(index, _)| index);

    let all_other_es_stops: Vec<_> = stops_indexes
        .map(|index| get_all_stops(rubber, index).unwrap())
        .flat_map(|stops| stops.into_iter())
        .collect();

    let all_es_stops = all_other_es_stops.into_iter().chain(
        stops.into_iter().cloned(),
    );

    let all_merged_stops = merge_stops(all_es_stops);
    let es_index_name = mimir::rubber::get_date_index_name(GLOBAL_STOP_INDEX_NAME);

    rubber.create_index(&es_index_name)?;

    let nb_stops_added = rubber
        .bulk_index(&es_index_name, all_merged_stops)
        .map_err(|e| e.to_string())?;
    info!("{} stops added in the global index", nb_stops_added);
    // create global index
    // fill structure for each stop indexes
    Ok(es_index_name)
}

// publish the global stop index
// alias the new index to the global stop alias, and remove the old index
fn publish_global_index(rubber: &mut Rubber, new_global_index: &str) -> Result<(), String> {
    let last_global_indexes: Vec<_> = rubber
        .get_all_aliased_index(GLOBAL_STOP_INDEX_NAME)?
        .into_iter()
        .map(|(k, _)| k)
        .filter(|k| k != new_global_index)
        .collect();
    rubber.alias(
        GLOBAL_STOP_INDEX_NAME,
        &vec![new_global_index.to_string()],
        &last_global_indexes,
    )?;

    for index in last_global_indexes {
        rubber.delete_index(&index)?;
    }
    Ok(())
}

fn main() {
    mimir::logger_init().unwrap();
    info!("Launching stops2mimir...");

    let args = Args::from_args();

    info!("creation of indexes");
    let mut rubber = Rubber::new(&args.connection_string);
    let mut rdr = csv::Reader::from_path(args.input).unwrap();

    let mut nb_stop_points = HashMap::new();

    let mut stops: Vec<mimir::Stop> = rdr.deserialize()
        .filter_map(|rc| {
            rc.map_err(|e| warn!("skip csv line because: {}", e)).ok()
        })
        .filter_map(|stop: GtfsStop| {
            stop.incr_stop_point(&mut nb_stop_points);
            stop.try_into()
        })
        .collect();

    attach_stops_to_admins(stops.iter_mut(), &mut rubber, args.city_level);

    finalize_stop_area(stops.iter_mut(), &nb_stop_points, &args.dataset);

    let global_index = update_global_stop_index(&mut rubber, stops.iter(), &args.dataset).unwrap();

    info!("Importing {} stops into Mimir", stops.len());
    let nb_stops = rubber.index(&args.dataset, stops.iter()).unwrap();
    info!("Nb of indexed stops: {}", nb_stops);

    publish_global_index(&mut rubber, &global_index).unwrap();
}

#[test]
fn test_load_stops() {
    use itertools::Itertools;
    let mut rdr = csv::Reader::from_path("./tests/fixtures/stops.txt".to_string()).unwrap();

    let mut nb_stop_points = HashMap::new();
    let stops: Vec<mimir::Stop> = rdr.deserialize()
        .filter_map(Result::ok)
        .filter_map(|stop: GtfsStop| {
            stop.incr_stop_point(&mut nb_stop_points);
            stop.try_into()
        })
        .collect();
    let ids: Vec<_> = stops.iter().map(|s| s.id.clone()).sorted();
    assert_eq!(
        ids,
        vec![
            "stop_area:SA:known_by_all_dataset",
            "stop_area:SA:main_station",
            "stop_area:SA:second_station",
            "stop_area:SA:station_no_city",
            "stop_area:SA:weight_1_station",
            "stop_area:SA:weight_3_station",
        ]
    );
    let weights: Vec<_> = ids.iter().map(|id| nb_stop_points.get(id)).collect();
    assert_eq!(
        weights,
        vec![None, Some(&1), Some(&1), None, Some(&1), Some(&3)]
    );
}
<commit_msg>stops2mimir: verify coordinates values and reject the line on error.<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate structopt_derive;
extern crate structopt;
extern crate csv;
extern crate mimir;
extern crate itertools;
extern crate mimirsbrunn;
#[macro_use]
extern crate log;

use std::rc::Rc;
use mimir::rubber::Rubber;
use mimirsbrunn::utils::{format_label, get_zip_codes_from_admins};
use mimir::MimirObject;
use mimirsbrunn::admin_geofinder::AdminGeoFinder;
use std::collections::HashMap;
use structopt::StructOpt;

const GLOBAL_STOP_INDEX_NAME: &'static str = "munin_global_stops";
const MAX_LAT: f64 = 90f64;
const MIN_LAT: f64 = -90f64;

const MAX_LON: f64 = 180f64;
const MIN_LON: f64 = -180f64;

#[derive(Debug, StructOpt)]
struct Args {
    /// NTFS stops.txt file.
    #[structopt(short = "i", long = "input")]
    input: String,
    /// Name of the dataset.
    #[structopt(short = "d", long = "dataset", default_value = "fr")]
    dataset: String,
    /// Elasticsearch parameters.
    #[structopt(short = "c", long = "connection-string",
                default_value = "http://localhost:9200/munin")]
    connection_string: String,
    /// City level to calculate weight.
    #[structopt(short = "C", long = "city-level", default_value = "8")]
    city_level: u32,
}

#[derive(Deserialize, Debug)]
enum StopConvErr {
    InvisibleStop,
    StopPoint,
    InvalidStop(String),
}

#[derive(Debug, Deserialize)]
struct GtfsStop {
    stop_id: String,
    stop_lat: f64,
    stop_lon: f64,
    stop_name: String,
    location_type: Option<i32>,
    visible: Option<i32>,
    parent_station: Option<String>,
}

impl GtfsStop {
    fn incr_stop_point(&self, nb_stop_points: &mut HashMap<String, u32>) {
        match (self.location_type, &self.parent_station) {
            (Some(0), &Some(ref id)) |
            (None, &Some(ref id)) if !id.is_empty() => {
                *nb_stop_points
                    .entry(format!("stop_area:{}", id))
                    .or_insert(0) += 1
            }
            _ => (),
        }
    }
    // to be moved when TryInto is stablilized
    fn try_into(self) -> Result<mimir::Stop, StopConvErr> {
        if self.location_type != Some(1) {
            Err(StopConvErr::StopPoint)
        } else if self.visible == Some(0){
        	Err(StopConvErr::InvisibleStop)
        } else if self.stop_lat <= MIN_LAT || self.stop_lat >= MAX_LAT || self.stop_lon <= MIN_LON || self.stop_lon >= MAX_LON {
         	//Here we return an error message
		  	Err(StopConvErr::InvalidStop(format!("Invalid lon or lat for stop {}", self.stop_name).into()))
        } else {
        	Ok(mimir::Stop {
                id: format!("stop_area:{}", self.stop_id), // prefix to match navitia's id
                coord: mimir::Coord::new(self.stop_lat, self.stop_lon),
                label: self.stop_name.clone(),
                weight: 0.,
                zip_codes: vec![],
                administrative_regions: vec![],
                name: self.stop_name,
                coverages: vec![],
            })
        }
    }
}

fn attach_stop(stop: &mut mimir::Stop, admins: Vec<Rc<mimir::Admin>>, city_level: u32) {
    stop.administrative_regions = admins;
    stop.label = format_label(&stop.administrative_regions, city_level, &stop.name);
    stop.zip_codes = get_zip_codes_from_admins(&stop.administrative_regions);
}

/// Attach the stops to administrative regions
///
/// The admins are loaded from Elasticsearch and stored in a quadtree
/// We attach a stop with all the admins that have a boundary containing
/// the coordinate of the stop
fn attach_stops_to_admins<'a, It: Iterator<Item = &'a mut mimir::Stop>>(
    stops: It,
    rubber: &mut Rubber,
    city_level: u32,
) {
    let admins = rubber.get_all_admins().unwrap_or_else(|_| {
        info!("Administratives regions not found in elasticsearch db");
        vec![]
    });

    info!("{} administrative regions loaded from mimir", admins.len());

    let admins_geofinder = admins.into_iter().collect::<AdminGeoFinder>();

    let mut nb_unmatched = 0u32;
    let mut nb_matched = 0u32;
    for mut stop in stops {
        let admins = admins_geofinder.get(&stop.coord);

        if admins.is_empty() {
            nb_unmatched += 1;
        } else {
            nb_matched += 1;
        }

        attach_stop(&mut stop, admins, city_level);
    }

    info!(
        "there is {}/{} stops without any admin",
        nb_unmatched,
        nb_matched + nb_unmatched
    );
}

// Update weight value for each stop_area from HashMap and stop's coverage
fn finalize_stop_area<'a, It: Iterator<Item = &'a mut mimir::Stop>>(
    stops: It,
    nb_stop_points: &HashMap<String, u32>,
    dataset: &str,
) {
    let max = *nb_stop_points.values().max().unwrap_or(&1) as f64;
    for stop in stops {
        if let Some(weight) = nb_stop_points.get(&stop.id) {
            stop.weight = *weight as f64 / max;
        }
        stop.coverages.push(dataset.to_string());
    }
}

/// merge the stops from all the different indexes
/// for the moment the merge is very simple and uses only the ID
/// (and we take the data from the first stop inserted)
fn merge_stops<It: IntoIterator<Item = mimir::Stop>>(
    stops: It,
) -> Box<Iterator<Item = mimir::Stop>> {
    let mut stops_by_id = HashMap::<String, mimir::Stop>::new();
    for mut stop in stops.into_iter() {
        let cov = std::mem::replace(&mut stop.coverages, vec![]);
        stops_by_id
            .entry(stop.id.clone())
            .or_insert(stop)
            .coverages
            .extend(cov.into_iter());
    }
    Box::new(stops_by_id.into_iter().map(|(_, v)| v))
}

fn get_all_stops(rubber: &mut Rubber, index: String) -> Result<Vec<mimir::Stop>, String> {
    rubber.get_all_objects_from_index(&index).map_err(
        |e| e.to_string(),
    )
}

fn update_global_stop_index<'a, It: Iterator<Item = &'a mimir::Stop>>(
    rubber: &mut Rubber,
    stops: It,
    dataset: &str,
) -> Result<String, String> {
    let dataset_index =
        mimir::rubber::get_main_type_and_dataset_index(mimir::Stop::doc_type(), dataset);
    let stops_indexes = rubber
        .get_all_aliased_index(&mimir::rubber::get_main_type_index(mimir::Stop::doc_type()))?
        .into_iter()
        .filter(|&(_, ref aliases)| !aliases.contains(&dataset_index))
        .map(|(index, _)| index);

    let all_other_es_stops: Vec<_> = stops_indexes
        .map(|index| get_all_stops(rubber, index).unwrap())
        .flat_map(|stops| stops.into_iter())
        .collect();

    let all_es_stops = all_other_es_stops.into_iter().chain(
        stops.into_iter().cloned(),
    );

    let all_merged_stops = merge_stops(all_es_stops);
    let es_index_name = mimir::rubber::get_date_index_name(GLOBAL_STOP_INDEX_NAME);

    rubber.create_index(&es_index_name)?;

    let nb_stops_added = rubber
        .bulk_index(&es_index_name, all_merged_stops)
        .map_err(|e| e.to_string())?;
    info!("{} stops added in the global index", nb_stops_added);
    // create global index
    // fill structure for each stop indexes
    Ok(es_index_name)
}

// publish the global stop index
// alias the new index to the global stop alias, and remove the old index
fn publish_global_index(rubber: &mut Rubber, new_global_index: &str) -> Result<(), String> {
    let last_global_indexes: Vec<_> = rubber
        .get_all_aliased_index(GLOBAL_STOP_INDEX_NAME)?
        .into_iter()
        .map(|(k, _)| k)
        .filter(|k| k != new_global_index)
        .collect();
    rubber.alias(
        GLOBAL_STOP_INDEX_NAME,
        &vec![new_global_index.to_string()],
        &last_global_indexes,
    )?;

    for index in last_global_indexes {
        rubber.delete_index(&index)?;
    }
    Ok(())
}

fn main() {
    mimir::logger_init().unwrap();
    info!("Launching stops2mimir...");

    let args = Args::from_args();

    info!("creation of indexes");
    let mut rubber = Rubber::new(&args.connection_string);
    let mut rdr = csv::Reader::from_path(args.input).unwrap();

    let mut nb_stop_points = HashMap::new();

    let mut stops: Vec<mimir::Stop> = rdr.deserialize()
        .filter_map(|rc| {
            rc.map_err(|e| warn!("skip csv line because: {}", e)).ok()
        })
        .filter_map(|stop: GtfsStop| {
            stop.incr_stop_point(&mut nb_stop_points);
            match stop.try_into() {
                Ok(s) => Some(s),
                Err(StopConvErr::InvisibleStop) | Err(StopConvErr::StopPoint) => None,
                Err(StopConvErr::InvalidStop(msg)) => {
                    warn!("skip csv line because: {}", msg);
                    None
                }
            }
        })
        .collect();

    attach_stops_to_admins(stops.iter_mut(), &mut rubber, args.city_level);

    finalize_stop_area(stops.iter_mut(), &nb_stop_points, &args.dataset);

    let global_index = update_global_stop_index(&mut rubber, stops.iter(), &args.dataset).unwrap();

    info!("Importing {} stops into Mimir", stops.len());
    let nb_stops = rubber.index(&args.dataset, stops.iter()).unwrap();
    info!("Nb of indexed stops: {}", nb_stops);

    publish_global_index(&mut rubber, &global_index).unwrap();
}

#[test]
fn test_load_stops() {
    use itertools::Itertools;
    let mut rdr = csv::Reader::from_path("./tests/fixtures/stops.txt".to_string()).unwrap();

    let mut nb_stop_points = HashMap::new();
    let stops: Vec<mimir::Stop> = rdr.deserialize()
        .filter_map(Result::ok)
        .filter_map(|stop: GtfsStop| {
            stop.incr_stop_point(&mut nb_stop_points);
            stop.try_into()
        })
        .collect();
    let ids: Vec<_> = stops.iter().map(|s| s.id.clone()).sorted();
    assert_eq!(
        ids,
        vec![
            "stop_area:SA:known_by_all_dataset",
            "stop_area:SA:main_station",
            "stop_area:SA:second_station",
            "stop_area:SA:station_no_city",
            "stop_area:SA:weight_1_station",
            "stop_area:SA:weight_3_station",
        ]
    );
    let weights: Vec<_> = ids.iter().map(|id| nb_stop_points.get(id)).collect();
    assert_eq!(
        weights,
        vec![None, Some(&1), Some(&1), None, Some(&1), Some(&3)]
    );
}
<|endoftext|>


<commit_before>use libc;
use std::thread;
use std::rc::Rc;
use std::sync::{Arc};
use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering};
use std::collections::BTreeSet;
use parking_lot::{Mutex, MutexGuard, RwLock, RwLockReadGuard};
use concurrent_hashmap::ConcHashMap;
use ram::schema::Schemas;
use ram::segs::{Segment, SEGMENT_SIZE};
use ram::cell::{Cell, ReadError, WriteError};
use server::ServerMeta;

pub struct Chunk {
    pub id: usize,
    pub addr: usize,
    pub index: ConcHashMap<u64, Mutex<usize>>,
    pub segs: Vec<Segment>,
    pub seg_round: AtomicUsize,
    pub meta: Rc<ServerMeta>,
    pub backup_storage: Option<String>,
}

pub struct Chunks {
    pub list: Vec<Chunk>,
}

impl Chunk {
    fn new (id: usize, size: usize, meta: Rc<ServerMeta>, back_storage: Option<String>) -> Chunk {
        let mem_ptr = unsafe {libc::malloc(size)} as usize;
        let seg_count = size / SEGMENT_SIZE;
        let mut segments = Vec::<Segment>::new();
        for seg_idx in 0..seg_count {
            let seg_addr = mem_ptr + seg_idx * SEGMENT_SIZE;
            segments.push(Segment {
                addr: seg_addr,
                id: seg_idx,
                bound: seg_addr + SEGMENT_SIZE,
                last: AtomicUsize::new(seg_addr),
                lock: RwLock::new(()),
                tombstones: Mutex::new(BTreeSet::new()),
            });
        }
        debug!("creating chunk at {}, segments {}", mem_ptr, seg_count + 1);
        Chunk {
            id: id,
            addr: mem_ptr,
            index: ConcHashMap::<u64, Mutex<usize>>::new(),
            meta: meta,
            segs: segments,
            seg_round: AtomicUsize::new(0),
            backup_storage: back_storage
        }
    }
    pub fn try_acquire(&self, size: usize) -> Option<(usize, RwLockReadGuard<()>)> {
        let mut retried = 0;
        loop {
            let n = self.seg_round.load(Ordering::Relaxed);
            let seg_id = n % self.segs.len();
            let seg_acquire = self.segs[seg_id].try_acquire(size);
            match seg_acquire {
                None => {
                    if retried > self.segs.len() * 2 {return None;}
                    self.seg_round.fetch_add(1, Ordering::Relaxed);
                    retried += 1;
                },
                _ => {return seg_acquire;}
            }
        }
    }
    fn locate_segment(&self, location: usize) -> &Segment {
        let offset = location - self.addr;
        let seg_id = offset / SEGMENT_SIZE;
        return &self.segs[seg_id];
    }
    fn location_of(&self, hash: u64) -> Option<MutexGuard<usize>> {
        match self.index.find(&hash) {
            Some(index) => {
                let index = index.get();
                let index_lock = index.lock();
                if *index_lock == 0 {
                    return None
                }
                return Some(index_lock);
            },
            None => None
        }
    }
    fn put_tombstone(&self, location: usize) {
        let seg = self.locate_segment(location);
        seg.put_cell_tombstone(location);
    }
    fn read_cell(&self, hash: u64) -> Result<Cell, ReadError> {
        match self.location_of(hash) {
            Some(loc) => {
                Cell::from_chunk_raw(*loc, self)
            },
            None => Err(ReadError::CellDoesNotExisted)
        }
    }
    fn write_cell(&self, cell: &mut Cell) -> Result<usize, WriteError> {
        let hash = cell.header.hash;
        if self.location_of(hash).is_some() {
            return Err(WriteError::CellAlreadyExisted);
        } else {
            let written = cell.write_to_chunk(self);
            let need_rollback = AtomicBool::new(false);
            if let Ok(loc) = written {
                self.index.upsert(
                    hash,
                    Mutex::new(loc),
                    &|m| {
                        let mut inserted_loc = m.lock();
                        if *inserted_loc == 0 {
                            *inserted_loc = loc
                        } else {
                            need_rollback.store(true, Ordering::Relaxed);
                        }
                    }
                );
                if need_rollback.load(Ordering::Relaxed) {
                    self.put_tombstone(loc);
                    return Err(WriteError::CellAlreadyExisted)
                }
            }
            return written
        }
    }
    fn update_cell(&self, cell: &mut Cell) -> Result<usize, WriteError> {
        let hash = cell.header.hash;
        if let Some(mut cell_location) = self.location_of(hash) {
            let written = cell.write_to_chunk(self);
            if let Ok(new_location) = written {
                let old_location = *cell_location;
                *cell_location = new_location;
                self.put_tombstone(old_location);
            }
            return written;
        } else {
            return Err(WriteError::CellDoesNotExisted)
        }
    }
    fn update_cell_by<U>(&self, hash: u64, update: U) -> Result<usize, WriteError>
        where U: Fn(Cell) -> Option<Cell> {
        if let Some(mut cell_location) = self.location_of(hash) {
            let cell = Cell::from_chunk_raw(*cell_location, self);
            match cell {
                Ok(cell) => {
                    let mut new_cell = update(cell);
                    if let Some(mut new_cell) = new_cell {
                        let written = new_cell.write_to_chunk(self);
                        if let Ok(new_location) = written {
                            let old_location = *cell_location;
                            *cell_location = new_location;
                            self.put_tombstone(old_location);
                        }
                        return written;
                    } else {
                        return Err(WriteError::UserCanceledUpdate);
                    }
                },
                Err(e) => Err(WriteError::ReadError(e))
            }
        } else {
            return Err(WriteError::CellDoesNotExisted)
        }
    }
    fn remove_cell(&self, hash: u64) -> Result<(), WriteError> {
        if let Some(cell_location) = self.location_of(hash) {
            self.index.remove(&hash);
            self.put_tombstone(*cell_location);
            return Ok(());
        } else {
            return Err(WriteError::CellDoesNotExisted);
        }
    }
    fn remove_cell_by<P>(&self, hash: u64, predict: P) -> Result<(), WriteError>
        where P: Fn(Cell) -> bool {
        if let Some(cell_location) = self.location_of(hash) {
            let cell = Cell::from_chunk_raw(*cell_location, self);
            match cell {
                Ok(cell) => {
                    if predict(cell) {
                        self.index.remove(&hash);
                        self.put_tombstone(*cell_location);
                        return Ok(());
                    } else {
                        return Err(WriteError::DeletionPredictionFailed);
                    }
                },
                Err(e) => Err(WriteError::ReadError(e))
            }
        } else {
            return Err(WriteError::CellDoesNotExisted);
        }
    }
    fn dispose (&mut self) {
        debug!("disposing chunk at {}", self.addr);
        unsafe {
            libc::free(self.addr as *mut libc::c_void)
        }
    }
}

impl Drop for Chunk {
    fn drop(&mut self) {
        self.dispose();
    }
}

impl Chunks {
    pub fn new (count: usize, size: usize, meta: Rc<ServerMeta>, backup_storage: Option<String>) -> Chunks {
        let chunk_size = size / count;
        let mut chunks = Vec::new();
        debug!("Creating {} chunks, total {} bytes", count, size);
        for i in 0..count {
            let backup_storage = match backup_storage {
                Some(ref dir) => Some(format!("{}/data-{}.bak", dir, i)),
                None => None
            };
            chunks.push(Chunk::new(i, chunk_size, meta.clone(), backup_storage));
        }
        Chunks {
            list: chunks
        }
    }

    pub fn new_dummy(count: usize, size: usize) -> Chunks {
        Chunks::new(count, size, Rc::<ServerMeta>::new(ServerMeta {
            schemas: Schemas::new(None)
        }), None)
    }
}<commit_msg>glue Chunk with Chunks<commit_after>use libc;
use std::thread;
use std::rc::Rc;
use std::sync::{Arc};
use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering};
use std::collections::BTreeSet;
use parking_lot::{Mutex, MutexGuard, RwLock, RwLockReadGuard};
use concurrent_hashmap::ConcHashMap;
use ram::schema::Schemas;
use ram::segs::{Segment, SEGMENT_SIZE};
use ram::cell::{Cell, ReadError, WriteError};
use server::ServerMeta;

pub struct Chunk {
    pub id: usize,
    pub addr: usize,
    pub index: ConcHashMap<u64, Mutex<usize>>,
    pub segs: Vec<Segment>,
    pub seg_round: AtomicUsize,
    pub meta: Rc<ServerMeta>,
    pub backup_storage: Option<String>,
}

pub struct Chunks {
    pub list: Vec<Chunk>,
}

impl Chunk {
    fn new (id: usize, size: usize, meta: Rc<ServerMeta>, back_storage: Option<String>) -> Chunk {
        let mem_ptr = unsafe {libc::malloc(size)} as usize;
        let seg_count = size / SEGMENT_SIZE;
        let mut segments = Vec::<Segment>::new();
        for seg_idx in 0..seg_count {
            let seg_addr = mem_ptr + seg_idx * SEGMENT_SIZE;
            segments.push(Segment {
                addr: seg_addr,
                id: seg_idx,
                bound: seg_addr + SEGMENT_SIZE,
                last: AtomicUsize::new(seg_addr),
                lock: RwLock::new(()),
                tombstones: Mutex::new(BTreeSet::new()),
            });
        }
        debug!("creating chunk at {}, segments {}", mem_ptr, seg_count + 1);
        Chunk {
            id: id,
            addr: mem_ptr,
            index: ConcHashMap::<u64, Mutex<usize>>::new(),
            meta: meta,
            segs: segments,
            seg_round: AtomicUsize::new(0),
            backup_storage: back_storage
        }
    }
    pub fn try_acquire(&self, size: usize) -> Option<(usize, RwLockReadGuard<()>)> {
        let mut retried = 0;
        loop {
            let n = self.seg_round.load(Ordering::Relaxed);
            let seg_id = n % self.segs.len();
            let seg_acquire = self.segs[seg_id].try_acquire(size);
            match seg_acquire {
                None => {
                    if retried > self.segs.len() * 2 {return None;}
                    self.seg_round.fetch_add(1, Ordering::Relaxed);
                    retried += 1;
                },
                _ => {return seg_acquire;}
            }
        }
    }
    fn locate_segment(&self, location: usize) -> &Segment {
        let offset = location - self.addr;
        let seg_id = offset / SEGMENT_SIZE;
        return &self.segs[seg_id];
    }
    fn location_of(&self, hash: u64) -> Option<MutexGuard<usize>> {
        match self.index.find(&hash) {
            Some(index) => {
                let index = index.get();
                let index_lock = index.lock();
                if *index_lock == 0 {
                    return None
                }
                return Some(index_lock);
            },
            None => None
        }
    }
    fn put_tombstone(&self, location: usize) {
        let seg = self.locate_segment(location);
        seg.put_cell_tombstone(location);
    }
    fn read_cell(&self, hash: u64) -> Result<Cell, ReadError> {
        match self.location_of(hash) {
            Some(loc) => {
                Cell::from_chunk_raw(*loc, self)
            },
            None => Err(ReadError::CellDoesNotExisted)
        }
    }
    fn write_cell(&self, cell: &mut Cell) -> Result<usize, WriteError> {
        let hash = cell.header.hash;
        if self.location_of(hash).is_some() {
            return Err(WriteError::CellAlreadyExisted);
        } else {
            let written = cell.write_to_chunk(self);
            let need_rollback = AtomicBool::new(false);
            if let Ok(loc) = written {
                self.index.upsert(
                    hash,
                    Mutex::new(loc),
                    &|m| {
                        let mut inserted_loc = m.lock();
                        if *inserted_loc == 0 {
                            *inserted_loc = loc
                        } else {
                            need_rollback.store(true, Ordering::Relaxed);
                        }
                    }
                );
                if need_rollback.load(Ordering::Relaxed) {
                    self.put_tombstone(loc);
                    return Err(WriteError::CellAlreadyExisted)
                }
            }
            return written
        }
    }
    fn update_cell(&self, cell: &mut Cell) -> Result<usize, WriteError> {
        let hash = cell.header.hash;
        if let Some(mut cell_location) = self.location_of(hash) {
            let written = cell.write_to_chunk(self);
            if let Ok(new_location) = written {
                let old_location = *cell_location;
                *cell_location = new_location;
                self.put_tombstone(old_location);
            }
            return written;
        } else {
            return Err(WriteError::CellDoesNotExisted)
        }
    }
    fn update_cell_by<U>(&self, hash: u64, update: U) -> Result<usize, WriteError>
        where U: Fn(Cell) -> Option<Cell> {
        if let Some(mut cell_location) = self.location_of(hash) {
            let cell = Cell::from_chunk_raw(*cell_location, self);
            match cell {
                Ok(cell) => {
                    let mut new_cell = update(cell);
                    if let Some(mut new_cell) = new_cell {
                        let written = new_cell.write_to_chunk(self);
                        if let Ok(new_location) = written {
                            let old_location = *cell_location;
                            *cell_location = new_location;
                            self.put_tombstone(old_location);
                        }
                        return written;
                    } else {
                        return Err(WriteError::UserCanceledUpdate);
                    }
                },
                Err(e) => Err(WriteError::ReadError(e))
            }
        } else {
            return Err(WriteError::CellDoesNotExisted)
        }
    }
    fn remove_cell(&self, hash: u64) -> Result<(), WriteError> {
        if let Some(cell_location) = self.location_of(hash) {
            self.index.remove(&hash);
            self.put_tombstone(*cell_location);
            return Ok(());
        } else {
            return Err(WriteError::CellDoesNotExisted);
        }
    }
    fn remove_cell_by<P>(&self, hash: u64, predict: P) -> Result<(), WriteError>
        where P: Fn(Cell) -> bool {
        if let Some(cell_location) = self.location_of(hash) {
            let cell = Cell::from_chunk_raw(*cell_location, self);
            match cell {
                Ok(cell) => {
                    if predict(cell) {
                        self.index.remove(&hash);
                        self.put_tombstone(*cell_location);
                        return Ok(());
                    } else {
                        return Err(WriteError::DeletionPredictionFailed);
                    }
                },
                Err(e) => Err(WriteError::ReadError(e))
            }
        } else {
            return Err(WriteError::CellDoesNotExisted);
        }
    }
    fn dispose (&mut self) {
        debug!("disposing chunk at {}", self.addr);
        unsafe {
            libc::free(self.addr as *mut libc::c_void)
        }
    }
}

impl Drop for Chunk {
    fn drop(&mut self) {
        self.dispose();
    }
}

impl Chunks {
    pub fn new (count: usize, size: usize, meta: Rc<ServerMeta>, backup_storage: Option<String>) -> Chunks {
        let chunk_size = size / count;
        let mut chunks = Vec::new();
        debug!("Creating {} chunks, total {} bytes", count, size);
        for i in 0..count {
            let backup_storage = match backup_storage {
                Some(ref dir) => Some(format!("{}/data-{}.bak", dir, i)),
                None => None
            };
            chunks.push(Chunk::new(i, chunk_size, meta.clone(), backup_storage));
        }
        Chunks {
            list: chunks
        }
    }
    pub fn new_dummy(count: usize, size: usize) -> Chunks {
        Chunks::new(count, size, Rc::<ServerMeta>::new(ServerMeta {
            schemas: Schemas::new(None)
        }), None)
    }
    fn locate_chunk_by_partition(&self, partition: u64) -> &Chunk {
        let chunk_id = partition as usize % self.list.len();
        return &self.list[chunk_id];
    }
    fn locate_chunk_by_key(&self, key: (u64, u64)) -> (&Chunk, u64) {
        let (partition, hash) = key;
        return (self.locate_chunk_by_partition(partition), hash);
    }
    pub fn read_cell(&self, key: (u64, u64)) -> Result<Cell, ReadError> {
        let (chunk, hash) = self.locate_chunk_by_key(key);
        return chunk.read_cell(hash);
    }
    pub fn write_cell(&self, cell: &mut Cell) -> Result<usize, WriteError> {
        let chunk = self.locate_chunk_by_partition(cell.header.partition);
        return chunk.write_cell(cell);
    }
    pub fn update_cell(&self, cell: &mut Cell) -> Result<usize, WriteError> {
        let chunk = self.locate_chunk_by_partition(cell.header.partition);
        return chunk.update_cell(cell);
    }
    pub fn update_cell_by<U>(&self, key: (u64, u64), update: U) -> Result<usize, WriteError>
        where U: Fn(Cell) -> Option<Cell>{
        let (chunk, hash) = self.locate_chunk_by_key(key);
        return chunk.update_cell_by(hash, update);
    }
    pub fn remove_cell(self, key: (u64, u64)) -> Result<(), WriteError> {
        let (chunk, hash) = self.locate_chunk_by_key(key);
        return chunk.remove_cell(hash);
    }
    pub fn remove_cell_by<P>(&self, key: (u64, u64), predict: P) -> Result<(), WriteError>
        where P: Fn(Cell) -> bool {
        let (chunk, hash) = self.locate_chunk_by_key(key);
        return chunk.remove_cell_by(hash, predict);
    }
}<|endoftext|>


<commit_before>use rustc::lint::*;
use rustc_front::hir::*;
use utils::span_lint;
use rustc::middle::ty::{TypeAndMut, TypeVariants};

declare_lint! {
    pub UNNECESSARY_MUT_PASSED,
    Warn,
    "an argument is passed as a mutable reference although the function only demands an \
     immutable reference"
}

#[derive(Copy,Clone)]
pub struct UnnecessaryMutPassed;

impl LintPass for UnnecessaryMutPassed {
    fn get_lints(&self) -> LintArray {
        lint_array!(UNNECESSARY_MUT_PASSED)
    }
}

impl LateLintPass for UnnecessaryMutPassed {
    fn check_expr(&mut self, cx: &LateContext, e: &Expr) {
        if let &ExprCall(ref fn_expr, ref arguments) = &e.node {
			let borrowed_table = cx.tcx.tables.borrow();
			let funtion_type = match borrowed_table.node_types.get(&fn_expr.id) {
				Some(funtion_type) => funtion_type,
				None => unreachable!(), // A function with unknown type is called.
				                        // If this happened the compiler would have aborted the
										// compilation long ago.
			};
            if let TypeVariants::TyBareFn(_, ref b) = funtion_type.sty {
                let parameters = b.sig.skip_binder().inputs.clone();
                for (argument, parameter) in arguments.iter().zip(parameters.iter()) {
                    match parameter.sty {
						TypeVariants::TyRef(_, TypeAndMut {ty: _, mutbl: MutImmutable}) |
						TypeVariants::TyRawPtr(TypeAndMut {ty: _, mutbl: MutImmutable}) => {
							if let Expr_::ExprAddrOf(MutMutable, _) = argument.node {
								if let ExprPath(_, path) = fn_expr.node.clone() {
									span_lint(cx, UNNECESSARY_MUT_PASSED,
                                              argument.span, &format!("This argument of the \
                                              function \"{}\" doesn't need to be mutable", path));
								}
							}
						},
						_ => {}
                    }
                }
            }
        }
    }
}
<commit_msg>Replace tabs by spaces<commit_after>use rustc::lint::*;
use rustc_front::hir::*;
use utils::span_lint;
use rustc::middle::ty::{TypeAndMut, TypeVariants};

declare_lint! {
    pub UNNECESSARY_MUT_PASSED,
    Warn,
    "an argument is passed as a mutable reference although the function only demands an \
     immutable reference"
}

#[derive(Copy,Clone)]
pub struct UnnecessaryMutPassed;

impl LintPass for UnnecessaryMutPassed {
    fn get_lints(&self) -> LintArray {
        lint_array!(UNNECESSARY_MUT_PASSED)
    }
}

impl LateLintPass for UnnecessaryMutPassed {
    fn check_expr(&mut self, cx: &LateContext, e: &Expr) {
        if let &ExprCall(ref fn_expr, ref arguments) = &e.node {
            let borrowed_table = cx.tcx.tables.borrow();
            let funtion_type = match borrowed_table.node_types.get(&fn_expr.id) {
                Some(funtion_type) => funtion_type,
                None => unreachable!(), // A function with unknown type is called.
                                        // If this happened the compiler would have aborted the
                                        // compilation long ago.
            };
            if let TypeVariants::TyBareFn(_, ref b) = funtion_type.sty {
                let parameters = b.sig.skip_binder().inputs.clone();
                for (argument, parameter) in arguments.iter().zip(parameters.iter()) {
                    match parameter.sty {
                        TypeVariants::TyRef(_, TypeAndMut {ty: _, mutbl: MutImmutable}) |
                        TypeVariants::TyRawPtr(TypeAndMut {ty: _, mutbl: MutImmutable}) => {
                            if let Expr_::ExprAddrOf(MutMutable, _) = argument.node {
                                if let ExprPath(_, path) = fn_expr.node.clone() {
                                    span_lint(cx, UNNECESSARY_MUT_PASSED,
                                              argument.span, &format!("This argument of the \
                                              function \"{}\" doesn't need to be mutable", path));
                                }
                            }
                        },
                        _ => {}
                    }
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>use engine::Name;
use macros::ProgPt;
use std::fmt::Debug;
//use std::hash::Hash;
use std::rc::Rc;
use std::collections::HashMap;

/// Reflected primitive data
pub trait Data : Debug { }

pub trait Reflect<T> : Debug {
  fn reflect (&self) -> T;
}

impl<S,T:Reflect<S>+Debug> Reflect<Option<S>> for Option<T> {
  fn reflect (&self) -> Option<S> {
    match *self {
      None => None,
      Some(ref x) => Some(x.reflect())
    }
  }
}

impl<'a,S,T:Reflect<S>+Debug> Reflect<S> for &'a Rc<T> {
  fn reflect (&self) -> S {
    (**self).reflect()
  }
}

impl<'a,S,T:Reflect<S>+Debug> Reflect<S> for Rc<T> {
  fn reflect (&self) -> S {
    (**self).reflect()
  }
}

/// Reflected value; Gives a syntax for inductive data type
/// constructors (`Constr`), named articulations (`Art`) and primitive
/// data (`Data).
#[derive(Debug,Clone)]
pub enum Val {
  /// Constructor, with value parameters.
  Constr(Name,Vec<Val>),
  /// Named articulation, and its content (an `Art` is either a named value, or a named computation).
  Art(Name,ArtContent),
  /// Primitive, immutable data.
  Data(Rc<Data>),

  /// Temporary; for marking places in code where we should produce a
  /// value, but don't yet have a good way to do so.
  ValTODO,
}

/// The content of an articulation: Either a cell holding a value, or
/// a thunk that has optionally produced a value.
#[derive(Debug,Clone)]
pub enum ArtContent {
  /// A cell holding a value
  Val(Rc<Val>),
  /// A thunk that, when forced, computes a value
  Comp(Option<Rc<Val>>),
}

/// `Reflected version of engine::Loc` A `Loc` is a particular
/// template for a `Name`: It is a path (a possibly-empty list of
/// `Name`s), followed by a distinguished `Name`.  A `Loc` can be
/// thought of roughly like a file path in UNIX (but Adapton has
/// nothing to do with files, or with UNIX, directly).
#[derive(PartialEq,Eq,Debug,Hash,Clone)]
pub struct Loc {
  /// The path of the `Loc` is a list of `Name`s.
  pub path: Path,
  /// The distinguished `Name` of the `Loc` (must be unique in the path).
  pub name: Name,
}

/// A `Path` here is just a `Vec` of `Name`s
pub type Path = Vec<Name>;

/// `Reflected version of engine::Effect`
#[derive(Debug,Clone)]
pub enum Effect {
  /// The effect consists of a thunk observing the value of another
  /// thunk or reference cell.  That is, the effect consists of
  /// **consuming** a value, by demanding its production.
  Force,
  /// The effect consists of a thunk allocating a value or thunk at a
  /// particular name.  That is, the effect consists of **producing**
  /// a value or computation.  If this content differs, then the
  /// allocation is a **reallocation**, and the engine dirties the old
  /// observers and allocators of the preceding content.
  Alloc,
}

/// Reflected version of `engine::Succ`
#[derive(Debug,Clone)]
pub struct Succ {
  /// Dirty invariant: If this edge is dirty, then all predecessors of
  /// the edge are dirty too.
  pub dirty:  bool,
  /// The target of the outgoing `Effect`
  pub loc:    Loc,
  /// The effect: either producing or consuming articulated content
  pub effect: Effect,
  /// The value either produced or consumed by this `Effect`
  pub value:  Val,
}

impl Reflect<Succ> for Succ {
  fn reflect (&self) -> Succ {
    self.clone()
  }
}

/// Reflected version of `engine::Pred`
#[derive(Debug)]
pub struct Pred {
  /// The predecessor of the node in question
  pub loc:    Loc,
  /// The effect that the predecessor has done to the node in question
  pub effect: Effect,
}

/// Reflected version of `engine::CompNode`
#[derive(Debug)]
pub struct CompNode {
  pub preds:   Vec<Pred>,
  pub succs:   Vec<Succ>,
  pub prog_pt: ProgPt,
  pub value:   Option<Val>,
}

/// Reflected version of `engine::MutNode`
#[derive(Debug)]
pub struct RefNode {
  pub preds: Vec<Pred>,
  pub value: Val,
}

/// Reflected version of `engine::PureNode`
#[derive(Debug)]
pub struct PureNode {
  pub value: Val,
}

/// Reflected version of `engine::GraphNode`
#[derive(Debug)]
pub enum GraphNode {
  Comp(CompNode),
  Ref(RefNode),
  Pure(PureNode),
}

/// Reflected version of `engine::Frame`
#[derive(Debug)]
pub struct Frame {
  pub loc:   Loc,
  pub succs: Vec<Succ>,
}

/// Reflected version of `engine::DCG`
#[derive(Debug)]
pub struct DCG {
  /// The current memo table, mapping `Loc`s to `GraphNode`s.

  pub table: HashMap<Loc, Box<GraphNode>>,
  pub stack: Vec<Frame>,
  pub path:  Vec<Name>,
}

#[derive(Debug)]
pub enum DCGAlloc {
  /// The allocation was **created** fresh; it was **not** reused.
  LocFresh,
  /// The allocation matched the location of a prior allocation. its
  /// content may or may not also match.  The `bool` indicates the two
  /// cases: `true` means same content, `false` means changed content.
  LocExists,
}

/// When the program performs a `force`, either the cache is either
/// empty (`CacheMiss`) or non-empth (`CacheHit`).  The cached value
/// may not be consistent without a cleaning.
#[derive(Debug)]
pub enum DCGForce {
  /// The DCG has no cached value for this computation; no prior computation will be reused.
  CacheMiss,
  /// The DCG has a cached value for this computation; it may not be consistent without a cleaning.
  CacheHit,
}

/// The effects of the DCG (including cleaning and dirtying) on one of
/// its edges.
#[derive(Debug)]
pub enum DCGEffect {
  /// Wrapper for Effect::Alloc; transition to DCG after the alloc.
  Alloc (DCGAlloc),

  /// Wrapper for Effect::Force; transition to DCG after the force.
  Force (DCGForce),

  /// Transition to this edge as **dirty** (potentially inconsistent).
  /// This transition may consist of marking other edges dirty.  The
  /// DCG's invariant about dirty edges is simple: If an edge is
  /// dirty, then all edges that force (demand/observe) the source of
  /// that edge must also be dirty.  This transitive closure property
  /// ensures that we do not accidentally reuse stale cached values by
  /// mistake (a dirty edge will always witness a potentially stale
  /// cached value)
  Dirty,

  /// Transition to this edge as **clean** (definitely consistent).
  /// This transition may consist of marking other edges clean,
  /// and/or, removing edges and replacing them via reexecution under
  /// the current DCG state.
  Clean,

  /// Transition to the DCG without this edge.  Perhaps it will be
  /// replaced via re-execution, sometime later.
  Remove,
}

/// An adge in the DCG, representing an effect of the incremental program.
#[derive(Debug)]
pub struct DCGEdge {
  /// The source of the directed edge; it is actively _doing_ the
  /// effect of `succ.effect` to `succ.loc`.  `None` means the doer is
  /// the **editor**, who is not identified by any location. (The editor
  /// is not a node in the DCG, but rather, an actor operating outside
  /// of it).
  pub loc: Option<Loc>,
  /// The effect and target of the directed edge.
  pub succ: Succ,
}

/// `DCGTrace`: A Rose-tree of DCG edge-effects.  This tree structure
/// allows the effects to have a a "time interval" that nests around
/// and within the time intervals of other effects.
#[derive(Debug)]
pub struct DCGTrace {
  /// The DCG effect (e.g., Alloc(MatchDiff), Dirty, Clean, etc.)
  pub effect:DCGEffect,

  /// The DCG edge on which this DCG effect takes place
  pub edge: DCGEdge,

  /// The DCG effects that occur subordinately as a result of this
  /// effect. (They begin after this effect begins, and the end before
  /// this effect ends).
  pub extent:DCGTraces,
}
/// A (boxed) vector of `DCGTrace`s.
pub type DCGTraces = Box<Vec<DCGTrace>>;
<commit_msg>refining the defs for dcg force effects<commit_after>use engine::Name;
use macros::ProgPt;
use std::fmt::Debug;
//use std::hash::Hash;
use std::rc::Rc;
use std::collections::HashMap;

/// Reflected primitive data
pub trait Data : Debug { }

pub trait Reflect<T> : Debug {
  fn reflect (&self) -> T;
}

impl<S,T:Reflect<S>+Debug> Reflect<Option<S>> for Option<T> {
  fn reflect (&self) -> Option<S> {
    match *self {
      None => None,
      Some(ref x) => Some(x.reflect())
    }
  }
}

impl<'a,S,T:Reflect<S>+Debug> Reflect<S> for &'a Rc<T> {
  fn reflect (&self) -> S {
    (**self).reflect()
  }
}

impl<'a,S,T:Reflect<S>+Debug> Reflect<S> for Rc<T> {
  fn reflect (&self) -> S {
    (**self).reflect()
  }
}

/// Reflected value; Gives a syntax for inductive data type
/// constructors (`Constr`), named articulations (`Art`) and primitive
/// data (`Data).
#[derive(Debug,Clone)]
pub enum Val {
  /// Constructor, with value parameters.
  Constr(Name,Vec<Val>),
  /// Named articulation, and its content (an `Art` is either a named value, or a named computation).
  Art(Name,ArtContent),
  /// Primitive, immutable data.
  Data(Rc<Data>),

  /// Temporary; for marking places in code where we should produce a
  /// value, but don't yet have a good way to do so.
  ValTODO,
}

/// The content of an articulation: Either a cell holding a value, or
/// a thunk that has optionally produced a value.
#[derive(Debug,Clone)]
pub enum ArtContent {
  /// A cell holding a value
  Val(Rc<Val>),
  /// A thunk that, when forced, computes a value
  Comp(Option<Rc<Val>>),
}

/// `Reflected version of engine::Loc` A `Loc` is a particular
/// template for a `Name`: It is a path (a possibly-empty list of
/// `Name`s), followed by a distinguished `Name`.  A `Loc` can be
/// thought of roughly like a file path in UNIX (but Adapton has
/// nothing to do with files, or with UNIX, directly).
#[derive(PartialEq,Eq,Debug,Hash,Clone)]
pub struct Loc {
  /// The path of the `Loc` is a list of `Name`s.
  pub path: Path,
  /// The distinguished `Name` of the `Loc` (must be unique in the path).
  pub name: Name,
}

/// A `Path` here is just a `Vec` of `Name`s
pub type Path = Vec<Name>;

/// `Reflected version of engine::Effect`
#[derive(Debug,Clone)]
pub enum Effect {
  /// The effect consists of a thunk observing the value of another
  /// thunk or reference cell.  That is, the effect consists of
  /// **consuming** a value, by demanding its production.
  Force,
  /// The effect consists of a thunk allocating a value or thunk at a
  /// particular name.  That is, the effect consists of **producing**
  /// a value or computation.  If this content differs, then the
  /// allocation is a **reallocation**, and the engine dirties the old
  /// observers and allocators of the preceding content.
  Alloc,
}

/// Reflected version of `engine::Succ`
#[derive(Debug,Clone)]
pub struct Succ {
  /// Dirty invariant: If this edge is dirty, then all predecessors of
  /// the edge are dirty too.
  pub dirty:  bool,
  /// The target of the outgoing `Effect`
  pub loc:    Loc,
  /// The effect: either producing or consuming articulated content
  pub effect: Effect,
  /// The value either produced or consumed by this `Effect`
  pub value:  Val,
}

impl Reflect<Succ> for Succ {
  fn reflect (&self) -> Succ {
    self.clone()
  }
}

/// Reflected version of `engine::Pred`
#[derive(Debug)]
pub struct Pred {
  /// The predecessor of the node in question
  pub loc:    Loc,
  /// The effect that the predecessor has done to the node in question
  pub effect: Effect,
}

/// Reflected version of `engine::CompNode`
#[derive(Debug)]
pub struct CompNode {
  pub preds:   Vec<Pred>,
  pub succs:   Vec<Succ>,
  pub prog_pt: ProgPt,
  pub value:   Option<Val>,
}

/// Reflected version of `engine::MutNode`
#[derive(Debug)]
pub struct RefNode {
  pub preds: Vec<Pred>,
  pub value: Val,
}

/// Reflected version of `engine::PureNode`
#[derive(Debug)]
pub struct PureNode {
  pub value: Val,
}

/// Reflected version of `engine::GraphNode`
#[derive(Debug)]
pub enum GraphNode {
  Comp(CompNode),
  Ref(RefNode),
  Pure(PureNode),
}

/// Reflected version of `engine::Frame`
#[derive(Debug)]
pub struct Frame {
  pub loc:   Loc,
  pub succs: Vec<Succ>,
}

/// Reflected version of `engine::DCG`
#[derive(Debug)]
pub struct DCG {
  /// The current memo table, mapping `Loc`s to `GraphNode`s.

  pub table: HashMap<Loc, Box<GraphNode>>,
  pub stack: Vec<Frame>,
  pub path:  Vec<Name>,
}

#[derive(Debug)]
pub enum DCGAlloc {
  /// The allocation was **created** fresh; it was **not** reused.
  LocFresh,
  /// The allocation matched the location of a prior allocation. its
  /// content may or may not also match.  The `bool` indicates the two
  /// cases: `true` means same content, `false` means changed content.
  LocExists,
}

/// When the program `force`s a computation, either the cache is
/// either empty (`CacheMiss`) or non-empty (`CacheHit`).  The cached
/// value may not be consistent without a cleaning.  When the program
/// `force`s a reference cell, it simply gets the current value.
#[derive(Debug)]
pub enum DCGForce {
  /// The DCG has no cached value for this computation; no prior
  /// computation will be reused.
  CompCacheMiss,
  /// The DCG has a cached value for this computation; it may not be
  /// consistent without a cleaning first.
  CompCacheHit,
  /// The forced node is a ref cell with a (mutable) value, and hence,
  /// no computation was necessary.  The `force` simply gets the
  /// current value.
  RefGet,
}

/// The effects of the DCG (including cleaning and dirtying) on one of
/// its edges.
#[derive(Debug)]
pub enum DCGEffect {
  /// Wrapper for Effect::Alloc; transition to DCG after the alloc.
  Alloc (DCGAlloc),

  /// Wrapper for Effect::Force; transition to DCG after the force.
  Force (DCGForce),

  /// Transition to this edge as **dirty** (potentially inconsistent).
  /// This transition may consist of marking other edges dirty.  The
  /// DCG's invariant about dirty edges is simple: If an edge is
  /// dirty, then all edges that force (demand/observe) the source of
  /// that edge must also be dirty.  This transitive closure property
  /// ensures that we do not accidentally reuse stale cached values by
  /// mistake (a dirty edge will always witness a potentially stale
  /// cached value).
  Dirty,

  /// Transition to this edge as **clean** (definitely consistent).
  /// This transition may consist of marking other edges clean,
  /// and/or, removing edges and replacing them via reexecution under
  /// the current DCG state.
  Clean,

  /// Transition to the DCG without this edge.  Perhaps it will be
  /// replaced via re-execution, sometime later.
  Remove,
}

/// An adge in the DCG, representing an effect of the incremental program.
#[derive(Debug)]
pub struct DCGEdge {
  /// The source of the directed edge; it is actively _doing_ the
  /// effect of `succ.effect` to `succ.loc`.  `None` means the doer is
  /// the **editor**, who is not identified by any location. (The editor
  /// is not a node in the DCG, but rather, an actor operating outside
  /// of it).
  pub loc: Option<Loc>,
  /// The effect and target of the directed edge.
  pub succ: Succ,
}

/// `DCGTrace`: A Rose-tree of DCG edge-effects.  This tree structure
/// allows the effects to have a a "time interval" that nests around
/// and within the time intervals of other effects.
#[derive(Debug)]
pub struct DCGTrace {
  /// The DCG effect (e.g., Alloc(MatchDiff), Dirty, Clean, etc.)
  pub effect:DCGEffect,

  /// The DCG edge on which this DCG effect takes place
  pub edge: DCGEdge,

  /// The DCG effects that occur subordinately as a result of this
  /// effect. (They begin after this effect begins, and the end before
  /// this effect ends).
  pub extent:DCGTraces,
}
/// A (boxed) vector of `DCGTrace`s.
pub type DCGTraces = Box<Vec<DCGTrace>>;
<|endoftext|>


<commit_before>pub fn hex_to_bytes(hex: &str) -> Vec<u8> {
	assert!(hex.len() % 2 == 0, "Hex string length is not even.");
	let mut result : Vec<u8> = Vec::new();

	let mut first_byte = true;

	for c in hex.chars() {
		let val = match c {
			'0' => 0,
			'1' => 1,
			'2' => 2,
			'3' => 3,
			'4' => 4,
			'5' => 5,
			'6' => 6,
			'7' => 7,
			'8' => 8,
			'9' => 9,
			'A' | 'a' => 10,
			'B' | 'b' => 11,
			'C' | 'c' => 12,
			'D' | 'd' => 13,
			'E' | 'e' => 14,
			'F' | 'f' => 15,
			_ => unreachable!(),
		};

		if first_byte {
			result.push(val << 4);
		}
		else
		{
			let length = result.len();
			result[length - 1] = result[length - 1] + val;
		}
		first_byte = !first_byte;
	}

	result
}

pub fn bytes_to_readable_text(bytes: &Vec<u8>) -> String {
	bytes.iter().map(|x| *x as char).collect()
}

pub fn readable_text_to_bytes(text: &str) -> Vec<u8> {
	text.bytes().collect()
}

pub fn bytes_to_hex(bytes: &Vec<u8>) -> String {
	let mut result : String = String::new();
	for byte in bytes {

		let bin_to_char = |x| match x {
			0 => '0',
			1 => '1',
			2 => '2',
			3 => '3',
			4 => '4',
			5 => '5',
			6 => '6',
			7 => '7',
			8 => '8',
			9 => '9',
			10 => 'a',
			11 => 'b',
			12 => 'c',
			13 => 'd',
			14 => 'e',
			15 => 'f',
			_ => unreachable!(),
		};

		result.push(bin_to_char(byte >> 4));
		result.push(bin_to_char(byte % 16));
	}

	result
}

/// Read up more on the base64 algorithm here:
/// https://en.wikibooks.org/wiki/Algorithm_Implementation/Miscellaneous/Base64
pub fn bytes_to_base64(bytes: &Vec<u8>) -> String {
	let base64chars: Vec<char> = vec!['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'];
	assert_eq!(64, base64chars.len());

	let mut padding = String::new();

	// Pad our bytes to a multiple of 3 in length
	let mut padded_bytes = bytes.clone();
	while padded_bytes.len() % 3 != 0 {
		padded_bytes.push(0);
		padding.push('=');
	}
	assert_eq!(0, padded_bytes.len() % 3);

	let mut result = String::new();

	// Step through our bytes 3 at a time
	let mut i = 0;
	while i < padded_bytes.len() {
		// We're using direct access as we know we won't be accessing invalid indices
		// Bring the three bytes into one larger word
		let mut chunk : u64 = 0;
		chunk +=	(padded_bytes[ i ] as u64) << 16;
		chunk += 	(padded_bytes[i+1] as u64) << 8;
		chunk += 	 padded_bytes[i+2] as u64;

		// And split it out into 4, appending to our string
		result.push(base64chars[((chunk >> 3*6) & 63) as usize]);
		result.push(base64chars[((chunk >> 2*6) & 63) as usize]);
		result.push(base64chars[((chunk >> 1*6) & 63) as usize]);
		result.push(base64chars[((chunk >> 0*6) & 63) as usize]);

		i += 3;
	}

	let truncated_bytes:Vec<u8> = result.bytes().take(result.len() - padding.len()).collect();
	let truncated_string = match String::from_utf8(truncated_bytes) {
		Ok(x) => x,
		Err(_) => unreachable!(),
	};

	let final_output = format!("{}{}", truncated_string, padding);
	assert_eq!(0, final_output.len() % 4);

	final_output
}

pub fn base64_to_bytes(base64: &str) -> Vec<u8> {
	// We use .bytes() and not .char() because base64 should be ASCII
	let mut bytes: Vec<u8> = base64.bytes().collect();

	// Reverse padding at the end from = to 0
	let mut length = bytes.len();
	if length == 0 {
		return vec![]
	}
	assert_eq!(0, length % 4); // Length should be a multiple of 4

	if bytes[length - 1] == 61 {
		bytes[length - 1] = 0;
	}
	if bytes[length - 2] == 61 {
		bytes[length - 2] = 0;
	}

	//
	let get_byte_from_char = |character:char| -> u64 {
		let r = match character {
			x @ 'A'...'Z' => x as u64 - 65 + 0,
			x @ 'a'...'z' => x as u64 - 97 + 26,
			x @ '0'...'9' => x as u64 - 49 + 53,
			'+' => 62,
			'/' => 63,
			'\0' => 0,
			e @ _ => { println!("unreachable:: {}, {}", e, e as u8); unreachable!() }
		};
		r
	};

	// Operate on the bytes 4 at a time
	let mut result: Vec<u8> = vec![];
	let mut i = 0;
	while i < bytes.len() {
		let mut chunk : u64 = 0;
		chunk += get_byte_from_char(bytes[i + 0 as usize] as char) << 18;
		chunk += get_byte_from_char(bytes[i + 1 as usize] as char) << 12;
		chunk += get_byte_from_char(bytes[i + 2 as usize] as char) << 06;
		chunk += get_byte_from_char(bytes[i + 3 as usize] as char) << 00;

		result.push((chunk >> 16 & 255) as u8);
		result.push((chunk >> 08 & 255) as u8);
		result.push((chunk >> 00 & 255) as u8);

		i += 4;
	}

	// Truncate the last two characters if they're not useful
	length = result.len();
	if result[length-1] == 0 {
		result.truncate(length - 1);
	}
	length = result.len();
	if result[length-1] == 0 {
		result.truncate(length - 1);
	}

	result
}

pub fn hex_to_base64(hex: &str) -> String {
	let bytes = hex_to_bytes(hex);
	bytes_to_base64(&bytes)
}

//-----------------------------------------------------------------------------

#[cfg(test)]
mod test {
	use super::*;

	#[test]
	fn hex_to_bytes_i_empty() {
		let x : Vec<u8> = vec![];
		assert_eq!(x, hex_to_bytes(""));
	}

	#[test]
	fn hex_to_bytes_i_simple() {
		let x : Vec<u8> = vec![73,74];
		assert_eq!(x, hex_to_bytes("494a"));
	}

	#[test]
	fn hex_to_bytes_i_simple_uppercase() {
		let x : Vec<u8> = vec![73,74];
		assert_eq!(x, hex_to_bytes("494A"));
	}

	#[test]
	fn bytes_to_readable_text_example() {
		let hex = "49276d206b696c6c696e6720796f757220627261696e206c696b65206120706f69736f6e6f7573206d757368726f6f6d";
		let bytes = hex_to_bytes(hex);
		let expected = "I'm killing your brain like a poisonous mushroom";
		assert_eq!(expected, bytes_to_readable_text(&bytes));
	}

	#[test]
	fn bytes_to_readable_symmetricality() {
		let text = "I'm killing your brain like a poisonous mushroom";
		let bytes = readable_text_to_bytes(text);
		let text_out = bytes_to_readable_text(&bytes);

		assert_eq!(text_out, text);
	}

	#[test]
	fn bytes_to_hex_i_empty() {
		assert_eq!("".to_string(), bytes_to_hex(&vec![]));
	}

	#[test]
	fn bytes_to_hex_i_simple() {
		let a : Vec<u8> = vec![73,74];
		assert_eq!("494a".to_string(), bytes_to_hex(&a));
	}

	#[test]
	fn bytes_and_hex_symmetricality() {
		let hex = "45ab";
		assert_eq!(hex, bytes_to_hex(&hex_to_bytes(hex)));
	}

	#[test]
	fn bytes_to_base64_i_empty() {
		let a : Vec<u8> = vec![];
		assert_eq!("".to_string(), bytes_to_base64(&a));
	}

	#[test]
	fn bytes_to_base64_i_simple() {
		let a : Vec<u8> = vec![97];
		let actual = bytes_to_base64(&a);
		assert_eq!("YQ==".to_string(), actual);
	}

	#[test]
	fn bytes_to_base64_i_fits() {
		let a : Vec<u8> = vec![72, 101, 108, 108, 111, 33];
		let actual = bytes_to_base64(&a);
		assert_eq!("SGVsbG8h".to_string(), actual);
	}

	#[test]
	fn bytes_to_base64_i_padded() {
		let a : Vec<u8> = vec![72, 101, 108, 108, 111];
		assert_eq!("SGVsbG8=".to_string(), bytes_to_base64(&a));
	}

	#[test]
	fn bytes_to_base64_i_double_padded() {
		let a : Vec<u8> = vec![72, 101, 108, 108];
		assert_eq!("SGVsbA==".to_string(), bytes_to_base64(&a));
	}

	#[test]
	fn base64_to_bytes_empty() {
		let a : Vec<u8> = vec![];
		assert_eq!(a, base64_to_bytes(""));
	}

	#[test]
	fn base64_to_bytes_simple() {
		let a : Vec<u8> = vec![97];
		assert_eq!(a, base64_to_bytes("YQ=="));
	}

	#[test]
	fn base64_to_bytes_i_fits() {
		let a : Vec<u8> = vec![72, 101, 108, 108, 111, 33];
		assert_eq!(a, base64_to_bytes("SGVsbG8h"));
	}

	#[test]
	fn bytes_and_base64_symmetricality_1() {
		let base64_1 = "aGVsbG8=";
		assert_eq!(base64_1, bytes_to_base64(&base64_to_bytes(base64_1)));
	}

	#[test]
	fn bytes_and_base64_symmetricality_2() {
		let base64_2 = "";
		assert_eq!(base64_2, bytes_to_base64(&base64_to_bytes(base64_2)));
	}

	#[test]
	fn bytes_and_base64_symmetricality_3() {
		let base64_3 = "SSdtIGtpbGxpbmcgeW91ciBicmFpbiBsaWtlIGEgcG9pc29ub3VzIG11c2hyb29t";
		assert_eq!(base64_3, bytes_to_base64(&base64_to_bytes(base64_3)));
	}

	#[test]
	fn hex_to_base64_i_standard() {
		let expected = "SSdtIGtpbGxpbmcgeW91ciBicmFpbiBsaWtlIGEgcG9pc29ub3VzIG11c2hyb29t";
		let actual: String = hex_to_base64("49276d206b696c6c696e6720796f757220627261696e206c696b65206120706f69736f6e6f7573206d757368726f6f6d");

		assert_eq!(expected, actual);
	}
}<commit_msg>Stop potential underflow operation<commit_after>pub fn hex_to_bytes(hex: &str) -> Vec<u8> {
	assert!(hex.len() % 2 == 0, "Hex string length is not even.");
	let mut result : Vec<u8> = Vec::new();

	let mut first_byte = true;

	for c in hex.chars() {
		let val = match c {
			'0' => 0,
			'1' => 1,
			'2' => 2,
			'3' => 3,
			'4' => 4,
			'5' => 5,
			'6' => 6,
			'7' => 7,
			'8' => 8,
			'9' => 9,
			'A' | 'a' => 10,
			'B' | 'b' => 11,
			'C' | 'c' => 12,
			'D' | 'd' => 13,
			'E' | 'e' => 14,
			'F' | 'f' => 15,
			_ => unreachable!(),
		};

		if first_byte {
			result.push(val << 4);
		}
		else
		{
			let length = result.len();
			result[length - 1] = result[length - 1] + val;
		}
		first_byte = !first_byte;
	}

	result
}

pub fn bytes_to_readable_text(bytes: &Vec<u8>) -> String {
	bytes.iter().map(|x| *x as char).collect()
}

pub fn readable_text_to_bytes(text: &str) -> Vec<u8> {
	text.bytes().collect()
}

pub fn bytes_to_hex(bytes: &Vec<u8>) -> String {
	let mut result : String = String::new();
	for byte in bytes {

		let bin_to_char = |x| match x {
			0 => '0',
			1 => '1',
			2 => '2',
			3 => '3',
			4 => '4',
			5 => '5',
			6 => '6',
			7 => '7',
			8 => '8',
			9 => '9',
			10 => 'a',
			11 => 'b',
			12 => 'c',
			13 => 'd',
			14 => 'e',
			15 => 'f',
			_ => unreachable!(),
		};

		result.push(bin_to_char(byte >> 4));
		result.push(bin_to_char(byte % 16));
	}

	result
}

/// Read up more on the base64 algorithm here:
/// https://en.wikibooks.org/wiki/Algorithm_Implementation/Miscellaneous/Base64
pub fn bytes_to_base64(bytes: &Vec<u8>) -> String {
	let base64chars: Vec<char> = vec!['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'];
	assert_eq!(64, base64chars.len());

	let mut padding = String::new();

	// Pad our bytes to a multiple of 3 in length
	let mut padded_bytes = bytes.clone();
	while padded_bytes.len() % 3 != 0 {
		padded_bytes.push(0);
		padding.push('=');
	}
	assert_eq!(0, padded_bytes.len() % 3);

	let mut result = String::new();

	// Step through our bytes 3 at a time
	let mut i = 0;
	while i < padded_bytes.len() {
		// We're using direct access as we know we won't be accessing invalid indices
		// Bring the three bytes into one larger word
		let mut chunk : u64 = 0;
		chunk +=	(padded_bytes[ i ] as u64) << 16;
		chunk += 	(padded_bytes[i+1] as u64) << 8;
		chunk += 	 padded_bytes[i+2] as u64;

		// And split it out into 4, appending to our string
		result.push(base64chars[((chunk >> 3*6) & 63) as usize]);
		result.push(base64chars[((chunk >> 2*6) & 63) as usize]);
		result.push(base64chars[((chunk >> 1*6) & 63) as usize]);
		result.push(base64chars[((chunk >> 0*6) & 63) as usize]);

		i += 3;
	}

	let truncated_bytes:Vec<u8> = result.bytes().take(result.len() - padding.len()).collect();
	let truncated_string = match String::from_utf8(truncated_bytes) {
		Ok(x) => x,
		Err(_) => unreachable!(),
	};

	let final_output = format!("{}{}", truncated_string, padding);
	assert_eq!(0, final_output.len() % 4);

	final_output
}

pub fn base64_to_bytes(base64: &str) -> Vec<u8> {
	// We use .bytes() and not .char() because base64 should be ASCII
	let mut bytes: Vec<u8> = base64.bytes().collect();

	// Reverse padding at the end from = to 0
	let mut length = bytes.len();
	if length == 0 {
		return vec![]
	}
	assert_eq!(0, length % 4); // Length should be a multiple of 4

	if bytes[length - 1] == 61 {
		bytes[length - 1] = 0;
	}
	if bytes[length - 2] == 61 {
		bytes[length - 2] = 0;
	}

	//
	let get_byte_from_char = |character:char| -> u64 {
		let r = match character {
			x @ 'A'...'Z' => x as u64 + 0  - 65,
			x @ 'a'...'z' => x as u64 + 26 - 97,
			x @ '0'...'9' => x as u64 + 53 - 49,
			'+' => 62,
			'/' => 63,
			'\0' => 0,
			'\n' => 0,
			e @ _ => { println!("unreachable:: {}, {}", e, e as u8); unreachable!() }
		};
		r
	};

	// Operate on the bytes 4 at a time
	let mut result: Vec<u8> = vec![];
	let mut i = 0;
	while i < bytes.len() {
		let mut chunk : u64 = 0;
		chunk += get_byte_from_char(bytes[i + 0 as usize] as char) << 18;
		chunk += get_byte_from_char(bytes[i + 1 as usize] as char) << 12;
		chunk += get_byte_from_char(bytes[i + 2 as usize] as char) << 06;
		chunk += get_byte_from_char(bytes[i + 3 as usize] as char) << 00;

		result.push((chunk >> 16 & 255) as u8);
		result.push((chunk >> 08 & 255) as u8);
		result.push((chunk >> 00 & 255) as u8);

		i += 4;
	}

	// Truncate the last two characters if they're not useful
	length = result.len();
	if result[length-1] == 0 {
		result.truncate(length - 1);
	}
	length = result.len();
	if result[length-1] == 0 {
		result.truncate(length - 1);
	}

	result
}

pub fn hex_to_base64(hex: &str) -> String {
	let bytes = hex_to_bytes(hex);
	bytes_to_base64(&bytes)
}

//-----------------------------------------------------------------------------

#[cfg(test)]
mod test {
	use super::*;

	#[test]
	fn hex_to_bytes_i_empty() {
		let x : Vec<u8> = vec![];
		assert_eq!(x, hex_to_bytes(""));
	}

	#[test]
	fn hex_to_bytes_i_simple() {
		let x : Vec<u8> = vec![73,74];
		assert_eq!(x, hex_to_bytes("494a"));
	}

	#[test]
	fn hex_to_bytes_i_simple_uppercase() {
		let x : Vec<u8> = vec![73,74];
		assert_eq!(x, hex_to_bytes("494A"));
	}

	#[test]
	fn bytes_to_readable_text_example() {
		let hex = "49276d206b696c6c696e6720796f757220627261696e206c696b65206120706f69736f6e6f7573206d757368726f6f6d";
		let bytes = hex_to_bytes(hex);
		let expected = "I'm killing your brain like a poisonous mushroom";
		assert_eq!(expected, bytes_to_readable_text(&bytes));
	}

	#[test]
	fn bytes_to_readable_symmetricality() {
		let text = "I'm killing your brain like a poisonous mushroom";
		let bytes = readable_text_to_bytes(text);
		let text_out = bytes_to_readable_text(&bytes);

		assert_eq!(text_out, text);
	}

	#[test]
	fn bytes_to_hex_i_empty() {
		assert_eq!("".to_string(), bytes_to_hex(&vec![]));
	}

	#[test]
	fn bytes_to_hex_i_simple() {
		let a : Vec<u8> = vec![73,74];
		assert_eq!("494a".to_string(), bytes_to_hex(&a));
	}

	#[test]
	fn bytes_and_hex_symmetricality() {
		let hex = "45ab";
		assert_eq!(hex, bytes_to_hex(&hex_to_bytes(hex)));
	}

	#[test]
	fn bytes_to_base64_i_empty() {
		let a : Vec<u8> = vec![];
		assert_eq!("".to_string(), bytes_to_base64(&a));
	}

	#[test]
	fn bytes_to_base64_i_simple() {
		let a : Vec<u8> = vec![97];
		let actual = bytes_to_base64(&a);
		assert_eq!("YQ==".to_string(), actual);
	}

	#[test]
	fn bytes_to_base64_i_fits() {
		let a : Vec<u8> = vec![72, 101, 108, 108, 111, 33];
		let actual = bytes_to_base64(&a);
		assert_eq!("SGVsbG8h".to_string(), actual);
	}

	#[test]
	fn bytes_to_base64_i_padded() {
		let a : Vec<u8> = vec![72, 101, 108, 108, 111];
		assert_eq!("SGVsbG8=".to_string(), bytes_to_base64(&a));
	}

	#[test]
	fn bytes_to_base64_i_double_padded() {
		let a : Vec<u8> = vec![72, 101, 108, 108];
		assert_eq!("SGVsbA==".to_string(), bytes_to_base64(&a));
	}

	#[test]
	fn base64_to_bytes_empty() {
		let a : Vec<u8> = vec![];
		assert_eq!(a, base64_to_bytes(""));
	}

	#[test]
	fn base64_to_bytes_simple() {
		let a : Vec<u8> = vec![97];
		assert_eq!(a, base64_to_bytes("YQ=="));
	}

	#[test]
	fn base64_to_bytes_i_fits() {
		let a : Vec<u8> = vec![72, 101, 108, 108, 111, 33];
		assert_eq!(a, base64_to_bytes("SGVsbG8h"));
	}

	#[test]
	fn bytes_and_base64_symmetricality_1() {
		let base64_1 = "aGVsbG8=";
		assert_eq!(base64_1, bytes_to_base64(&base64_to_bytes(base64_1)));
	}

	#[test]
	fn bytes_and_base64_symmetricality_2() {
		let base64_2 = "";
		assert_eq!(base64_2, bytes_to_base64(&base64_to_bytes(base64_2)));
	}

	#[test]
	fn bytes_and_base64_symmetricality_3() {
		let base64_3 = "SSdtIGtpbGxpbmcgeW91ciBicmFpbiBsaWtlIGEgcG9pc29ub3VzIG11c2hyb29t";
		assert_eq!(base64_3, bytes_to_base64(&base64_to_bytes(base64_3)));
	}

	#[test]
	fn hex_to_base64_i_standard() {
		let expected = "SSdtIGtpbGxpbmcgeW91ciBicmFpbiBsaWtlIGEgcG9pc29ub3VzIG11c2hyb29t";
		let actual: String = hex_to_base64("49276d206b696c6c696e6720796f757220627261696e206c696b65206120706f69736f6e6f7573206d757368726f6f6d");

		assert_eq!(expected, actual);
	}
}<|endoftext|>


<commit_before>use serde::{ser};
use std::mem::transmute;
use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
use std::fmt::{Display,Debug};
use std::fmt;
use serialize::{
    serialize_u64
};

// Option to display references as nested
pub static mut NESTED_REFERENCE: bool = false;

#[derive(Serialize, Debug)]
pub struct MftEnumReference{
    #[serde(serialize_with = "serialize_u64")]
    reference: u64,
    entry: u64,
    sequence: u16
}

// Represents a MFT Reference struct
// https://msdn.microsoft.com/en-us/library/bb470211(v=vs.85).aspx
// https://jmharkness.wordpress.com/2011/01/27/mft-file-reference-number/
pub struct MftReference(pub u64);
impl MftReference{
    pub fn from_entry_and_seq(&mut self, entry: u64, sequence: u16){
        let entry_buffer: [u8; 8] = unsafe {
            transmute(entry.to_le())
        };
        let seq_buffer: [u8; 2] = unsafe {
            transmute(sequence.to_le())
        };
        let mut ref_buffer = vec![];
        ref_buffer.extend_from_slice(&entry_buffer[0..6]);
        ref_buffer.extend_from_slice(&seq_buffer);

        self.0 = LittleEndian::read_u64(&ref_buffer[0..8]);
    }
    pub fn get_from_entry_and_seq(entry: u64, sequence: u16) -> MftReference{
        let entry_buffer: [u8; 8] = unsafe {
            transmute(entry.to_le())
        };
        let seq_buffer: [u8; 2] = unsafe {
            transmute(sequence.to_le())
        };
        let mut ref_buffer = vec![];
        ref_buffer.extend_from_slice(&entry_buffer[0..6]);
        ref_buffer.extend_from_slice(&seq_buffer);

        MftReference(LittleEndian::read_u64(&ref_buffer[0..8]))
    }
    pub fn get_enum_ref(&self)->MftEnumReference{
        let mut raw_buffer = vec![];
        raw_buffer.write_u64::<LittleEndian>(self.0).unwrap();
        MftEnumReference{
            reference: LittleEndian::read_u64(&raw_buffer[0..8]),
            entry: LittleEndian::read_u64(
                &[&raw_buffer[0..6], &[0,0]].concat()
            ),
            sequence: LittleEndian::read_u16(&raw_buffer[6..8])
        }
    }
}
impl Display for MftReference {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f,"{}",self.0)
    }
}
impl Debug for MftReference {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f,"{}",self.0)
    }
}
impl ser::Serialize for MftReference {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
        where S: ser::Serializer
    {
        // If NESTED_REFERENCE we need to serialize the enumerated structure
        if unsafe{NESTED_REFERENCE} {
            serializer.serialize_newtype_struct("mft_reference",&self.get_enum_ref())
        } else {
            // Just serialize the u64 version
            serialize_u64(&self.0,serializer)
        }
    }
}

#[test]
fn test_reference() {
    use std::mem;
    let raw_reference: &[u8] = &[0x73,0x00,0x00,0x00,0x00,0x00,0x68,0x91];

    let mft_reference = MftReference(
        LittleEndian::read_u64(&raw_reference[0..8])
    );
    assert_eq!(mft_reference.0,10477624533077459059);
    assert_eq!(format!("{}", mft_reference),"10477624533077459059");
    // assert_eq!(mft_reference.sequence,37224);

    let mft_reference_01 = MftReference::get_from_entry_and_seq(
        115,
        37224
    );
    assert_eq!(mft_reference_01.0,10477624533077459059);

    let mut mft_reference_02: MftReference = unsafe {
        mem::zeroed()
    };
    assert_eq!(mft_reference_02.0,0);
    mft_reference_02.from_entry_and_seq(
        115,
        37224
    );
    assert_eq!(mft_reference_02.0,10477624533077459059);
}
<commit_msg>allow MftReference to be hash key<commit_after>use serde::{ser};
use std::mem::transmute;
use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
use std::fmt::{Display,Debug};
use std::fmt;
use serialize::{
    serialize_u64
};

// Option to display references as nested
pub static mut NESTED_REFERENCE: bool = false;

#[derive(Serialize, Debug)]
pub struct MftEnumReference{
    #[serde(serialize_with = "serialize_u64")]
    reference: u64,
    entry: u64,
    sequence: u16
}

// Represents a MFT Reference struct
// https://msdn.microsoft.com/en-us/library/bb470211(v=vs.85).aspx
// https://jmharkness.wordpress.com/2011/01/27/mft-file-reference-number/
#[derive(Hash, Eq, PartialEq, Debug)]
pub struct MftReference(pub u64);
impl MftReference{
    pub fn from_entry_and_seq(&mut self, entry: u64, sequence: u16){
        let entry_buffer: [u8; 8] = unsafe {
            transmute(entry.to_le())
        };
        let seq_buffer: [u8; 2] = unsafe {
            transmute(sequence.to_le())
        };
        let mut ref_buffer = vec![];
        ref_buffer.extend_from_slice(&entry_buffer[0..6]);
        ref_buffer.extend_from_slice(&seq_buffer);

        self.0 = LittleEndian::read_u64(&ref_buffer[0..8]);
    }
    pub fn get_from_entry_and_seq(entry: u64, sequence: u16) -> MftReference{
        let entry_buffer: [u8; 8] = unsafe {
            transmute(entry.to_le())
        };
        let seq_buffer: [u8; 2] = unsafe {
            transmute(sequence.to_le())
        };
        let mut ref_buffer = vec![];
        ref_buffer.extend_from_slice(&entry_buffer[0..6]);
        ref_buffer.extend_from_slice(&seq_buffer);

        MftReference(LittleEndian::read_u64(&ref_buffer[0..8]))
    }
    pub fn get_enum_ref(&self)->MftEnumReference{
        let mut raw_buffer = vec![];
        raw_buffer.write_u64::<LittleEndian>(self.0).unwrap();
        MftEnumReference{
            reference: LittleEndian::read_u64(&raw_buffer[0..8]),
            entry: LittleEndian::read_u64(
                &[&raw_buffer[0..6], &[0,0]].concat()
            ),
            sequence: LittleEndian::read_u16(&raw_buffer[6..8])
        }
    }
}
impl Display for MftReference {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f,"{}",self.0)
    }
}
impl Debug for MftReference {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f,"{}",self.0)
    }
}
impl ser::Serialize for MftReference {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
        where S: ser::Serializer
    {
        // If NESTED_REFERENCE we need to serialize the enumerated structure
        if unsafe{NESTED_REFERENCE} {
            serializer.serialize_newtype_struct("mft_reference",&self.get_enum_ref())
        } else {
            // Just serialize the u64 version
            serialize_u64(&self.0,serializer)
        }
    }
}

#[test]
fn test_reference() {
    use std::mem;
    let raw_reference: &[u8] = &[0x73,0x00,0x00,0x00,0x00,0x00,0x68,0x91];

    let mft_reference = MftReference(
        LittleEndian::read_u64(&raw_reference[0..8])
    );
    assert_eq!(mft_reference.0,10477624533077459059);
    assert_eq!(format!("{}", mft_reference),"10477624533077459059");
    // assert_eq!(mft_reference.sequence,37224);

    let mft_reference_01 = MftReference::get_from_entry_and_seq(
        115,
        37224
    );
    assert_eq!(mft_reference_01.0,10477624533077459059);

    let mut mft_reference_02: MftReference = unsafe {
        mem::zeroed()
    };
    assert_eq!(mft_reference_02.0,0);
    mft_reference_02.from_entry_and_seq(
        115,
        37224
    );
    assert_eq!(mft_reference_02.0,10477624533077459059);
}
<|endoftext|>