<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate log;
extern crate osmpbfreader;
extern crate rustc_serialize;
extern crate docopt;
extern crate mimir;
extern crate rs_es;

use std::collections::HashSet;
use std::collections::HashMap;
use mimir::rubber::Rubber;
use mimir::objects::{Polygon, MultiPolygon};

pub type AdminsVec = Vec<mimir::Admin>;
pub type StreetsVec = Vec<mimir::Street>;
pub type OsmPbfReader = osmpbfreader::OsmPbfReader<std::fs::File>;

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_level: Vec<u32>,
    flag_connection_string: String,
    flag_import_way: bool,
}

static USAGE: &'static str = "
Usage:
    osm2mimir --help
    osm2mimir --input=<file> [--connection-string=<connection-string>] --level=<level> ... [--import-way]

Options:
    -h, --help            Show this message.
    -i, --input=<file>    OSM PBF file.
    -l, --level=<level>   Admin levels to keep.
    -w, --import-way      By default, streets are not imported
    -c, --connection-string=<connection-string>
                          Elasticsearch parameters, [default: http://localhost:9200/munin]
";

#[derive(Debug)]
struct AdminMatcher {
    admin_levels: HashSet<u32>,
}
impl AdminMatcher {
    pub fn new(levels: HashSet<u32>) -> AdminMatcher {
        AdminMatcher { admin_levels: levels }
    }

    pub fn is_admin(&self, obj: &osmpbfreader::OsmObj) -> bool {
        match *obj {
            osmpbfreader::OsmObj::Relation(ref rel) => {
                rel.tags.get("boundary").map_or(false, |v| v == "administrative") &&
                rel.tags.get("admin_level").map_or(false, |lvl| {
                    self.admin_levels.contains(&lvl.parse::<u32>().unwrap_or(0))
                })
            }
            _ => false,
        }
    }
}

struct BoundaryPart {
    nodes: Vec<osmpbfreader::Node>,
}

impl BoundaryPart {
    pub fn new(nodes: Vec<osmpbfreader::Node>) -> BoundaryPart {
        BoundaryPart { nodes: nodes }
    }
    pub fn first(&self) -> i64 {
        self.nodes.first().unwrap().id
    }
    pub fn last(&self) -> i64 {
        self.nodes.last().unwrap().id
    }
}

fn get_nodes(way: &osmpbfreader::Way,
             objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
             -> Vec<osmpbfreader::Node> {
    way.nodes
       .iter()
       .filter_map(|node_id| objects.get(&osmpbfreader::OsmId::Node(*node_id)))
       .filter_map(|node_obj| if let &osmpbfreader::OsmObj::Node(ref node) = node_obj {
           Some(node.clone())
       } else {
           None
       })
       .collect()
}

fn build_boundary(relation: &osmpbfreader::Relation,
                  objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
                  -> Option<MultiPolygon> {
    let mut boundary_parts: Vec<BoundaryPart> =
        relation.refs
                .iter()
                .filter(|rf| rf.role == "outer" || rf.role == "" || rf.role == "enclave")
                .filter_map(|refe| {
                    objects.get(&refe.member).or_else(|| {
                        warn!("missing element for relation {}", relation.id);
                        None
                    })
                })
                .filter_map(|way_obj| if let &osmpbfreader::OsmObj::Way(ref way) = way_obj {
                    Some(way)
                } else {
                    None
                })
                .map(|way| get_nodes(&way, objects))
                .filter(|nodes| nodes.len() > 1)
                .map(|nodes| BoundaryPart::new(nodes))
                .collect();
    let mut multipoly = MultiPolygon { polygons: Vec::new() };
    // we want to try build a polygon for a least each way
    while !boundary_parts.is_empty() {
        let mut current = -1;
        let mut first = 0;
        let mut nb_try = 0;

        let mut outer: Vec<osmpbfreader::Node> = Vec::new();
        // we try to close the polygon, if we can't we want to at least have tried one time per
        // way. We could improve that latter by trying to attach the way to both side of the
        // polygon
        let max_try = boundary_parts.len();
        while current != first && nb_try < max_try {
            let mut i = 0;
            while i < boundary_parts.len() {
                if outer.is_empty() {
                    // our polygon is empty, we initialise it with the current way
                    first = boundary_parts[i].first();
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    // this way has been used, we remove it from the pool
                    boundary_parts.remove(i);
                    continue;
                }
                if current == boundary_parts[i].first() {
                    // the start of current way touch the polygon, we add it and remove it from the
                    // pool
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else if current == boundary_parts[i].last() {
                    // the end of the current way touch the polygon, we reverse the way and add it
                    current = boundary_parts[i].first();
                    boundary_parts[i].nodes.reverse();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else {
                    i += 1;
                    // didnt do anything, we want to explore the next way, if we had do something we
                    // will have removed the current way and there will be no need to increment
                }
                if current == first {
                    // our polygon is closed, we create it and add it to the multipolygon
                    let polygon = Polygon::new(outer.iter()
                                                    .map(|n| {
                                                        mimir::Coord {
                                                            lat: n.lat,
                                                            lon: n.lon,
                                                        }
                                                    })
                                                    .collect());
                    multipoly.polygons.push(polygon);
                    break;
                }
            }
            nb_try += 1;
        }
    }
    debug!("polygon for relation {};{}", relation.id, multipoly.to_wkt());
    if multipoly.polygons.is_empty() {
        None
    } else {
        Some(multipoly)
    }
}

fn parse_osm_pbf(path: &str) -> OsmPbfReader {
    let path = std::path::Path::new(&path);
    osmpbfreader::OsmPbfReader::new(std::fs::File::open(&path).unwrap())
}

fn administrative_regions(pbf: &mut OsmPbfReader, levels: HashSet<u32>) -> AdminsVec {
    let mut administrative_regions = AdminsVec::new();
    let matcher = AdminMatcher::new(levels);
    let objects = osmpbfreader::get_objs_and_deps(pbf, |o| matcher.is_admin(o)).unwrap();
    // load administratives regions
    for (_, obj) in &objects {
        if !matcher.is_admin(&obj) {
            continue;
        }
        if let &osmpbfreader::OsmObj::Relation(ref relation) = obj {
            let level = relation.tags
                                .get("admin_level")
                                .and_then(|s| s.parse().ok());
            let level = match level {
                None => {
                    info!("invalid admin_level for relation {}: admin_level {:?}",
                          relation.id,
                          relation.tags.get("admin_level"));
                    continue;
                }
                Some(l) => l,
            };
            // administrative region with name ?
            let name = match relation.tags.get("name") {
                Some(val) => val,
                None => {
                    warn!("adminstrative region without name for relation {}:  admin_level {} \
                           ignored.",
                          relation.id,
                          level);
                    continue;
                }
            };
            // admininstrative region without coordinates
            let coord_centre = relation.refs
                                       .iter()
                                       .find(|rf| rf.role == "admin_centre")
                                       .and_then(|r| {
                                           objects.get(&r.member).and_then(|value| {
                                               match value {
                                                   &osmpbfreader::OsmObj::Node(ref node) => {
                                                       Some(mimir::Coord {
                                                           lat: node.lat,
                                                           lon: node.lon,
                                                       })
                                                   }
                                                   _ => None,
                                               }
                                           })
                                       });

            let admin_id = match relation.tags.get("ref:INSEE") {
                Some(val) => format!("admin:fr:{}", val.trim_left_matches('0')),
                None => format!("admin:osm:{}", relation.id),
            };
            let zip_code = match relation.tags.get("addr:postcode") {
                Some(val) => &val[..],
                None => "",
            };
            let boundary = build_boundary(&relation, &objects);
            let admin = mimir::Admin {
                id: admin_id,
                level: level,
                name: name.to_string(),
                zip_code: zip_code.to_string(),
                // TODO weight value ?
                weight: 1,
                coord: coord_centre,
                boundary: boundary,
            };
            administrative_regions.push(admin);
        }
    }
    return administrative_regions;
}

fn streets(pbf: &mut OsmPbfReader) -> StreetsVec {

    let is_valid_obj = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Way(ref way) => {
                way.tags.get("highway").map_or(false, |x| !x.is_empty()) &&
                way.tags.get("name").map_or(false, |x| !x.is_empty())
            }
            osmpbfreader::OsmObj::Relation(ref rel) => {
                rel.tags.get("type").map_or(false, |v| v == "associatedStreet")
            }
            _ => false,
        }
    };

    let mut objs_map = osmpbfreader::get_objs_and_deps(pbf, is_valid_obj).unwrap();
    // Sometimes, streets can be devided into several "way"s that still have the same street name.
    // The reason why a street is devided may be that a part of the street become a bridge/tunne/etc.
    // In this case, a "relation" tagged with (type = associatedStreet) is used to group all these "way"s.
    // In order not to have doublons in autocompleion, we should keep only one "way" in the relation
    // and remove all the rest way whose street name is the same.
    let mut objs_to_remove = Vec::<osmpbfreader::OsmId>::new();

    for (_, rel_obj) in &objs_map {
        if let &osmpbfreader::OsmObj::Relation(ref rel) = rel_obj {
            let mut found = false;
            for ref_obj in &rel.refs {
                if let &osmpbfreader::OsmId::Way(_) = &ref_obj.member {
                    if !found {
                        found = true;
                        continue;
                    }
                    objs_to_remove.push(ref_obj.member.clone());
                };
            }
        };
    }

    for osm_id in objs_to_remove {
        objs_map.remove(&osm_id);
    }

    objs_map.iter()
            .filter_map(|(_, obj)| {
                if let &osmpbfreader::OsmObj::Way(ref way) = obj {
                    way.tags.get("name").and_then(|way_name| {
                        Some(mimir::Street {
                            id: way.id.to_string(),
                            street_name: way_name.to_string(),
                            name: way_name.to_string(),
                            weight: 1,
                            administrative_region: None,
                        })
                    })
                } else {
                    None
                }
            })
            .collect()

}
// TODO: template these two functions
fn index_admin(es_cnx_string: &str, admins: AdminsVec) {
    let mut rubber = Rubber::new(es_cnx_string);
    match rubber.clean_db_by_doc_type(&["admin"]) {
        Err(e) => panic!("failed to clean data for admin: {}", e),
        Ok(nb) => info!("Nb of cleaned indexed admins: {}", nb),
    }
    match rubber.bulk_index(admins.iter()) {
        Err(e) => panic!("failed to index admins of osm because: {}", e),
        Ok(nb) => info!("Nb of indexed adminstrative regions: {}", nb),
    }
}

fn index_ways(es_cnx_string: &str, streets: StreetsVec) {
    let mut rubber = Rubber::new(es_cnx_string);
    match rubber.clean_db_by_doc_type(&["street"]) {
        Err(e) => panic!("failed to clean data for street: {}", e),
        Ok(nb) => info!("Nb of cleaned indexed streets: {}", nb),
    }
    match rubber.bulk_index(streets.iter()) {
        Err(e) => panic!("failed to index streets of osm because: {}", e),
        Ok(nb) => info!("Nb of indexed street: {}", nb),
    }
}

fn index_settings(es_cnx_string: &str) {
    let mut rubber = Rubber::new(es_cnx_string);
    rubber.create_index();
}

fn main() {
    mimir::logger_init().unwrap();
    debug!("importing adminstrative region into Mimir");
    let args: Args = docopt::Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());

    let levels = args.flag_level.iter().cloned().collect();
    let mut parsed_pbf = parse_osm_pbf(&args.flag_input);

    let admins = administrative_regions(&mut parsed_pbf, levels);

    index_settings(&args.flag_connection_string);
    index_admin(&args.flag_connection_string, admins);

    let mut optinoal_streets: Option<StreetsVec> = None;
    if args.flag_import_way {
        optinoal_streets = Some(streets(&mut parsed_pbf));
    }

    if let Some(streets) = optinoal_streets {
        index_ways(&args.flag_connection_string, streets);
    }
}
<commit_msg>rename flag<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate log;
extern crate osmpbfreader;
extern crate rustc_serialize;
extern crate docopt;
extern crate mimir;
extern crate rs_es;

use std::collections::HashSet;
use std::collections::HashMap;
use mimir::rubber::Rubber;
use mimir::objects::{Polygon, MultiPolygon};

pub type AdminsVec = Vec<mimir::Admin>;
pub type StreetsVec = Vec<mimir::Street>;
pub type OsmPbfReader = osmpbfreader::OsmPbfReader<std::fs::File>;

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_level: Vec<u32>,
    flag_connection_string: String,
    flag_way: bool,
}

static USAGE: &'static str = "
Usage:
    osm2mimir --help
    osm2mimir --input=<file> [--connection-string=<connection-string>] --level=<level> ... [--way]

Options:
    -h, --help            Show this message.
    -i, --input=<file>    OSM PBF file.
    -l, --level=<level>   Admin levels to keep.
    -w, --way             Import ways
    -c, --connection-string=<connection-string>
                          Elasticsearch parameters, [default: http://localhost:9200/munin]
";

#[derive(Debug)]
struct AdminMatcher {
    admin_levels: HashSet<u32>,
}
impl AdminMatcher {
    pub fn new(levels: HashSet<u32>) -> AdminMatcher {
        AdminMatcher { admin_levels: levels }
    }

    pub fn is_admin(&self, obj: &osmpbfreader::OsmObj) -> bool {
        match *obj {
            osmpbfreader::OsmObj::Relation(ref rel) => {
                rel.tags.get("boundary").map_or(false, |v| v == "administrative") &&
                rel.tags.get("admin_level").map_or(false, |lvl| {
                    self.admin_levels.contains(&lvl.parse::<u32>().unwrap_or(0))
                })
            }
            _ => false,
        }
    }
}

struct BoundaryPart {
    nodes: Vec<osmpbfreader::Node>,
}

impl BoundaryPart {
    pub fn new(nodes: Vec<osmpbfreader::Node>) -> BoundaryPart {
        BoundaryPart { nodes: nodes }
    }
    pub fn first(&self) -> i64 {
        self.nodes.first().unwrap().id
    }
    pub fn last(&self) -> i64 {
        self.nodes.last().unwrap().id
    }
}

fn get_nodes(way: &osmpbfreader::Way,
             objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
             -> Vec<osmpbfreader::Node> {
    way.nodes
       .iter()
       .filter_map(|node_id| objects.get(&osmpbfreader::OsmId::Node(*node_id)))
       .filter_map(|node_obj| {
           if let &osmpbfreader::OsmObj::Node(ref node) = node_obj {
               Some(node.clone())
           } else {
               None
           }
       })
       .collect()
}

fn build_boundary(relation: &osmpbfreader::Relation,
                  objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
                  -> Option<MultiPolygon> {
    let mut boundary_parts: Vec<BoundaryPart> =
        relation.refs
                .iter()
                .filter(|rf| rf.role == "outer" || rf.role == "" || rf.role == "enclave")
                .filter_map(|refe| {
                    objects.get(&refe.member).or_else(|| {
                        warn!("missing element for relation {}", relation.id);
                        None
                    })
                })
                .filter_map(|way_obj| {
                    if let &osmpbfreader::OsmObj::Way(ref way) = way_obj {
                        Some(way)
                    } else {
                        None
                    }
                })
                .map(|way| get_nodes(&way, objects))
                .filter(|nodes| nodes.len() > 1)
                .map(|nodes| BoundaryPart::new(nodes))
                .collect();
    let mut multipoly = MultiPolygon { polygons: Vec::new() };
    // we want to try build a polygon for a least each way
    while !boundary_parts.is_empty() {
        let mut current = -1;
        let mut first = 0;
        let mut nb_try = 0;

        let mut outer: Vec<osmpbfreader::Node> = Vec::new();
        // we try to close the polygon, if we can't we want to at least have tried one time per
        // way. We could improve that latter by trying to attach the way to both side of the
        // polygon
        let max_try = boundary_parts.len();
        while current != first && nb_try < max_try {
            let mut i = 0;
            while i < boundary_parts.len() {
                if outer.is_empty() {
                    // our polygon is empty, we initialise it with the current way
                    first = boundary_parts[i].first();
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    // this way has been used, we remove it from the pool
                    boundary_parts.remove(i);
                    continue;
                }
                if current == boundary_parts[i].first() {
                    // the start of current way touch the polygon, we add it and remove it from the
                    // pool
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else if current == boundary_parts[i].last() {
                    // the end of the current way touch the polygon, we reverse the way and add it
                    current = boundary_parts[i].first();
                    boundary_parts[i].nodes.reverse();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else {
                    i += 1;
                    // didnt do anything, we want to explore the next way, if we had do something we
                    // will have removed the current way and there will be no need to increment
                }
                if current == first {
                    // our polygon is closed, we create it and add it to the multipolygon
                    let polygon = Polygon::new(outer.iter()
                                                    .map(|n| {
                                                        mimir::Coord {
                                                            lat: n.lat,
                                                            lon: n.lon,
                                                        }
                                                    })
                                                    .collect());
                    multipoly.polygons.push(polygon);
                    break;
                }
            }
            nb_try += 1;
        }
    }
    debug!("polygon for relation {};{}",
           relation.id,
           multipoly.to_wkt());
    if multipoly.polygons.is_empty() {
        None
    } else {
        Some(multipoly)
    }
}

fn parse_osm_pbf(path: &str) -> OsmPbfReader {
    let path = std::path::Path::new(&path);
    osmpbfreader::OsmPbfReader::new(std::fs::File::open(&path).unwrap())
}

fn administrative_regions(pbf: &mut OsmPbfReader, levels: HashSet<u32>) -> AdminsVec {
    let mut administrative_regions = AdminsVec::new();
    let matcher = AdminMatcher::new(levels);
    let objects = osmpbfreader::get_objs_and_deps(pbf, |o| matcher.is_admin(o)).unwrap();
    // load administratives regions
    for (_, obj) in &objects {
        if !matcher.is_admin(&obj) {
            continue;
        }
        if let &osmpbfreader::OsmObj::Relation(ref relation) = obj {
            let level = relation.tags
                                .get("admin_level")
                                .and_then(|s| s.parse().ok());
            let level = match level {
                None => {
                    info!("invalid admin_level for relation {}: admin_level {:?}",
                          relation.id,
                          relation.tags.get("admin_level"));
                    continue;
                }
                Some(l) => l,
            };
            // administrative region with name ?
            let name = match relation.tags.get("name") {
                Some(val) => val,
                None => {
                    warn!("adminstrative region without name for relation {}:  admin_level {} \
                           ignored.",
                          relation.id,
                          level);
                    continue;
                }
            };
            // admininstrative region without coordinates
            let coord_centre = relation.refs
                                       .iter()
                                       .find(|rf| rf.role == "admin_centre")
                                       .and_then(|r| {
                                           objects.get(&r.member).and_then(|value| {
                                               match value {
                                                   &osmpbfreader::OsmObj::Node(ref node) => {
                                                       Some(mimir::Coord {
                                                           lat: node.lat,
                                                           lon: node.lon,
                                                       })
                                                   }
                                                   _ => None,
                                               }
                                           })
                                       });

            let admin_id = match relation.tags.get("ref:INSEE") {
                Some(val) => format!("admin:fr:{}", val.trim_left_matches('0')),
                None => format!("admin:osm:{}", relation.id),
            };
            let zip_code = match relation.tags.get("addr:postcode") {
                Some(val) => &val[..],
                None => "",
            };
            let boundary = build_boundary(&relation, &objects);
            let admin = mimir::Admin {
                id: admin_id,
                level: level,
                name: name.to_string(),
                zip_code: zip_code.to_string(),
                // TODO weight value ?
                weight: 1,
                coord: coord_centre,
                boundary: boundary,
            };
            administrative_regions.push(admin);
        }
    }
    return administrative_regions;
}

fn streets(pbf: &mut OsmPbfReader) -> StreetsVec {

    let is_valid_obj = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Way(ref way) => {
                way.tags.get("highway").map_or(false, |x| !x.is_empty()) &&
                way.tags.get("name").map_or(false, |x| !x.is_empty())
            }
            osmpbfreader::OsmObj::Relation(ref rel) => {
                rel.tags.get("type").map_or(false, |v| v == "associatedStreet")
            }
            _ => false,
        }
    };

    let mut objs_map = osmpbfreader::get_objs_and_deps(pbf, is_valid_obj).unwrap();
    // Sometimes, streets can be devided into several "way"s that still have the same street name.
    // The reason why a street is devided may be that a part of the street become a bridge/tunne/etc.
    // In this case, a "relation" tagged with (type = associatedStreet) is used to group all these "way"s.
    // In order not to have doublons in autocompleion, we should keep only one "way" in the relation
    // and remove all the rest way whose street name is the same.
    let mut objs_to_remove = Vec::<osmpbfreader::OsmId>::new();

    for (_, rel_obj) in &objs_map {
        if let &osmpbfreader::OsmObj::Relation(ref rel) = rel_obj {
            let mut found = false;
            for ref_obj in &rel.refs {
                if let &osmpbfreader::OsmId::Way(_) = &ref_obj.member {
                    if !found {
                        found = true;
                        continue;
                    }
                    objs_to_remove.push(ref_obj.member.clone());
                };
            }
        };
    }

    for osm_id in objs_to_remove {
        objs_map.remove(&osm_id);
    }

    objs_map.iter()
            .filter_map(|(_, obj)| {
                if let &osmpbfreader::OsmObj::Way(ref way) = obj {
                    way.tags.get("name").and_then(|way_name| {
                        Some(mimir::Street {
                            id: way.id.to_string(),
                            street_name: way_name.to_string(),
                            name: way_name.to_string(),
                            weight: 1,
                            administrative_region: None,
                        })
                    })
                } else {
                    None
                }
            })
            .collect()

}
// TODO: template these two functions
fn index_admin(es_cnx_string: &str, admins: AdminsVec) {
    let mut rubber = Rubber::new(es_cnx_string);
    match rubber.clean_db_by_doc_type(&["admin"]) {
        Err(e) => panic!("failed to clean data for admin: {}", e),
        Ok(nb) => info!("Nb of cleaned indexed admins: {}", nb),
    }
    match rubber.bulk_index(admins.iter()) {
        Err(e) => panic!("failed to index admins of osm because: {}", e),
        Ok(nb) => info!("Nb of indexed adminstrative regions: {}", nb),
    }
}

fn index_ways(es_cnx_string: &str, streets: StreetsVec) {
    let mut rubber = Rubber::new(es_cnx_string);
    match rubber.clean_db_by_doc_type(&["street"]) {
        Err(e) => panic!("failed to clean data for street: {}", e),
        Ok(nb) => info!("Nb of cleaned indexed streets: {}", nb),
    }
    match rubber.bulk_index(streets.iter()) {
        Err(e) => panic!("failed to index streets of osm because: {}", e),
        Ok(nb) => info!("Nb of indexed street: {}", nb),
    }
}

fn index_settings(es_cnx_string: &str) {
    let mut rubber = Rubber::new(es_cnx_string);
    rubber.create_index();
}

fn main() {
    mimir::logger_init().unwrap();
    debug!("importing adminstrative region into Mimir");
    let args: Args = docopt::Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());

    let levels = args.flag_level.iter().cloned().collect();
    let mut parsed_pbf = parse_osm_pbf(&args.flag_input);

    let admins = administrative_regions(&mut parsed_pbf, levels);

    index_settings(&args.flag_connection_string);
    index_admin(&args.flag_connection_string, admins);

    if args.flag_way {
        index_ways(&args.flag_connection_string, streets(&mut parsed_pbf));
    }
}
<|endoftext|>


<commit_before>// Implements http://rosettacode.org/wiki/Binary_digits
use std::iter::range_inclusive;

trait BinaryString {
    fn to_binary_string(&self) -> String;
}

impl BinaryString for uint {
    fn to_binary_string(&self) -> String {
        self.to_string()
    }
}

#[cfg(not(test))]
fn main() {
    for s in range_inclusive(0, 16u) {
        println!("{}", s.to_binary_string());
    }
}

#[test]
fn test_digits() {
    let expected = ["0", "1", "10", "11",
                    "100", "101", "110", "111",
                    "1000", "1001", "1010", "1011",
                    "1100", "1101", "1110", "1111"];

    for (n, expected) in range_inclusive(0, 16u).zip(expected.iter()) {
        assert_eq!(n.to_binary_string().as_slice(), *expected);
    }
}
<commit_msg>fixed binary_digits<commit_after>// Implements http://rosettacode.org/wiki/Binary_digits
use std::iter::range_inclusive;

trait BinaryString {
    fn to_binary_string(&self) -> String;
}

impl BinaryString for uint {
    fn to_binary_string(&self) -> String {
        format!("{:b}", *self)
    }
}

#[cfg(not(test))]
fn main() {
    for s in range_inclusive(0, 16u) {
        println!("{}", s.to_binary_string());
    }
}

#[test]
fn test_digits() {
    let expected = ["0", "1", "10", "11",
                    "100", "101", "110", "111",
                    "1000", "1001", "1010", "1011",
                    "1100", "1101", "1110", "1111"];

    for (n, expected) in range_inclusive(0, 16u).zip(expected.iter()) {
        assert_eq!(n.to_binary_string().as_slice(), *expected);
    }
}
<|endoftext|>


<commit_before>extern crate luthor;

use std::rc::Rc;
use std::cell::RefCell;
use std::fs::File;
use std::io;
use std::io::{Read, Write};
use std::path::PathBuf;
use super::{GapBuffer, gap_buffer, Position, Range, Cursor, cursor, type_detection, operations};
use super::operations::Operation;
use self::luthor::token::Token;
use self::luthor::lexers;

/// A UTF-8 buffer with bounds-checked cursor management and persistence.
pub struct Buffer {
    data: Rc<RefCell<GapBuffer>>,
    lexer: fn(&str) -> Vec<Token>,
    pub path: Option<PathBuf>,
    pub cursor: Cursor,
    data_cache: Option<String>,
    token_cache: Option<Vec<Token>>,
    history: Vec<Box<Operation>>,
}

impl Buffer {
    /// Returns the contents of the buffer as a string. Caches
    /// this string representation to make subsequent requests
    /// to an unchanged buffer as fast as possible.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// assert_eq!(buffer.data(), "scribe");
    /// ```
    pub fn data(&mut self) -> String {
        match self.data_cache {
            Some(ref cache) => cache.clone(), // Cache hit; return a copy.
            None => {
                // Cache miss; update the cache w/ fresh data and return a copy.
                self.data_cache = Some(self.data.borrow().to_string());
                self.data_cache.clone().unwrap()
            }
        }
    }

    /// Writes the contents of the buffer to its path.
    ///
    /// # Examples
    ///
    /// ```
    /// # use std::path::PathBuf;
    /// # use std::path::Path;
    /// # use std::fs::File;
    /// # use std::io::Read;
    ///
    /// // Set up a buffer and point it to a path.
    /// let mut buffer = scribe::buffer::new();
    /// let write_path = PathBuf::from("my_doc");
    /// buffer.path = Some(write_path.clone());
    ///
    /// // Put some data into the buffer and save it.
    /// buffer.insert("scribe");
    /// buffer.save();
    ///
    /// # let mut saved_data = String::new();
    /// # File::open(Path::new("my_doc")).unwrap().
    /// #   read_to_string(&mut saved_data).unwrap();
    /// # assert_eq!(saved_data, "scribe");
    ///
    /// # std::fs::remove_file(&write_path);
    /// ```
    pub fn save(&mut self) -> Option<io::Error> {
        let path = match self.path.clone() {
            Some(p) => p,
            None => PathBuf::new(),
        };

        // Try to open and write to the file, returning any errors encountered.
        let mut file = match File::create(&path) {
            Ok(f) => f,
            Err(error) => return Some(error),
        };

        // We use to_string here because we don't want to write the gap contents.
        match file.write_all(self.data().to_string().as_bytes()) {
            Ok(_) => (),
            Err(error) => return Some(error),
        }

        return None
    }

    /// Inserts `data` into the buffer at the cursor position.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// assert_eq!(buffer.data(), "scribe");
    /// ```
    pub fn insert(&mut self, data: &str) {
        // Build and run an insert operation.
        let mut op = operations::insert::new(data.to_string(), self.cursor.position.clone());
        op.run(&mut *self.data.borrow_mut());

        // Store the operation in the history
        // object so that it can be undone.
        self.history.push(Box::new(op));

        // Caches are invalid as the buffer has changed.
        self.clear_caches();
    }

    /// Deletes a character at the cursor position. If at the end
    /// of the current line, it'll try to delete a newline character
    /// (joining the lines), succeeding if there's a line below.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// buffer.delete();
    /// assert_eq!(buffer.data(), "cribe");
    /// ```
    pub fn delete(&mut self) {
        // We need to specify a range to delete, so start at
        // the current offset and delete the character to the right.
        let mut end = Position{ line: self.cursor.line, offset: self.cursor.offset + 1 };

        // If there isn't a character to the right,
        // delete the newline by jumping to the start
        // of the next line. If it doesn't exist, that's okay;
        // these values are bounds-checked by delete() anyway.
        if !self.data.borrow().in_bounds(&end) {
            end.line += 1;
            end.offset = 0;
        }

        // Delete the data.
        self.data.borrow_mut().delete(&Range{ start: *self.cursor, end: end});

        // Caches are invalid as the buffer has changed.
        self.clear_caches();
    }

    /// Produces a set of tokens based on the buffer data
    /// suitable for colorized display, using a lexer for the
    /// buffer data's language and/or format. Caches this
    /// lexed representation to make subsequent requests
    /// to an unchanged buffer as fast as possible.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe data");
    ///
    /// // Build the buffer data string back by combining its token lexemes.
    /// let mut data = String::new();
    /// for token in buffer.tokens().iter() {
    ///     data.push_str(&token.lexeme);
    /// }
    /// assert_eq!(data, "scribe data");
    /// ```
    pub fn tokens(&mut self) -> Vec<Token> {
        match self.token_cache {
            Some(ref cache) => cache.clone(), // Cache hit; return a copy.
            None => {
                // Cache miss; update the cache w/ fresh tokens and return a copy.
                self.token_cache = Some((self.lexer)(&self.data()));
                self.token_cache.clone().unwrap()
            }
        }
    }

    /// Returns the file name portion of the buffer's path, if
    /// the path is set and its file name is a valid UTF-8 sequence.
    ///
    /// # Examples
    ///
    /// ```
    /// use std::path::PathBuf;
    ///
    /// let file_path = PathBuf::from("tests/sample/file");
    /// let buffer = scribe::buffer::from_file(file_path).unwrap();
    /// assert_eq!(buffer.file_name().unwrap(), "file");
    /// ```
    pub fn file_name(&self) -> Option<String> {
        match self.path {
            Some(ref path) => {
                match path.file_name() {
                    Some(file_name) => {
                        match file_name.to_str() {
                            Some(utf8_file_name) => Some(utf8_file_name.to_string()),
                            None => None,
                        }
                    },
                    None => None,
                }
            },
            None => None,
        }
    }

    /// Reverses the last modification to the buffer.
    ///
    /// # Examples
    ///
    /// ```
    /// use scribe::buffer;
    ///
    /// let mut buffer = buffer::new();
    /// // Run an initial insert operation.
    /// buffer.insert("scribe");
    /// buffer.cursor.move_to(buffer::Position{ line: 0, offset: 6});
    ///
    /// // Run a second insert operation.
    /// buffer.insert(" library");
    /// assert_eq!("scribe library", buffer.data());
    ///
    /// // Undo the second operation.
    /// buffer.undo();
    /// assert_eq!("scribe", buffer.data());
    ///
    /// // Undo the first operation.
    /// buffer.undo();
    /// assert_eq!("", buffer.data());
    /// ```
    pub fn undo(&mut self) {
        match self.history.pop() {
            Some(mut operation) => {
                // Reverse the last operation.
                operation.reverse(&mut self.data.borrow_mut());

                // Reversing the operation will have modified
                // the buffer, so we'll want to clear the cache.
                self.clear_caches();
            },
            None => (),
        }
    }

    /// Called when caches are invalidated via buffer modifications.
    fn clear_caches(&mut self) {
        self.data_cache = None;
        self.token_cache = None;
    }
}

/// Creates a new empty buffer. The buffer's cursor is set to the beginning of the buffer.
///
/// # Examples
///
/// ```
/// let buffer = scribe::buffer::new();
/// # assert_eq!(buffer.cursor.line, 0);
/// # assert_eq!(buffer.cursor.offset, 0);
/// ```
pub fn new() -> Buffer {
    let data = Rc::new(RefCell::new(gap_buffer::new(String::new())));
    let cursor = cursor::new(data.clone(), 0, 0);

    Buffer{
        data: data.clone(),
        path: None,
        cursor: cursor,
        lexer: lexers::default::lex as fn(&str) -> Vec<Token>,
        data_cache: None,
        token_cache: None,
        history: Vec::new(),
    }
}

/// Creates a new buffer by reading the UTF-8 interpreted file contents of the specified path.
/// The buffer's cursor is set to the beginning of the buffer. The buffer data's type will be
/// inferred based on its extension, and an appropriate lexer will be used, if available (see
/// tokens method for further information on why this happens).
///
/// # Examples
///
/// ```
/// use std::path::PathBuf;
///
/// let file_path = PathBuf::from("tests/sample/file");
/// let mut buffer = scribe::buffer::from_file(file_path).unwrap();
/// assert_eq!(buffer.data(), "it works!\n");
/// # assert_eq!(buffer.cursor.line, 0);
/// # assert_eq!(buffer.cursor.offset, 0);
/// ```
pub fn from_file(path: PathBuf) -> io::Result<Buffer> {
    // Try to open and read the file, returning any errors encountered.
    let mut file = match File::open(path.clone()) {
        Ok(f) => f,
        Err(error) => return Err(error),
    };
    let mut data = String::new();
    match file.read_to_string(&mut data) {
        Ok(_) => (),
        Err(error) => return Err(error),
    };

    let data = Rc::new(RefCell::new(gap_buffer::new(data)));
    let cursor = cursor::new(data.clone(), 0, 0);

    // Detect the file type and use its corresponding lexer, if available.
    let lexer = match type_detection::from_path(&path) {
        Some(type_detection::Type::JSON) => lexers::json::lex as fn(&str) -> Vec<Token>,
        Some(type_detection::Type::XML) => lexers::xml::lex as fn(&str) -> Vec<Token>,
        Some(type_detection::Type::Ruby) => lexers::ruby::lex as fn(&str) -> Vec<Token>,
        _ => lexers::default::lex as fn(&str) -> Vec<Token>,
    };

    // Create a new buffer using the loaded data, path, and other defaults.
    Ok(
        Buffer{
            data: data.clone(),
            path: Some(path),
            cursor: cursor,
            lexer: lexer,
            data_cache: None,
            token_cache: None,
            history: Vec::new(),
        }
    )
}

#[cfg(test)]
mod tests {
    use super::new;
    use super::luthor::token::{Token, Category};

    #[test]
    fn tokens_returns_result_of_lexer() {
        let mut buffer = new();
        buffer.insert("scribe data");
        let expected_tokens = vec![
            Token{ lexeme: "scribe".to_string(), category: Category::Text },
            Token{ lexeme: " ".to_string(), category: Category::Whitespace },
            Token{ lexeme: "data".to_string(), category: Category::Text },
        ];
        assert_eq!(buffer.tokens(), expected_tokens);
    }

    #[test]
    fn delete_joins_lines_when_invoked_at_end_of_line() {
        let mut buffer = new();
        buffer.insert("scribe\n library");
        buffer.cursor.move_to_end_of_line();
        buffer.delete();
        assert_eq!(buffer.data(), "scribe library");
    }

    #[test]
    fn delete_does_nothing_when_invoked_at_the_end_of_the_document() {
        let mut buffer = new();
        buffer.insert("scribe\n library");
        buffer.cursor.move_down();
        buffer.cursor.move_to_end_of_line();
        buffer.delete();
        assert_eq!(buffer.data(), "scribe\n library");
    }

    #[test]
    fn insert_clears_data_and_token_caches() {
        let mut buffer = new();
        buffer.insert("scribe");

        // Trigger data and token cache storage.
        assert_eq!("scribe", buffer.data());
        assert_eq!(
            vec![Token{ lexeme: "scribe".to_string(), category: Category::Text }],
            buffer.tokens()
        );

        // Change the buffer contents using delete.
        buffer.insert("test_");

        // Ensure the cache has been busted.
        assert_eq!("test_scribe", buffer.data());
        assert_eq!(
            vec![Token{ lexeme: "test_scribe".to_string(), category: Category::Text }],
            buffer.tokens()
        );
    }

    #[test]
    fn delete_clears_data_and_token_caches() {
        let mut buffer = new();
        buffer.insert("scribe");

        // Trigger data and token cache storage.
        assert_eq!("scribe", buffer.data());
        assert_eq!(
            vec![Token{ lexeme: "scribe".to_string(), category: Category::Text }],
            buffer.tokens()
        );

        // Change the buffer contents using delete.
        buffer.delete();

        // Ensure the cache has been busted.
        assert_eq!("cribe", buffer.data());
        assert_eq!(
            vec![Token{ lexeme: "cribe".to_string(), category: Category::Text }],
            buffer.tokens()
        );
    }

    #[test]
    fn insert_is_undoable() {
        let mut buffer = new();
        buffer.insert("scribe");
        assert_eq!("scribe", buffer.data());
        buffer.undo();
        assert_eq!("", buffer.data());
    }
}
<commit_msg>Update buffer delete method to use delete operation; adds undo support.<commit_after>extern crate luthor;

use std::rc::Rc;
use std::cell::RefCell;
use std::fs::File;
use std::io;
use std::io::{Read, Write};
use std::path::PathBuf;
use super::{GapBuffer, gap_buffer, Position, Range, Cursor, cursor, type_detection, operations};
use super::operations::Operation;
use self::luthor::token::Token;
use self::luthor::lexers;

/// A UTF-8 buffer with bounds-checked cursor management and persistence.
pub struct Buffer {
    data: Rc<RefCell<GapBuffer>>,
    lexer: fn(&str) -> Vec<Token>,
    pub path: Option<PathBuf>,
    pub cursor: Cursor,
    data_cache: Option<String>,
    token_cache: Option<Vec<Token>>,
    history: Vec<Box<Operation>>,
}

impl Buffer {
    /// Returns the contents of the buffer as a string. Caches
    /// this string representation to make subsequent requests
    /// to an unchanged buffer as fast as possible.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// assert_eq!(buffer.data(), "scribe");
    /// ```
    pub fn data(&mut self) -> String {
        match self.data_cache {
            Some(ref cache) => cache.clone(), // Cache hit; return a copy.
            None => {
                // Cache miss; update the cache w/ fresh data and return a copy.
                self.data_cache = Some(self.data.borrow().to_string());
                self.data_cache.clone().unwrap()
            }
        }
    }

    /// Writes the contents of the buffer to its path.
    ///
    /// # Examples
    ///
    /// ```
    /// # use std::path::PathBuf;
    /// # use std::path::Path;
    /// # use std::fs::File;
    /// # use std::io::Read;
    ///
    /// // Set up a buffer and point it to a path.
    /// let mut buffer = scribe::buffer::new();
    /// let write_path = PathBuf::from("my_doc");
    /// buffer.path = Some(write_path.clone());
    ///
    /// // Put some data into the buffer and save it.
    /// buffer.insert("scribe");
    /// buffer.save();
    ///
    /// # let mut saved_data = String::new();
    /// # File::open(Path::new("my_doc")).unwrap().
    /// #   read_to_string(&mut saved_data).unwrap();
    /// # assert_eq!(saved_data, "scribe");
    ///
    /// # std::fs::remove_file(&write_path);
    /// ```
    pub fn save(&mut self) -> Option<io::Error> {
        let path = match self.path.clone() {
            Some(p) => p,
            None => PathBuf::new(),
        };

        // Try to open and write to the file, returning any errors encountered.
        let mut file = match File::create(&path) {
            Ok(f) => f,
            Err(error) => return Some(error),
        };

        // We use to_string here because we don't want to write the gap contents.
        match file.write_all(self.data().to_string().as_bytes()) {
            Ok(_) => (),
            Err(error) => return Some(error),
        }

        return None
    }

    /// Inserts `data` into the buffer at the cursor position.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// assert_eq!(buffer.data(), "scribe");
    /// ```
    pub fn insert(&mut self, data: &str) {
        // Build and run an insert operation.
        let mut op = operations::insert::new(data.to_string(), self.cursor.position.clone());
        op.run(&mut *self.data.borrow_mut());

        // Store the operation in the history
        // object so that it can be undone.
        self.history.push(Box::new(op));

        // Caches are invalid as the buffer has changed.
        self.clear_caches();
    }

    /// Deletes a character at the cursor position. If at the end
    /// of the current line, it'll try to delete a newline character
    /// (joining the lines), succeeding if there's a line below.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe");
    /// buffer.delete();
    /// assert_eq!(buffer.data(), "cribe");
    /// ```
    pub fn delete(&mut self) {
        // We need to specify a range to delete, so start at
        // the current offset and delete the character to the right.
        let mut end = Position{ line: self.cursor.line, offset: self.cursor.offset + 1 };

        // If there isn't a character to the right,
        // delete the newline by jumping to the start
        // of the next line. If it doesn't exist, that's okay;
        // these values are bounds-checked by delete() anyway.
        if !self.data.borrow().in_bounds(&end) {
            end.line += 1;
            end.offset = 0;
        }

        // Build and run a delete operation.
        let mut op = operations::delete::new(Range{ start: self.cursor.position.clone(), end: end });
        op.run(&mut *self.data.borrow_mut());

        // Store the operation in the history
        // object so that it can be undone.
        self.history.push(Box::new(op));

        // Caches are invalid as the buffer has changed.
        self.clear_caches();
    }

    /// Produces a set of tokens based on the buffer data
    /// suitable for colorized display, using a lexer for the
    /// buffer data's language and/or format. Caches this
    /// lexed representation to make subsequent requests
    /// to an unchanged buffer as fast as possible.
    ///
    /// # Examples
    ///
    /// ```
    /// let mut buffer = scribe::buffer::new();
    /// buffer.insert("scribe data");
    ///
    /// // Build the buffer data string back by combining its token lexemes.
    /// let mut data = String::new();
    /// for token in buffer.tokens().iter() {
    ///     data.push_str(&token.lexeme);
    /// }
    /// assert_eq!(data, "scribe data");
    /// ```
    pub fn tokens(&mut self) -> Vec<Token> {
        match self.token_cache {
            Some(ref cache) => cache.clone(), // Cache hit; return a copy.
            None => {
                // Cache miss; update the cache w/ fresh tokens and return a copy.
                self.token_cache = Some((self.lexer)(&self.data()));
                self.token_cache.clone().unwrap()
            }
        }
    }

    /// Returns the file name portion of the buffer's path, if
    /// the path is set and its file name is a valid UTF-8 sequence.
    ///
    /// # Examples
    ///
    /// ```
    /// use std::path::PathBuf;
    ///
    /// let file_path = PathBuf::from("tests/sample/file");
    /// let buffer = scribe::buffer::from_file(file_path).unwrap();
    /// assert_eq!(buffer.file_name().unwrap(), "file");
    /// ```
    pub fn file_name(&self) -> Option<String> {
        match self.path {
            Some(ref path) => {
                match path.file_name() {
                    Some(file_name) => {
                        match file_name.to_str() {
                            Some(utf8_file_name) => Some(utf8_file_name.to_string()),
                            None => None,
                        }
                    },
                    None => None,
                }
            },
            None => None,
        }
    }

    /// Reverses the last modification to the buffer.
    ///
    /// # Examples
    ///
    /// ```
    /// use scribe::buffer;
    ///
    /// let mut buffer = buffer::new();
    /// // Run an initial insert operation.
    /// buffer.insert("scribe");
    /// buffer.cursor.move_to(buffer::Position{ line: 0, offset: 6});
    ///
    /// // Run a second insert operation.
    /// buffer.insert(" library");
    /// assert_eq!("scribe library", buffer.data());
    ///
    /// // Undo the second operation.
    /// buffer.undo();
    /// assert_eq!("scribe", buffer.data());
    ///
    /// // Undo the first operation.
    /// buffer.undo();
    /// assert_eq!("", buffer.data());
    /// ```
    pub fn undo(&mut self) {
        match self.history.pop() {
            Some(mut operation) => {
                // Reverse the last operation.
                operation.reverse(&mut self.data.borrow_mut());

                // Reversing the operation will have modified
                // the buffer, so we'll want to clear the cache.
                self.clear_caches();
            },
            None => (),
        }
    }

    /// Called when caches are invalidated via buffer modifications.
    fn clear_caches(&mut self) {
        self.data_cache = None;
        self.token_cache = None;
    }
}

/// Creates a new empty buffer. The buffer's cursor is set to the beginning of the buffer.
///
/// # Examples
///
/// ```
/// let buffer = scribe::buffer::new();
/// # assert_eq!(buffer.cursor.line, 0);
/// # assert_eq!(buffer.cursor.offset, 0);
/// ```
pub fn new() -> Buffer {
    let data = Rc::new(RefCell::new(gap_buffer::new(String::new())));
    let cursor = cursor::new(data.clone(), 0, 0);

    Buffer{
        data: data.clone(),
        path: None,
        cursor: cursor,
        lexer: lexers::default::lex as fn(&str) -> Vec<Token>,
        data_cache: None,
        token_cache: None,
        history: Vec::new(),
    }
}

/// Creates a new buffer by reading the UTF-8 interpreted file contents of the specified path.
/// The buffer's cursor is set to the beginning of the buffer. The buffer data's type will be
/// inferred based on its extension, and an appropriate lexer will be used, if available (see
/// tokens method for further information on why this happens).
///
/// # Examples
///
/// ```
/// use std::path::PathBuf;
///
/// let file_path = PathBuf::from("tests/sample/file");
/// let mut buffer = scribe::buffer::from_file(file_path).unwrap();
/// assert_eq!(buffer.data(), "it works!\n");
/// # assert_eq!(buffer.cursor.line, 0);
/// # assert_eq!(buffer.cursor.offset, 0);
/// ```
pub fn from_file(path: PathBuf) -> io::Result<Buffer> {
    // Try to open and read the file, returning any errors encountered.
    let mut file = match File::open(path.clone()) {
        Ok(f) => f,
        Err(error) => return Err(error),
    };
    let mut data = String::new();
    match file.read_to_string(&mut data) {
        Ok(_) => (),
        Err(error) => return Err(error),
    };

    let data = Rc::new(RefCell::new(gap_buffer::new(data)));
    let cursor = cursor::new(data.clone(), 0, 0);

    // Detect the file type and use its corresponding lexer, if available.
    let lexer = match type_detection::from_path(&path) {
        Some(type_detection::Type::JSON) => lexers::json::lex as fn(&str) -> Vec<Token>,
        Some(type_detection::Type::XML) => lexers::xml::lex as fn(&str) -> Vec<Token>,
        Some(type_detection::Type::Ruby) => lexers::ruby::lex as fn(&str) -> Vec<Token>,
        _ => lexers::default::lex as fn(&str) -> Vec<Token>,
    };

    // Create a new buffer using the loaded data, path, and other defaults.
    Ok(
        Buffer{
            data: data.clone(),
            path: Some(path),
            cursor: cursor,
            lexer: lexer,
            data_cache: None,
            token_cache: None,
            history: Vec::new(),
        }
    )
}

#[cfg(test)]
mod tests {
    use super::new;
    use buffer::Position;
    use super::luthor::token::{Token, Category};

    #[test]
    fn tokens_returns_result_of_lexer() {
        let mut buffer = new();
        buffer.insert("scribe data");
        let expected_tokens = vec![
            Token{ lexeme: "scribe".to_string(), category: Category::Text },
            Token{ lexeme: " ".to_string(), category: Category::Whitespace },
            Token{ lexeme: "data".to_string(), category: Category::Text },
        ];
        assert_eq!(buffer.tokens(), expected_tokens);
    }

    #[test]
    fn delete_joins_lines_when_invoked_at_end_of_line() {
        let mut buffer = new();
        buffer.insert("scribe\n library");
        buffer.cursor.move_to_end_of_line();
        buffer.delete();
        assert_eq!(buffer.data(), "scribe library");
    }

    #[test]
    fn delete_does_nothing_when_invoked_at_the_end_of_the_document() {
        let mut buffer = new();
        buffer.insert("scribe\n library");
        buffer.cursor.move_down();
        buffer.cursor.move_to_end_of_line();
        buffer.delete();
        assert_eq!(buffer.data(), "scribe\n library");
    }

    #[test]
    fn insert_clears_data_and_token_caches() {
        let mut buffer = new();
        buffer.insert("scribe");

        // Trigger data and token cache storage.
        assert_eq!("scribe", buffer.data());
        assert_eq!(
            vec![Token{ lexeme: "scribe".to_string(), category: Category::Text }],
            buffer.tokens()
        );

        // Change the buffer contents using delete.
        buffer.insert("test_");

        // Ensure the cache has been busted.
        assert_eq!("test_scribe", buffer.data());
        assert_eq!(
            vec![Token{ lexeme: "test_scribe".to_string(), category: Category::Text }],
            buffer.tokens()
        );
    }

    #[test]
    fn delete_clears_data_and_token_caches() {
        let mut buffer = new();
        buffer.insert("scribe");

        // Trigger data and token cache storage.
        assert_eq!("scribe", buffer.data());
        assert_eq!(
            vec![Token{ lexeme: "scribe".to_string(), category: Category::Text }],
            buffer.tokens()
        );

        // Change the buffer contents using delete.
        buffer.delete();

        // Ensure the cache has been busted.
        assert_eq!("cribe", buffer.data());
        assert_eq!(
            vec![Token{ lexeme: "cribe".to_string(), category: Category::Text }],
            buffer.tokens()
        );
    }

    #[test]
    fn insert_is_undoable() {
        let mut buffer = new();
        buffer.insert("scribe");
        assert_eq!("scribe", buffer.data());
        buffer.undo();
        assert_eq!("", buffer.data());
    }

    #[test]
    fn delete_is_undoable() {
        let mut buffer = new();
        buffer.insert("scribe");
        assert_eq!("scribe", buffer.data());

        buffer.cursor.move_to(Position{ line: 0, offset: 0 });
        buffer.delete();
        assert_eq!("cribe", buffer.data());

        buffer.undo();
        assert_eq!("scribe", buffer.data());
    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate gluon_codegen;
extern crate gluon;
#[macro_use]
extern crate gluon_vm;

extern crate env_logger;
#[macro_use]
extern crate serde_derive;

use std::collections::HashMap;

use gluon::base::types::ArcType;

use gluon::vm::api::generic::{L, R};
use gluon::vm::api::{self, FunctionRef, Generic, OpaqueValue, IO};
use gluon::vm::thread::Context;
use gluon::vm::{self, ExternModule};
use gluon::{import, new_vm, Compiler, Result, RootedThread, Thread};

#[derive(Debug, Deserialize, Serialize)]
enum Enum {
    A,
    B(i32),
    C(String, String),
}

impl api::VmType for Enum {
    type Type = Self;
    fn make_type(thread: &Thread) -> ArcType {
        thread
            .find_type_info("examples.enum.Enum")
            .unwrap()
            .clone()
            .into_type()
    }
}

impl<'vm> api::Pushable<'vm> for Enum {
    fn push(self, thread: &'vm Thread, context: &mut Context) -> vm::Result<()> {
        api::ser::Ser(self).push(thread, context)
    }
}

impl<'vm> api::Getable<'vm> for Enum {
    fn from_value(thread: &'vm Thread, value: vm::Variants) -> Self {
        api::de::De::from_value(thread, value).0
    }
}

field_decl!{ unwrap_b, value, key }

// we define Either with type parameters, just like in Gluon
#[derive(Getable, Pushable, VmType)]
#[gluon(vm_type = "examples.either.Either")]
enum Either<L, R> {
    Left(L),
    Right(R),
}

fn marshal_enum() -> Result<()> {
    let thread = new_vm();

    let enum_source = api::typ::make_source::<Enum>(&thread)?;
    Compiler::new().load_script(&thread, "examples.enum", &enum_source)?;

    let source = r#"
        let { Enum } = import! "examples/enum.glu"

        let unwrap_b x =
            match x with
            | B y -> y
            | _ -> error "Expected B"

        {
            unwrap_b,
            value = C "hello" "world"
        }
    "#;
    type SourceType<'thread> = record_type! {
        unwrap_b => api::FunctionRef<'thread, fn (Enum) -> i32>,
        value => Enum
    };
    let (record_p! { mut unwrap_b, value }, _) =
        Compiler::new().run_expr::<SourceType>(&thread, "example", source)?;
    match value {
        Enum::C(ref a, ref b) => {
            assert_eq!(a, "hello");
            assert_eq!(b, "world");
            println!("`value` evaluated to: {:?}", value)
        }
        _ => panic!("Unexpected result returned"),
    }

    let x = unwrap_b.call(Enum::B(123))?;
    assert_eq!(x, 123);
    println!("`unwrap_b` returned: {}", x);

    Ok(())
}

fn marshal_map<I>(iterable: I) -> Result<()>
where
    I: IntoIterator<Item = (String, String)>,
{
    let thread = new_vm();

    // Load std.map so that we can retrieve the `Map` type through the `VmType` trait
    Compiler::new().run_expr::<()>(&thread, "example", "let _ = import! std.map in ()")?;

    let config_example = r#"
        let array = import! std.array
        let string = import! std.string
        let map @ { Map } = import! std.map

        let string_map = map.make string.ord

        let run config_array =
            let f m entry : Map String String -> (String, String) -> _ =
                string_map.insert entry._0 entry._1 m
            array.foldable.foldl f string_map.empty config_array
        run
        "#;
    let mut make_map: FunctionRef<
        fn(Vec<(String, String)>) -> OpaqueValue<RootedThread, api::Map<String, String>>,
    > = Compiler::new()
        .run_expr(&thread, "example", config_example)?
        .0;

    let entries: Vec<_> = iterable.into_iter().collect();
    let map = make_map.call(entries)?;

    let config_query_example = r#"
        let string = import! std.string
        let map = import! std.map

        let string_map = map.make string.ord

        let run config_map =
            (string_map.find "key" config_map, string_map.find "undefined" config_map)
        run
        "#;
    let mut query_map: FunctionRef<
        fn(OpaqueValue<RootedThread, api::Map<String, String>>) -> (Option<String>, Option<String>),
    > = Compiler::new()
        .run_expr(&thread, "example", config_query_example)?
        .0;

    let tuple = query_map.call(map)?;
    assert_eq!(tuple, (Some("value".to_string()), None));
    println!("Querying the map returned: {:?}", tuple);

    Ok(())
}

// the function takes an Either instantiated with the `Generic` struct,
// which will handle the generic Gluon values for us
fn flip(either: Either<Generic<L>, Generic<R>>) -> Either<Generic<R>, Generic<L>> {
    match either {
        Either::Left(val) => Either::Right(val),
        Either::Right(val) => Either::Left(val),
    }
}

fn marshal_generic() -> Result<()> {
    let vm = new_vm();
    let mut compiler = Compiler::new();

    // define the gluon type that maps to the rust Either
    let src = r#"
        type Either l r = | Left l | Right r
        { Either }
    "#;

    // load the type and then the module containing the rust function
    fn load_mod(vm: &Thread) -> vm::Result<ExternModule> {
        let module = record! {
            flip => primitive!(1 flip),
        };

        ExternModule::new(vm, module)
    }

    compiler.load_script(&vm, "examples.either", src).unwrap();
    import::add_extern_module(&vm, "examples.prim", load_mod);

    let script = r#"
        let { Either } = import! examples.either
        let { flip } = import! examples.prim
        let { (<>) } = import! std.semigroup
        let io @ { flat_map } = import! std.io

        // Either is defined as:
        // type Either l r = | Left l | Right r
        let either: forall r . Either String r = Left "hello rust!"

        // we can pass the generic Either to the Rust function without an issue
        do _ =
            match flip either with
            | Left _ -> error "unreachable!"
            | Right val -> io.println ("Right is: " <> val)

        // using an Int instead also works
        let either: forall r . Either Int r = Left 42

        match flip either with
        | Left _ -> error "also unreachable!"
        | Right 42 -> io.println "this is the right answer"
        | Right _ -> error "wrong answer!"
    "#;

    compiler
        .run_io(true)
        .run_expr::<IO<()>>(&vm, "example", script)?;

    Ok(())
}

fn main() {
    env_logger::init();

    if let Err(err) = marshal_enum() {
        eprintln!("{}", err)
    }

    let mut map = HashMap::new();
    map.insert("key".to_string(), "value".to_string());
    map.insert("key2".to_string(), "value2".to_string());

    if let Err(err) = marshal_map(map) {
        eprintln!("{}", err)
    }

    if let Err(err) = marshal_generic() {
        eprintln!("{}", err)
    }
}
<commit_msg>Add example for manual VmType, Getable and Pushable impl<commit_after>#[macro_use]
extern crate gluon_codegen;
extern crate gluon;
#[macro_use]
extern crate gluon_vm;

extern crate env_logger;
#[macro_use]
extern crate serde_derive;

use std::collections::HashMap;

use gluon::base::types::ArcType;

use gluon::base::types::{AppVec, Type};
use gluon::vm::api::generic::{A, L, R};
use gluon::vm::api::{
    self, FunctionRef, Generic, Getable, OpaqueValue, Pushable, ValueRef, VmType, IO,
};
use gluon::vm::thread::Context;
use gluon::vm::{self, ExternModule, Variants};
use gluon::{import, new_vm, Compiler, Result, RootedThread, Thread};

#[derive(Debug, Deserialize, Serialize)]
enum Enum {
    A,
    B(i32),
    C(String, String),
}

impl api::VmType for Enum {
    type Type = Self;
    fn make_type(thread: &Thread) -> ArcType {
        thread
            .find_type_info("examples.enum.Enum")
            .unwrap()
            .clone()
            .into_type()
    }
}

impl<'vm> api::Pushable<'vm> for Enum {
    fn push(self, thread: &'vm Thread, context: &mut Context) -> vm::Result<()> {
        api::ser::Ser(self).push(thread, context)
    }
}

impl<'vm> api::Getable<'vm> for Enum {
    fn from_value(thread: &'vm Thread, value: vm::Variants) -> Self {
        api::de::De::from_value(thread, value).0
    }
}

field_decl!{ unwrap_b, value, key }

// we define Either with type parameters, just like in Gluon
#[derive(Getable, Pushable, VmType)]
#[gluon(vm_type = "examples.either.Either")]
enum Either<L, R> {
    Left(L),
    Right(R),
}

fn marshal_enum() -> Result<()> {
    let thread = new_vm();

    let enum_source = api::typ::make_source::<Enum>(&thread)?;
    Compiler::new().load_script(&thread, "examples.enum", &enum_source)?;

    let source = r#"
        let { Enum } = import! "examples/enum.glu"

        let unwrap_b x =
            match x with
            | B y -> y
            | _ -> error "Expected B"

        {
            unwrap_b,
            value = C "hello" "world"
        }
    "#;
    type SourceType<'thread> = record_type! {
        unwrap_b => api::FunctionRef<'thread, fn (Enum) -> i32>,
        value => Enum
    };
    let (record_p! { mut unwrap_b, value }, _) =
        Compiler::new().run_expr::<SourceType>(&thread, "example", source)?;
    match value {
        Enum::C(ref a, ref b) => {
            assert_eq!(a, "hello");
            assert_eq!(b, "world");
            println!("`value` evaluated to: {:?}", value)
        }
        _ => panic!("Unexpected result returned"),
    }

    let x = unwrap_b.call(Enum::B(123))?;
    assert_eq!(x, 123);
    println!("`unwrap_b` returned: {}", x);

    Ok(())
}

fn marshal_map<I>(iterable: I) -> Result<()>
where
    I: IntoIterator<Item = (String, String)>,
{
    let thread = new_vm();

    // Load std.map so that we can retrieve the `Map` type through the `VmType` trait
    Compiler::new().run_expr::<()>(&thread, "example", "let _ = import! std.map in ()")?;

    let config_example = r#"
        let array = import! std.array
        let string = import! std.string
        let map @ { Map } = import! std.map

        let string_map = map.make string.ord

        let run config_array =
            let f m entry : Map String String -> (String, String) -> _ =
                string_map.insert entry._0 entry._1 m
            array.foldable.foldl f string_map.empty config_array
        run
        "#;
    let mut make_map: FunctionRef<
        fn(Vec<(String, String)>) -> OpaqueValue<RootedThread, api::Map<String, String>>,
    > = Compiler::new()
        .run_expr(&thread, "example", config_example)?
        .0;

    let entries: Vec<_> = iterable.into_iter().collect();
    let map = make_map.call(entries)?;

    let config_query_example = r#"
        let string = import! std.string
        let map = import! std.map

        let string_map = map.make string.ord

        let run config_map =
            (string_map.find "key" config_map, string_map.find "undefined" config_map)
        run
        "#;
    let mut query_map: FunctionRef<
        fn(OpaqueValue<RootedThread, api::Map<String, String>>) -> (Option<String>, Option<String>),
    > = Compiler::new()
        .run_expr(&thread, "example", config_query_example)?
        .0;

    let tuple = query_map.call(map)?;
    assert_eq!(tuple, (Some("value".to_string()), None));
    println!("Querying the map returned: {:?}", tuple);

    Ok(())
}

// the function takes an Either instantiated with the `Generic` struct,
// which will handle the generic Gluon values for us
fn flip(either: Either<Generic<L>, Generic<R>>) -> Either<Generic<R>, Generic<L>> {
    match either {
        Either::Left(val) => Either::Right(val),
        Either::Right(val) => Either::Left(val),
    }
}

fn marshal_generic() -> Result<()> {
    let vm = new_vm();
    let mut compiler = Compiler::new();

    // define the gluon type that maps to the rust Either
    let src = r#"
        type Either l r = | Left l | Right r
        { Either }
    "#;

    // load the type and then the module containing the rust function
    fn load_mod(vm: &Thread) -> vm::Result<ExternModule> {
        let module = record! {
            flip => primitive!(1 flip),
        };

        ExternModule::new(vm, module)
    }

    compiler.load_script(&vm, "examples.either", src).unwrap();
    import::add_extern_module(&vm, "examples.prim", load_mod);

    let script = r#"
        let { Either } = import! examples.either
        let { flip } = import! examples.prim
        let { (<>) } = import! std.semigroup
        let io @ { flat_map } = import! std.io

        // Either is defined as:
        // type Either l r = | Left l | Right r
        let either: forall r . Either String r = Left "hello rust!"

        // we can pass the generic Either to the Rust function without an issue
        do _ =
            match flip either with
            | Left _ -> error "unreachable!"
            | Right val -> io.println ("Right is: " <> val)

        // using an Int instead also works
        let either: forall r . Either Int r = Left 42

        match flip either with
        | Left _ -> error "also unreachable!"
        | Right 42 -> io.println "this is the right answer"
        | Right _ -> error "wrong answer!"
    "#;

    compiler
        .run_io(true)
        .run_expr::<IO<()>>(&vm, "example", script)?;

    Ok(())
}

// if this were a foreign type, we would not be able to
// directly implement traits for it
#[derive(Debug)]
struct User<T> {
    name: String,
    age: u32,
    data: T,
}

// but it's possible to wrap it and manually implement traits for it
struct GluonUser<T> {
    inner: User<T>,
}

impl<T> VmType for GluonUser<T>
where
    T: 'static + VmType,
{
    type Type = Self;

    fn make_type(vm: &Thread) -> ArcType {
        // get the type defined in Gluon
        let ty = vm.find_type_info("examples.wrapper.User")
            .expect("Could not find type")
            .into_type();

        // apply all generic parameters to the type
        let mut vec = AppVec::new();
        vec.push(T::make_type(vm));
        Type::app(ty, vec)
    }
}

impl<'vm, T> Pushable<'vm> for GluonUser<T>
where
    T: Pushable<'vm>,
{
    fn push(self, vm: &'vm Thread, ctx: &mut Context) -> vm::Result<()> {
        // push the fields to the stack in reverse order
        self.inner.data.push(vm, ctx)?;
        self.inner.age.push(vm, ctx)?;
        self.inner.name.push(vm, ctx)?;

        // Pop the values and create a complex type from them,
        // then push it onto the stack
        let fields = [ctx.stack.pop(), ctx.stack.pop(), ctx.stack.pop()];

        // the second arg in this case doesn't matter, if this were an enum
        // it would be the tag of the current variant
        let val = ctx.new_data(vm, 0, &fields)?;
        ctx.stack.push(val);

        Ok(())
    }
}

impl<'vm, T> Getable<'vm> for GluonUser<T>
where
    T: Getable<'vm>,
{
    fn from_value(vm: &'vm Thread, data: Variants) -> GluonUser<T> {
        // get the data, it must be a complex type
        let data = match data.as_ref() {
            ValueRef::Data(data) => data,
            _ => panic!("Value is not a complex type"),
        };

        GluonUser {
            inner: User {
                // lookup the fields by name; in case of a tuple variant we
                // would use Data::get_variant instead
                name: String::from_value(vm, data.lookup_field(vm, "name").unwrap()),
                age: u32::from_value(vm, data.lookup_field(vm, "age").unwrap()),
                data: T::from_value(vm, data.lookup_field(vm, "data").unwrap()),
            },
        }
    }
}

fn marshal_wrapper() -> Result<()> {
    let vm = new_vm();
    let mut compiler = Compiler::new();

    let src = r#"
        type User a = { name: String, age: Int, data: a }
        { User }
    "#;

    fn load_mod(vm: &Thread) -> vm::Result<ExternModule> {
        let module = record! {
            roundtrip => primitive!(1 |user: GluonUser<Generic<A>>| {
                println!("name: {}, age: {}", user.inner.name, user.inner.age);
                user
            }),
        };

        ExternModule::new(vm, module)
    }

    compiler.load_script(&vm, "examples.wrapper", src).unwrap();
    import::add_extern_module(&vm, "examples.prim", load_mod);

    let script = r#"
        let { User } = import! examples.wrapper
        let { roundtrip } = import! examples.prim
        let { assert } = import! std.test

        let actual = { name = "Bob", age = 11, data = True }
        let expected = roundtrip actual

        assert (actual.name == expected.name)
        assert (actual.age == expected.age)
        assert (actual.data == expected.data)
    "#;

    compiler.run_expr::<()>(&vm, "example", script)?;

    Ok(())
}

fn main() {
    env_logger::init();

    if let Err(err) = marshal_enum() {
        eprintln!("{}", err)
    }

    let mut map = HashMap::new();
    map.insert("key".to_string(), "value".to_string());
    map.insert("key2".to_string(), "value2".to_string());

    if let Err(err) = marshal_map(map) {
        eprintln!("{}", err)
    }

    if let Err(err) = marshal_generic() {
        eprintln!("{}", err)
    }

    if let Err(err) = marshal_wrapper() {
        eprintln!("{}", err)
    }
}
<|endoftext|>


<commit_before>//! # Diesel
//!
//! Diesel is an ORM and query builder designed to reduce the boilerplate for database interactions.
//!
//! * [Homepage](https://diesel.rs/)
//! * [Getting Started Guide](https://diesel.rs/guides/getting-started/)
//! * [Source Code](https://github.com/diesel-rs/diesel/)
//!
//! ## Crates
//!
//! * **Diesel**: core libary, documented here
//! * [**Diesel CLI**](https://github.com/diesel-rs/diesel/tree/master/diesel_cli): command line tool that aids in managing your database schema
//!
#![cfg_attr(feature = "unstable", feature(specialization))]
// Built-in Lints
#![deny(warnings, missing_debug_implementations, missing_copy_implementations)]
// Clippy lints
#![cfg_attr(feature = "clippy", allow(unstable_features))]
#![cfg_attr(feature = "clippy", feature(plugin))]
#![cfg_attr(feature = "clippy", plugin(clippy(conf_file = "../../clippy.toml")))]
#![cfg_attr(feature = "clippy",
            allow(option_map_unwrap_or_else, option_map_unwrap_or, match_same_arms,
                  type_complexity))]
#![cfg_attr(feature = "clippy",
            warn(option_unwrap_used, result_unwrap_used, print_stdout,
                 wrong_pub_self_convention, mut_mut, non_ascii_literal, similar_names,
                 unicode_not_nfc, enum_glob_use, if_not_else, items_after_statements,
                 used_underscore_binding))]
#![cfg_attr(all(test, feature = "clippy"), allow(option_unwrap_used, result_unwrap_used))]

#[cfg(feature = "postgres")]
#[macro_use]
extern crate bitflags;
extern crate byteorder;
// This is required to make `diesel_derives` re-export, but clippy thinks its unused
#[cfg_attr(feature = "clippy", allow(useless_attribute))]
#[allow(unused_imports)]
#[macro_use]
extern crate diesel_derives;
#[doc(hidden)]
pub use diesel_derives::*;

#[macro_use]
mod macros;

#[cfg(test)]
#[macro_use]
extern crate cfg_if;

#[cfg(test)]
pub mod test_helpers;

#[deny(missing_docs)]
pub mod associations;
#[deny(missing_docs)]
pub mod backend;
#[doc(hidden)]
pub mod connection;
#[macro_use]
#[deny(missing_docs)]
pub mod expression;
#[deny(missing_docs)]
pub mod expression_methods;
#[doc(hidden)]
pub mod insertable;
#[deny(missing_docs)]
pub mod query_builder;
#[deny(missing_docs)]
#[macro_use]
pub mod types;

#[cfg(feature = "mysql")]
#[deny(missing_docs)]
pub mod mysql;
#[cfg(feature = "postgres")]
#[deny(missing_docs)]
pub mod pg;
#[cfg(feature = "sqlite")]
#[deny(missing_docs)]
pub mod sqlite;

#[deny(missing_docs)]
pub mod query_dsl;
#[deny(missing_docs)]
pub mod query_source;
#[deny(missing_docs)]
pub mod result;
#[deny(missing_docs)]
pub mod row;
mod util;

pub mod dsl {
    //! Includes various helper types and bare functions which are named too
    //! generically to be included in prelude, but are often used when using Diesel.

    #[doc(inline)]
    pub use helper_types::*;

    #[doc(inline)]
    pub use expression::dsl::*;
}

pub mod helper_types {
    //! Provide helper types for concisely writing the return type of functions.
    //! As with iterators, it is unfortunately difficult to return a partially
    //! constructed query without exposing the exact implementation of the
    //! function. Without higher kinded types, these various DSLs can't be
    //! combined into a single trait for boxing purposes.
    //!
    //! All types here are in the form `<FirstType as
    //! DslName<OtherTypes>>::Output`. So the return type of
    //! `users.filter(first_name.eq("John")).order(last_name.asc()).limit(10)` would
    //! be `Limit<Order<FindBy<users, first_name, &str>, Asc<last_name>>>`
    use super::query_dsl::*;
    use super::query_dsl::methods::*;
    use super::query_source::joins;

    #[doc(inline)]
    pub use expression::helper_types::*;

    /// Represents the return type of `.select(selection)`
    pub type Select<Source, Selection> = <Source as SelectDsl<Selection>>::Output;

    /// Represents the return type of `.filter(predicate)`
    pub type Filter<Source, Predicate> = <Source as FilterDsl<Predicate>>::Output;

    /// Represents the return type of `.filter(lhs.eq(rhs))`
    pub type FindBy<Source, Column, Value> = Filter<Source, Eq<Column, Value>>;

    /// Represents the return type of `.for_update()`
    pub type ForUpdate<Source> = <Source as ForUpdateDsl>::Output;

    /// Represents the return type of `.find(pk)`
    pub type Find<Source, PK> = <Source as FindDsl<PK>>::Output;

    /// Represents the return type of `.order(ordering)`
    pub type Order<Source, Ordering> = <Source as OrderDsl<Ordering>>::Output;

    /// Represents the return type of `.limit()`
    pub type Limit<Source> = <Source as LimitDsl>::Output;

    /// Represents the return type of `.offset()`
    pub type Offset<Source> = <Source as OffsetDsl>::Output;

    /// Represents the return type of `.inner_join(rhs)`
    pub type InnerJoin<Source, Rhs> =
        <Source as JoinWithImplicitOnClause<Rhs, joins::Inner>>::Output;

    /// Represents the return type of `.left_join(rhs)`
    pub type LeftJoin<Source, Rhs> =
        <Source as JoinWithImplicitOnClause<Rhs, joins::LeftOuter>>::Output;

    use super::associations::HasTable;
    use super::query_builder::{AsChangeset, IntoUpdateTarget, UpdateStatement};
    /// Represents the return type of `update(lhs).set(rhs)`
    pub type Update<Target, Changes> = UpdateStatement<
        <Target as HasTable>::Table,
        <Target as IntoUpdateTarget>::WhereClause,
        <Changes as AsChangeset>::Changeset,
    >;

    /// Represents the return type of `.into_boxed::<'a, DB>()`
    pub type IntoBoxed<'a, Source, DB> = <Source as BoxedDsl<'a, DB>>::Output;

    /// Represents the return type of `.distinct()`
    pub type Distinct<Source> = <Source as DistinctDsl>::Output;

    /// Represents the return type of `.distinct_on(expr)`
    #[cfg(feature = "postgres")]
    pub type DistinctOn<Source, Expr> = <Source as DistinctOnDsl<Expr>>::Output;
}

pub mod prelude {
    //! Re-exports important traits and types. Meant to be glob imported when using Diesel.
    pub use associations::{GroupedBy, Identifiable};
    pub use connection::Connection;
    pub use expression::{AppearsOnTable, BoxableExpression, Expression, IntoSql,
                         SelectableExpression};
    pub use expression_methods::*;
    #[doc(inline)]
    pub use insertable::Insertable;
    #[doc(hidden)]
    pub use query_dsl::GroupByDsl;
    pub use query_dsl::{BelongingToDsl, JoinOnDsl, QueryDsl, RunQueryDsl, SaveChangesDsl};

    pub use query_source::{Column, JoinTo, QuerySource, Queryable, Table};
    pub use result::{ConnectionError, ConnectionResult, OptionalExtension, QueryResult};

    #[cfg(feature = "postgres")]
    pub use pg::PgConnection;
    #[cfg(feature = "sqlite")]
    pub use sqlite::SqliteConnection;
    #[cfg(feature = "mysql")]
    pub use mysql::MysqlConnection;
}

pub use prelude::*;
#[doc(inline)]
pub use query_builder::debug_query;
#[doc(inline)]
pub use query_builder::functions::{delete, insert_into, replace_into, select, sql_query, update};
pub use result::Error::NotFound;
#[doc(inline)]
pub use types::structs::data_types;
<commit_msg>Rewrite the root docs<commit_after>//! # Diesel
//!
//! Diesel is an ORM and query builder designed to reduce the boilerplate for database interactions.
//! If this is your first time reading this documentation,
//! we recommend you start with the [getting started guide].
//! We also have [many other long form guides].
//!
//! [getting started guide]: https://diesel.rs/guides/getting-started/
//! [many other long form guides]: https://diesel.rs/guides
//!
//! # Where to find things
//!
//! ## Declaring your schema
//!
//! So Diesel is able to validate your queries at compile time,
//! it requires you to specify your schema in your code,
//! which you can do with [the `table!` macro][`table!`].
//! `diesel print-schema` or `infer_schema!` can be used
//! to automatically generate these macro calls
//! (by connecting to your database and querying its schema).
//!
//! [`table!`]: macro.table.html
//!
//! ## Getting started
//!
//! Queries usually start from either a table, or a function like [`update`].
//! Those functions can be found [here](#functions).
//!
//! Diesel provides a [`prelude` module](prelude),
//! which exports most of the typically used traits and types.
//! We are conservative about what goes in this module,
//! and avoid anything which has a generic name.
//! Files which use Diesel are expected to have `use diesel::prelude::*;`.
//!
//! [`update`]: fn.update.html
//!
//! ## Constructing a query
//!
//! The tools the query builder gives you can be put into these three categories:
//!
//! - "Query builder methods" are things that map to portions of a whole query
//!   (such as `ORDER` and `WHERE`). These methods usually have the same name
//!   as the SQL they map to, except for `WHERE` which is called `filter` in Diesel
//!   (To not conflict with the Rust keyword).
//!   These methods live in [the `query_dsl` module](query_dsl).
//! - "Expression methods" are things you would call on columns
//!   or other individual values.
//!   These methods live in [the `expression_methods` module](expression_methods)
//!   You can often find these by thinking "what would this be called"
//!   if it were a method
//!   and typing that into the search bar
//!   (e.g. `LIKE` is called `like` in Diesel).
//!   Most operators are named based on the Rust function which maps to that
//!   operator in [`std::ops`]
//!   (For example `==` is called `.eq`, and `!=` is called `.ne`).
//! - "Bare functions" are normal SQL functions
//!   such as `sum`.
//!   They live in [the `dsl` module](dsl).
//!   Diesel only supports a very small number of these functions.
//!   You can declare additional functions you want to use
//!   with [the `sql_function!` macro][`sql_function!`].
//!
//! [`std::ops`]: //doc.rust-lang.org/stable/std/ops/index.html
//! [`sql_function!`]: macro.sql_function.html
//!
//! ## Serializing and Deserializing
//!
//! Types which represent the result of a SQL query implement
//! a trait called [`Queryable`].
//!
//! Diesel maps "Rust types" (e.g. `i32`) to and from "SQL types"
//! (e.g. [`diesel::types::Integer`]).
//! You can find all the types supported by Diesel in [the `types` module](types).
//! These types are only used to represent a SQL type.
//! You should never put them on your `Queryable` structs.
//!
//! To find all the Rust types which can be used with a given SQL type,
//! see the documentation for that SQL type.
//!
//! To find all the SQL types which can be used with a Rust type,
//! go to the docs for either [`ToSql`] or [`FromSql`],
//! go to the "Implementors" section,
//! and find the Rust type you want to use.
//!
//! [`Queryable`]: query_source/trait.Queryable.html
//! [`diesel::types::Integer`]: types/struct.Integer.html
//! [`ToSql`]: types/trait.ToSql.html
//! [`FromSql`]: types/trait.FromSql.html
//!
//! ## Getting help
//!
//! If you run into problems, Diesel has a very active Gitter room.
//! You can come ask for help at
//! [gitter.im/diesel-rs/diesel](https://gitter.im/diesel-rs/diesel)

#![cfg_attr(feature = "unstable", feature(specialization))]
// Built-in Lints
#![deny(warnings, missing_debug_implementations, missing_copy_implementations)]
// Clippy lints
#![cfg_attr(feature = "clippy", allow(unstable_features))]
#![cfg_attr(feature = "clippy", feature(plugin))]
#![cfg_attr(feature = "clippy", plugin(clippy(conf_file = "../../clippy.toml")))]
#![cfg_attr(feature = "clippy",
            allow(option_map_unwrap_or_else, option_map_unwrap_or, match_same_arms,
                  type_complexity))]
#![cfg_attr(feature = "clippy",
            warn(option_unwrap_used, result_unwrap_used, print_stdout,
                 wrong_pub_self_convention, mut_mut, non_ascii_literal, similar_names,
                 unicode_not_nfc, enum_glob_use, if_not_else, items_after_statements,
                 used_underscore_binding))]
#![cfg_attr(all(test, feature = "clippy"), allow(option_unwrap_used, result_unwrap_used))]

#[cfg(feature = "postgres")]
#[macro_use]
extern crate bitflags;
extern crate byteorder;
// This is required to make `diesel_derives` re-export, but clippy thinks its unused
#[cfg_attr(feature = "clippy", allow(useless_attribute))]
#[allow(unused_imports)]
#[macro_use]
extern crate diesel_derives;
#[doc(hidden)]
pub use diesel_derives::*;

#[macro_use]
mod macros;

#[cfg(test)]
#[macro_use]
extern crate cfg_if;

#[cfg(test)]
pub mod test_helpers;

#[deny(missing_docs)]
pub mod associations;
#[deny(missing_docs)]
pub mod backend;
#[doc(hidden)]
pub mod connection;
#[macro_use]
#[deny(missing_docs)]
pub mod expression;
#[deny(missing_docs)]
pub mod expression_methods;
#[doc(hidden)]
pub mod insertable;
#[deny(missing_docs)]
pub mod query_builder;
#[deny(missing_docs)]
#[macro_use]
pub mod types;

#[cfg(feature = "mysql")]
#[deny(missing_docs)]
pub mod mysql;
#[cfg(feature = "postgres")]
#[deny(missing_docs)]
pub mod pg;
#[cfg(feature = "sqlite")]
#[deny(missing_docs)]
pub mod sqlite;

#[deny(missing_docs)]
pub mod query_dsl;
#[deny(missing_docs)]
pub mod query_source;
#[deny(missing_docs)]
pub mod result;
#[deny(missing_docs)]
pub mod row;
mod util;

pub mod dsl {
    //! Includes various helper types and bare functions which are named too
    //! generically to be included in prelude, but are often used when using Diesel.

    #[doc(inline)]
    pub use helper_types::*;

    #[doc(inline)]
    pub use expression::dsl::*;
}

pub mod helper_types {
    //! Provide helper types for concisely writing the return type of functions.
    //! As with iterators, it is unfortunately difficult to return a partially
    //! constructed query without exposing the exact implementation of the
    //! function. Without higher kinded types, these various DSLs can't be
    //! combined into a single trait for boxing purposes.
    //!
    //! All types here are in the form `<FirstType as
    //! DslName<OtherTypes>>::Output`. So the return type of
    //! `users.filter(first_name.eq("John")).order(last_name.asc()).limit(10)` would
    //! be `Limit<Order<FindBy<users, first_name, &str>, Asc<last_name>>>`
    use super::query_dsl::*;
    use super::query_dsl::methods::*;
    use super::query_source::joins;

    #[doc(inline)]
    pub use expression::helper_types::*;

    /// Represents the return type of `.select(selection)`
    pub type Select<Source, Selection> = <Source as SelectDsl<Selection>>::Output;

    /// Represents the return type of `.filter(predicate)`
    pub type Filter<Source, Predicate> = <Source as FilterDsl<Predicate>>::Output;

    /// Represents the return type of `.filter(lhs.eq(rhs))`
    pub type FindBy<Source, Column, Value> = Filter<Source, Eq<Column, Value>>;

    /// Represents the return type of `.for_update()`
    pub type ForUpdate<Source> = <Source as ForUpdateDsl>::Output;

    /// Represents the return type of `.find(pk)`
    pub type Find<Source, PK> = <Source as FindDsl<PK>>::Output;

    /// Represents the return type of `.order(ordering)`
    pub type Order<Source, Ordering> = <Source as OrderDsl<Ordering>>::Output;

    /// Represents the return type of `.limit()`
    pub type Limit<Source> = <Source as LimitDsl>::Output;

    /// Represents the return type of `.offset()`
    pub type Offset<Source> = <Source as OffsetDsl>::Output;

    /// Represents the return type of `.inner_join(rhs)`
    pub type InnerJoin<Source, Rhs> =
        <Source as JoinWithImplicitOnClause<Rhs, joins::Inner>>::Output;

    /// Represents the return type of `.left_join(rhs)`
    pub type LeftJoin<Source, Rhs> =
        <Source as JoinWithImplicitOnClause<Rhs, joins::LeftOuter>>::Output;

    use super::associations::HasTable;
    use super::query_builder::{AsChangeset, IntoUpdateTarget, UpdateStatement};
    /// Represents the return type of `update(lhs).set(rhs)`
    pub type Update<Target, Changes> = UpdateStatement<
        <Target as HasTable>::Table,
        <Target as IntoUpdateTarget>::WhereClause,
        <Changes as AsChangeset>::Changeset,
    >;

    /// Represents the return type of `.into_boxed::<'a, DB>()`
    pub type IntoBoxed<'a, Source, DB> = <Source as BoxedDsl<'a, DB>>::Output;

    /// Represents the return type of `.distinct()`
    pub type Distinct<Source> = <Source as DistinctDsl>::Output;

    /// Represents the return type of `.distinct_on(expr)`
    #[cfg(feature = "postgres")]
    pub type DistinctOn<Source, Expr> = <Source as DistinctOnDsl<Expr>>::Output;
}

pub mod prelude {
    //! Re-exports important traits and types. Meant to be glob imported when using Diesel.
    pub use associations::{GroupedBy, Identifiable};
    pub use connection::Connection;
    pub use expression::{AppearsOnTable, BoxableExpression, Expression, IntoSql,
                         SelectableExpression};
    pub use expression_methods::*;
    #[doc(inline)]
    pub use insertable::Insertable;
    #[doc(hidden)]
    pub use query_dsl::GroupByDsl;
    pub use query_dsl::{BelongingToDsl, JoinOnDsl, QueryDsl, RunQueryDsl, SaveChangesDsl};

    pub use query_source::{Column, JoinTo, QuerySource, Queryable, Table};
    pub use result::{ConnectionError, ConnectionResult, OptionalExtension, QueryResult};

    #[cfg(feature = "postgres")]
    pub use pg::PgConnection;
    #[cfg(feature = "sqlite")]
    pub use sqlite::SqliteConnection;
    #[cfg(feature = "mysql")]
    pub use mysql::MysqlConnection;
}

pub use prelude::*;
#[doc(inline)]
pub use query_builder::debug_query;
#[doc(inline)]
pub use query_builder::functions::{delete, insert_into, replace_into, select, sql_query, update};
pub use result::Error::NotFound;
#[doc(inline)]
pub use types::structs::data_types;
<|endoftext|>