<commit_before>// Copyright 2015 The GeoRust Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::collections::BTreeMap;

use rustc_serialize::json::{self, ToJson};

use ::{Bbox, Crs, Error, LineStringType, PointType, PolygonType, FromObject, util};

/// The underlying Geometry value
#[derive(Clone, Debug, PartialEq)]
pub enum Value {
    /// Point
    ///
    /// [GeoJSON Format Specification § 2.1.2]
    /// (http://geojson.org/geojson-spec.html#point)
    Point(PointType),

    /// MultiPoint
    ///
    /// [GeoJSON Format Specification § 2.1.3]
    /// (http://geojson.org/geojson-spec.html#multipoint)
    MultiPoint(Vec<PointType>),

    /// LineString
    ///
    /// [GeoJSON Format Specification § 2.1.4]
    /// (http://geojson.org/geojson-spec.html#linestring)
    LineString(LineStringType),

    /// MultiLineString
    ///
    /// [GeoJSON Format Specification § 2.1.5]
    /// (http://geojson.org/geojson-spec.html#multilinestring)
    MultiLineString(Vec<LineStringType>),

    /// Polygon
    ///
    /// [GeoJSON Format Specification § 2.1.6]
    /// (http://geojson.org/geojson-spec.html#polygon)
    Polygon(PolygonType),

    /// MultiPolygon
    ///
    /// [GeoJSON Format Specification § 2.1.7]
    /// (http://geojson.org/geojson-spec.html#multipolygon)
    MultiPolygon(Vec<PolygonType>),

    /// GeometryCollection
    ///
    /// [GeoJSON Format Specification § 2.1.8]
    /// (http://geojson.org/geojson-spec.html#geometry-collection)
    GeometryCollection(Vec<Geometry>),
}

impl ToJson for Value {
    fn to_json(&self) -> json::Json {
        return match *self {
            Value::Point(ref x) => x.to_json(),
            Value::MultiPoint(ref x) => x.to_json(),
            Value::LineString(ref x) => x.to_json(),
            Value::MultiLineString(ref x) => x.to_json(),
            Value::Polygon(ref x) => x.to_json(),
            Value::MultiPolygon(ref x) => x.to_json(),
            Value::GeometryCollection(ref x) => x.to_json(),
        };
    }
}

/// Geometry Objects
///
/// [GeoJSON Format Specification § 2.1]
/// (http://geojson.org/geojson-spec.html#geometry-objects)
#[derive(Clone, Debug, PartialEq)]
pub struct Geometry {
    pub bbox: Option<Bbox>,
    pub value: Value,
    pub crs: Option<Crs>,
}

impl<'a> From<&'a Geometry> for json::Object {
    fn from(geometry: &'a Geometry) -> json::Object {
        let mut map = BTreeMap::new();
        if let Some(ref crs) = geometry.crs {
            map.insert(String::from("crs"), crs.to_json());
        }
        if let Some(ref bbox) = geometry.bbox {
            map.insert(String::from("bbox"), bbox.to_json());
        }

        map.insert(String::from("type"), match geometry.value {
            Value::Point(..) => "Point",
            Value::MultiPoint(..) => "MultiPoint",
            Value::LineString(..) => "LineString",
            Value::MultiLineString(..) => "MultiLineString",
            Value::Polygon(..) => "Polygon",
            Value::MultiPolygon(..) => "MultiPolygon",
            Value::GeometryCollection(..) => "GeometryCollection",
        }.to_json());

        map.insert(String::from(match geometry.value {
            Value::GeometryCollection(..) => "geometries",
            _ => "coordinates",
        }), geometry.value.to_json());
        return map;
    }
}

impl FromObject for Geometry {
    fn from_object(object: &json::Object) -> Result<Self, Error> {
        let type_ = expect_type!(object);
        let value = match type_ {
            "Point" =>
                Value::Point(try!(util::get_coords_one_pos(object))),
            "MultiPoint" =>
                Value::MultiPoint(try!(util::get_coords_1d_pos(object))),
            "LineString" =>
                Value::LineString(try!(util::get_coords_1d_pos(object))),
            "MultiLineString" =>
                Value::MultiLineString(try!(util::get_coords_2d_pos(object))),
            "Polygon" =>
                Value::Polygon(try!(util::get_coords_2d_pos(object))),
            "MultiPolygon" =>
                Value::MultiPolygon(try!(util::get_coords_3d_pos(object))),
            "GeometryCollection" =>
                Value::GeometryCollection(try!(util::get_geometries(object))),
            _ => return Err(Error::new("Unknown geometry type")),
        };

        let bbox = try!(util::get_bbox(object));
        let crs = try!(util::get_crs(object));

        return Ok(Geometry {
            bbox: bbox,
            value: value,
            crs: crs,
        });
    }
}

impl ToJson for Geometry {
    fn to_json(&self) -> json::Json {
        return json::Json::Object(self.into());
    }
}

#[cfg(test)]
mod tests {
    use rustc_serialize::json::{self, ToJson};
    use super::super::{GeoJson, Geometry, Value};

    #[test]
    fn encode_decode_geometry() {
        let geometry_json_str = "{\"coordinates\":[1.0,2.0],\"type\":\"Point\"}";
        let geometry = Geometry {
            value: Value::Point(vec![1., 2.]),
            crs: None,
            bbox: None,
        };

        // Test encode
        let json_string = json::encode(&geometry.to_json()).unwrap();
        assert_eq!(json_string, geometry_json_str);

        // Test decode
        let decoded_geometry = match json_string.parse() {
            Ok(GeoJson::Geometry(g)) => g,
            _ => unreachable!(),
        };
        assert_eq!(decoded_geometry, geometry);
    }
}
<commit_msg>Create helper function to create a Geometry<commit_after>// Copyright 2015 The GeoRust Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::collections::BTreeMap;

use rustc_serialize::json::{self, ToJson};

use ::{Bbox, Crs, Error, LineStringType, PointType, PolygonType, FromObject, util};

/// The underlying Geometry value
#[derive(Clone, Debug, PartialEq)]
pub enum Value {
    /// Point
    ///
    /// [GeoJSON Format Specification § 2.1.2]
    /// (http://geojson.org/geojson-spec.html#point)
    Point(PointType),

    /// MultiPoint
    ///
    /// [GeoJSON Format Specification § 2.1.3]
    /// (http://geojson.org/geojson-spec.html#multipoint)
    MultiPoint(Vec<PointType>),

    /// LineString
    ///
    /// [GeoJSON Format Specification § 2.1.4]
    /// (http://geojson.org/geojson-spec.html#linestring)
    LineString(LineStringType),

    /// MultiLineString
    ///
    /// [GeoJSON Format Specification § 2.1.5]
    /// (http://geojson.org/geojson-spec.html#multilinestring)
    MultiLineString(Vec<LineStringType>),

    /// Polygon
    ///
    /// [GeoJSON Format Specification § 2.1.6]
    /// (http://geojson.org/geojson-spec.html#polygon)
    Polygon(PolygonType),

    /// MultiPolygon
    ///
    /// [GeoJSON Format Specification § 2.1.7]
    /// (http://geojson.org/geojson-spec.html#multipolygon)
    MultiPolygon(Vec<PolygonType>),

    /// GeometryCollection
    ///
    /// [GeoJSON Format Specification § 2.1.8]
    /// (http://geojson.org/geojson-spec.html#geometry-collection)
    GeometryCollection(Vec<Geometry>),
}

impl ToJson for Value {
    fn to_json(&self) -> json::Json {
        return match *self {
            Value::Point(ref x) => x.to_json(),
            Value::MultiPoint(ref x) => x.to_json(),
            Value::LineString(ref x) => x.to_json(),
            Value::MultiLineString(ref x) => x.to_json(),
            Value::Polygon(ref x) => x.to_json(),
            Value::MultiPolygon(ref x) => x.to_json(),
            Value::GeometryCollection(ref x) => x.to_json(),
        };
    }
}

/// Geometry Objects
///
/// [GeoJSON Format Specification § 2.1]
/// (http://geojson.org/geojson-spec.html#geometry-objects)
#[derive(Clone, Debug, PartialEq)]
pub struct Geometry {
    pub bbox: Option<Bbox>,
    pub value: Value,
    pub crs: Option<Crs>,
}

impl Geometry {
    /// Returns a new `Geometry` with the specified `value`. `bbox` and `crs` will be set to
    /// `None`.
    pub fn new(value: Value) -> Self {
        Geometry {
            bbox: None,
            value: value,
            crs: None,
        }
    }
}

impl<'a> From<&'a Geometry> for json::Object {
    fn from(geometry: &'a Geometry) -> json::Object {
        let mut map = BTreeMap::new();
        if let Some(ref crs) = geometry.crs {
            map.insert(String::from("crs"), crs.to_json());
        }
        if let Some(ref bbox) = geometry.bbox {
            map.insert(String::from("bbox"), bbox.to_json());
        }

        map.insert(String::from("type"), match geometry.value {
            Value::Point(..) => "Point",
            Value::MultiPoint(..) => "MultiPoint",
            Value::LineString(..) => "LineString",
            Value::MultiLineString(..) => "MultiLineString",
            Value::Polygon(..) => "Polygon",
            Value::MultiPolygon(..) => "MultiPolygon",
            Value::GeometryCollection(..) => "GeometryCollection",
        }.to_json());

        map.insert(String::from(match geometry.value {
            Value::GeometryCollection(..) => "geometries",
            _ => "coordinates",
        }), geometry.value.to_json());
        return map;
    }
}

impl FromObject for Geometry {
    fn from_object(object: &json::Object) -> Result<Self, Error> {
        let type_ = expect_type!(object);
        let value = match type_ {
            "Point" =>
                Value::Point(try!(util::get_coords_one_pos(object))),
            "MultiPoint" =>
                Value::MultiPoint(try!(util::get_coords_1d_pos(object))),
            "LineString" =>
                Value::LineString(try!(util::get_coords_1d_pos(object))),
            "MultiLineString" =>
                Value::MultiLineString(try!(util::get_coords_2d_pos(object))),
            "Polygon" =>
                Value::Polygon(try!(util::get_coords_2d_pos(object))),
            "MultiPolygon" =>
                Value::MultiPolygon(try!(util::get_coords_3d_pos(object))),
            "GeometryCollection" =>
                Value::GeometryCollection(try!(util::get_geometries(object))),
            _ => return Err(Error::new("Unknown geometry type")),
        };

        let bbox = try!(util::get_bbox(object));
        let crs = try!(util::get_crs(object));

        return Ok(Geometry {
            bbox: bbox,
            value: value,
            crs: crs,
        });
    }
}

impl ToJson for Geometry {
    fn to_json(&self) -> json::Json {
        return json::Json::Object(self.into());
    }
}

#[cfg(test)]
mod tests {
    use rustc_serialize::json::{self, ToJson};
    use super::super::{GeoJson, Geometry, Value};

    #[test]
    fn encode_decode_geometry() {
        let geometry_json_str = "{\"coordinates\":[1.0,2.0],\"type\":\"Point\"}";
        let geometry = Geometry {
            value: Value::Point(vec![1., 2.]),
            crs: None,
            bbox: None,
        };

        // Test encode
        let json_string = json::encode(&geometry.to_json()).unwrap();
        assert_eq!(json_string, geometry_json_str);

        // Test decode
        let decoded_geometry = match json_string.parse() {
            Ok(GeoJson::Geometry(g)) => g,
            _ => unreachable!(),
        };
        assert_eq!(decoded_geometry, geometry);
    }
}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io
#![allow(
    clippy::unused_unit,
    clippy::needless_return,
    clippy::never_loop,
    clippy::option_map_unit_fn
)]
use super::osm_utils::get_way_coord;
use super::OsmPbfReader;
use crate::admin_geofinder::AdminGeoFinder;
use crate::{labels, settings, utils, Error};
use cosmogony::ZoneType;
use failure::ResultExt;
use osmpbfreader::StoreObjs;
use slog_scope::info;
use std::collections::{BTreeMap, HashSet};
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;

use super::osm_store::{Getter, ObjWrapper};

#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
#[allow(dead_code)]
pub enum Kind {
    Node = 0,
    Way = 1,
    Relation = 2,
}

pub fn streets(
    pbf: &mut OsmPbfReader,
    admins_geofinder: &AdminGeoFinder,
    db_file: &Option<PathBuf>,
    db_buffer_size: usize,
    settings: &settings::Settings,
) -> Result<Vec<mimir::Street>, Error> {
    let invalid_highways = settings
        .street
        .as_ref()
        .and_then(|street| street.exclusion.highways.as_deref())
        .unwrap_or(&[]);

    let is_valid_highway = |tag: &str| -> bool { !invalid_highways.iter().any(|k| k == tag) };

    // For the object to be a valid street, it needs to be an osm highway of a valid type,
    // or a relation of type associatedStreet.
    let is_valid_obj = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Way(ref way) => {
                way.tags
                    .get("highway")
                    .map_or(false, |v| !v.is_empty() && is_valid_highway(v))
                    && way.tags.get("name").map_or(false, |v| !v.is_empty())
            }
            osmpbfreader::OsmObj::Relation(ref rel) => rel
                .tags
                .get("type")
                .map_or(false, |v| v == "associatedStreet"),
            _ => false,
        }
    };

    info!("reading pbf...");
    let mut objs_map = ObjWrapper::new(db_file, db_buffer_size)?;
    pbf.get_objs_and_deps_store(is_valid_obj, &mut objs_map)
        .context("Error occurred when reading pbf")?;
    info!("reading pbf done.");

    // Builder for street object
    let build_street =
        |id: String, name: String, coord: mimir::Coord, admins: Vec<Arc<mimir::Admin>>| {
            let admins_iter = admins.iter().map(Deref::deref);
            let country_codes = utils::find_country_codes(admins_iter.clone());
            mimir::Street {
                id,
                label: labels::format_street_label(&name, admins_iter, &country_codes),
                name,
                weight: 0.,
                zip_codes: utils::get_zip_codes_from_admins(&admins),
                administrative_regions: admins,
                coord,
                approx_coord: Some(coord.into()),
                distance: None,
                country_codes,
                context: None,
            }
        };

    // Return an iterator giving documents that will be inserted for a given street: one for each
    // hierarchy of admins
    let pois_with_admin = move |name: String, id, kind, mut admins: Vec<Vec<_>>, coord| {
        admins.sort_unstable(); // sort admins to make id deterministic
        admins.into_iter().enumerate().map(move |(i, admins)| {
            let doc_id = {
                if admins.len() <= 1 {
                    format!("street:osm:{}:{}", kind, id)
                } else {
                    format!("street:osm:{}:{}-{}", kind, id, i)
                }
            };

            build_street(doc_id, name.clone(), coord, admins)
        })
    };

    // List of outputed streets
    let mut street_list = Vec::new();

    // Sometimes, streets can be divided into several "way"s that still have the same street name.
    // The reason why a street is divided may be that a part of the street become
    // a bridge/tunnel/etc. In this case, a "relation" tagged with (type = associatedStreet) is used
    // to group all these "way"s. In order not to have duplicates in autocompletion, we should tag
    // the osm ways in the relation not to index them twice.
    let mut street_in_relation = HashSet::new();

    objs_map.for_each_filter(Kind::Relation, |obj| {
        let rel = obj.relation().expect("invalid relation filter");
        let rel_name = rel.tags.get("name");

        // Add osmid of all the relation members in the set.
        // Then, we won't create any street for the ways that belong to this relation.
        for ref_obj in &rel.refs {
            street_in_relation.insert(ref_obj.member);
        }

        let rel_street = rel
            .refs
            .iter()
            .filter(|ref_obj| ref_obj.member.is_way() && ref_obj.role == "street")
            .filter_map(|ref_obj| {
                let obj = objs_map.get(&ref_obj.member)?;
                let way = obj.way()?;
                let coord = get_way_coord(&objs_map, &way);
                let name = rel_name.or_else(|| way.tags.get("name"))?;

                Some(pois_with_admin(
                    name.to_string(),
                    rel.id.0,
                    "relation",
                    get_street_admin(admins_geofinder, &objs_map, &way),
                    coord,
                ))
            })
            .next();

        if let Some(street) = rel_street {
            street_list.extend(street);
        }
    });

    // We merge all the ways with same `way_name` and `admin list of level(=city_level)`
    // We use a Map to keep track of the way of smallest Id for a given pair of "name + cities list"
    let mut name_admin_map = BTreeMap::new();

    objs_map.for_each_filter(Kind::Way, |obj| {
        let osmid = obj.id();
        let way = obj.way().expect("invalid way filter");

        if street_in_relation.contains(&osmid) {
            return;
        }

        if let Some(name) = way.tags.get("name") {
            for admins in get_street_admin(admins_geofinder, &objs_map, way) {
                // Discriminate ways with same names by city
                if let Some(city) = admins
                    .iter()
                    .find(|admin| admin.is_city())
                    .map(|city| city.id.to_string())
                {
                    name_admin_map
                        .entry((name.to_string(), city))
                        .and_modify(|(stored_id, stored_admins)| {
                            if *stored_id > osmid {
                                *stored_id = std::cmp::min(*stored_id, osmid);
                                *stored_admins = admins.clone();
                            }
                        })
                        .or_insert((osmid, admins));
                }
            }
        }
    });

    // Create a street for each way with osmid present in objs_map
    street_list.extend(
        name_admin_map
            .into_iter()
            .filter_map(|(_, (min_id, admins))| {
                let obj = objs_map.get(&min_id)?;
                let way = obj.way()?;

                Some(pois_with_admin(
                    way.tags.get("name")?.to_string(),
                    way.id.0,
                    "way",
                    vec![admins],
                    get_way_coord(&objs_map, way),
                ))
            })
            .flatten(),
    );

    Ok(street_list)
}

/// Returns branches of admins encompassing the street `way`.
fn get_street_admin<T: StoreObjs + Getter>(
    admins_geofinder: &AdminGeoFinder,
    obj_map: &T,
    way: &osmpbfreader::objects::Way,
) -> Vec<Vec<Arc<mimir::Admin>>> {
    let nb_nodes = way.nodes.len();

    // To avoid corner cases where the ends of the way are near
    // administrative boundaries, the geofinder is called
    // preferably on a middle node.
    let (nodes_left, nodes_right) = (
        way.nodes[..nb_nodes / 2].iter(),
        way.nodes[nb_nodes / 2..].iter(),
    );

    nodes_right
        .chain(nodes_left)
        .filter_map(|node_id| obj_map.get(&(*node_id).into()))
        .filter_map(|node_obj| {
            node_obj.node().map(|node| geo_types::Coordinate {
                x: node.lon(),
                y: node.lat(),
            })
        })
        .next()
        .map_or_else(Vec::new, |coord| {
            // If the coords are part of several cities or suburbs, they are
            // all part of the output together with their parents. For
            // performance reasons, if the admin hierarchy is built of zones
            // bigger than cities, at most one result will belong to the output.
            admins_geofinder.get_admins_if(&coord, |admin| {
                admin
                    .zone_type
                    .map(|zt| zt <= ZoneType::City)
                    .unwrap_or(false)
            })
        })
}

pub fn compute_street_weight(streets: &mut Vec<mimir::Street>) {
    for st in streets {
        for admin in &mut st.administrative_regions {
            if admin.is_city() {
                st.weight = admin.weight;
                break;
            }
        }
    }
}
<commit_msg>streets: minor fixes<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io
#![allow(
    clippy::unused_unit,
    clippy::needless_return,
    clippy::never_loop,
    clippy::option_map_unit_fn
)]
use super::osm_utils::get_way_coord;
use super::OsmPbfReader;
use crate::admin_geofinder::AdminGeoFinder;
use crate::{labels, settings, utils, Error};
use cosmogony::ZoneType;
use failure::ResultExt;
use osmpbfreader::{OsmId, StoreObjs};
use slog_scope::info;
use std::collections::{BTreeMap, HashSet};
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;

use super::osm_store::{Getter, ObjWrapper};

#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
#[allow(dead_code)]
pub enum Kind {
    Node = 0,
    Way = 1,
    Relation = 2,
}

pub fn streets(
    pbf: &mut OsmPbfReader,
    admins_geofinder: &AdminGeoFinder,
    db_file: &Option<PathBuf>,
    db_buffer_size: usize,
    settings: &settings::Settings,
) -> Result<Vec<mimir::Street>, Error> {
    let invalid_highways = settings
        .street
        .as_ref()
        .and_then(|street| street.exclusion.highways.as_deref())
        .unwrap_or(&[]);

    let is_valid_highway = |tag: &str| -> bool { !invalid_highways.iter().any(|k| k == tag) };

    // For the object to be a valid street, it needs to be an osm highway of a valid type,
    // or a relation of type associatedStreet.
    let is_valid_obj = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Way(ref way) => {
                way.tags
                    .get("highway")
                    .map_or(false, |v| !v.is_empty() && is_valid_highway(v))
                    && way.tags.get("name").map_or(false, |v| !v.is_empty())
            }
            osmpbfreader::OsmObj::Relation(ref rel) => rel
                .tags
                .get("type")
                .map_or(false, |v| v == "associatedStreet"),
            _ => false,
        }
    };

    info!("reading pbf...");
    let mut objs_map = ObjWrapper::new(db_file, db_buffer_size)?;
    pbf.get_objs_and_deps_store(is_valid_obj, &mut objs_map)
        .context("Error occurred when reading pbf")?;
    info!("reading pbf done.");

    // Builder for street object
    let build_street =
        |id: String, name: String, coord: mimir::Coord, admins: Vec<Arc<mimir::Admin>>| {
            let admins_iter = admins.iter().map(Deref::deref);
            let country_codes = utils::find_country_codes(admins_iter.clone());
            mimir::Street {
                id,
                label: labels::format_street_label(&name, admins_iter, &country_codes),
                name,
                weight: 0.,
                zip_codes: utils::get_zip_codes_from_admins(&admins),
                administrative_regions: admins,
                coord,
                approx_coord: Some(coord.into()),
                distance: None,
                country_codes,
                context: None,
            }
        };

    // Return an iterator giving documents that will be inserted for a given
    // street: one for each hierarchy of admins.
    let build_streets_for_admins = move |name: String, id, kind, mut admins: Vec<Vec<_>>, coord| {
        admins.sort_unstable(); // sort admins to make id deterministic
        admins.into_iter().enumerate().map(move |(i, admins)| {
            let doc_id = {
                if admins.len() <= 1 {
                    format!("street:osm:{}:{}", kind, id)
                } else {
                    format!("street:osm:{}:{}-{}", kind, id, i)
                }
            };

            build_street(doc_id, name.clone(), coord, admins)
        })
    };

    // List of outputed streets
    let mut street_list = Vec::new();

    // Sometimes, streets can be divided into several "way"s that still have the same street name.
    // The reason why a street is divided may be that a part of the street become
    // a bridge/tunnel/etc. In this case, a "relation" tagged with (type = associatedStreet) is used
    // to group all these "way"s. In order not to have duplicates in autocompletion, we should tag
    // the osm ways in the relation not to index them twice.
    let mut street_in_relation = HashSet::new();

    objs_map.for_each_filter(Kind::Relation, |obj| {
        let rel = obj.relation().expect("invalid relation filter");
        let rel_name = rel.tags.get("name");

        // Add osmid of all the relation members in the set.
        // Then, we won't create any street for the ways that belong to this relation.
        street_in_relation.extend(
            rel.refs
                .iter()
                .map(|ref_obj| ref_obj.member)
                .filter(OsmId::is_way),
        );

        let rel_street = rel
            .refs
            .iter()
            .filter(|ref_obj| ref_obj.member.is_way() && ref_obj.role == "street")
            .filter_map(|ref_obj| {
                let obj = objs_map.get(&ref_obj.member)?;
                let way = obj.way()?;
                let coord = get_way_coord(&objs_map, &way);
                let name = rel_name.or_else(|| way.tags.get("name"))?;

                Some(build_streets_for_admins(
                    name.to_string(),
                    rel.id.0,
                    "relation",
                    get_street_admin(admins_geofinder, &objs_map, &way),
                    coord,
                ))
            })
            .next();

        if let Some(street) = rel_street {
            street_list.extend(street);
        }
    });

    // We merge all the ways with same `way_name` and `admin list of level(=city_level)`
    // We use a Map to keep track of the way of smallest Id for a given pair of "name + cities list"
    let mut name_admin_map = BTreeMap::new();

    objs_map.for_each_filter(Kind::Way, |obj| {
        let osmid = obj.id();
        let way = obj.way().expect("invalid way filter");

        if street_in_relation.contains(&osmid) {
            return;
        }

        if let Some(name) = way.tags.get("name") {
            for admins in get_street_admin(admins_geofinder, &objs_map, way) {
                // Discriminate ways with same names by city
                if let Some(city) = admins
                    .iter()
                    .find(|admin| admin.is_city())
                    .map(|city| city.id.to_string())
                {
                    name_admin_map
                        .entry((name.to_string(), city))
                        .and_modify(|(stored_id, stored_admins)| {
                            if *stored_id > osmid {
                                *stored_id = std::cmp::min(*stored_id, osmid);
                                *stored_admins = admins.clone();
                            }
                        })
                        .or_insert((osmid, admins));
                }
            }
        }
    });

    // Create a street for each way with osmid present in objs_map
    street_list.extend(
        name_admin_map
            .into_iter()
            .filter_map(|(_, (min_id, admins))| {
                let obj = objs_map.get(&min_id)?;
                let way = obj.way()?;

                Some(build_streets_for_admins(
                    way.tags.get("name")?.to_string(),
                    way.id.0,
                    "way",
                    vec![admins],
                    get_way_coord(&objs_map, way),
                ))
            })
            .flatten(),
    );

    Ok(street_list)
}

/// Returns branches of admins encompassing the street `way`.
fn get_street_admin<T: StoreObjs + Getter>(
    admins_geofinder: &AdminGeoFinder,
    obj_map: &T,
    way: &osmpbfreader::objects::Way,
) -> Vec<Vec<Arc<mimir::Admin>>> {
    let nb_nodes = way.nodes.len();

    // To avoid corner cases where the ends of the way are near
    // administrative boundaries, the geofinder is called
    // preferably on a middle node.
    let (nodes_left, nodes_right) = (
        way.nodes[..nb_nodes / 2].iter(),
        way.nodes[nb_nodes / 2..].iter(),
    );

    nodes_right
        .chain(nodes_left)
        .filter_map(|node_id| obj_map.get(&(*node_id).into()))
        .filter_map(|node_obj| {
            node_obj.node().map(|node| geo_types::Coordinate {
                x: node.lon(),
                y: node.lat(),
            })
        })
        .next()
        .map_or_else(Vec::new, |coord| {
            // If the coords are part of several cities or suburbs, they are
            // all part of the output together with their parents. For
            // performance reasons, if the admin hierarchy is built of zones
            // bigger than cities, at most one result will belong to the output.
            admins_geofinder.get_admins_if(&coord, |admin| {
                admin
                    .zone_type
                    .map(|zt| zt <= ZoneType::City)
                    .unwrap_or(false)
            })
        })
}

pub fn compute_street_weight(streets: &mut Vec<mimir::Street>) {
    for st in streets {
        for admin in &mut st.administrative_regions {
            if admin.is_city() {
                st.weight = admin.weight;
                break;
            }
        }
    }
}
<|endoftext|>


<commit_before>use crate::tensor::Tensor;
use crate::Float;
use std::cmp::Ordering;
use std::collections::binary_heap::BinaryHeap;
use std::fmt;
use std::mem;
use std::rc::Rc;

struct GradInfo<'a, T: Float + 'a> {
    node: &'a Tensor<T>, // information of this node
    has_gradient: bool,
    grad_called: bool,
    computed_grads: Vec<Tensor<T>>,
    default_grad: Option<&'a Tensor<T>>,
}

impl<'a, T: Float> fmt::Display for GradInfo<'a, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "name={}", self.node.op.name(),)
    }
}

impl<'a, T: Float> GradInfo<'a, T> {
    #[inline]
    fn new(
        t: &'a Tensor<T>,
        has_gradient: bool,
        default_grad: Option<&'a Tensor<T>>,
    ) -> GradInfo<'a, T> {
        GradInfo {
            node: t,
            has_gradient,
            computed_grads: Vec::new(),
            grad_called: false,
            default_grad,
        }
    }
}

impl<'a, T: Float> fmt::Debug for GradInfo<'a, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", self.node.op.name())
    }
}

impl<T: Float> fmt::Debug for Tensor<T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", self.op.name())
    }
}

macro_rules! access_grad_info_of {
    ($node:expr, $path:expr) => {
        $path[$node.resource_lookup_key.get()]
    };
}

#[inline]
fn has_marked_child<T: Float>(parent: &Tensor<T>, path: &Vec<GradInfo<T>>) -> bool {
    let mut it = parent.get_backprop_inputs().iter();
    while let Some(child) = it.next() {
        if access_grad_info_of!(child, path).has_gradient {
            return true;
        }
    }
    false
}

#[inline]
fn visited<T: Float>(node: &Tensor<T>, path: &Vec<GradInfo<T>>) -> bool {
    let k = node.resource_lookup_key.get();
    k < path.len() && Rc::ptr_eq(node, path[k].node)
}

#[inline]
fn is_wrt<T: Float>(node: &Tensor<T>, wrt: &[&Tensor<T>]) -> bool {
    wrt.contains(&node)
}

// Marks `has_gradient` if each node is on the gradient propagation path.
//
// Strategy
//   1. Record all nodes that are reachable from `ys` into `path`.
//   2. Mark the path between `ys` and `xs` as `has_gradient`.
fn mark_gradient_path<'a, T: Float>(
    ys: &[&'a Tensor<T>],
    wrt: &[&'a Tensor<T>],
) -> Vec<GradInfo<'a, T>> {
    // Randomly accessible by use of each node's lookup key.
    let mut path: Vec<GradInfo<'a, T>> = Vec::new();

    // Builds GradInfo while performing depth-first-search.
    // `has_gradient` properties are filled at the same time.

    // dfs_stack: (node, should_visit)
    let mut dfs_stack: Vec<(&Tensor<T>, bool)> = ys.iter().map(|&y| (y, false)).collect();
    while let Some((node, should_visit)) = dfs_stack.pop() {
        if should_visit {
            let marker =
                node.is_differentiable && (is_wrt(node, wrt) || has_marked_child(node, &path));
            node.resource_lookup_key.set(path.len());
            path.push(GradInfo::new(node, marker, None));
        } else {
            // Put self on the stack top (should visit next time)
            dfs_stack.push((node, true));
            // Push children as necessary
            for child in node.get_backprop_inputs() {
                if !visited(child, &path) {
                    if child.is_source() || !child.is_differentiable {
                        // Add to result, but don't allow any more recursive search
                        // because there will be no `wrt` nodes in this direction....
                        child.resource_lookup_key.set(path.len());
                        path.push(GradInfo::new(
                            child,
                            child.is_differentiable && is_wrt(child, wrt),
                            None,
                        ));
                    } else {
                        // Recurse
                        dfs_stack.push((child, false));
                    }
                }
            }
        }
    }
    path
}

#[test]
fn test_gradient_path() {
    // dummy graph
    // y = 3 * x1 * x1 + 5 * x2 + x3;
    let ref x1 = crate::ops::placeholder(&[]);
    let ref x2 = crate::ops::placeholder(&[]);
    let ref x3 = crate::ops::placeholder(&[]);
    let ref a = 3. * x1; // rank 1
    let ref b = a * x1; // rank 2
    let ref c = 5. * x2; // rank 1
    let ref d = b + c; // rank 3
    let ref y = d + x3; // rank 4
    let path = mark_gradient_path(&[y], &[x1, x2]);
    let path_: Vec<&Tensor<f64>> = path.iter().map(|a| a.node).collect();

    assert!(path_.contains(&x1));
    assert!(path_.contains(&x2));
    assert!(path_.contains(&x3));
    assert!(path_.contains(&a));
    assert!(path_.contains(&b));
    assert!(path_.contains(&c));
    assert!(path_.contains(&d));
    assert!(path_.contains(&y));
    assert_eq!(path_.len(), 10); // number of nodes in the grad path

    // Topological ordering test
    let ix1 = path_.iter().position(|x| Rc::ptr_eq(x, x1)).unwrap();
    let ix2 = path_.iter().position(|x| Rc::ptr_eq(x, x2)).unwrap();
    let ix3 = path_.iter().position(|x| Rc::ptr_eq(x, x3)).unwrap();
    let ia = path_.iter().position(|x| Rc::ptr_eq(x, a)).unwrap();
    let ic = path_.iter().position(|x| Rc::ptr_eq(x, c)).unwrap();
    let ib = path_.iter().position(|x| Rc::ptr_eq(x, b)).unwrap();
    let id = path_.iter().position(|x| Rc::ptr_eq(x, d)).unwrap();
    let iy = path_.iter().position(|x| Rc::ptr_eq(x, y)).unwrap();
    assert!(ix1 < ia);
    assert!(ix2 < ic);
    assert!(ix3 < iy);
    assert!(ib < id);
    assert!(id < iy);

    // Ensure continuity of keys
    for (i, node) in path_.iter().enumerate() {
        assert_eq!(i, node.resource_lookup_key.get());
    }

    // Connection test
    use std::collections::btree_set::BTreeSet;
    let all = (0..10).collect::<BTreeSet<usize>>();
    let should_be_has_gradient = [ix1, ix2, ia, ic, ib, id, iy]
        .into_iter()
        .cloned()
        .collect::<BTreeSet<usize>>();
    for &id in should_be_has_gradient.iter() {
        if !path[id].has_gradient {
            panic!("{} is not has_gradient", path[id].node.op.name());
        }
    }
    for &id in all.difference(&should_be_has_gradient).into_iter() {
        if path[id].has_gradient {
            panic!("{} should not be has_gradient", path[id].node.op.name());
        }
    }
}

/// Returns symbolic gradient tensors of `xs`.
///
/// This computes partial derivatives of `ys` with `xs` and returns the
/// gradients. This is achieved by building a subgraph between `ys` and
/// `xs` in reverse order from user's graph definition.
/// `gys` are already known gradients of `ys`'s outputs.
///
/// NOTE: Nodes that do not have gradients won't be included in the subgraph to avoid
/// unnecessary computation.
pub fn symbolic_gradients<T: Float>(
    ys: &[&Tensor<T>],
    wrt: &[&Tensor<T>],
    gys: &[&Tensor<T>],
) -> Vec<Tensor<T>> {
    assert_eq!(ys.len(), gys.len(), "`ys.len()` must match `gys.len()`");

    // Setup gradient path.
    let mut path = mark_gradient_path(ys, wrt);

    // Set default grads.
    for (y, gy) in ys.iter().zip(gys) {
        let mut y_info = &mut access_grad_info_of!(y, path);
        y_info.default_grad = Some(gy);
    }

    // Prepare a heap with given ys.
    let mut heap = ys
        .into_iter()
        .map(|y| y.wrapped())
        .collect::<BinaryHeap<TensorWrapper<T>>>();

    // Backprop.
    // Starts with `ys`.
    while let Some(y) = heap.pop() {
        let gxs = {
            let info = &mut access_grad_info_of!(y.inner, path);
            let gy = if let Some(def) = info.default_grad {
                def
            } else {
                let gys = &mut info.computed_grads;
                accumulate_grads_if_needed(gys);
                &gys[0]
            };
            // Call Op::grad
            let xs = y.inner.get_input_refs();
            let gxs = y.inner.op.grad(gy, xs.as_slice(), y.inner);
            debug_assert_eq!(xs.len(), gxs.len());
            gxs
        };
        // Register computed gradients
        let xs = y.inner.get_backprop_inputs();
        for (gx, x) in gxs.into_iter().zip(xs) {
            let mut x_info = &mut access_grad_info_of!(x, path);
            if x_info.has_gradient {
                if let Some(gx) = gx {
                    x_info.computed_grads.push(gx);
                    // update heap
                    if !x.is_source() && !x_info.grad_called {
                        x_info.grad_called = true;
                        heap.push(x.wrapped());
                    }
                }
            }
        }
    }

    // Aggregate and return xs's gradients
    wrt.iter()
        .map(|x| {
            let xk = x.resource_lookup_key.get();
            assert!(
                xk < path.len() && {
                    let info = &path[xk];
                    Rc::ptr_eq(x, info.node) && info.has_gradient
                },
                "Not differentiable with given tensor(s)."
            );
            let info = &mut path[xk];
            assert!(
                info.default_grad.is_none(),
                "Can't differentiate with objective itself"
            );
            let gxs = &mut info.computed_grads;
            accumulate_grads_if_needed(gxs);
            gxs.remove(0)
        })
        .collect::<Vec<Tensor<T>>>()
}

struct TensorWrapper<'a, T: Float + 'a> {
    inner: &'a Tensor<T>,
}

impl<'a, T: Float> Ord for TensorWrapper<'a, T> {
    // Compares the ranks in topological ordering
    fn cmp(&self, other: &Self) -> Ordering {
        self.inner.top_rank.cmp(&other.inner.top_rank)
    }
}

impl<'a, T: Float> PartialOrd for TensorWrapper<'a, T> {
    #[inline]
    // Compares the ranks in topological ordering
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
        Some(self.inner.top_rank.cmp(&other.inner.top_rank))
    }
}

impl<'a, T: Float> Eq for TensorWrapper<'a, T> {}

impl<'a, T: Float> PartialEq for TensorWrapper<'a, T> {
    #[inline]
    fn eq(&self, other: &TensorWrapper<'a, T>) -> bool {
        Rc::ptr_eq(&self.inner, &other.inner)
    }
}

impl<T: Float> Tensor<T> {
    #[inline]
    fn wrapped(&self) -> TensorWrapper<T> {
        TensorWrapper { inner: self }
    }
}

#[inline]
fn accumulate_grads_if_needed<T: Float>(grads: &mut Vec<Tensor<T>>) {
    if grads.len() > 1 {
        let mut acc = {
            let refs = grads.iter().map(|a| a).collect::<Vec<_>>();
            crate::ops::add_n(refs.as_slice())
        };
        mem::swap(&mut acc, &mut grads[0]);
        grads.truncate(1)
    }
}
<commit_msg>Use `slice::iter` instead of `into_iter` to avoid future breakage<commit_after>use crate::tensor::Tensor;
use crate::Float;
use std::cmp::Ordering;
use std::collections::binary_heap::BinaryHeap;
use std::fmt;
use std::mem;
use std::rc::Rc;

struct GradInfo<'a, T: Float + 'a> {
    node: &'a Tensor<T>, // information of this node
    has_gradient: bool,
    grad_called: bool,
    computed_grads: Vec<Tensor<T>>,
    default_grad: Option<&'a Tensor<T>>,
}

impl<'a, T: Float> fmt::Display for GradInfo<'a, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "name={}", self.node.op.name(),)
    }
}

impl<'a, T: Float> GradInfo<'a, T> {
    #[inline]
    fn new(
        t: &'a Tensor<T>,
        has_gradient: bool,
        default_grad: Option<&'a Tensor<T>>,
    ) -> GradInfo<'a, T> {
        GradInfo {
            node: t,
            has_gradient,
            computed_grads: Vec::new(),
            grad_called: false,
            default_grad,
        }
    }
}

impl<'a, T: Float> fmt::Debug for GradInfo<'a, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", self.node.op.name())
    }
}

impl<T: Float> fmt::Debug for Tensor<T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", self.op.name())
    }
}

macro_rules! access_grad_info_of {
    ($node:expr, $path:expr) => {
        $path[$node.resource_lookup_key.get()]
    };
}

#[inline]
fn has_marked_child<T: Float>(parent: &Tensor<T>, path: &Vec<GradInfo<T>>) -> bool {
    let mut it = parent.get_backprop_inputs().iter();
    while let Some(child) = it.next() {
        if access_grad_info_of!(child, path).has_gradient {
            return true;
        }
    }
    false
}

#[inline]
fn visited<T: Float>(node: &Tensor<T>, path: &Vec<GradInfo<T>>) -> bool {
    let k = node.resource_lookup_key.get();
    k < path.len() && Rc::ptr_eq(node, path[k].node)
}

#[inline]
fn is_wrt<T: Float>(node: &Tensor<T>, wrt: &[&Tensor<T>]) -> bool {
    wrt.contains(&node)
}

// Marks `has_gradient` if each node is on the gradient propagation path.
//
// Strategy
//   1. Record all nodes that are reachable from `ys` into `path`.
//   2. Mark the path between `ys` and `xs` as `has_gradient`.
fn mark_gradient_path<'a, T: Float>(
    ys: &[&'a Tensor<T>],
    wrt: &[&'a Tensor<T>],
) -> Vec<GradInfo<'a, T>> {
    // Randomly accessible by use of each node's lookup key.
    let mut path: Vec<GradInfo<'a, T>> = Vec::new();

    // Builds GradInfo while performing depth-first-search.
    // `has_gradient` properties are filled at the same time.

    // dfs_stack: (node, should_visit)
    let mut dfs_stack: Vec<(&Tensor<T>, bool)> = ys.iter().map(|&y| (y, false)).collect();
    while let Some((node, should_visit)) = dfs_stack.pop() {
        if should_visit {
            let marker =
                node.is_differentiable && (is_wrt(node, wrt) || has_marked_child(node, &path));
            node.resource_lookup_key.set(path.len());
            path.push(GradInfo::new(node, marker, None));
        } else {
            // Put self on the stack top (should visit next time)
            dfs_stack.push((node, true));
            // Push children as necessary
            for child in node.get_backprop_inputs() {
                if !visited(child, &path) {
                    if child.is_source() || !child.is_differentiable {
                        // Add to result, but don't allow any more recursive search
                        // because there will be no `wrt` nodes in this direction....
                        child.resource_lookup_key.set(path.len());
                        path.push(GradInfo::new(
                            child,
                            child.is_differentiable && is_wrt(child, wrt),
                            None,
                        ));
                    } else {
                        // Recurse
                        dfs_stack.push((child, false));
                    }
                }
            }
        }
    }
    path
}

#[test]
fn test_gradient_path() {
    // dummy graph
    // y = 3 * x1 * x1 + 5 * x2 + x3;
    let ref x1 = crate::ops::placeholder(&[]);
    let ref x2 = crate::ops::placeholder(&[]);
    let ref x3 = crate::ops::placeholder(&[]);
    let ref a = 3. * x1; // rank 1
    let ref b = a * x1; // rank 2
    let ref c = 5. * x2; // rank 1
    let ref d = b + c; // rank 3
    let ref y = d + x3; // rank 4
    let path = mark_gradient_path(&[y], &[x1, x2]);
    let path_: Vec<&Tensor<f64>> = path.iter().map(|a| a.node).collect();

    assert!(path_.contains(&x1));
    assert!(path_.contains(&x2));
    assert!(path_.contains(&x3));
    assert!(path_.contains(&a));
    assert!(path_.contains(&b));
    assert!(path_.contains(&c));
    assert!(path_.contains(&d));
    assert!(path_.contains(&y));
    assert_eq!(path_.len(), 10); // number of nodes in the grad path

    // Topological ordering test
    let ix1 = path_.iter().position(|x| Rc::ptr_eq(x, x1)).unwrap();
    let ix2 = path_.iter().position(|x| Rc::ptr_eq(x, x2)).unwrap();
    let ix3 = path_.iter().position(|x| Rc::ptr_eq(x, x3)).unwrap();
    let ia = path_.iter().position(|x| Rc::ptr_eq(x, a)).unwrap();
    let ic = path_.iter().position(|x| Rc::ptr_eq(x, c)).unwrap();
    let ib = path_.iter().position(|x| Rc::ptr_eq(x, b)).unwrap();
    let id = path_.iter().position(|x| Rc::ptr_eq(x, d)).unwrap();
    let iy = path_.iter().position(|x| Rc::ptr_eq(x, y)).unwrap();
    assert!(ix1 < ia);
    assert!(ix2 < ic);
    assert!(ix3 < iy);
    assert!(ib < id);
    assert!(id < iy);

    // Ensure continuity of keys
    for (i, node) in path_.iter().enumerate() {
        assert_eq!(i, node.resource_lookup_key.get());
    }

    // Connection test
    use std::collections::btree_set::BTreeSet;
    let all = (0..10).collect::<BTreeSet<usize>>();
    let should_be_has_gradient = [ix1, ix2, ia, ic, ib, id, iy]
        .iter()
        .cloned()
        .collect::<BTreeSet<usize>>();
    for &id in should_be_has_gradient.iter() {
        if !path[id].has_gradient {
            panic!("{} is not has_gradient", path[id].node.op.name());
        }
    }
    for &id in all.difference(&should_be_has_gradient).into_iter() {
        if path[id].has_gradient {
            panic!("{} should not be has_gradient", path[id].node.op.name());
        }
    }
}

/// Returns symbolic gradient tensors of `xs`.
///
/// This computes partial derivatives of `ys` with `xs` and returns the
/// gradients. This is achieved by building a subgraph between `ys` and
/// `xs` in reverse order from user's graph definition.
/// `gys` are already known gradients of `ys`'s outputs.
///
/// NOTE: Nodes that do not have gradients won't be included in the subgraph to avoid
/// unnecessary computation.
pub fn symbolic_gradients<T: Float>(
    ys: &[&Tensor<T>],
    wrt: &[&Tensor<T>],
    gys: &[&Tensor<T>],
) -> Vec<Tensor<T>> {
    assert_eq!(ys.len(), gys.len(), "`ys.len()` must match `gys.len()`");

    // Setup gradient path.
    let mut path = mark_gradient_path(ys, wrt);

    // Set default grads.
    for (y, gy) in ys.iter().zip(gys) {
        let mut y_info = &mut access_grad_info_of!(y, path);
        y_info.default_grad = Some(gy);
    }

    // Prepare a heap with given ys.
    let mut heap = ys
        .into_iter()
        .map(|y| y.wrapped())
        .collect::<BinaryHeap<TensorWrapper<T>>>();

    // Backprop.
    // Starts with `ys`.
    while let Some(y) = heap.pop() {
        let gxs = {
            let info = &mut access_grad_info_of!(y.inner, path);
            let gy = if let Some(def) = info.default_grad {
                def
            } else {
                let gys = &mut info.computed_grads;
                accumulate_grads_if_needed(gys);
                &gys[0]
            };
            // Call Op::grad
            let xs = y.inner.get_input_refs();
            let gxs = y.inner.op.grad(gy, xs.as_slice(), y.inner);
            debug_assert_eq!(xs.len(), gxs.len());
            gxs
        };
        // Register computed gradients
        let xs = y.inner.get_backprop_inputs();
        for (gx, x) in gxs.into_iter().zip(xs) {
            let mut x_info = &mut access_grad_info_of!(x, path);
            if x_info.has_gradient {
                if let Some(gx) = gx {
                    x_info.computed_grads.push(gx);
                    // update heap
                    if !x.is_source() && !x_info.grad_called {
                        x_info.grad_called = true;
                        heap.push(x.wrapped());
                    }
                }
            }
        }
    }

    // Aggregate and return xs's gradients
    wrt.iter()
        .map(|x| {
            let xk = x.resource_lookup_key.get();
            assert!(
                xk < path.len() && {
                    let info = &path[xk];
                    Rc::ptr_eq(x, info.node) && info.has_gradient
                },
                "Not differentiable with given tensor(s)."
            );
            let info = &mut path[xk];
            assert!(
                info.default_grad.is_none(),
                "Can't differentiate with objective itself"
            );
            let gxs = &mut info.computed_grads;
            accumulate_grads_if_needed(gxs);
            gxs.remove(0)
        })
        .collect::<Vec<Tensor<T>>>()
}

struct TensorWrapper<'a, T: Float + 'a> {
    inner: &'a Tensor<T>,
}

impl<'a, T: Float> Ord for TensorWrapper<'a, T> {
    // Compares the ranks in topological ordering
    fn cmp(&self, other: &Self) -> Ordering {
        self.inner.top_rank.cmp(&other.inner.top_rank)
    }
}

impl<'a, T: Float> PartialOrd for TensorWrapper<'a, T> {
    #[inline]
    // Compares the ranks in topological ordering
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
        Some(self.inner.top_rank.cmp(&other.inner.top_rank))
    }
}

impl<'a, T: Float> Eq for TensorWrapper<'a, T> {}

impl<'a, T: Float> PartialEq for TensorWrapper<'a, T> {
    #[inline]
    fn eq(&self, other: &TensorWrapper<'a, T>) -> bool {
        Rc::ptr_eq(&self.inner, &other.inner)
    }
}

impl<T: Float> Tensor<T> {
    #[inline]
    fn wrapped(&self) -> TensorWrapper<T> {
        TensorWrapper { inner: self }
    }
}

#[inline]
fn accumulate_grads_if_needed<T: Float>(grads: &mut Vec<Tensor<T>>) {
    if grads.len() > 1 {
        let mut acc = {
            let refs = grads.iter().map(|a| a).collect::<Vec<_>>();
            crate::ops::add_n(refs.as_slice())
        };
        mem::swap(&mut acc, &mut grads[0]);
        grads.truncate(1)
    }
}
<|endoftext|>


<commit_before>// Handle incoming requests.
extern crate iron;
extern crate redis;
extern crate rustc_serialize;
extern crate hyper;
extern crate url;

extern crate router;

use std::vec::Vec;
use rustc_serialize::json::Json;

use iron::modifiers::Redirect;
use iron::headers::{CacheControl, CacheDirective};
use iron::prelude::*;
use iron::status;
use iron::Url as iUrl;

use hyper::client::Client;

use router::Router;

use std::slice::SliceConcatExt;
use redis::{Commands, Value};

use helpers::{setup_redis, fetch, get_status_or,  local_redir, set_redis_cache};
use github::schedule_update as schedule_github_update;

// The base URL for our badges. We aren't actually compiling them ourselfes,
// but are reusing the great shields.io service.
static BADGE_URL_BASE: &'static str = "https://img.shields.io/badge/";

// Github Finder
// Expand a branch name into the hash, cache the redirect for 5min
// `/github/:user/:repo/badge.svg => /github/:user/:repo/:sha/badge.svg`
pub fn github_finder(req: &mut Request) -> IronResult<Response> {

    // Learn the parameters given to the request
    let router = req.extensions.get::<Router>().unwrap();
    let redis: redis::Connection = setup_redis();
    let hyper_client: Client = Client::new();

    let user = router.find("user").unwrap();
    let repo = router.find("repo").unwrap();
    let branch = router.find("branch").unwrap_or("master");
    let method = router.find("method").unwrap_or("badge.svg");

    // And the cache key we use to keep the map from branch->SHA
    let redis_key = format!("cached-sha/github/{0}/{1}:{2}", user, repo, branch);

    // Let's see if redis has this key. If it does, redirect the request
    // directly
    match redis.get(redis_key.to_owned()) {
        Ok(Value::Data(sha)) => {
            local_redir(&format!("/github/sha/{0}/{1}/{2}/{3}",
                                 user,
                                 repo,
                                 String::from_utf8(sha).unwrap(),
                                 method),
                        &req.url)
        }
        // otherwise, we need to look up the current SHA for the branch
        _ => {
            let github_url = format!("https://api.github.com/repos/{0}/{1}/git/refs/heads/{2}",
                                     user,
                                     repo,
                                     branch);
            // Fetch the content API request for the Github URL,
            // Parse its JSON and try to find the `SHA`-key.
            if let Some(body) = fetch(&hyper_client, &github_url) {
                if let Ok(json) = Json::from_str(&body) {
                    if let Some(&Json::String(ref sha)) = json.find_path(&["object", "sha"]) {
                        // Once found, store the SHA in the cache and redirect
                        // the request to
                        set_redis_cache(&redis, &redis_key, &sha);
                        local_redir(&format!("/github/sha/{0}/{1}/{2}/{3}",
                                             user,
                                             repo,
                                             sha,
                                             method),
                                    &req.url)
                    } else {
                        // If we couldn't find the SHA, then there is a problem
                        // we need to inform the user about. Usually this means
                        // they did a typo or the content moved – either way, we
                        // fire a 404 – Not Found.
                        warn!("{}: SHA not found in JSON: {}", &github_url, &json);
                        Ok(Response::with((status::NotFound,
                                           format!("Couldn't find on Github {}", &github_url))))
                    }
                } else {
                    warn!("{}: Couldn't parse Githubs JSON response: {}",
                          &github_url,
                          &body);
                    Ok(Response::with((status::InternalServerError,
                                       "Couldn't parse Githubs JSON response")))
                }
            } else {
                Ok(Response::with((status::NotFound,
                                   format!("Couldn't find on Github {}", &github_url))))
            }
        }
    }
}

// ## Github Handler
// Handle the request for a status report of a user-repo-sha combination.
// Usually the request ends up here after having been redirected via the
// `github_finder`-handler.
// In this request is where the actual sausage is done.
pub fn github_handler(req: &mut Request) -> IronResult<Response> {

    // First extract all the request information
    let router = req.extensions.get::<Router>().unwrap();
    let redis: redis::Connection = setup_redis();

    let user = router.find("user").unwrap();
    let repo = router.find("repo").unwrap();
    let sha = router.find("sha").unwrap();
    let filename: Vec<&str> = router.find("method")
                                    .unwrap_or("badge.svg")
                                    .rsplitn(2, '.')
                                    .collect();
    let (method, ext) = match filename.len() {
        2 => (filename[1], filename[0]),
        _ => (filename[0], ""),
    };

    // Use `get_status_or` to look up and map the cached result
    // or trigger a `schedule_github_update` if that isn't found yet
    let result_key = format!("result/github/{0}/{1}:{2}", user, repo, sha);
    let (text, color): (String, String) = get_status_or(
        redis.get(result_key.to_owned()),
        || schedule_github_update(&user, &repo, &sha));

    // Then render the response
    let mut response = match method {
        // If this is a simple request for status, just return the result
        "status" => Response::with((status::Ok, text.to_owned())),
        // for the badge, put text, color, base URL and query-parameters from the
        // incoming requests together to the URL we need to forward it to
        "badge" => {
            let target_badge = match req.url.clone().query {
                Some(query) => format!("{}clippy-{}-{}.{}?{}", BADGE_URL_BASE, text, color, ext, query),
                _ => format!("{}clippy-{}-{}.{}", BADGE_URL_BASE, text, color, ext),
            };
            // while linting, use only temporary redirects, so that the actual
            // result will be asked for later
            Response::with((match text.as_str() {
                    "linting" => status::Found,
                    _ => status::PermanentRedirect
                }, Redirect(iUrl::parse(&target_badge).unwrap())))
        },
        // emojibadge and fullemojibadge do the same as the request for `badge`,
        // except that they replace the status with appropriate emoji
        "emojibadge" => {
            let emoji = match text.as_str() {
                "linting" => "👷".to_string(),
                "failed" => "😱".to_string(),
                "success" => "👌".to_string(),
                _ => text.replace("errors", "🤕").replace("warnings", "😟")
            };

            let target_badge = match req.url.clone().query {
                Some(query) => format!("{}clippy-{}-{}.{}?{}", BADGE_URL_BASE, emoji, color, ext, query),
                _ => format!("{}clippy-{}-{}.{}", BADGE_URL_BASE, emoji, color, ext),
            };
            Response::with((match color.as_str() {
                    "blue" => status::Found,
                    _ => status::PermanentRedirect
                }, Redirect(iUrl::parse(&target_badge).unwrap())))
        },
        "fullemojibadge" => {
            let emoji = match text.as_str() {
                "linting" => "👷".to_string(),
                "failed" => "😱".to_string(),
                "success" => "👌".to_string(),
                _ => text.replace("errors", "🤕").replace("warnings", "😟")
            };

            let target_badge = match req.url.clone().query {
                Some(query) => format!("{}📎-{}-{}.{}?{}", BADGE_URL_BASE, emoji, color, ext, query),
                _ => format!("{}📎-{}-{}.{}", BADGE_URL_BASE, emoji, color, ext),
            };
            Response::with((match color.as_str() {
                    "blue" => status::Found,
                    _ => status::PermanentRedirect
                }, Redirect(iUrl::parse(&target_badge).unwrap())))
        },
        // If the request is asking for the logs, fetch those. This isn't particularly
        // simple as the Redis library makes the unwrapping a little bit tricky and hard
        // for rust to guess the proper types. So we have to specify the types and iterator
        // rather explictly at times.
        "log" => {
            let log_key = format!("log/github/{0}/{1}:{2}", user, repo, sha);
            match redis.lrange(log_key.to_owned(), 0, -1) {
                Ok(Some(Value::Bulk(logs))) => {
                    let logs: Vec<String> = logs.iter()
                                                .map(|ref v| {
                                                    match **v {
                                                        Value::Data(ref val) => {
                                        String::from_utf8(val.to_owned())
                                            .unwrap()
                                            .to_owned()
                                    }
                                                        _ => "".to_owned(),
                                                    }
                                                })
                                                .collect();
                    Response::with((status::Ok, logs.join("\n")))
                }
                // if there aren't any logs found, we might just started the
                // process. Let the request know.
                _ => {
                    Response::with((status::Ok, "Started. Please refresh"))
                }
            }
        },
        // Nothing else is supported – but in rust, we have to return all things
        // of the same type. So let's return a `BadRequst` :) .
        _ => Response::with((status::BadRequest, format!("{} Not Implemented.", method))),
    };

    response.headers.set(CacheControl(vec![CacheDirective::NoCache]));
    Ok(response)
}
<commit_msg>And switch to a moved Permanently (301) rather than Permanent Redirect (308), refs #16<commit_after>// Handle incoming requests.
extern crate iron;
extern crate redis;
extern crate rustc_serialize;
extern crate hyper;
extern crate url;

extern crate router;

use std::vec::Vec;
use rustc_serialize::json::Json;

use iron::modifiers::Redirect;
use iron::headers::{CacheControl, CacheDirective};
use iron::prelude::*;
use iron::status;
use iron::Url as iUrl;

use hyper::client::Client;

use router::Router;

use std::slice::SliceConcatExt;
use redis::{Commands, Value};

use helpers::{setup_redis, fetch, get_status_or,  local_redir, set_redis_cache};
use github::schedule_update as schedule_github_update;

// The base URL for our badges. We aren't actually compiling them ourselfes,
// but are reusing the great shields.io service.
static BADGE_URL_BASE: &'static str = "https://img.shields.io/badge/";

// Github Finder
// Expand a branch name into the hash, cache the redirect for 5min
// `/github/:user/:repo/badge.svg => /github/:user/:repo/:sha/badge.svg`
pub fn github_finder(req: &mut Request) -> IronResult<Response> {

    // Learn the parameters given to the request
    let router = req.extensions.get::<Router>().unwrap();
    let redis: redis::Connection = setup_redis();
    let hyper_client: Client = Client::new();

    let user = router.find("user").unwrap();
    let repo = router.find("repo").unwrap();
    let branch = router.find("branch").unwrap_or("master");
    let method = router.find("method").unwrap_or("badge.svg");

    // And the cache key we use to keep the map from branch->SHA
    let redis_key = format!("cached-sha/github/{0}/{1}:{2}", user, repo, branch);

    // Let's see if redis has this key. If it does, redirect the request
    // directly
    match redis.get(redis_key.to_owned()) {
        Ok(Value::Data(sha)) => {
            local_redir(&format!("/github/sha/{0}/{1}/{2}/{3}",
                                 user,
                                 repo,
                                 String::from_utf8(sha).unwrap(),
                                 method),
                        &req.url)
        }
        // otherwise, we need to look up the current SHA for the branch
        _ => {
            let github_url = format!("https://api.github.com/repos/{0}/{1}/git/refs/heads/{2}",
                                     user,
                                     repo,
                                     branch);
            // Fetch the content API request for the Github URL,
            // Parse its JSON and try to find the `SHA`-key.
            if let Some(body) = fetch(&hyper_client, &github_url) {
                if let Ok(json) = Json::from_str(&body) {
                    if let Some(&Json::String(ref sha)) = json.find_path(&["object", "sha"]) {
                        // Once found, store the SHA in the cache and redirect
                        // the request to
                        set_redis_cache(&redis, &redis_key, &sha);
                        local_redir(&format!("/github/sha/{0}/{1}/{2}/{3}",
                                             user,
                                             repo,
                                             sha,
                                             method),
                                    &req.url)
                    } else {
                        // If we couldn't find the SHA, then there is a problem
                        // we need to inform the user about. Usually this means
                        // they did a typo or the content moved – either way, we
                        // fire a 404 – Not Found.
                        warn!("{}: SHA not found in JSON: {}", &github_url, &json);
                        Ok(Response::with((status::NotFound,
                                           format!("Couldn't find on Github {}", &github_url))))
                    }
                } else {
                    warn!("{}: Couldn't parse Githubs JSON response: {}",
                          &github_url,
                          &body);
                    Ok(Response::with((status::InternalServerError,
                                       "Couldn't parse Githubs JSON response")))
                }
            } else {
                Ok(Response::with((status::NotFound,
                                   format!("Couldn't find on Github {}", &github_url))))
            }
        }
    }
}

// ## Github Handler
// Handle the request for a status report of a user-repo-sha combination.
// Usually the request ends up here after having been redirected via the
// `github_finder`-handler.
// In this request is where the actual sausage is done.
pub fn github_handler(req: &mut Request) -> IronResult<Response> {

    // First extract all the request information
    let router = req.extensions.get::<Router>().unwrap();
    let redis: redis::Connection = setup_redis();

    let user = router.find("user").unwrap();
    let repo = router.find("repo").unwrap();
    let sha = router.find("sha").unwrap();
    let filename: Vec<&str> = router.find("method")
                                    .unwrap_or("badge.svg")
                                    .rsplitn(2, '.')
                                    .collect();
    let (method, ext) = match filename.len() {
        2 => (filename[1], filename[0]),
        _ => (filename[0], ""),
    };

    // Use `get_status_or` to look up and map the cached result
    // or trigger a `schedule_github_update` if that isn't found yet
    let result_key = format!("result/github/{0}/{1}:{2}", user, repo, sha);
    let (text, color): (String, String) = get_status_or(
        redis.get(result_key.to_owned()),
        || schedule_github_update(&user, &repo, &sha));

    // Then render the response
    let mut response = match method {
        // If this is a simple request for status, just return the result
        "status" => Response::with((status::Ok, text.to_owned())),
        // for the badge, put text, color, base URL and query-parameters from the
        // incoming requests together to the URL we need to forward it to
        "badge" => {
            let target_badge = match req.url.clone().query {
                Some(query) => format!("{}clippy-{}-{}.{}?{}", BADGE_URL_BASE, text, color, ext, query),
                _ => format!("{}clippy-{}-{}.{}", BADGE_URL_BASE, text, color, ext),
            };
            // while linting, use only temporary redirects, so that the actual
            // result will be asked for later
            Response::with((match text.as_str() {
                    "linting" => status::Found,
                    _ => status::MovedPermanently
                }, Redirect(iUrl::parse(&target_badge).unwrap())))
        },
        // emojibadge and fullemojibadge do the same as the request for `badge`,
        // except that they replace the status with appropriate emoji
        "emojibadge" => {
            let emoji = match text.as_str() {
                "linting" => "👷".to_string(),
                "failed" => "😱".to_string(),
                "success" => "👌".to_string(),
                _ => text.replace("errors", "🤕").replace("warnings", "😟")
            };

            let target_badge = match req.url.clone().query {
                Some(query) => format!("{}clippy-{}-{}.{}?{}", BADGE_URL_BASE, emoji, color, ext, query),
                _ => format!("{}clippy-{}-{}.{}", BADGE_URL_BASE, emoji, color, ext),
            };
            Response::with((match color.as_str() {
                    "blue" => status::Found,
                    _ => status::MovedPermanently
                }, Redirect(iUrl::parse(&target_badge).unwrap())))
        },
        "fullemojibadge" => {
            let emoji = match text.as_str() {
                "linting" => "👷".to_string(),
                "failed" => "😱".to_string(),
                "success" => "👌".to_string(),
                _ => text.replace("errors", "🤕").replace("warnings", "😟")
            };

            let target_badge = match req.url.clone().query {
                Some(query) => format!("{}📎-{}-{}.{}?{}", BADGE_URL_BASE, emoji, color, ext, query),
                _ => format!("{}📎-{}-{}.{}", BADGE_URL_BASE, emoji, color, ext),
            };
            Response::with((match color.as_str() {
                    "blue" => status::Found,
                    _ => status::MovedPermanently
                }, Redirect(iUrl::parse(&target_badge).unwrap())))
        },
        // If the request is asking for the logs, fetch those. This isn't particularly
        // simple as the Redis library makes the unwrapping a little bit tricky and hard
        // for rust to guess the proper types. So we have to specify the types and iterator
        // rather explictly at times.
        "log" => {
            let log_key = format!("log/github/{0}/{1}:{2}", user, repo, sha);
            match redis.lrange(log_key.to_owned(), 0, -1) {
                Ok(Some(Value::Bulk(logs))) => {
                    let logs: Vec<String> = logs.iter()
                                                .map(|ref v| {
                                                    match **v {
                                                        Value::Data(ref val) => {
                                        String::from_utf8(val.to_owned())
                                            .unwrap()
                                            .to_owned()
                                    }
                                                        _ => "".to_owned(),
                                                    }
                                                })
                                                .collect();
                    Response::with((status::Ok, logs.join("\n")))
                }
                // if there aren't any logs found, we might just started the
                // process. Let the request know.
                _ => {
                    Response::with((status::Ok, "Started. Please refresh"))
                }
            }
        },
        // Nothing else is supported – but in rust, we have to return all things
        // of the same type. So let's return a `BadRequst` :) .
        _ => Response::with((status::BadRequest, format!("{} Not Implemented.", method))),
    };

    response.headers.set(CacheControl(vec![CacheDirective::NoCache]));
    Ok(response)
}
<|endoftext|>


<commit_before>// force-host

#![feature(plugin_registrar)]
#![feature(box_syntax, rustc_private)]

// Load rustc as a plugin to get macros.
#[macro_use]
extern crate rustc;
extern crate rustc_plugin;

use rustc::hir;
use rustc::lint::{LateContext, LintContext, LintPass, LateLintPass, LateLintPassObject, LintArray};
use rustc_plugin::Registry;

declare_lint!(TEST_LINT, Warn, "Warn about items named 'lintme'");

declare_lint!(PLEASE_LINT, Warn, "Warn about items named 'pleaselintme'");

struct Pass;

impl LintPass for Pass {
    fn name(&self) -> &'static str {
        "Pass"
    }

    fn get_lints(&self) -> LintArray {
        lint_array!(TEST_LINT, PLEASE_LINT)
    }
}

impl<'a, 'tcx> LateLintPass<'a, 'tcx> for Pass {
    fn check_item(&mut self, cx: &LateContext, it: &hir::Item) {
        match &*it.ident.as_str() {
            "lintme" => cx.span_lint(TEST_LINT, it.span, "item is named 'lintme'"),
            "pleaselintme" => cx.span_lint(PLEASE_LINT, it.span, "item is named 'pleaselintme'"),
            _ => {}
        }
    }
}

#[plugin_registrar]
pub fn plugin_registrar(reg: &mut Registry) {
    reg.register_late_lint_pass(box Pass);
    reg.register_lint_group("lint_me", None, vec![TEST_LINT, PLEASE_LINT]);
}
<commit_msg>cleanup: Remove compile-fail-fulldeps directory again<commit_after><|endoftext|>


<commit_before>use std::cmp::{max, min};

use serde::Deserialize;
use serde_with::{serde_as, CommaSeparator, DisplayFromStr, StringWithSeparator, TryFromInto};
use shakmaty::{
    fen::Fen,
    uci::Uci,
    variant::{Variant, VariantPosition},
    zobrist::Zobrist,
    CastlingMode, Color,
};

use crate::{
    api::{Error, LaxFen, LilaVariant},
    model::{Mode, Month, RatingGroup, Speed, UserName, Year},
    opening::{Opening, Openings},
};

#[serde_as]
#[derive(Deserialize, Debug)]
pub struct MastersQuery {
    #[serde(flatten)]
    pub play: Play,
    #[serde_as(as = "TryFromInto<u16>")]
    #[serde(default)]
    pub since: Year,
    #[serde_as(as = "TryFromInto<u16>")]
    #[serde(default = "Year::max_value")]
    pub until: Year,
    #[serde(flatten)]
    pub limits: Limits,
}

#[serde_as]
#[derive(Deserialize, Debug)]
pub struct LichessQuery {
    #[serde(flatten)]
    pub play: Play,
    #[serde(flatten)]
    pub limits: Limits,
    #[serde(flatten)]
    pub filter: LichessQueryFilter,
}

#[serde_as]
#[derive(Deserialize, Debug)]
pub struct LichessQueryFilter {
    #[serde_as(as = "Option<StringWithSeparator<CommaSeparator, Speed>>")]
    #[serde(default)]
    pub speeds: Option<Vec<Speed>>,
    #[serde_as(as = "Option<StringWithSeparator<CommaSeparator, RatingGroup>>")]
    #[serde(default)]
    pub ratings: Option<Vec<RatingGroup>>,
    #[serde_as(as = "DisplayFromStr")]
    #[serde(default)]
    pub since: Month,
    #[serde_as(as = "DisplayFromStr")]
    #[serde(default = "Month::max_value")]
    pub until: Month,
}

impl LichessQueryFilter {
    pub fn contains_speed(&self, speed: Speed) -> bool {
        self.speeds
            .as_ref()
            .map_or(true, |speeds| speeds.contains(&speed))
    }

    pub fn contains_rating_group(&self, rating_group: RatingGroup) -> bool {
        self.ratings.as_ref().map_or(true, |ratings| {
            ratings.contains(&max(
                RatingGroup::Group1600,
                min(rating_group, RatingGroup::Group2500),
            ))
        })
    }

    pub fn top_group(&self) -> Option<RatingGroup> {
        let mut top_group = None;
        for group in RatingGroup::ALL.into_iter().rev() {
            if !self.contains_rating_group(group) || group < RatingGroup::Group2000 {
                break;
            } else {
                top_group = Some(group);
            }
        }
        top_group
    }
}

#[serde_as]
#[derive(Deserialize, Debug)]
pub struct PlayerQuery {
    #[serde(flatten)]
    pub play: Play,
    #[serde_as(as = "DisplayFromStr")]
    pub player: UserName,
    #[serde_as(as = "DisplayFromStr")]
    pub color: Color,
    #[serde(flatten)]
    pub filter: PlayerQueryFilter,
    #[serde(flatten)]
    pub limits: Limits,
}

#[serde_as]
#[derive(Deserialize, Debug)]
pub struct PlayerQueryFilter {
    #[serde_as(as = "Option<StringWithSeparator<CommaSeparator, Mode>>")]
    #[serde(default)]
    pub modes: Option<Vec<Mode>>,
    #[serde_as(as = "Option<StringWithSeparator<CommaSeparator, Speed>>")]
    #[serde(default)]
    pub speeds: Option<Vec<Speed>>,
    #[serde_as(as = "DisplayFromStr")]
    #[serde(default)]
    pub since: Month,
    #[serde_as(as = "DisplayFromStr")]
    #[serde(default = "Month::max_value")]
    pub until: Month,
}

#[serde_as]
#[derive(Deserialize, Debug)]
pub struct Play {
    #[serde(default)]
    pub variant: LilaVariant,
    #[serde_as(as = "Option<DisplayFromStr>")]
    #[serde(default)]
    pub fen: Option<LaxFen>,
    #[serde_as(as = "StringWithSeparator<CommaSeparator, Uci>")]
    #[serde(default)]
    pub play: Vec<Uci>,
}

impl Play {
    pub fn position(
        self,
        openings: &Openings,
    ) -> Result<(Variant, Zobrist<VariantPosition, u128>, Option<&Opening>), Error> {
        let variant = Variant::from(self.variant);
        let mut pos = Zobrist::new(match self.fen {
            Some(fen) => {
                VariantPosition::from_setup(variant, &Fen::from(fen), CastlingMode::Chess960)?
            }
            None => VariantPosition::new(variant),
        });
        let opening = openings.classify_and_play(&mut pos, self.play)?;
        Ok((variant, pos, opening))
    }
}

#[serde_as]
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct Limits {
    #[serde_as(as = "DisplayFromStr")]
    #[serde(default = "usize::max_value")]
    pub top_games: usize,
    #[serde_as(as = "DisplayFromStr")]
    #[serde(default = "usize::max_value")]
    pub recent_games: usize,
    #[serde_as(as = "Option<DisplayFromStr>")]
    #[serde(default)]
    pub moves: Option<usize>,
}
<commit_msg>relax position validation for queries<commit_after>use std::cmp::{max, min};

use serde::Deserialize;
use serde_with::{serde_as, CommaSeparator, DisplayFromStr, StringWithSeparator, TryFromInto};
use shakmaty::{
    fen::Fen,
    uci::Uci,
    variant::{Variant, VariantPosition},
    zobrist::Zobrist,
    CastlingMode, Color, PositionError,
};

use crate::{
    api::{Error, LaxFen, LilaVariant},
    model::{Mode, Month, RatingGroup, Speed, UserName, Year},
    opening::{Opening, Openings},
};

#[serde_as]
#[derive(Deserialize, Debug)]
pub struct MastersQuery {
    #[serde(flatten)]
    pub play: Play,
    #[serde_as(as = "TryFromInto<u16>")]
    #[serde(default)]
    pub since: Year,
    #[serde_as(as = "TryFromInto<u16>")]
    #[serde(default = "Year::max_value")]
    pub until: Year,
    #[serde(flatten)]
    pub limits: Limits,
}

#[serde_as]
#[derive(Deserialize, Debug)]
pub struct LichessQuery {
    #[serde(flatten)]
    pub play: Play,
    #[serde(flatten)]
    pub limits: Limits,
    #[serde(flatten)]
    pub filter: LichessQueryFilter,
}

#[serde_as]
#[derive(Deserialize, Debug)]
pub struct LichessQueryFilter {
    #[serde_as(as = "Option<StringWithSeparator<CommaSeparator, Speed>>")]
    #[serde(default)]
    pub speeds: Option<Vec<Speed>>,
    #[serde_as(as = "Option<StringWithSeparator<CommaSeparator, RatingGroup>>")]
    #[serde(default)]
    pub ratings: Option<Vec<RatingGroup>>,
    #[serde_as(as = "DisplayFromStr")]
    #[serde(default)]
    pub since: Month,
    #[serde_as(as = "DisplayFromStr")]
    #[serde(default = "Month::max_value")]
    pub until: Month,
}

impl LichessQueryFilter {
    pub fn contains_speed(&self, speed: Speed) -> bool {
        self.speeds
            .as_ref()
            .map_or(true, |speeds| speeds.contains(&speed))
    }

    pub fn contains_rating_group(&self, rating_group: RatingGroup) -> bool {
        self.ratings.as_ref().map_or(true, |ratings| {
            ratings.contains(&max(
                RatingGroup::Group1600,
                min(rating_group, RatingGroup::Group2500),
            ))
        })
    }

    pub fn top_group(&self) -> Option<RatingGroup> {
        let mut top_group = None;
        for group in RatingGroup::ALL.into_iter().rev() {
            if !self.contains_rating_group(group) || group < RatingGroup::Group2000 {
                break;
            } else {
                top_group = Some(group);
            }
        }
        top_group
    }
}

#[serde_as]
#[derive(Deserialize, Debug)]
pub struct PlayerQuery {
    #[serde(flatten)]
    pub play: Play,
    #[serde_as(as = "DisplayFromStr")]
    pub player: UserName,
    #[serde_as(as = "DisplayFromStr")]
    pub color: Color,
    #[serde(flatten)]
    pub filter: PlayerQueryFilter,
    #[serde(flatten)]
    pub limits: Limits,
}

#[serde_as]
#[derive(Deserialize, Debug)]
pub struct PlayerQueryFilter {
    #[serde_as(as = "Option<StringWithSeparator<CommaSeparator, Mode>>")]
    #[serde(default)]
    pub modes: Option<Vec<Mode>>,
    #[serde_as(as = "Option<StringWithSeparator<CommaSeparator, Speed>>")]
    #[serde(default)]
    pub speeds: Option<Vec<Speed>>,
    #[serde_as(as = "DisplayFromStr")]
    #[serde(default)]
    pub since: Month,
    #[serde_as(as = "DisplayFromStr")]
    #[serde(default = "Month::max_value")]
    pub until: Month,
}

#[serde_as]
#[derive(Deserialize, Debug)]
pub struct Play {
    #[serde(default)]
    pub variant: LilaVariant,
    #[serde_as(as = "Option<DisplayFromStr>")]
    #[serde(default)]
    pub fen: Option<LaxFen>,
    #[serde_as(as = "StringWithSeparator<CommaSeparator, Uci>")]
    #[serde(default)]
    pub play: Vec<Uci>,
}

impl Play {
    pub fn position(
        self,
        openings: &Openings,
    ) -> Result<(Variant, Zobrist<VariantPosition, u128>, Option<&Opening>), Error> {
        let variant = Variant::from(self.variant);
        let mut pos = Zobrist::new(match self.fen {
            Some(fen) => {
                VariantPosition::from_setup(variant, &Fen::from(fen), CastlingMode::Chess960)
                    .or_else(PositionError::ignore_invalid_castling_rights)
                    .or_else(PositionError::ignore_invalid_ep_square)
                    .or_else(PositionError::ignore_impossible_material)?
            }
            None => VariantPosition::new(variant),
        });
        let opening = openings.classify_and_play(&mut pos, self.play)?;
        Ok((variant, pos, opening))
    }
}

#[serde_as]
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct Limits {
    #[serde_as(as = "DisplayFromStr")]
    #[serde(default = "usize::max_value")]
    pub top_games: usize,
    #[serde_as(as = "DisplayFromStr")]
    #[serde(default = "usize::max_value")]
    pub recent_games: usize,
    #[serde_as(as = "Option<DisplayFromStr>")]
    #[serde(default)]
    pub moves: Option<usize>,
}
<|endoftext|>