<commit_before>//! Extensions to `std::process` for Windows.

#![stable(feature = "process_extensions", since = "1.2.0")]

use crate::ffi::OsStr;
use crate::os::windows::io::{AsRawHandle, FromRawHandle, IntoRawHandle, RawHandle};
use crate::process;
use crate::sealed::Sealed;
use crate::sys;
use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner};

#[stable(feature = "process_extensions", since = "1.2.0")]
impl FromRawHandle for process::Stdio {
    unsafe fn from_raw_handle(handle: RawHandle) -> process::Stdio {
        let handle = sys::handle::Handle::new(handle as *mut _);
        let io = sys::process::Stdio::Handle(handle);
        process::Stdio::from_inner(io)
    }
}

#[stable(feature = "process_extensions", since = "1.2.0")]
impl AsRawHandle for process::Child {
    #[inline]
    fn as_raw_handle(&self) -> RawHandle {
        self.as_inner().handle().raw() as *mut _
    }
}

#[stable(feature = "into_raw_os", since = "1.4.0")]
impl IntoRawHandle for process::Child {
    fn into_raw_handle(self) -> RawHandle {
        self.into_inner().into_handle().into_raw() as *mut _
    }
}

#[stable(feature = "process_extensions", since = "1.2.0")]
impl AsRawHandle for process::ChildStdin {
    #[inline]
    fn as_raw_handle(&self) -> RawHandle {
        self.as_inner().handle().raw() as *mut _
    }
}

#[stable(feature = "process_extensions", since = "1.2.0")]
impl AsRawHandle for process::ChildStdout {
    #[inline]
    fn as_raw_handle(&self) -> RawHandle {
        self.as_inner().handle().raw() as *mut _
    }
}

#[stable(feature = "process_extensions", since = "1.2.0")]
impl AsRawHandle for process::ChildStderr {
    #[inline]
    fn as_raw_handle(&self) -> RawHandle {
        self.as_inner().handle().raw() as *mut _
    }
}

#[stable(feature = "into_raw_os", since = "1.4.0")]
impl IntoRawHandle for process::ChildStdin {
    fn into_raw_handle(self) -> RawHandle {
        self.into_inner().into_handle().into_raw() as *mut _
    }
}

#[stable(feature = "into_raw_os", since = "1.4.0")]
impl IntoRawHandle for process::ChildStdout {
    fn into_raw_handle(self) -> RawHandle {
        self.into_inner().into_handle().into_raw() as *mut _
    }
}

#[stable(feature = "into_raw_os", since = "1.4.0")]
impl IntoRawHandle for process::ChildStderr {
    fn into_raw_handle(self) -> RawHandle {
        self.into_inner().into_handle().into_raw() as *mut _
    }
}

/// Windows-specific extensions to [`process::ExitStatus`].
///
/// This trait is sealed: it cannot be implemented outside the standard library.
/// This is so that future additional methods are not breaking changes.
#[stable(feature = "exit_status_from", since = "1.12.0")]
pub trait ExitStatusExt: Sealed {
    /// Creates a new `ExitStatus` from the raw underlying `u32` return value of
    /// a process.
    #[stable(feature = "exit_status_from", since = "1.12.0")]
    fn from_raw(raw: u32) -> Self;
}

#[stable(feature = "exit_status_from", since = "1.12.0")]
impl ExitStatusExt for process::ExitStatus {
    fn from_raw(raw: u32) -> Self {
        process::ExitStatus::from_inner(From::from(raw))
    }
}

/// Windows-specific extensions to the [`process::Command`] builder.
///
/// This trait is sealed: it cannot be implemented outside the standard library.
/// This is so that future additional methods are not breaking changes.
#[stable(feature = "windows_process_extensions", since = "1.16.0")]
pub trait CommandExt: Sealed {
    /// Sets the [process creation flags][1] to be passed to `CreateProcess`.
    ///
    /// These will always be ORed with `CREATE_UNICODE_ENVIRONMENT`.
    ///
    /// [1]: https://docs.microsoft.com/en-us/windows/win32/procthread/process-creation-flags
    #[stable(feature = "windows_process_extensions", since = "1.16.0")]
    fn creation_flags(&mut self, flags: u32) -> &mut process::Command;

    /// Forces all arguments to be wrapped in quote (`"`) characters.
    ///
    /// This is useful for passing arguments to [MSYS2/Cygwin][1] based
    /// executables: these programs will expand unquoted arguments containing
    /// wildcard characters (`?` and `*`) by searching for any file paths
    /// matching the wildcard pattern.
    ///
    /// Adding quotes has no effect when passing arguments to programs
    /// that use [msvcrt][2]. This includes programs built with both
    /// MinGW and MSVC.
    ///
    /// [1]: <https://github.com/msys2/MSYS2-packages/issues/2176>
    /// [2]: <https://msdn.microsoft.com/en-us/library/17w5ykft.aspx>
    #[unstable(feature = "windows_process_extensions_force_quotes", issue = "82227")]
    fn force_quotes(&mut self, enabled: bool) -> &mut process::Command;

    /// Append literal text to the command line without any quoting or escaping.
    ///
    /// This is useful for passing arguments to `cmd.exe /c`, which doesn't follow
    /// `CommandLineToArgvW` escaping rules.
    #[unstable(feature = "windows_process_extensions_raw_arg", issue = "29494")]
    fn raw_arg(&mut self, text_to_append_as_is: &OsStr) -> &mut process::Command;
}

#[stable(feature = "windows_process_extensions", since = "1.16.0")]
impl CommandExt for process::Command {
    fn creation_flags(&mut self, flags: u32) -> &mut process::Command {
        self.as_inner_mut().creation_flags(flags);
        self
    }

    fn force_quotes(&mut self, enabled: bool) -> &mut process::Command {
        self.as_inner_mut().force_quotes(enabled);
        self
    }

    fn raw_arg(&mut self, raw_text: &OsStr) -> &mut process::Command {
        self.as_inner_mut().raw_arg(raw_text);
        self
    }
}
<commit_msg>Use AsRef in CommandExt for raw_arg<commit_after>//! Extensions to `std::process` for Windows.

#![stable(feature = "process_extensions", since = "1.2.0")]

use crate::ffi::OsStr;
use crate::os::windows::io::{AsRawHandle, FromRawHandle, IntoRawHandle, RawHandle};
use crate::process;
use crate::sealed::Sealed;
use crate::sys;
use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner};

#[stable(feature = "process_extensions", since = "1.2.0")]
impl FromRawHandle for process::Stdio {
    unsafe fn from_raw_handle(handle: RawHandle) -> process::Stdio {
        let handle = sys::handle::Handle::new(handle as *mut _);
        let io = sys::process::Stdio::Handle(handle);
        process::Stdio::from_inner(io)
    }
}

#[stable(feature = "process_extensions", since = "1.2.0")]
impl AsRawHandle for process::Child {
    #[inline]
    fn as_raw_handle(&self) -> RawHandle {
        self.as_inner().handle().raw() as *mut _
    }
}

#[stable(feature = "into_raw_os", since = "1.4.0")]
impl IntoRawHandle for process::Child {
    fn into_raw_handle(self) -> RawHandle {
        self.into_inner().into_handle().into_raw() as *mut _
    }
}

#[stable(feature = "process_extensions", since = "1.2.0")]
impl AsRawHandle for process::ChildStdin {
    #[inline]
    fn as_raw_handle(&self) -> RawHandle {
        self.as_inner().handle().raw() as *mut _
    }
}

#[stable(feature = "process_extensions", since = "1.2.0")]
impl AsRawHandle for process::ChildStdout {
    #[inline]
    fn as_raw_handle(&self) -> RawHandle {
        self.as_inner().handle().raw() as *mut _
    }
}

#[stable(feature = "process_extensions", since = "1.2.0")]
impl AsRawHandle for process::ChildStderr {
    #[inline]
    fn as_raw_handle(&self) -> RawHandle {
        self.as_inner().handle().raw() as *mut _
    }
}

#[stable(feature = "into_raw_os", since = "1.4.0")]
impl IntoRawHandle for process::ChildStdin {
    fn into_raw_handle(self) -> RawHandle {
        self.into_inner().into_handle().into_raw() as *mut _
    }
}

#[stable(feature = "into_raw_os", since = "1.4.0")]
impl IntoRawHandle for process::ChildStdout {
    fn into_raw_handle(self) -> RawHandle {
        self.into_inner().into_handle().into_raw() as *mut _
    }
}

#[stable(feature = "into_raw_os", since = "1.4.0")]
impl IntoRawHandle for process::ChildStderr {
    fn into_raw_handle(self) -> RawHandle {
        self.into_inner().into_handle().into_raw() as *mut _
    }
}

/// Windows-specific extensions to [`process::ExitStatus`].
///
/// This trait is sealed: it cannot be implemented outside the standard library.
/// This is so that future additional methods are not breaking changes.
#[stable(feature = "exit_status_from", since = "1.12.0")]
pub trait ExitStatusExt: Sealed {
    /// Creates a new `ExitStatus` from the raw underlying `u32` return value of
    /// a process.
    #[stable(feature = "exit_status_from", since = "1.12.0")]
    fn from_raw(raw: u32) -> Self;
}

#[stable(feature = "exit_status_from", since = "1.12.0")]
impl ExitStatusExt for process::ExitStatus {
    fn from_raw(raw: u32) -> Self {
        process::ExitStatus::from_inner(From::from(raw))
    }
}

/// Windows-specific extensions to the [`process::Command`] builder.
///
/// This trait is sealed: it cannot be implemented outside the standard library.
/// This is so that future additional methods are not breaking changes.
#[stable(feature = "windows_process_extensions", since = "1.16.0")]
pub trait CommandExt: Sealed {
    /// Sets the [process creation flags][1] to be passed to `CreateProcess`.
    ///
    /// These will always be ORed with `CREATE_UNICODE_ENVIRONMENT`.
    ///
    /// [1]: https://docs.microsoft.com/en-us/windows/win32/procthread/process-creation-flags
    #[stable(feature = "windows_process_extensions", since = "1.16.0")]
    fn creation_flags(&mut self, flags: u32) -> &mut process::Command;

    /// Forces all arguments to be wrapped in quote (`"`) characters.
    ///
    /// This is useful for passing arguments to [MSYS2/Cygwin][1] based
    /// executables: these programs will expand unquoted arguments containing
    /// wildcard characters (`?` and `*`) by searching for any file paths
    /// matching the wildcard pattern.
    ///
    /// Adding quotes has no effect when passing arguments to programs
    /// that use [msvcrt][2]. This includes programs built with both
    /// MinGW and MSVC.
    ///
    /// [1]: <https://github.com/msys2/MSYS2-packages/issues/2176>
    /// [2]: <https://msdn.microsoft.com/en-us/library/17w5ykft.aspx>
    #[unstable(feature = "windows_process_extensions_force_quotes", issue = "82227")]
    fn force_quotes(&mut self, enabled: bool) -> &mut process::Command;

    /// Append literal text to the command line without any quoting or escaping.
    ///
    /// This is useful for passing arguments to `cmd.exe /c`, which doesn't follow
    /// `CommandLineToArgvW` escaping rules.
    #[unstable(feature = "windows_process_extensions_raw_arg", issue = "29494")]
    fn raw_arg<S: AsRef<OsStr>>(&mut self, text_to_append_as_is: S) -> &mut process::Command;
}

#[stable(feature = "windows_process_extensions", since = "1.16.0")]
impl CommandExt for process::Command {
    fn creation_flags(&mut self, flags: u32) -> &mut process::Command {
        self.as_inner_mut().creation_flags(flags);
        self
    }

    fn force_quotes(&mut self, enabled: bool) -> &mut process::Command {
        self.as_inner_mut().force_quotes(enabled);
        self
    }

    fn raw_arg<S: AsRef<OsStr>>(&mut self, raw_text: S) -> &mut process::Command {
        self.as_inner_mut().raw_arg(raw_text.as_ref());
        self
    }
}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io
#![allow(
    clippy::unused_unit,
    clippy::needless_return,
    clippy::never_loop,
    clippy::option_map_unit_fn
)]
use super::osm_utils::get_way_coord;
use super::OsmPbfReader;
use crate::admin_geofinder::AdminGeoFinder;
use crate::{labels, settings, utils, Error};
use cosmogony::ZoneType;
use failure::ResultExt;
use osmpbfreader::{OsmId, StoreObjs};
use slog_scope::info;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::ops::Deref;
use std::sync::Arc;

use super::osm_store::{Getter, ObjWrapper};

#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
#[allow(dead_code)]
pub enum Kind {
    Node = 0,
    Way = 1,
    Relation = 2,
}

pub fn streets(
    pbf: &mut OsmPbfReader,
    admins_geofinder: &AdminGeoFinder,
    settings: &settings::osm2mimir::Settings,
) -> Result<Vec<mimir::Street>, Error> {
    let invalid_highways = settings
        .street
        .as_ref()
        .and_then(|street| street.exclusion.highway.as_deref())
        .unwrap_or(&[]);

    let is_valid_highway = |tag: &str| -> bool { !invalid_highways.iter().any(|k| k == tag) };

    let invalid_public_transports = settings
        .street
        .as_ref()
        .and_then(|street| street.exclusion.public_transport.as_deref())
        .unwrap_or(&[]);

    let is_valid_public_transport =
        |tag: &str| -> bool { !invalid_public_transports.iter().any(|k| k == tag) };

    // For the object to be a valid street, it needs to be an osm highway of a valid type,
    // or a relation of type associatedStreet.
    let is_valid_obj = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Way(ref way) => {
                let has_valid_highway_tag = way
                    .tags
                    .get("highway")
                    .map_or(false, |v| !v.is_empty() && is_valid_highway(v));
                let has_valid_public_transport_tag = way
                    .tags
                    .get("public_transport")
                    .map_or(false, |v| !v.is_empty() && is_valid_public_transport(v));
                let has_valid_name_tag = way.tags.get("name").map_or(false, |v| !v.is_empty());
                has_valid_name_tag && (has_valid_highway_tag || has_valid_public_transport_tag)
            }
            osmpbfreader::OsmObj::Relation(ref rel) => rel
                .tags
                .get("type")
                .map_or(false, |v| v == "associatedStreet"),
            _ => false,
        }
    };

    info!("reading pbf...");
    #[cfg(feature = "db-storage")]
    let mut objs_map = ObjWrapper::new(&settings.database)?;
    #[cfg(not(feature = "db-storage"))]
    let mut objs_map = ObjWrapper::new()?;

    pbf.get_objs_and_deps_store(is_valid_obj, &mut objs_map)
        .context("Error occurred when reading pbf")?;
    info!("reading pbf done.");

    // Builder for street object
    let build_street =
        |id: String, name: String, coord: mimir::Coord, admins: Vec<Arc<mimir::Admin>>| {
            let admins_iter = admins.iter().map(Deref::deref);
            let country_codes = utils::find_country_codes(admins_iter.clone());
            mimir::Street {
                id,
                label: labels::format_street_label(&name, admins_iter, &country_codes),
                name,
                weight: 0.,
                zip_codes: utils::get_zip_codes_from_admins(&admins),
                administrative_regions: admins,
                coord,
                approx_coord: Some(coord.into()),
                distance: None,
                country_codes,
                context: None,
            }
        };

    // Return an iterator giving documents that will be inserted for a given
    // street: one for each hierarchy of admins.
    let build_streets_for_admins =
        move |name: String, id, kind, mut all_admins: Vec<Vec<_>>, coord| {
            let single_output = all_admins.len() <= 1;
            all_admins.sort_unstable(); // sort admins to make id deterministic
            all_admins.into_iter().enumerate().map(move |(i, admins)| {
                let doc_id = {
                    if single_output {
                        format!("street:osm:{}:{}", kind, id)
                    } else {
                        format!("street:osm:{}:{}-{}", kind, id, i)
                    }
                };

                build_street(doc_id, name.clone(), coord, admins)
            })
        };

    // List of outputed streets
    let mut street_list = Vec::new();

    // Sometimes, streets can be divided into several "way"s that still have the same street name.
    // The reason why a street is divided may be that a part of the street become
    // a bridge/tunnel/etc. In this case, a "relation" tagged with (type = associatedStreet) is used
    // to group all these "way"s. In order not to have duplicates in autocompletion, we should tag
    // the osm ways in the relation not to index them twice.
    let mut street_in_relation = HashSet::new();

    objs_map.for_each_filter(Kind::Relation, |obj| {
        let rel = obj.relation().expect("invalid relation filter");
        let rel_name = rel.tags.get("name");

        // Add osmid of all the relation members in the set.
        // Then, we won't create any street for the ways that belong to this relation.

        street_in_relation.extend(
            rel.refs
                .iter()
                .map(|ref_obj| ref_obj.member)
                .filter(OsmId::is_way),
        );

        let rel_street = rel
            .refs
            .iter()
            .filter(|ref_obj| ref_obj.member.is_way() && &ref_obj.role == "street")
            .filter_map(|ref_obj| {
                let obj = objs_map.get(&ref_obj.member)?;
                let way = obj.way()?;
                let coord = get_way_coord(&objs_map, &way);
                let name = rel_name.or_else(|| way.tags.get("name"))?;

                Some(build_streets_for_admins(
                    name.to_string(),
                    rel.id.0,
                    "relation",
                    get_street_admin(admins_geofinder, &objs_map, &way),
                    coord,
                ))
            })
            .next();

        if let Some(street) = rel_street {
            street_list.extend(street);
        }
    });

    // We merge all the ways with same `way_name` and `admin list of level(=city_level)`
    // We use a Map to keep track of the way of smallest Id for a given pair of "name + cities list"
    let mut name_admin_map = BTreeMap::new();

    objs_map.for_each_filter(Kind::Way, |obj| {
        let osmid = obj.id();
        let way = obj.way().expect("invalid way filter");

        if street_in_relation.contains(&osmid) {
            return;
        }

        if let Some(name) = way.tags.get("name") {
            for admins in get_street_admin(admins_geofinder, &objs_map, way) {
                // Discriminate ways with same names by city
                if let Some(city) = admins
                    .iter()
                    .find(|admin| admin.is_city())
                    .map(|city| city.id.to_string())
                {
                    name_admin_map
                        .entry((name.to_string(), city))
                        .and_modify(|(stored_id, stored_admins)| {
                            if *stored_id > osmid {
                                *stored_id = std::cmp::min(*stored_id, osmid);
                                *stored_admins = admins.clone();
                            }
                        })
                        .or_insert((osmid, admins));
                }
            }
        }
    });

    // For each street, construct the list of admins it will be added in.
    // This step ensure that documents have distinguishable IDs if they are
    // added for the same street but different admins.
    let mut all_admins_for_street = HashMap::new();

    for (_, (min_id, admins)) in name_admin_map {
        all_admins_for_street
            .entry(min_id)
            .or_insert_with(Vec::new)
            .push(admins);
    }

    street_list.extend(
        all_admins_for_street
            .into_iter()
            .filter_map(|(id, all_admins)| {
                let obj = objs_map.get(&id)?;
                let way = obj.way()?;

                Some(build_streets_for_admins(
                    way.tags.get("name")?.to_string(),
                    way.id.0,
                    "way",
                    all_admins,
                    get_way_coord(&objs_map, way),
                ))
            })
            .flatten(),
    );

    Ok(street_list)
}

/// Returns branches of admins encompassing the street `way`.
fn get_street_admin<T: StoreObjs + Getter>(
    admins_geofinder: &AdminGeoFinder,
    obj_map: &T,
    way: &osmpbfreader::objects::Way,
) -> Vec<Vec<Arc<mimir::Admin>>> {
    let nb_nodes = way.nodes.len();

    // To avoid corner cases where the ends of the way are near
    // administrative boundaries, the geofinder is called
    // preferably on a middle node.
    let (nodes_left, nodes_right) = (
        way.nodes[..nb_nodes / 2].iter(),
        way.nodes[nb_nodes / 2..].iter(),
    );

    nodes_right
        .chain(nodes_left)
        .filter_map(|node_id| obj_map.get(&(*node_id).into()))
        .filter_map(|node_obj| {
            node_obj.node().map(|node| geo_types::Coordinate {
                x: node.lon(),
                y: node.lat(),
            })
        })
        .next()
        .map_or_else(Vec::new, |coord| {
            // If the coords are part of several cities or suburbs, they are
            // all part of the output together with their parents. For
            // performance reasons, if the admin hierarchy is built of zones
            // bigger than cities, at most one result will belong to the output.
            admins_geofinder.get_admins_if(&coord, |admin| {
                admin
                    .zone_type
                    .map(|zt| zt <= ZoneType::City)
                    .unwrap_or(false)
            })
        })
}

pub fn compute_street_weight(streets: &mut Vec<mimir::Street>) {
    for st in streets {
        for admin in &mut st.administrative_regions {
            if admin.is_city() {
                st.weight = admin.weight;
                break;
            }
        }
    }
}
<commit_msg>[Fixed] Allow import of valid highways without public transport<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io
#![allow(
    clippy::unused_unit,
    clippy::needless_return,
    clippy::never_loop,
    clippy::option_map_unit_fn
)]
use super::osm_utils::get_way_coord;
use super::OsmPbfReader;
use crate::admin_geofinder::AdminGeoFinder;
use crate::{labels, settings, utils, Error};
use cosmogony::ZoneType;
use failure::ResultExt;
use osmpbfreader::{OsmId, StoreObjs};
use slog_scope::info;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::ops::Deref;
use std::sync::Arc;

use super::osm_store::{Getter, ObjWrapper};

#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
#[allow(dead_code)]
pub enum Kind {
    Node = 0,
    Way = 1,
    Relation = 2,
}

pub fn streets(
    pbf: &mut OsmPbfReader,
    admins_geofinder: &AdminGeoFinder,
    settings: &settings::osm2mimir::Settings,
) -> Result<Vec<mimir::Street>, Error> {
    let invalid_highways = settings
        .street
        .as_ref()
        .and_then(|street| street.exclusion.highway.as_deref())
        .unwrap_or(&[]);

    let is_valid_highway = |tag: &str| -> bool { !invalid_highways.iter().any(|k| k == tag) };

    let invalid_public_transports = settings
        .street
        .as_ref()
        .and_then(|street| street.exclusion.public_transport.as_deref())
        .unwrap_or(&[]);

    let is_valid_public_transport =
        |tag: &str| -> bool { !invalid_public_transports.iter().any(|k| k == tag) };

    // For the object to be a valid street, it needs to be an osm highway of a valid type,
    // or a relation of type associatedStreet.
    let is_valid_obj = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Way(ref way) => {
                let has_valid_highway_tag = way
                    .tags
                    .get("highway")
                    .map_or(false, |v| !v.is_empty() && is_valid_highway(v));
                let has_no_excluded_public_transport_tag = way
                    .tags
                    .get("public_transport")
                    .map_or(true, |v| is_valid_public_transport(v));
                let has_valid_name_tag = way.tags.get("name").map_or(false, |v| !v.is_empty());
                has_valid_name_tag
                    && (has_valid_highway_tag || has_no_excluded_public_transport_tag)
            }
            osmpbfreader::OsmObj::Relation(ref rel) => rel
                .tags
                .get("type")
                .map_or(false, |v| v == "associatedStreet"),
            _ => false,
        }
    };

    info!("reading pbf...");
    #[cfg(feature = "db-storage")]
    let mut objs_map = ObjWrapper::new(&settings.database)?;
    #[cfg(not(feature = "db-storage"))]
    let mut objs_map = ObjWrapper::new()?;

    pbf.get_objs_and_deps_store(is_valid_obj, &mut objs_map)
        .context("Error occurred when reading pbf")?;
    info!("reading pbf done.");

    // Builder for street object
    let build_street =
        |id: String, name: String, coord: mimir::Coord, admins: Vec<Arc<mimir::Admin>>| {
            let admins_iter = admins.iter().map(Deref::deref);
            let country_codes = utils::find_country_codes(admins_iter.clone());
            mimir::Street {
                id,
                label: labels::format_street_label(&name, admins_iter, &country_codes),
                name,
                weight: 0.,
                zip_codes: utils::get_zip_codes_from_admins(&admins),
                administrative_regions: admins,
                coord,
                approx_coord: Some(coord.into()),
                distance: None,
                country_codes,
                context: None,
            }
        };

    // Return an iterator giving documents that will be inserted for a given
    // street: one for each hierarchy of admins.
    let build_streets_for_admins =
        move |name: String, id, kind, mut all_admins: Vec<Vec<_>>, coord| {
            let single_output = all_admins.len() <= 1;
            all_admins.sort_unstable(); // sort admins to make id deterministic
            all_admins.into_iter().enumerate().map(move |(i, admins)| {
                let doc_id = {
                    if single_output {
                        format!("street:osm:{}:{}", kind, id)
                    } else {
                        format!("street:osm:{}:{}-{}", kind, id, i)
                    }
                };

                build_street(doc_id, name.clone(), coord, admins)
            })
        };

    // List of outputed streets
    let mut street_list = Vec::new();

    // Sometimes, streets can be divided into several "way"s that still have the same street name.
    // The reason why a street is divided may be that a part of the street become
    // a bridge/tunnel/etc. In this case, a "relation" tagged with (type = associatedStreet) is used
    // to group all these "way"s. In order not to have duplicates in autocompletion, we should tag
    // the osm ways in the relation not to index them twice.
    let mut street_in_relation = HashSet::new();

    objs_map.for_each_filter(Kind::Relation, |obj| {
        let rel = obj.relation().expect("invalid relation filter");
        let rel_name = rel.tags.get("name");

        // Add osmid of all the relation members in the set.
        // Then, we won't create any street for the ways that belong to this relation.

        street_in_relation.extend(
            rel.refs
                .iter()
                .map(|ref_obj| ref_obj.member)
                .filter(OsmId::is_way),
        );

        let rel_street = rel
            .refs
            .iter()
            .filter(|ref_obj| ref_obj.member.is_way() && &ref_obj.role == "street")
            .filter_map(|ref_obj| {
                let obj = objs_map.get(&ref_obj.member)?;
                let way = obj.way()?;
                let coord = get_way_coord(&objs_map, &way);
                let name = rel_name.or_else(|| way.tags.get("name"))?;

                Some(build_streets_for_admins(
                    name.to_string(),
                    rel.id.0,
                    "relation",
                    get_street_admin(admins_geofinder, &objs_map, &way),
                    coord,
                ))
            })
            .next();

        if let Some(street) = rel_street {
            street_list.extend(street);
        }
    });

    // We merge all the ways with same `way_name` and `admin list of level(=city_level)`
    // We use a Map to keep track of the way of smallest Id for a given pair of "name + cities list"
    let mut name_admin_map = BTreeMap::new();

    objs_map.for_each_filter(Kind::Way, |obj| {
        let osmid = obj.id();
        let way = obj.way().expect("invalid way filter");

        if street_in_relation.contains(&osmid) {
            return;
        }

        if let Some(name) = way.tags.get("name") {
            for admins in get_street_admin(admins_geofinder, &objs_map, way) {
                // Discriminate ways with same names by city
                if let Some(city) = admins
                    .iter()
                    .find(|admin| admin.is_city())
                    .map(|city| city.id.to_string())
                {
                    name_admin_map
                        .entry((name.to_string(), city))
                        .and_modify(|(stored_id, stored_admins)| {
                            if *stored_id > osmid {
                                *stored_id = std::cmp::min(*stored_id, osmid);
                                *stored_admins = admins.clone();
                            }
                        })
                        .or_insert((osmid, admins));
                }
            }
        }
    });

    // For each street, construct the list of admins it will be added in.
    // This step ensure that documents have distinguishable IDs if they are
    // added for the same street but different admins.
    let mut all_admins_for_street = HashMap::new();

    for (_, (min_id, admins)) in name_admin_map {
        all_admins_for_street
            .entry(min_id)
            .or_insert_with(Vec::new)
            .push(admins);
    }

    street_list.extend(
        all_admins_for_street
            .into_iter()
            .filter_map(|(id, all_admins)| {
                let obj = objs_map.get(&id)?;
                let way = obj.way()?;

                Some(build_streets_for_admins(
                    way.tags.get("name")?.to_string(),
                    way.id.0,
                    "way",
                    all_admins,
                    get_way_coord(&objs_map, way),
                ))
            })
            .flatten(),
    );

    Ok(street_list)
}

/// Returns branches of admins encompassing the street `way`.
fn get_street_admin<T: StoreObjs + Getter>(
    admins_geofinder: &AdminGeoFinder,
    obj_map: &T,
    way: &osmpbfreader::objects::Way,
) -> Vec<Vec<Arc<mimir::Admin>>> {
    let nb_nodes = way.nodes.len();

    // To avoid corner cases where the ends of the way are near
    // administrative boundaries, the geofinder is called
    // preferably on a middle node.
    let (nodes_left, nodes_right) = (
        way.nodes[..nb_nodes / 2].iter(),
        way.nodes[nb_nodes / 2..].iter(),
    );

    nodes_right
        .chain(nodes_left)
        .filter_map(|node_id| obj_map.get(&(*node_id).into()))
        .filter_map(|node_obj| {
            node_obj.node().map(|node| geo_types::Coordinate {
                x: node.lon(),
                y: node.lat(),
            })
        })
        .next()
        .map_or_else(Vec::new, |coord| {
            // If the coords are part of several cities or suburbs, they are
            // all part of the output together with their parents. For
            // performance reasons, if the admin hierarchy is built of zones
            // bigger than cities, at most one result will belong to the output.
            admins_geofinder.get_admins_if(&coord, |admin| {
                admin
                    .zone_type
                    .map(|zt| zt <= ZoneType::City)
                    .unwrap_or(false)
            })
        })
}

pub fn compute_street_weight(streets: &mut Vec<mimir::Street>) {
    for st in streets {
        for admin in &mut st.administrative_regions {
            if admin.is_city() {
                st.weight = admin.weight;
                break;
            }
        }
    }
}
<|endoftext|>


<commit_before>mod keys;
mod operator;
mod primitive;

pub use self::{
    keys::{Key, KeyBuf, KeyIterator, TypeError},
    operator::Operator,
    primitive::Primitive,
};

/// Given an valid assignment expression, this will split it into `keys`,
/// `operator`, `values`.
pub fn assignment_lexer(statement: &str) -> (Option<&str>, Option<Operator>, Option<&str>) {
    let statement = statement.trim();
    if statement.is_empty() {
        return (None, None, None);
    }

    let (mut read, mut start) = (0, 0);
    let as_bytes = statement.as_bytes();
    let mut bytes = statement.bytes().peekable();
    let mut operator = None;

    while let Some(byte) = bytes.next() {
        operator = Some(Operator::Equal);
        if b'=' == byte {
            if bytes.peek().is_none() {
                return (Some(&statement[..read].trim()), Some(Operator::Equal), None);
            }
            start = read;
            read += 1;
            break;
        }

        if let Some((op, found)) = find_operator(as_bytes, read) {
            operator = Some(op);
            start = read;
            read = found;
            break;
        }

        read += 1;
    }

    if statement.len() == read {
        return (Some(statement.trim()), None, None);
    }

    let keys = statement[..start].trim_right();

    let values = &statement[read..];
    (Some(keys), operator, Some(values.trim()))
}

fn find_operator(bytes: &[u8], read: usize) -> Option<(Operator, usize)> {
    if bytes.len() < read + 3 {
        None
    } else if bytes[read + 1] == b'=' {
        Operator::parse_single(bytes[read]).map(|op| (op, read + 2))
    } else if bytes[read + 2] == b'=' {
        Operator::parse_double(&bytes[read..=read + 1]).map(|op| (op, read + 3))
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn assignment_splitting() {
        assert_eq!(assignment_lexer(""), (None, None, None));
        assert_eq!(assignment_lexer("abc"), (Some("abc"), None, None));

        assert_eq!(assignment_lexer("abc+=def"), (Some("abc"), Some(Operator::Add), Some("def")));

        assert_eq!(assignment_lexer("a+=b"), (Some("a"), Some(Operator::Add), Some("b")));

        assert_eq!(assignment_lexer("a=b"), (Some("a"), Some(Operator::Equal), Some("b")));

        assert_eq!(assignment_lexer("abc ="), (Some("abc"), Some(Operator::Equal), None));

        assert_eq!(assignment_lexer("abc =  "), (Some("abc"), Some(Operator::Equal), None));

        assert_eq!(
            assignment_lexer("abc = def"),
            (Some("abc"), Some(Operator::Equal), Some("def"))
        );

        assert_eq!(assignment_lexer("abc=def"), (Some("abc"), Some(Operator::Equal), Some("def")));

        assert_eq!(
            assignment_lexer("def ghi += 124 523"),
            (Some("def ghi"), Some(Operator::Add), Some("124 523"))
        )
    }

    #[test]
    fn assignment_assignments() {
        assert_eq!(
            assignment_lexer("a ?= b"),
            (Some("a"), Some(Operator::OptionalEqual), Some("b"))
        );

        assert_eq!(
            assignment_lexer("abc def ?= 123 456"),
            (Some("abc def"), Some(Operator::OptionalEqual), Some("123 456"))
        );
    }

    #[test]
    fn arithmetic_assignments() {
        assert_eq!(
            assignment_lexer("abc //= def"),
            (Some("abc"), Some(Operator::IntegerDivide), Some("def"))
        );

        assert_eq!(
            assignment_lexer("abc **= def"),
            (Some("abc"), Some(Operator::Exponent), Some("def"))
        );

        assert_eq!(assignment_lexer("abc += def"), (Some("abc"), Some(Operator::Add), Some("def")));

        assert_eq!(
            assignment_lexer("abc -= def"),
            (Some("abc"), Some(Operator::Subtract), Some("def"))
        );

        assert_eq!(
            assignment_lexer("abc /= def"),
            (Some("abc"), Some(Operator::Divide), Some("def"))
        );

        assert_eq!(
            assignment_lexer("abc *= def"),
            (Some("abc"), Some(Operator::Multiply), Some("def"))
        );
    }

    #[test]
    fn concatenate_assignments() {
        assert_eq!(
            assignment_lexer("abc ++= def"),
            (Some("abc"), Some(Operator::Concatenate), Some("def"))
        );

        assert_eq!(
            assignment_lexer("abc::=def"),
            (Some("abc"), Some(Operator::ConcatenateHead), Some("def"))
        );
    }

    #[test]
    fn filter_assignment() {
        assert_eq!(
            assignment_lexer("abc \\\\= def"),
            (Some("abc"), Some(Operator::Filter), Some("def"))
        )
    }
}
<commit_msg>Resolve "Out of bounds panics on map assignments"<commit_after>mod keys;
mod operator;
mod primitive;

pub use self::{
    keys::{Key, KeyBuf, KeyIterator, TypeError},
    operator::Operator,
    primitive::Primitive,
};

/// Given an valid assignment expression, this will split it into `keys`,
/// `operator`, `values`.
pub fn assignment_lexer(statement: &str) -> (Option<&str>, Option<Operator>, Option<&str>) {
    let statement = statement.trim();
    if statement.is_empty() {
        return (None, None, None);
    }

    let (mut read, mut start) = (0, 0);
    let as_bytes = statement.as_bytes();
    let mut bytes = statement.bytes().peekable();
    let mut operator = None;
    let mut delimiter_stack = Vec::new();

    while let Some(byte) = bytes.next() {
        operator = Some(Operator::Equal);

        if is_open_delimiter(byte) {
            delimiter_stack.push(byte);
        } else if delimiter_stack.last().map_or(false, |open| delimiters_match(*open, byte)) {
            delimiter_stack.pop();
        } else if delimiter_stack.is_empty() {
            if b'=' == byte {
                if bytes.peek().is_none() {
                    return (Some(&statement[..read].trim()), Some(Operator::Equal), None);
                }
                start = read;
                read += 1;
                break;
            }

            if let Some((op, found)) = find_operator(as_bytes, read) {
                operator = Some(op);
                start = read;
                read = found;
                break;
            }
        }

        read += 1;
    }

    if statement.len() == read {
        return (Some(statement.trim()), None, None);
    }

    let keys = statement[..start].trim_right();

    let values = &statement[read..];
    (Some(keys), operator, Some(values.trim()))
}

fn find_operator(bytes: &[u8], read: usize) -> Option<(Operator, usize)> {
    if bytes.len() < read + 3 {
        None
    } else if bytes[read + 1] == b'=' {
        Operator::parse_single(bytes[read]).map(|op| (op, read + 2))
    } else if bytes[read + 2] == b'=' {
        Operator::parse_double(&bytes[read..=read + 1]).map(|op| (op, read + 3))
    } else {
        None
    }
}

fn is_open_delimiter(byte: u8) -> bool { byte == b'[' }

fn delimiters_match(open: u8, close: u8) -> bool {
    match (open, close) {
        (b'[', b']') => true,
        _ => false,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn assignment_splitting() {
        assert_eq!(assignment_lexer(""), (None, None, None));
        assert_eq!(assignment_lexer("abc"), (Some("abc"), None, None));

        assert_eq!(assignment_lexer("abc+=def"), (Some("abc"), Some(Operator::Add), Some("def")));

        assert_eq!(assignment_lexer("a+=b"), (Some("a"), Some(Operator::Add), Some("b")));

        assert_eq!(assignment_lexer("a=b"), (Some("a"), Some(Operator::Equal), Some("b")));

        assert_eq!(assignment_lexer("abc ="), (Some("abc"), Some(Operator::Equal), None));

        assert_eq!(assignment_lexer("abc =  "), (Some("abc"), Some(Operator::Equal), None));

        assert_eq!(
            assignment_lexer("abc = def"),
            (Some("abc"), Some(Operator::Equal), Some("def"))
        );

        assert_eq!(assignment_lexer("abc=def"), (Some("abc"), Some(Operator::Equal), Some("def")));

        assert_eq!(
            assignment_lexer("def ghi += 124 523"),
            (Some("def ghi"), Some(Operator::Add), Some("124 523"))
        )
    }

    #[test]
    fn assignment_assignments() {
        assert_eq!(
            assignment_lexer("a ?= b"),
            (Some("a"), Some(Operator::OptionalEqual), Some("b"))
        );

        assert_eq!(
            assignment_lexer("abc def ?= 123 456"),
            (Some("abc def"), Some(Operator::OptionalEqual), Some("123 456"))
        );
    }

    #[test]
    fn arithmetic_assignments() {
        assert_eq!(
            assignment_lexer("abc //= def"),
            (Some("abc"), Some(Operator::IntegerDivide), Some("def"))
        );

        assert_eq!(
            assignment_lexer("abc **= def"),
            (Some("abc"), Some(Operator::Exponent), Some("def"))
        );

        assert_eq!(assignment_lexer("abc += def"), (Some("abc"), Some(Operator::Add), Some("def")));

        assert_eq!(
            assignment_lexer("abc -= def"),
            (Some("abc"), Some(Operator::Subtract), Some("def"))
        );

        assert_eq!(
            assignment_lexer("abc /= def"),
            (Some("abc"), Some(Operator::Divide), Some("def"))
        );

        assert_eq!(
            assignment_lexer("abc *= def"),
            (Some("abc"), Some(Operator::Multiply), Some("def"))
        );
    }

    #[test]
    fn concatenate_assignments() {
        assert_eq!(
            assignment_lexer("abc ++= def"),
            (Some("abc"), Some(Operator::Concatenate), Some("def"))
        );

        assert_eq!(
            assignment_lexer("abc::=def"),
            (Some("abc"), Some(Operator::ConcatenateHead), Some("def"))
        );
    }

    #[test]
    fn filter_assignment() {
        assert_eq!(
            assignment_lexer("abc \\\\= def"),
            (Some("abc"), Some(Operator::Filter), Some("def"))
        )
    }

    #[test]
    fn map_assignment() {
        assert_eq!(assignment_lexer("abc[=]"), (Some("abc[=]"), None, None));

        assert_eq!(
            assignment_lexer("abc['='] = '='"),
            (Some("abc['=']"), Some(Operator::Equal), Some("'='"))
        );

        assert_eq!(
            assignment_lexer("abc[=] = []=[]"),
            (Some("abc[=]"), Some(Operator::Equal), Some("[]=[]"))
        );
    }
}
<|endoftext|>


<commit_before>#![feature(plugin, never_type)]
#![plugin(clippy)]
#![deny(diverging_sub_expression)]

#[allow(empty_loop)]
fn diverge() -> ! { loop {} }

struct A;

impl A {
    fn foo(&self) -> ! { diverge() }
}

#[allow(unused_variables, unnecessary_operation)]
fn main() {
    let b = true;
    b || diverge(); //~ ERROR sub-expression diverges
    b || A.foo(); //~ ERROR sub-expression diverges
    let y = (5, diverge(), 6); //~ ERROR sub-expression diverges
    println!("{}", y.1);
}
<commit_msg>also test match statements, return, continue and break<commit_after>#![feature(plugin, never_type)]
#![plugin(clippy)]
#![deny(diverging_sub_expression)]

#[allow(empty_loop)]
fn diverge() -> ! { loop {} }

struct A;

impl A {
    fn foo(&self) -> ! { diverge() }
}

#[allow(unused_variables, unnecessary_operation)]
fn main() {
    let b = true;
    b || diverge(); //~ ERROR sub-expression diverges
    b || A.foo(); //~ ERROR sub-expression diverges
    let y = (5, diverge(), 6); //~ ERROR sub-expression diverges
    println!("{}", y.1);
}

#[allow(dead_code, unused_variables)]
fn foobar() {
    loop {
        let x = match 5 {
            4 => return,
            5 => continue,
            6 => (println!("foo"), return), //~ ERROR sub-expression diverges
            7 => (println!("bar"), continue), //~ ERROR sub-expression diverges
            8 => break,
            9 => diverge(),
            3 => (println!("moo"), diverge()), //~ ERROR sub-expression diverges
            _ => (println!("boo"), break), //~ ERROR sub-expression diverges
        };
    }
}
<|endoftext|>


<commit_before>//! Types for the *m.room.guest_access* event.

use StateEvent;

/// Controls whether guest users are allowed to join rooms.
///
/// This event controls whether guest users are allowed to join rooms. If this event is absent,
/// servers should act as if it is present and has the value `GuestAccess::Forbidden`.
pub type GuestAccessEvent = StateEvent<GuestAccessEventContent, ()>;

/// The payload of a `GuestAccessEvent`.
#[derive(Debug, Deserialize, Serialize)]
pub struct GuestAccessEventContent {
    /// A policy for guest user access to a room.
    pub guest_access: GuestAccess,
}

/// A policy for guest user access to a room.
#[derive(Debug, Deserialize, Serialize)]
pub enum GuestAccess {
    /// Guests are allowed to join the room.
    CanJoin,
    /// Guests are not allowed to join the room.
    Forbidden,
}
<commit_msg>Implement Serialize and Deserialize for GuestAccess.<commit_after>//! Types for the *m.room.guest_access* event.

use std::fmt::{Display, Formatter, Error as FmtError};
use std::str::FromStr;

use serde::{Deserialize, Deserializer, Error as SerdeError, Serialize, Serializer};
use serde::de::Visitor;

use StateEvent;

/// Controls whether guest users are allowed to join rooms.
///
/// This event controls whether guest users are allowed to join rooms. If this event is absent,
/// servers should act as if it is present and has the value `GuestAccess::Forbidden`.
pub type GuestAccessEvent = StateEvent<GuestAccessEventContent, ()>;

/// The payload of a `GuestAccessEvent`.
#[derive(Debug, Deserialize, Serialize)]
pub struct GuestAccessEventContent {
    /// A policy for guest user access to a room.
    pub guest_access: GuestAccess,
}

/// A policy for guest user access to a room.
#[derive(Debug, PartialEq)]
pub enum GuestAccess {
    /// Guests are allowed to join the room.
    CanJoin,

    /// Guests are not allowed to join the room.
    Forbidden,
}

/// An error when attempting to parse an invalid `PresenceState` from a string.
pub struct GuestAccessParseError;

impl Display for GuestAccess {
    fn fmt(&self, f: &mut Formatter) -> Result<(), FmtError> {
        let guest_access_str = match *self {
            GuestAccess::CanJoin => "can_join",
            GuestAccess::Forbidden => "forbidden",
        };

        write!(f, "{}", guest_access_str)
    }
}

impl FromStr for GuestAccess {
    type Err = GuestAccessParseError;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        match s {
            "can_join" => Ok(GuestAccess::CanJoin),
            "forbidden" => Ok(GuestAccess::Forbidden),
            _ => Err(GuestAccessParseError),
        }
    }
}

impl Serialize for GuestAccess {
    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error> where S: Serializer {
        serializer.serialize_str(&self.to_string())
    }
}

impl Deserialize for GuestAccess {
    fn deserialize<D>(deserializer: &mut D) -> Result<Self, D::Error> where D: Deserializer {
        struct GuestAccessVisitor;

        impl Visitor for GuestAccessVisitor {
            type Value = GuestAccess;

            fn visit_str<E>(&mut self, v: &str) -> Result<Self::Value, E> where E: SerdeError {
                v.parse().map_err(|_| {
                    E::invalid_value(v)
                })
            }
        }

        deserializer.deserialize_str(GuestAccessVisitor)
    }
}

#[cfg(test)]
mod tests {
    use serde_json::{from_str, to_string};

    use super::GuestAccess;

    #[test]
    fn guest_access_serializes_to_display_form() {
        assert_eq!(
            to_string(&GuestAccess::CanJoin).unwrap(),
            r#""can_join""#
        );
    }

    #[test]
    fn guest_access_deserializes_from_display_form() {
        assert_eq!(
            from_str::<GuestAccess>(r#""can_join""#).unwrap(),
            GuestAccess::CanJoin
        );
    }

    #[test]
    fn invalid_guest_access_fails_deserialization() {
        assert!(from_str::<GuestAccess>(r#""bad""#).is_err());
    }
}
<|endoftext|>


<commit_before>use chrono::{DateTime, Utc};

use super::resource::{ResourceKind, CResourceUpdate, SResourceUpdate};
use super::criterion::Criterion;

/// Client -> server messages, deserialize only
#[derive(Debug, Deserialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
#[serde(tag = "type")]
#[serde(deny_unknown_fields)]
pub enum CMessage {
    // Standard messages
    GetResources { serial: u64, ids: Vec<String> },
    Subscribe { serial: u64, ids: Vec<String> },
    Unsubscribe { serial: u64, ids: Vec<String> },
    UpdateResource {
        serial: u64,
        resource: CResourceUpdate,
    },
    RemoveResource { serial: u64, id: String },
    FilterSubscribe {
        serial: u64,
        #[serde(default)]
        kind: ResourceKind,
        criteria: Vec<Criterion>,
    },
    FilterUnsubscribe { serial: u64, filter_serial: u64 },

    // Special messages
    UploadTorrent {
        serial: u64,
        size: u64,
        path: Option<String>,
    },
    UploadMagnet {
        serial: u64,
        uri: String,
        path: Option<String>,
    },
    UploadFiles {
        serial: u64,
        size: u64,
        path: String,
    },
    DownloadFile { serial: u64, id: String },
}

/// Server -> client message, serialize only
#[derive(Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
#[serde(tag = "type")]
pub enum SMessage<'a> {
    // Standard messages
    ResourcesExtant { serial: u64, ids: Vec<&'a str> },
    ResourcesRemoved { serial: u64, ids: Vec<String> },
    UpdateResources { resources: Vec<SResourceUpdate<'a>> },

    // Special messages
    TransferOffer {
        serial: u64,
        expires: DateTime<Utc>,
        token: String,
        size: u64,
    },

    // Error messages
    UnknownResource(Error),
    InvalidResource(Error),
    // InvalidMessage(Error),
    InvalidSchema(Error),
    // InvalidRequest(Error),
    PermissionDenied(Error),
    TransferFailed(Error),
    // ServerError(Error),
}

#[derive(Serialize)]
pub struct Error {
    pub serial: Option<u64>,
    pub reason: String,
}

#[cfg(test)]
mod tests {
    use super::*;
    use super::super::criterion;
    use serde_json;

    #[test]
    fn test_json_repr() {
        let data = r#"
            {
                "type": "FILTER_SUBSCRIBE",
                "serial": 0,
                "criteria": [
                    { "field": "id", "op": "in", "value": [1,2,3] }
                ]
            }
            "#;
        let m = serde_json::from_str(data).unwrap();
        if let CMessage::FilterSubscribe {
            kind: criterion::ResourceKind::Torrent,
            serial: 0,
            criteria: c,
        } = m
        {
            assert_eq!(c[0].field, "id");
            assert_eq!(c[0].op, criterion::Operation::In);
            assert_eq!(c[0].value, criterion::Value::AN(vec![1, 2, 3]));
        } else {
            unreachable!();
        }
    }
}
<commit_msg>Fix failing test<commit_after>use chrono::{DateTime, Utc};

use super::resource::{ResourceKind, CResourceUpdate, SResourceUpdate};
use super::criterion::Criterion;

/// Client -> server messages, deserialize only
#[derive(Debug, Deserialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
#[serde(tag = "type")]
#[serde(deny_unknown_fields)]
pub enum CMessage {
    // Standard messages
    GetResources { serial: u64, ids: Vec<String> },
    Subscribe { serial: u64, ids: Vec<String> },
    Unsubscribe { serial: u64, ids: Vec<String> },
    UpdateResource {
        serial: u64,
        resource: CResourceUpdate,
    },
    RemoveResource { serial: u64, id: String },
    FilterSubscribe {
        serial: u64,
        #[serde(default)]
        kind: ResourceKind,
        criteria: Vec<Criterion>,
    },
    FilterUnsubscribe { serial: u64, filter_serial: u64 },

    // Special messages
    UploadTorrent {
        serial: u64,
        size: u64,
        path: Option<String>,
    },
    UploadMagnet {
        serial: u64,
        uri: String,
        path: Option<String>,
    },
    UploadFiles {
        serial: u64,
        size: u64,
        path: String,
    },
    DownloadFile { serial: u64, id: String },
}

/// Server -> client message, serialize only
#[derive(Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
#[serde(tag = "type")]
pub enum SMessage<'a> {
    // Standard messages
    ResourcesExtant { serial: u64, ids: Vec<&'a str> },
    ResourcesRemoved { serial: u64, ids: Vec<String> },
    UpdateResources { resources: Vec<SResourceUpdate<'a>> },

    // Special messages
    TransferOffer {
        serial: u64,
        expires: DateTime<Utc>,
        token: String,
        size: u64,
    },

    // Error messages
    UnknownResource(Error),
    InvalidResource(Error),
    // InvalidMessage(Error),
    InvalidSchema(Error),
    // InvalidRequest(Error),
    PermissionDenied(Error),
    TransferFailed(Error),
    // ServerError(Error),
}

#[derive(Serialize)]
pub struct Error {
    pub serial: Option<u64>,
    pub reason: String,
}

#[cfg(test)]
mod tests {
    use super::*;
    use super::super::{resource, criterion};
    use serde_json;

    #[test]
    fn test_json_repr() {
        let data = r#"
            {
                "type": "FILTER_SUBSCRIBE",
                "serial": 0,
                "criteria": [
                    { "field": "id", "op": "in", "value": [1,2,3] }
                ]
            }
            "#;
        let m = serde_json::from_str(data).unwrap();
        if let CMessage::FilterSubscribe {
            kind: resource::ResourceKind::Torrent,
            serial: 0,
            criteria: c,
        } = m
        {
            assert_eq!(c[0].field, "id");
            assert_eq!(c[0].op, criterion::Operation::In);
            assert_eq!(c[0].value, criterion::Value::AN(vec![1, 2, 3]));
        } else {
            unreachable!();
        }
    }
}
<|endoftext|>


<commit_before>// The MIT License (MIT)

// Copyright (c) 2015 Y. T. Chung <zonyitoo@gmail.com>

// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

//! Processing unit of a thread

use std::cell::UnsafeCell;
use std::io;
#[cfg(target_os = "linux")]
use std::os::unix::io::AsRawFd;
#[cfg(target_os = "linux")]
use std::convert::From;
// use std::sync::Arc;
use std::thread;
use std::mem;
use std::sync::mpsc::{self, Sender, Receiver};
use std::boxed::FnBox;

use mio::{EventLoop, Evented, Handler, Token, EventSet, PollOpt};
use mio::util::Slab;
#[cfg(target_os = "linux")]
use mio::Io;

// use mio::util::BoundedQueue;

use deque::{BufferPool, Stolen, Worker, Stealer};

use rand;

use chrono::NaiveDateTime;

use scheduler::Scheduler;
use coroutine::{self, Coroutine, State, Handle, SendableCoroutinePtr};
use options::Options;
use runtime::timer::Timer;

const MAX_TOKEN_NUM: usize = 102400;

thread_local!(static PROCESSOR: UnsafeCell<Processor> = UnsafeCell::new(Processor::new()));

/// Processing unit of a thread
pub struct Processor {
    event_loop: EventLoop<Processor>,

    main_coro: Handle,
    cur_running: Option<*mut Coroutine>,
    last_result: Option<coroutine::Result<State>>,

    queue_worker: Worker<SendableCoroutinePtr>,
    queue_stealer: Stealer<SendableCoroutinePtr>,
    neighbor_stealers: Vec<Stealer<SendableCoroutinePtr>>,

    chan_sender: Sender<ProcMessage>,
    chan_receiver: Receiver<ProcMessage>,

    is_scheduling: bool,
    has_ready_tasks: bool,

    coroutine_timer: Timer,

    #[cfg(any(target_os = "linux",
              target_os = "android"))]
    io_slabs: Slab<(*mut Coroutine, Io)>,
    #[cfg(any(target_os = "macos",
              target_os = "freebsd",
              target_os = "dragonfly",
              target_os = "ios",
              target_os = "bitrig",
              target_os = "openbsd"))]
    io_slabs: Slab<*mut Coroutine>,
}

impl Processor {
    fn new() -> Processor {
        Processor::new_with_neighbors(Vec::new())
    }

    fn new_with_neighbors(neigh: Vec<Stealer<SendableCoroutinePtr>>) -> Processor {
        let main_coro = unsafe {
            Coroutine::empty()
        };

        let (worker, stealer) = BufferPool::new().deque();
        let (tx, rx) = mpsc::channel();

        Processor {
            event_loop: EventLoop::new().expect("Unable to create the eventloop"),

            main_coro: main_coro,
            cur_running: None,
            last_result: None,

            queue_worker: worker,
            queue_stealer: stealer,
            neighbor_stealers: neigh,

            chan_sender: tx,
            chan_receiver: rx,

            is_scheduling: false,
            has_ready_tasks: false,

            coroutine_timer: Timer::new(),

            io_slabs: Slab::new(MAX_TOKEN_NUM),
        }
    }

    #[inline]
    pub unsafe fn running(&mut self) -> Option<*mut Coroutine> {
        self.cur_running
    }

    #[inline]
    pub fn set_neighbors(&mut self, neigh: Vec<Stealer<SendableCoroutinePtr>>) {
        self.neighbor_stealers = neigh;
    }

    /// Get the thread local processor
    #[inline]
    pub fn current() -> &'static mut Processor {
        PROCESSOR.with(|p| unsafe { &mut *p.get() })
    }

    #[inline]
    pub fn stealer(&self) -> Stealer<SendableCoroutinePtr> {
        self.queue_stealer.clone()
    }

    #[inline]
    pub fn handle(&self) -> Sender<ProcMessage> {
        self.chan_sender.clone()
    }

    #[inline]
    pub unsafe fn ready(&mut self, coro_ptr: *mut Coroutine) {
        self.has_ready_tasks = true;
        self.queue_worker.push(SendableCoroutinePtr(coro_ptr));
    }

    #[inline]
    pub fn spawn_opts<F>(&mut self, f: Box<F>, opts: Options)
        where F: FnBox() + 'static
    {
        let coro = Coroutine::spawn_opts(f, opts);

        unsafe {
            self.ready(mem::transmute(coro));
        }
        self.sched();
    }

    #[inline]
    pub fn set_last_result(&mut self, r: coroutine::Result<State>) {
        self.last_result = Some(r);
    }

    #[inline]
    unsafe fn run_with_all_local_tasks(&mut self, coro_ptr: *mut Coroutine) {
        let mut hdl = coro_ptr;
        loop {
            let is_suspended = match self.resume(hdl) {
                Ok(State::Suspended) => {
                    true
                },
                Ok(State::Finished) | Err(..) => {
                    Scheduler::finished(hdl);
                    false
                },
                Ok(..) => {
                    false
                }
            };

            // Try to fetch one task from the local queue
            let next_hdl = self.queue_worker.pop();
            if is_suspended {
                // If the current task has to be suspended, then
                // push it back to the local work queue
                self.ready(hdl);
            }

            match next_hdl {
                Some(h) => hdl = h.0,
                None => break
            }
        }
    }

    /// Run the processor
    pub fn schedule(&mut self) -> io::Result<()> {
        self.is_scheduling = true;

        'outerloop:
        loop {
            // 0. Check the mailbox
            while let Ok(msg) = self.chan_receiver.try_recv() {
                match msg {
                    ProcMessage::NewNeighbor(nei) => self.neighbor_stealers.push(nei),
                }
            }

            // 1. Run all tasks in local queue
            if let Some(hdl) = self.queue_worker.pop() {
                unsafe {
                    self.run_with_all_local_tasks(hdl.0);
                }
            } else {
                self.has_ready_tasks = false;
            }

            // 2. Get work from timer
            unsafe {
                while let Some(coro_ptr) = self.coroutine_timer.try_awake() {
                    self.run_with_all_local_tasks(coro_ptr);
                }
            }

            // 3. Well, check if there is some work could be waked up
            if self.io_slabs.count() != 0 {
                // Make the borrow checker happy.
                let proc_ptr: *mut Processor = self;
                if let Err(err) = self.event_loop.run_once(unsafe { &mut *proc_ptr }) {
                    self.is_scheduling = false;
                    error!("EventLoop failed with {:?}", err);
                    return Err(err);
                }

                if self.has_ready_tasks {
                    continue;
                }
            } else if Scheduler::instance().work_count() == 0 {
                break;
            } else {
                // We don't have active tasks in the local queue
                // And we don't have any activated tasks from event loop
            }

            // 4. Randomly steal from neighbors
            let rand_idx = rand::random::<usize>();
            let total_stealers = self.neighbor_stealers.len();
            for idx in (0..self.neighbor_stealers.len()).map(|x| (x + rand_idx) % total_stealers) {
                if let Stolen::Data(hdl) = self.neighbor_stealers[idx].steal() {
                    unsafe {
                        self.run_with_all_local_tasks(hdl.0);
                    }
                    continue 'outerloop;
                }
            }

            thread::sleep_ms(100);
        }

        self.is_scheduling = false;

        Ok(())
    }

    #[inline]
    pub fn resume(&mut self, coro_ptr: *mut Coroutine) -> coroutine::Result<State> {
        self.cur_running = Some(coro_ptr);
        unsafe {
            self.main_coro.yield_to(&mut *coro_ptr);
        }

        match self.last_result.take() {
            None => Ok(State::Suspended),
            Some(r) => r,
        }
    }

    /// Block the current coroutine until the specific time
    #[inline]
    pub fn sleep_until(&mut self, target_time: NaiveDateTime) {
        if let Some(coro_ptr) = unsafe { self.running() } {
            // Set into the timer
            self.coroutine_timer.wait_until(coro_ptr, target_time);

            // Just block it, the handle has already been sent to
            // the timer.
            self.block();
        } else {
            warn!("Called `wait_until` without running coroutine");
        }
    }

    /// Suspended the current running coroutine, equivalent to `Scheduler::sched`
    #[inline]
    pub fn sched(&mut self) {
        match self.cur_running.take() {
            None => {},
            Some(coro_ptr) => unsafe {
                self.set_last_result(Ok(State::Suspended));
                (&mut *coro_ptr).yield_to(&*self.main_coro)
            }
        }
    }

    /// Block the current running coroutine, equivalent to `Scheduler::block`
    #[inline]
    pub fn block(&mut self) {
        match self.cur_running.take() {
            None => {},
            Some(coro_ptr) => unsafe {
                self.set_last_result(Ok(State::Blocked));
                (&mut *coro_ptr).yield_to(&*self.main_coro)
            }
        }
    }

    /// Yield the current running coroutine with specified result
    #[inline]
    pub fn yield_with(&mut self, r: coroutine::Result<State>) {
        match self.cur_running.take() {
            None => {},
            Some(coro_ptr) => unsafe {
                self.set_last_result(r);
                (&mut *coro_ptr).yield_to(&*self.main_coro)
            }
        }
    }
}

#[cfg(any(target_os = "linux",
          target_os = "android"))]
impl Processor {
    /// Register and wait I/O
    pub fn wait_event<E: Evented + AsRawFd>(&mut self, fd: &E, interest: EventSet) -> io::Result<()> {
        let token = self.io_slabs.insert((unsafe { Processor::current().running().unwrap() },
                                               From::from(fd.as_raw_fd()))).unwrap();
        try!(self.event_loop.register_opt(fd, token, interest,
                                          PollOpt::edge()|PollOpt::oneshot()));

        debug!("wait_event: Blocked current Coroutine ...; token={:?}", token);
        self.block();
        debug!("wait_event: Waked up; token={:?}", token);

        Ok(())
    }
}

#[cfg(any(target_os = "linux",
          target_os = "android"))]
impl Handler for Processor {
    type Timeout = ();
    type Message = ();

    fn ready(&mut self, event_loop: &mut EventLoop<Self>, token: Token, events: EventSet) {
        debug!("Got {:?} for {:?}", events, token);

        match self.io_slabs.remove(token) {
            Some((hdl, fd)) => {
                // Linux EPoll needs to explicit EPOLL_CTL_DEL the fd
                event_loop.deregister(&fd).unwrap();
                mem::forget(fd);
                unsafe {
                    self.ready(hdl);
                }
            },
            None => {
                warn!("No coroutine is waiting on readable {:?}", token);
            }
        }
    }
}

#[cfg(any(target_os = "macos",
          target_os = "freebsd",
          target_os = "dragonfly",
          target_os = "ios",
          target_os = "bitrig",
          target_os = "openbsd"))]
impl Processor {
    /// Register and wait I/O
    pub fn wait_event<E: Evented>(&mut self, fd: &E, interest: EventSet) -> io::Result<()> {
        let token = self.io_slabs.insert(unsafe { Processor::current().running().unwrap() }).unwrap();
        try!(self.event_loop.register_opt(fd, token, interest,
                                          PollOpt::edge()|PollOpt::oneshot()));

        debug!("wait_event: Blocked current Coroutine ...; token={:?}", token);
        self.block();
        debug!("wait_event: Waked up; token={:?}", token);

        Ok(())
    }
}

#[cfg(any(target_os = "macos",
          target_os = "freebsd",
          target_os = "dragonfly",
          target_os = "ios",
          target_os = "bitrig",
          target_os = "openbsd"))]
impl Handler for Processor {
    type Timeout = ();
    type Message = ();

    fn ready(&mut self, _: &mut EventLoop<Self>, token: Token, events: EventSet) {
        debug!("Got {:?} for {:?}", events, token);

        match self.io_slabs.remove(token) {
            Some(hdl) => {
                unsafe {
                    self.ready(hdl);
                }
            },
            None => {
                warn!("No coroutine is waiting on readable {:?}", token);
            }
        }
    }
}

pub enum ProcMessage {
    NewNeighbor(Stealer<SendableCoroutinePtr>),
}
<commit_msg>wake up as many as possible<commit_after>// The MIT License (MIT)

// Copyright (c) 2015 Y. T. Chung <zonyitoo@gmail.com>

// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

//! Processing unit of a thread

use std::cell::UnsafeCell;
use std::io;
#[cfg(target_os = "linux")]
use std::os::unix::io::AsRawFd;
#[cfg(target_os = "linux")]
use std::convert::From;
// use std::sync::Arc;
use std::thread;
use std::mem;
use std::sync::mpsc::{self, Sender, Receiver};
use std::boxed::FnBox;

use mio::{EventLoop, Evented, Handler, Token, EventSet, PollOpt};
use mio::util::Slab;
#[cfg(target_os = "linux")]
use mio::Io;

// use mio::util::BoundedQueue;

use deque::{BufferPool, Stolen, Worker, Stealer};

use rand;

use chrono::NaiveDateTime;

use scheduler::Scheduler;
use coroutine::{self, Coroutine, State, Handle, SendableCoroutinePtr};
use options::Options;
use runtime::timer::Timer;

const MAX_TOKEN_NUM: usize = 102400;

thread_local!(static PROCESSOR: UnsafeCell<Processor> = UnsafeCell::new(Processor::new()));

/// Processing unit of a thread
pub struct Processor {
    event_loop: EventLoop<Processor>,

    main_coro: Handle,
    cur_running: Option<*mut Coroutine>,
    last_result: Option<coroutine::Result<State>>,

    queue_worker: Worker<SendableCoroutinePtr>,
    queue_stealer: Stealer<SendableCoroutinePtr>,
    neighbor_stealers: Vec<Stealer<SendableCoroutinePtr>>,

    chan_sender: Sender<ProcMessage>,
    chan_receiver: Receiver<ProcMessage>,

    is_scheduling: bool,
    has_ready_tasks: bool,

    coroutine_timer: Timer,

    #[cfg(any(target_os = "linux",
              target_os = "android"))]
    io_slabs: Slab<(*mut Coroutine, Io)>,
    #[cfg(any(target_os = "macos",
              target_os = "freebsd",
              target_os = "dragonfly",
              target_os = "ios",
              target_os = "bitrig",
              target_os = "openbsd"))]
    io_slabs: Slab<*mut Coroutine>,
}

impl Processor {
    fn new() -> Processor {
        Processor::new_with_neighbors(Vec::new())
    }

    fn new_with_neighbors(neigh: Vec<Stealer<SendableCoroutinePtr>>) -> Processor {
        let main_coro = unsafe {
            Coroutine::empty()
        };

        let (worker, stealer) = BufferPool::new().deque();
        let (tx, rx) = mpsc::channel();

        Processor {
            event_loop: EventLoop::new().expect("Unable to create the eventloop"),

            main_coro: main_coro,
            cur_running: None,
            last_result: None,

            queue_worker: worker,
            queue_stealer: stealer,
            neighbor_stealers: neigh,

            chan_sender: tx,
            chan_receiver: rx,

            is_scheduling: false,
            has_ready_tasks: false,

            coroutine_timer: Timer::new(),

            io_slabs: Slab::new(MAX_TOKEN_NUM),
        }
    }

    #[inline]
    pub unsafe fn running(&mut self) -> Option<*mut Coroutine> {
        self.cur_running
    }

    #[inline]
    pub fn set_neighbors(&mut self, neigh: Vec<Stealer<SendableCoroutinePtr>>) {
        self.neighbor_stealers = neigh;
    }

    /// Get the thread local processor
    #[inline]
    pub fn current() -> &'static mut Processor {
        PROCESSOR.with(|p| unsafe { &mut *p.get() })
    }

    #[inline]
    pub fn stealer(&self) -> Stealer<SendableCoroutinePtr> {
        self.queue_stealer.clone()
    }

    #[inline]
    pub fn handle(&self) -> Sender<ProcMessage> {
        self.chan_sender.clone()
    }

    #[inline]
    pub unsafe fn ready(&mut self, coro_ptr: *mut Coroutine) {
        self.has_ready_tasks = true;
        self.queue_worker.push(SendableCoroutinePtr(coro_ptr));
    }

    #[inline]
    pub fn spawn_opts<F>(&mut self, f: Box<F>, opts: Options)
        where F: FnBox() + 'static
    {
        let coro = Coroutine::spawn_opts(f, opts);

        unsafe {
            self.ready(mem::transmute(coro));
        }
        self.sched();
    }

    #[inline]
    pub fn set_last_result(&mut self, r: coroutine::Result<State>) {
        self.last_result = Some(r);
    }

    #[inline]
    unsafe fn run_with_all_local_tasks(&mut self, coro_ptr: *mut Coroutine) {
        let mut hdl = coro_ptr;
        loop {
            let is_suspended = match self.resume(hdl) {
                Ok(State::Suspended) => {
                    true
                },
                Ok(State::Finished) | Err(..) => {
                    Scheduler::finished(hdl);
                    false
                },
                Ok(..) => {
                    false
                }
            };

            // Try to fetch one task from the local queue
            let next_hdl = self.queue_worker.pop();
            if is_suspended {
                // If the current task has to be suspended, then
                // push it back to the local work queue
                self.ready(hdl);
            }

            match next_hdl {
                Some(h) => hdl = h.0,
                None => break
            }
        }
    }

    /// Run the processor
    pub fn schedule(&mut self) -> io::Result<()> {
        self.is_scheduling = true;

        'outerloop:
        loop {
            // 1. Run all tasks in local queue
            if let Some(hdl) = self.queue_worker.pop() {
                unsafe {
                    self.run_with_all_local_tasks(hdl.0);
                }
            } else {
                self.has_ready_tasks = false;
            }

            // 2. Get work from timer
            unsafe {
                let mut has_waked_up_tasks = false;
                // Wake up as many as possible
                while let Some(coro_ptr) = self.coroutine_timer.try_awake() {
                    self.ready(coro_ptr);
                    has_waked_up_tasks = true;
                }

                if has_waked_up_tasks {
                    continue 'outerloop;
                }
            }

            // 3. Well, check if there are any works could be waked up
            if self.io_slabs.count() != 0 {
                // Make the borrow checker happy.
                let proc_ptr: *mut Processor = self;
                if let Err(err) = self.event_loop.run_once(unsafe { &mut *proc_ptr }) {
                    self.is_scheduling = false;
                    error!("EventLoop failed with {:?}", err);
                    return Err(err);
                }

                if self.has_ready_tasks {
                    continue 'outerloop;
                }
            } else if Scheduler::instance().work_count() == 0 {
                break;
            } else {
                // We don't have active tasks in the local queue
                // And we don't have any activated tasks from event loop
            }

            // 4. Randomly steal from neighbors
            let rand_idx = rand::random::<usize>();
            let total_stealers = self.neighbor_stealers.len();
            for idx in (0..self.neighbor_stealers.len()).map(|x| (x + rand_idx) % total_stealers) {
                if let Stolen::Data(hdl) = self.neighbor_stealers[idx].steal() {
                    unsafe {
                        self.run_with_all_local_tasks(hdl.0);
                    }
                    continue 'outerloop;
                }
            }

            // 5. Check the mailbox
            while let Ok(msg) = self.chan_receiver.try_recv() {
                match msg {
                    ProcMessage::NewNeighbor(nei) => self.neighbor_stealers.push(nei),
                }
            }

            thread::sleep_ms(100);
        }

        self.is_scheduling = false;

        Ok(())
    }

    #[inline]
    pub fn resume(&mut self, coro_ptr: *mut Coroutine) -> coroutine::Result<State> {
        self.cur_running = Some(coro_ptr);
        unsafe {
            self.main_coro.yield_to(&mut *coro_ptr);
        }

        match self.last_result.take() {
            None => Ok(State::Suspended),
            Some(r) => r,
        }
    }

    /// Block the current coroutine until the specific time
    #[inline]
    pub fn sleep_until(&mut self, target_time: NaiveDateTime) {
        if let Some(coro_ptr) = unsafe { self.running() } {
            // Set into the timer
            self.coroutine_timer.wait_until(coro_ptr, target_time);

            // Just block it, the handle has already been sent to
            // the timer.
            self.block();
        } else {
            warn!("Called `wait_until` without running coroutine");
        }
    }

    /// Suspended the current running coroutine, equivalent to `Scheduler::sched`
    #[inline]
    pub fn sched(&mut self) {
        match self.cur_running.take() {
            None => {},
            Some(coro_ptr) => unsafe {
                self.set_last_result(Ok(State::Suspended));
                (&mut *coro_ptr).yield_to(&*self.main_coro)
            }
        }
    }

    /// Block the current running coroutine, equivalent to `Scheduler::block`
    #[inline]
    pub fn block(&mut self) {
        match self.cur_running.take() {
            None => {},
            Some(coro_ptr) => unsafe {
                self.set_last_result(Ok(State::Blocked));
                (&mut *coro_ptr).yield_to(&*self.main_coro)
            }
        }
    }

    /// Yield the current running coroutine with specified result
    #[inline]
    pub fn yield_with(&mut self, r: coroutine::Result<State>) {
        match self.cur_running.take() {
            None => {},
            Some(coro_ptr) => unsafe {
                self.set_last_result(r);
                (&mut *coro_ptr).yield_to(&*self.main_coro)
            }
        }
    }
}

#[cfg(any(target_os = "linux",
          target_os = "android"))]
impl Processor {
    /// Register and wait I/O
    pub fn wait_event<E: Evented + AsRawFd>(&mut self, fd: &E, interest: EventSet) -> io::Result<()> {
        let token = self.io_slabs.insert((unsafe { Processor::current().running().unwrap() },
                                               From::from(fd.as_raw_fd()))).unwrap();
        try!(self.event_loop.register_opt(fd, token, interest,
                                          PollOpt::edge()|PollOpt::oneshot()));

        debug!("wait_event: Blocked current Coroutine ...; token={:?}", token);
        self.block();
        debug!("wait_event: Waked up; token={:?}", token);

        Ok(())
    }
}

#[cfg(any(target_os = "linux",
          target_os = "android"))]
impl Handler for Processor {
    type Timeout = ();
    type Message = ();

    fn ready(&mut self, event_loop: &mut EventLoop<Self>, token: Token, events: EventSet) {
        debug!("Got {:?} for {:?}", events, token);

        match self.io_slabs.remove(token) {
            Some((hdl, fd)) => {
                // Linux EPoll needs to explicit EPOLL_CTL_DEL the fd
                event_loop.deregister(&fd).unwrap();
                mem::forget(fd);
                unsafe {
                    self.ready(hdl);
                }
            },
            None => {
                warn!("No coroutine is waiting on readable {:?}", token);
            }
        }
    }
}

#[cfg(any(target_os = "macos",
          target_os = "freebsd",
          target_os = "dragonfly",
          target_os = "ios",
          target_os = "bitrig",
          target_os = "openbsd"))]
impl Processor {
    /// Register and wait I/O
    pub fn wait_event<E: Evented>(&mut self, fd: &E, interest: EventSet) -> io::Result<()> {
        let token = self.io_slabs.insert(unsafe { Processor::current().running().unwrap() }).unwrap();
        try!(self.event_loop.register_opt(fd, token, interest,
                                          PollOpt::edge()|PollOpt::oneshot()));

        debug!("wait_event: Blocked current Coroutine ...; token={:?}", token);
        self.block();
        debug!("wait_event: Waked up; token={:?}", token);

        Ok(())
    }
}

#[cfg(any(target_os = "macos",
          target_os = "freebsd",
          target_os = "dragonfly",
          target_os = "ios",
          target_os = "bitrig",
          target_os = "openbsd"))]
impl Handler for Processor {
    type Timeout = ();
    type Message = ();

    fn ready(&mut self, _: &mut EventLoop<Self>, token: Token, events: EventSet) {
        debug!("Got {:?} for {:?}", events, token);

        match self.io_slabs.remove(token) {
            Some(hdl) => {
                unsafe {
                    self.ready(hdl);
                }
            },
            None => {
                warn!("No coroutine is waiting on readable {:?}", token);
            }
        }
    }
}

pub enum ProcMessage {
    NewNeighbor(Stealer<SendableCoroutinePtr>),
}
<|endoftext|>


<commit_before>use std::ptr;
use glib::Error;
use glib::object::{Object, Upcast, Wrapper, Ref};
use glib::types::{StaticType, Type};
use glib::translate::*;
use glib::glib_container::GlibContainer;
use secret_service::SecretService;
use secret_item::SecretItem;
use SecretResult;
use util::{lock_object, unlock_object};
use Lock;
use ffi;

/// SecretCollection represents a collection of secret items stored in the Secret Service.
/// A collection can be in a locked or unlocked state. Use `Lock::lock()` or `Lock::unlock()` to lock or unlock the collection.
/// Use `get_items()` to lookup the items in the collection. There may not be any items exposed when the collection is locked.
pub struct SecretCollection(Ref);

impl SecretCollection {

    /// Lookup which collection is assigned to this alias.
    /// Aliases help determine well known collections, such as 'default'.
    /// Returns the collection, or NULL if none assigned to the alias.
    pub fn for_alias(alias: &str) -> SecretResult<SecretCollection>{
        let mut err = ptr::null_mut();
        let ptr = unsafe{ffi::secret_collection_for_alias_sync(ptr::null_mut(), alias.to_glib_none().0, SECRET_COLLECTION_LOAD_ITEMS, ptr::null_mut(), &mut err)};
        if err.is_null(){
            Ok(unsafe { from_glib_full(ptr) })
        } else {
            Err(Error::wrap(err))
        }
    }

    /// Create a new collection in the secret service.
    /// If you specify an alias and a collection with that alias already exists, then a new collection will not be created. The previous one will be returned instead.
    /// Returns the created Collection.
    pub fn create(label: &str, alias: Option<&str>) -> SecretResult<SecretCollection> {
        let mut err = ptr::null_mut();
        let ptr = unsafe{ffi::secret_collection_create_sync(ptr::null_mut(), label.to_glib_none().0, alias.to_glib_none().0, 0, ptr::null_mut(), &mut err)};
        if err.is_null(){
            Ok(unsafe { from_glib_full(ptr) })
        } else {
            Err(Error::wrap(err))
        }
    }

    /// Delete this collection.
    /// The Secret Service may prompt the user.
    pub fn delete(&self) -> SecretResult<()>{
        let mut err = ptr::null_mut();
        unsafe{ffi::secret_collection_delete_sync(self.to_glib_none().0, ptr::null_mut(), &mut err)};
        if err.is_null(){
            Ok(())
        } else {
            Err(Error::wrap(err))
        }
    }

    /// Get the created date and time of the collection.
    /// The return value is the number of seconds since the unix epoch, January 1st 1970.
    pub fn get_created(&self) -> u64 {
        unsafe {ffi::secret_collection_get_created(self.to_glib_none().0)}
    }

    /// Get the modified date and time of the collection.
    /// The return value is the number of seconds since the unix epoch, January 1st 1970.
    pub fn get_modified(&self) -> u64 {
        unsafe {ffi::secret_collection_get_modified(self.to_glib_none().0)}
    }

    /// Get the Secret Service object that this collection was created with.
    pub fn get_service(&self) -> SecretService { //TODO find out if this can return null
        unsafe {
            let ptr = ffi::secret_collection_get_service(self.to_glib_none().0);
            from_glib_none(ptr)
        }
    }

    /// Get if the items are currently loaded. Use `load_items()` to load them.
    pub fn are_items_loaded(&self) -> bool {
        let flags = unsafe {ffi::secret_collection_get_flags(self.to_glib_none().0)};
        flags & SECRET_COLLECTION_LOAD_ITEMS != 0
    }

    /// Get the label of this collection.
    pub fn get_label(&self) -> String {
        unsafe{
            let ptr = ffi::secret_collection_get_label(self.to_glib_none().0);
            from_glib_full(ptr)
        }
    }

    /// Get the SecretItems of the collection
    pub fn get_items(&self) -> Vec<SecretItem> {
        unsafe {
            let glist = ffi::secret_collection_get_items(self.to_glib_none().0);
            Vec::from_glib_full(glist)
        }
    }

    /// Ensure that the SecretCollection proxy has loaded all the items present in the Secret Service.
    pub fn load_items(&self) -> SecretResult<()>{
        unsafe {
            let mut err = ptr::null_mut();
            ffi::secret_collection_load_items_sync(self.to_glib_none().0, ptr::null_mut(), &mut err);
            if err.is_null() {
                Ok(())
            } else {
                Err(Error::wrap(err))
            }
        }
    }

    /// Assign the collection to this alias. Aliases help determine well known collections, such as 'default'.
    pub fn set_alias(&self, alias: &str) -> SecretResult<()>{
        unsafe {
            let mut err = ptr::null_mut();
            ffi::secret_service_set_alias_sync(ptr::null_mut(), alias.to_glib_none().0, self.to_glib_none().0, ptr::null_mut(), &mut err);
            if err.is_null() {
                Ok(())
            } else {
                Err(Error::wrap(err))
            }
        }
    }

    /// Check if the collection is currently locked.
    pub fn is_locked(&self) -> bool {
        let gbool = unsafe {
            ffi::secret_collection_get_locked(self.to_glib_none().0)
        };
        from_glib(gbool)
    }
}

impl StaticType for SecretCollection {
    fn static_type() -> Type{
        unsafe { from_glib(ffi::secret_collection_get_type()) }
    }
}

unsafe impl Upcast<Object> for SecretCollection { }

impl Wrapper for SecretCollection {
    type GlibType = ffi::SecretCollection;
    unsafe fn wrap(r: Ref) -> Self{
        SecretCollection(r)
    }

    fn as_ref(&self) -> &Ref{
        &self.0
    }

    fn unwrap(self) -> Ref{
        self.0
    }
}

impl Lock<SecretCollection> for SecretCollection {

    fn lock(&self) -> SecretResult<Vec<SecretCollection>>{
        lock_object::<SecretCollection>(self)
    }

    fn unlock(&self) -> SecretResult<Vec<SecretCollection>>{
        unlock_object::<SecretCollection>(self)
    }
}

#[allow(dead_code)]
const SECRET_COLLECTION_NONE: i32        = 0;
const SECRET_COLLECTION_LOAD_ITEMS: i32  = 1 << 1;
<commit_msg>Testcases for SecretCollection<commit_after>use std::ptr;
use glib::Error;
use glib::object::{Object, Upcast, Wrapper, Ref};
use glib::types::{StaticType, Type};
use glib::translate::*;
use glib::glib_container::GlibContainer;
use secret_service::SecretService;
use secret_item::SecretItem;
use SecretResult;
use util::{lock_object, unlock_object};
use Lock;
use ffi;

/// SecretCollection represents a collection of secret items stored in the Secret Service.
/// A collection can be in a locked or unlocked state. Use `Lock::lock()` or `Lock::unlock()` to lock or unlock the collection.
/// Use `get_items()` to lookup the items in the collection. There may not be any items exposed when the collection is locked.
pub struct SecretCollection(Ref);

impl SecretCollection {

    /// Lookup which collection is assigned to this alias.
    /// Aliases help determine well known collections, such as 'default'.
    /// Returns the collection, or NULL if none assigned to the alias.
    pub fn for_alias(alias: &str) -> SecretResult<SecretCollection>{
        let mut err = ptr::null_mut();
        let ptr = unsafe{ffi::secret_collection_for_alias_sync(ptr::null_mut(), alias.to_glib_none().0, SECRET_COLLECTION_LOAD_ITEMS, ptr::null_mut(), &mut err)};
        if err.is_null(){
            Ok(unsafe { from_glib_full(ptr) })
        } else {
            Err(Error::wrap(err))
        }
    }

    /// Create a new collection in the secret service.
    /// If you specify an alias and a collection with that alias already exists, then a new collection will not be created. The previous one will be returned instead.
    /// Returns the created Collection.
    pub fn create(label: &str, alias: Option<&str>) -> SecretResult<SecretCollection> {
        let mut err = ptr::null_mut();
        let ptr = unsafe{ffi::secret_collection_create_sync(ptr::null_mut(), label.to_glib_none().0, alias.to_glib_none().0, 0, ptr::null_mut(), &mut err)};
        if err.is_null(){
            Ok(unsafe { from_glib_full(ptr) })
        } else {
            Err(Error::wrap(err))
        }
    }

    /// Delete this collection.
    /// The Secret Service may prompt the user.
    pub fn delete(&self) -> SecretResult<()>{
        let mut err = ptr::null_mut();
        unsafe{ffi::secret_collection_delete_sync(self.to_glib_none().0, ptr::null_mut(), &mut err)};
        if err.is_null(){
            Ok(())
        } else {
            Err(Error::wrap(err))
        }
    }

    /// Get the created date and time of the collection.
    /// The return value is the number of seconds since the unix epoch, January 1st 1970.
    pub fn get_created(&self) -> u64 {
        unsafe {ffi::secret_collection_get_created(self.to_glib_none().0)}
    }

    /// Get the modified date and time of the collection.
    /// The return value is the number of seconds since the unix epoch, January 1st 1970.
    pub fn get_modified(&self) -> u64 {
        unsafe {ffi::secret_collection_get_modified(self.to_glib_none().0)}
    }

    /// Get the Secret Service object that this collection was created with.
    pub fn get_service(&self) -> SecretService {
        unsafe {
            let ptr = ffi::secret_collection_get_service(self.to_glib_none().0);
            from_glib_none(ptr)
        }
    }

    /// Get if the items are currently loaded. Use `load_items()` to load them.
    pub fn are_items_loaded(&self) -> bool {
        let flags = unsafe {ffi::secret_collection_get_flags(self.to_glib_none().0)};
        flags & SECRET_COLLECTION_LOAD_ITEMS != 0
    }

    /// Get the label of this collection.
    pub fn get_label(&self) -> String {
        unsafe{
            let ptr = ffi::secret_collection_get_label(self.to_glib_none().0);
            from_glib_full(ptr)
        }
    }

    /// Get the SecretItems of the collection
    pub fn get_items(&self) -> Vec<SecretItem> {
        unsafe {
            let glist = ffi::secret_collection_get_items(self.to_glib_none().0);
            Vec::from_glib_full(glist)
        }
    }

    /// Ensure that the SecretCollection proxy has loaded all the items present in the Secret Service.
    pub fn load_items(&self) -> SecretResult<()>{
        unsafe {
            let mut err = ptr::null_mut();
            ffi::secret_collection_load_items_sync(self.to_glib_none().0, ptr::null_mut(), &mut err);
            if err.is_null() {
                Ok(())
            } else {
                Err(Error::wrap(err))
            }
        }
    }

    /// Assign the collection to this alias. Aliases help determine well known collections, such as 'default'.
    pub fn set_alias(&self, alias: &str) -> SecretResult<()>{
        unsafe {
            let mut err = ptr::null_mut();
            ffi::secret_service_set_alias_sync(ptr::null_mut(), alias.to_glib_none().0, self.to_glib_none().0, ptr::null_mut(), &mut err);
            if err.is_null() {
                Ok(())
            } else {
                Err(Error::wrap(err))
            }
        }
    }

    /// Check if the collection is currently locked.
    pub fn is_locked(&self) -> bool {
        let gbool = unsafe {
            ffi::secret_collection_get_locked(self.to_glib_none().0)
        };
        from_glib(gbool)
    }
}

impl StaticType for SecretCollection {
    fn static_type() -> Type{
        unsafe { from_glib(ffi::secret_collection_get_type()) }
    }
}

unsafe impl Upcast<Object> for SecretCollection { }

impl Wrapper for SecretCollection {
    type GlibType = ffi::SecretCollection;
    unsafe fn wrap(r: Ref) -> Self{
        SecretCollection(r)
    }

    fn as_ref(&self) -> &Ref{
        &self.0
    }

    fn unwrap(self) -> Ref{
        self.0
    }
}

impl Lock<SecretCollection> for SecretCollection {

    fn lock(&self) -> SecretResult<Vec<SecretCollection>>{
        lock_object::<SecretCollection>(self)
    }

    fn unlock(&self) -> SecretResult<Vec<SecretCollection>>{
        unlock_object::<SecretCollection>(self)
    }
}

#[allow(dead_code)]
const SECRET_COLLECTION_NONE: i32        = 0;
const SECRET_COLLECTION_LOAD_ITEMS: i32  = 1 << 1;

#[cfg(test)]
mod test {
    use glib::types::{StaticType, Type};
    use super::SecretCollection;

    /*
    #[test]
    fn test_sc_create_delete() {
        let sc = SecretCollection::create("cool_label", None).unwrap();
        assert!(!sc.is_locked());
        assert!(sc.are_items_loaded());
        assert_eq!(sc.get_label(), "cool_label");
        assert_eq!(sc.get_items().len(), 0);
        sc.delete().ok().unwrap();
    } */

    #[test]
    pub fn test_sc_static_type() {
        match SecretCollection::static_type() {
            Type::Other(_) => {},
            _ => panic!("Expected Type::Other")
        }
    }
}
<|endoftext|>