<commit_before>use super::{
    child_properties::ChildProperties, imports::Imports, info_base::InfoBase,
    signatures::Signatures, *,
};
use crate::{config::gobjects::GObject, env::Env, library, nameutil::*, traits::*};
use log::info;
use std::ops::Deref;

#[derive(Debug, Default)]
pub struct Info {
    pub base: InfoBase,
    pub c_type: String,
    pub c_class_type: Option<String>,
    pub rust_class_type: Option<String>,
    pub get_type: String,
    pub is_interface: bool,
    pub supertypes: Vec<general::StatusedTypeId>,
    pub final_type: bool,
    pub generate_trait: bool,
    pub trait_name: String,
    pub has_constructors: bool,
    pub has_methods: bool,
    pub has_functions: bool,
    pub signals: Vec<signals::Info>,
    pub notify_signals: Vec<signals::Info>,
    pub properties: Vec<properties::Property>,
    pub builder_properties: Vec<properties::Property>,
    pub child_properties: ChildProperties,
    pub signatures: Signatures,
}

impl Info {
    pub fn has_signals(&self) -> bool {
        self.signals.iter().any(|s| s.trampoline.is_ok())
            || self.notify_signals.iter().any(|s| s.trampoline.is_ok())
    }

    pub fn has_action_signals(&self) -> bool {
        self.signals.iter().any(|s| s.action_emit_name.is_some())
    }
}

impl Deref for Info {
    type Target = InfoBase;

    fn deref(&self) -> &InfoBase {
        &self.base
    }
}

pub fn class(env: &Env, obj: &GObject, deps: &[library::TypeId]) -> Option<Info> {
    info!("Analyzing class {}", obj.name);
    let full_name = obj.name.clone();

    let class_tid = match env.library.find_type(0, &full_name) {
        Some(tid) => tid,
        None => return None,
    };

    let type_ = env.type_(class_tid);

    let name: String = split_namespace_name(&full_name).1.into();

    let klass: &library::Class = match type_.maybe_ref() {
        Some(klass) => klass,
        None => return None,
    };

    let mut imports = Imports::with_defined(&env.library, &name);
    imports.add("glib::translate::*");
    imports.add(env.main_sys_crate_name());
    if obj.generate_display_trait {
        imports.add("std::fmt");
    }

    let supertypes = supertypes::analyze(env, class_tid, &mut imports);

    let final_type = klass.final_type;
    let trait_name = obj
        .trait_name
        .as_ref()
        .cloned()
        .unwrap_or_else(|| format!("{}Ext", name));

    let mut signatures = Signatures::with_capacity(klass.functions.len());

    let mut functions = functions::analyze(
        env,
        &klass.functions,
        class_tid,
        !final_type,
        false,
        obj,
        &mut imports,
        Some(&mut signatures),
        Some(deps),
    );
    let mut specials = special_functions::extract(&mut functions);
    // `copy` will duplicate an object while `clone` just adds a reference
    special_functions::unhide(&mut functions, &specials, special_functions::Type::Copy);
    // these are all automatically derived on objects and compare by pointer. If such functions
    // exist they will provide additional functionality
    for t in &[
        special_functions::Type::Hash,
        special_functions::Type::Equal,
        special_functions::Type::Compare,
        special_functions::Type::ToString,
    ] {
        special_functions::unhide(&mut functions, &specials, *t);
        specials.remove(t);
    }
    special_functions::analyze_imports(&specials, &mut imports);

    let signals = signals::analyze(
        env,
        &klass.signals,
        class_tid,
        !final_type,
        obj,
        &mut imports,
    );
    let (properties, notify_signals) = properties::analyze(
        env,
        &klass.properties,
        class_tid,
        !final_type,
        obj,
        &mut imports,
        &signatures,
        deps,
    );

    let builder_properties =
        class_builder::analyze(env, &klass.properties, class_tid, obj, &mut imports);

    let (version, deprecated_version) = info_base::versions(
        env,
        obj,
        &functions,
        klass.version,
        klass.deprecated_version,
    );

    let child_properties =
        child_properties::analyze(env, obj.child_properties.as_ref(), class_tid, &mut imports);

    let has_methods = functions
        .iter()
        .any(|f| f.kind == library::FunctionKind::Method);
    let has_signals = signals.iter().any(|s| s.trampoline.is_ok())
        || notify_signals.iter().any(|s| s.trampoline.is_ok());

    // There's no point in generating a trait if there are no signals, methods, properties
    // and child properties: it would be empty
    //
    // There's also no point in generating a trait for final types: there are no possible subtypes
    let generate_trait = !final_type
        && (has_signals || has_methods || !properties.is_empty() || !child_properties.is_empty());

    if !builder_properties.is_empty() {
        imports.add("glib::object::Cast");
        imports.add("glib::StaticType");
        imports.add("glib::ToValue");
    }

    if generate_trait {
        imports.add("glib::object::IsA");
    }

    if obj.concurrency == library::Concurrency::SendUnique {
        imports.add("glib::ObjectExt");
    }

    let rust_class_type = Some(format!("{}Class", name));

    let base = InfoBase {
        full_name,
        type_id: class_tid,
        name,
        functions,
        specials,
        imports,
        version,
        deprecated_version,
        cfg_condition: obj.cfg_condition.clone(),
        concurrency: obj.concurrency,
    };

    // patch up trait methods in the symbol table
    if generate_trait {
        let mut symbols = env.symbols.borrow_mut();
        for func in base.methods() {
            if let Some(symbol) = symbols.by_c_name_mut(&func.glib_name) {
                symbol.make_trait_method(&trait_name);
            }
        }
    }

    let has_constructors = !base.constructors().is_empty();
    let has_functions = !base.functions().is_empty();

    let info = Info {
        base,
        c_type: klass.c_type.clone(),
        c_class_type: klass.c_class_type.clone(),
        rust_class_type,
        get_type: klass.glib_get_type.clone(),
        is_interface: false,
        supertypes,
        final_type,
        generate_trait,
        trait_name,
        has_constructors,
        has_methods,
        has_functions,
        signals,
        notify_signals,
        properties,
        builder_properties,
        child_properties,
        signatures,
    };

    Some(info)
}

pub fn interface(env: &Env, obj: &GObject, deps: &[library::TypeId]) -> Option<Info> {
    info!("Analyzing interface {}", obj.name);
    let full_name = obj.name.clone();

    let iface_tid = match env.library.find_type(0, &full_name) {
        Some(tid) => tid,
        None => return None,
    };

    let type_ = env.type_(iface_tid);

    let name: String = split_namespace_name(&full_name).1.into();

    let iface: &library::Interface = match type_.maybe_ref() {
        Some(iface) => iface,
        None => return None,
    };

    let mut imports = Imports::with_defined(&env.library, &name);
    imports.add("glib::translate::*");
    imports.add(env.main_sys_crate_name());
    imports.add("glib::object::IsA");
    if obj.generate_display_trait {
        imports.add("std::fmt");
    }

    let supertypes = supertypes::analyze(env, iface_tid, &mut imports);

    let trait_name = obj
        .trait_name
        .as_ref()
        .cloned()
        .unwrap_or_else(|| format!("{}Ext", name));

    let mut signatures = Signatures::with_capacity(iface.functions.len());

    let functions = functions::analyze(
        env,
        &iface.functions,
        iface_tid,
        true,
        false,
        obj,
        &mut imports,
        Some(&mut signatures),
        Some(deps),
    );

    let signals = signals::analyze(env, &iface.signals, iface_tid, true, obj, &mut imports);
    let (properties, notify_signals) = properties::analyze(
        env,
        &iface.properties,
        iface_tid,
        true,
        obj,
        &mut imports,
        &signatures,
        deps,
    );

    let (version, deprecated_version) = info_base::versions(
        env,
        obj,
        &functions,
        iface.version,
        iface.deprecated_version,
    );

    if obj.concurrency == library::Concurrency::SendUnique {
        imports.add("glib::ObjectExt");
    }

    let base = InfoBase {
        full_name,
        type_id: iface_tid,
        name,
        functions,
        specials: Default::default(),
        imports,
        version,
        deprecated_version,
        cfg_condition: obj.cfg_condition.clone(),
        concurrency: obj.concurrency,
    };

    let has_methods = !base.methods().is_empty();
    let has_functions = !base.functions().is_empty();

    let info = Info {
        base,
        c_type: iface.c_type.clone(),
        c_class_type: None,
        rust_class_type: None,
        get_type: iface.glib_get_type.clone(),
        is_interface: true,
        supertypes,
        final_type: false,
        generate_trait: true,
        trait_name,
        has_methods,
        has_functions,
        signals,
        notify_signals,
        properties,
        signatures,
        ..Default::default()
    };

    Some(info)
}
<commit_msg>analysis: Pass through new builder_postprocess attribute<commit_after>use super::{
    child_properties::ChildProperties, imports::Imports, info_base::InfoBase,
    signatures::Signatures, *,
};
use crate::{config::gobjects::GObject, env::Env, library, nameutil::*, traits::*};
use log::info;
use std::ops::Deref;

#[derive(Debug, Default)]
pub struct Info {
    pub base: InfoBase,
    pub c_type: String,
    pub c_class_type: Option<String>,
    pub rust_class_type: Option<String>,
    pub get_type: String,
    pub is_interface: bool,
    pub supertypes: Vec<general::StatusedTypeId>,
    pub final_type: bool,
    pub generate_trait: bool,
    pub trait_name: String,
    pub has_constructors: bool,
    pub has_methods: bool,
    pub has_functions: bool,
    pub signals: Vec<signals::Info>,
    pub notify_signals: Vec<signals::Info>,
    pub properties: Vec<properties::Property>,
    pub builder_properties: Vec<properties::Property>,
    pub builder_postprocess: Option<String>,
    pub child_properties: ChildProperties,
    pub signatures: Signatures,
}

impl Info {
    pub fn has_signals(&self) -> bool {
        self.signals.iter().any(|s| s.trampoline.is_ok())
            || self.notify_signals.iter().any(|s| s.trampoline.is_ok())
    }

    pub fn has_action_signals(&self) -> bool {
        self.signals.iter().any(|s| s.action_emit_name.is_some())
    }
}

impl Deref for Info {
    type Target = InfoBase;

    fn deref(&self) -> &InfoBase {
        &self.base
    }
}

pub fn class(env: &Env, obj: &GObject, deps: &[library::TypeId]) -> Option<Info> {
    info!("Analyzing class {}", obj.name);
    let full_name = obj.name.clone();

    let class_tid = match env.library.find_type(0, &full_name) {
        Some(tid) => tid,
        None => return None,
    };

    let type_ = env.type_(class_tid);

    let name: String = split_namespace_name(&full_name).1.into();

    let klass: &library::Class = match type_.maybe_ref() {
        Some(klass) => klass,
        None => return None,
    };

    let mut imports = Imports::with_defined(&env.library, &name);
    imports.add("glib::translate::*");
    imports.add(env.main_sys_crate_name());
    if obj.generate_display_trait {
        imports.add("std::fmt");
    }

    let supertypes = supertypes::analyze(env, class_tid, &mut imports);

    let final_type = klass.final_type;
    let trait_name = obj
        .trait_name
        .as_ref()
        .cloned()
        .unwrap_or_else(|| format!("{}Ext", name));

    let mut signatures = Signatures::with_capacity(klass.functions.len());

    let mut functions = functions::analyze(
        env,
        &klass.functions,
        class_tid,
        !final_type,
        false,
        obj,
        &mut imports,
        Some(&mut signatures),
        Some(deps),
    );
    let mut specials = special_functions::extract(&mut functions);
    // `copy` will duplicate an object while `clone` just adds a reference
    special_functions::unhide(&mut functions, &specials, special_functions::Type::Copy);
    // these are all automatically derived on objects and compare by pointer. If such functions
    // exist they will provide additional functionality
    for t in &[
        special_functions::Type::Hash,
        special_functions::Type::Equal,
        special_functions::Type::Compare,
        special_functions::Type::ToString,
    ] {
        special_functions::unhide(&mut functions, &specials, *t);
        specials.remove(t);
    }
    special_functions::analyze_imports(&specials, &mut imports);

    let signals = signals::analyze(
        env,
        &klass.signals,
        class_tid,
        !final_type,
        obj,
        &mut imports,
    );
    let (properties, notify_signals) = properties::analyze(
        env,
        &klass.properties,
        class_tid,
        !final_type,
        obj,
        &mut imports,
        &signatures,
        deps,
    );

    let builder_properties =
        class_builder::analyze(env, &klass.properties, class_tid, obj, &mut imports);

    let (version, deprecated_version) = info_base::versions(
        env,
        obj,
        &functions,
        klass.version,
        klass.deprecated_version,
    );

    let child_properties =
        child_properties::analyze(env, obj.child_properties.as_ref(), class_tid, &mut imports);

    let has_methods = functions
        .iter()
        .any(|f| f.kind == library::FunctionKind::Method);
    let has_signals = signals.iter().any(|s| s.trampoline.is_ok())
        || notify_signals.iter().any(|s| s.trampoline.is_ok());

    // There's no point in generating a trait if there are no signals, methods, properties
    // and child properties: it would be empty
    //
    // There's also no point in generating a trait for final types: there are no possible subtypes
    let generate_trait = !final_type
        && (has_signals || has_methods || !properties.is_empty() || !child_properties.is_empty());

    if !builder_properties.is_empty() {
        imports.add("glib::object::Cast");
        imports.add("glib::StaticType");
        imports.add("glib::ToValue");
    }

    if generate_trait {
        imports.add("glib::object::IsA");
    }

    if obj.concurrency == library::Concurrency::SendUnique {
        imports.add("glib::ObjectExt");
    }

    let rust_class_type = Some(format!("{}Class", name));

    let base = InfoBase {
        full_name,
        type_id: class_tid,
        name,
        functions,
        specials,
        imports,
        version,
        deprecated_version,
        cfg_condition: obj.cfg_condition.clone(),
        concurrency: obj.concurrency,
    };

    // patch up trait methods in the symbol table
    if generate_trait {
        let mut symbols = env.symbols.borrow_mut();
        for func in base.methods() {
            if let Some(symbol) = symbols.by_c_name_mut(&func.glib_name) {
                symbol.make_trait_method(&trait_name);
            }
        }
    }

    let has_constructors = !base.constructors().is_empty();
    let has_functions = !base.functions().is_empty();

    let info = Info {
        base,
        c_type: klass.c_type.clone(),
        c_class_type: klass.c_class_type.clone(),
        rust_class_type,
        get_type: klass.glib_get_type.clone(),
        is_interface: false,
        supertypes,
        final_type,
        generate_trait,
        trait_name,
        has_constructors,
        has_methods,
        has_functions,
        signals,
        notify_signals,
        properties,
        builder_properties,
        builder_postprocess: obj.builder_postprocess.clone(),
        child_properties,
        signatures,
    };

    Some(info)
}

pub fn interface(env: &Env, obj: &GObject, deps: &[library::TypeId]) -> Option<Info> {
    info!("Analyzing interface {}", obj.name);
    let full_name = obj.name.clone();

    let iface_tid = match env.library.find_type(0, &full_name) {
        Some(tid) => tid,
        None => return None,
    };

    let type_ = env.type_(iface_tid);

    let name: String = split_namespace_name(&full_name).1.into();

    let iface: &library::Interface = match type_.maybe_ref() {
        Some(iface) => iface,
        None => return None,
    };

    let mut imports = Imports::with_defined(&env.library, &name);
    imports.add("glib::translate::*");
    imports.add(env.main_sys_crate_name());
    imports.add("glib::object::IsA");
    if obj.generate_display_trait {
        imports.add("std::fmt");
    }

    let supertypes = supertypes::analyze(env, iface_tid, &mut imports);

    let trait_name = obj
        .trait_name
        .as_ref()
        .cloned()
        .unwrap_or_else(|| format!("{}Ext", name));

    let mut signatures = Signatures::with_capacity(iface.functions.len());

    let functions = functions::analyze(
        env,
        &iface.functions,
        iface_tid,
        true,
        false,
        obj,
        &mut imports,
        Some(&mut signatures),
        Some(deps),
    );

    let signals = signals::analyze(env, &iface.signals, iface_tid, true, obj, &mut imports);
    let (properties, notify_signals) = properties::analyze(
        env,
        &iface.properties,
        iface_tid,
        true,
        obj,
        &mut imports,
        &signatures,
        deps,
    );

    let (version, deprecated_version) = info_base::versions(
        env,
        obj,
        &functions,
        iface.version,
        iface.deprecated_version,
    );

    if obj.concurrency == library::Concurrency::SendUnique {
        imports.add("glib::ObjectExt");
    }

    let base = InfoBase {
        full_name,
        type_id: iface_tid,
        name,
        functions,
        specials: Default::default(),
        imports,
        version,
        deprecated_version,
        cfg_condition: obj.cfg_condition.clone(),
        concurrency: obj.concurrency,
    };

    let has_methods = !base.methods().is_empty();
    let has_functions = !base.functions().is_empty();

    let info = Info {
        base,
        c_type: iface.c_type.clone(),
        c_class_type: None,
        rust_class_type: None,
        get_type: iface.glib_get_type.clone(),
        is_interface: true,
        supertypes,
        final_type: false,
        generate_trait: true,
        trait_name,
        has_methods,
        has_functions,
        signals,
        notify_signals,
        properties,
        signatures,
        ..Default::default()
    };

    Some(info)
}
<|endoftext|>


<commit_before>#![deny(warnings)]

extern crate raw_cpuid;
extern crate syscall;

use std::env;
use std::fs::File;
use std::io::{self, Read, Write};

// std::fmt::Write conflicts with std::io::Write, hence the alias
use std::fmt::Write as FmtWrite;

const SECONDS_PER_MINUTE: i64 = 60;
const SECONDS_PER_HOUR: i64 = 3600;
const SECONDS_PER_DAY: i64 = 86400;

fn main() {
    let user = env::var("USER").unwrap_or(String::new());
    let mut hostname = String::new();
    if let Ok(mut file) = File::open("/etc/hostname") {
        let _ = file.read_to_string(&mut hostname);
    }

    let mut uptime_str = String::new();

    let mut ts = syscall::TimeSpec::default();
    if syscall::clock_gettime(syscall::CLOCK_MONOTONIC, &mut ts).is_ok() {
        let uptime = ts.tv_sec;
        let uptime_secs = uptime % 60;
        let uptime_mins = (uptime / SECONDS_PER_MINUTE) % 60;
        let uptime_hours = (uptime / SECONDS_PER_HOUR) % 24;
        let uptime_days = (uptime / SECONDS_PER_DAY) % 365;

        let fmt_result;
        if uptime_days > 0 {
            fmt_result = write!(&mut uptime_str, "{}d {}h {}m {}s", uptime_days,
                                uptime_hours, uptime_mins, uptime_secs);
        } else if uptime_hours > 0 {
            fmt_result = write!(&mut uptime_str, "{}h {}m {}s", uptime_hours,
                                uptime_mins, uptime_secs);
        } else if uptime_mins > 0 {
            fmt_result = write!(&mut uptime_str, "{}m {}s", uptime_mins,
                                uptime_secs);
        } else {
            fmt_result = write!(&mut uptime_str, "{}s", uptime_secs);
        }

        if let Err(_) = fmt_result {
            // We probably don't want to panic! on this error
            println!("error: couldn't parse uptime");
        }
    }

    let mut width = 0;
    let mut height = 0;
    if let Ok(display_name) = env::var("DISPLAY") {
        if let Ok(display) = syscall::open(&display_name, syscall::O_STAT) {
            let mut buf: [u8; 4096] = [0; 4096];
            if let Ok(count) = syscall::fpath(display, &mut buf) {
                let path = unsafe { String::from_utf8_unchecked(Vec::from(&buf[..count])) };
                let res = path.split(":").nth(1).unwrap_or("");
                width = res.split("/").nth(1).unwrap_or("").parse::<i32>().unwrap_or(0);
                height = res.split("/").nth(2).unwrap_or("").parse::<i32>().unwrap_or(0);
            }
            let _ = syscall::close(display);
        }
    }

    let mut cpu = String::new();
    {
        let cpuid = raw_cpuid::CpuId::new();
        if let Some(info) = cpuid.get_extended_function_info() {
            if let Some(brand) = info.processor_brand_string() {
                cpu = brand.trim().to_string();
            }
        }
    }

    let mut ram = String::new();
    {
        let mut stat = syscall::StatVfs::default();
        if let Ok(fd) = syscall::open("memory:", syscall::O_STAT) {
            if syscall::fstatvfs(fd, &mut stat).is_ok() {
                let size = stat.f_blocks * stat.f_bsize as u64;
                let used = (stat.f_blocks - stat.f_bfree) * stat.f_bsize as u64;

                ram = format!("{}MB / {}MB", (used + 1048575)/1048576, (size + 1048575)/1048576);
            }
            let _ = syscall::close(fd);
        }
    }

    let left = [
        "              :+yMMMMy+:              ",
        "         .+dddNMMMMMMMMNddd+.         ",
        "       sydMMMMo/sMMMMs/oMMMMdys       ",
        "    .oMMMdso              osdMMMo.    ",
        "   .+MMd/`   -:::::::`      `/dMM+.   ",
        "  +dMMN.     NMMNNNMMdyo`     .NMMd+  ",
        "  yMMN       NM+   oomMMN       NMMy  ",
        " hNMd.       NM+     `oMN       .dMNh ",
        " dMMh        NM+     `oMN        hMMd ",
        " dMMh        NM+-oooodMMN        hMMd ",
        " dMMh        NM+/MMMMdhs`        hMMd ",
        " hNMd.       NM+`/mMMm+         .dMNh ",
        "  yMMN       NM+   oNMd+        NMMy  ",
        "  +dMMN.     NM+    omMMm     .NMMd+  ",
        "   .+MMd/`   --.     .---   `/dMM+.   ",
        "    .oMMMdso              osdMMMo.    ",
        "       sydMMMMo////////oMMMMdys       ",
        "         .+dddNMMMMMMMMNddd+.         ",
        "              :++++++++:              ",
    ];

    let right = [
        format!("\x1B[1;38;5;75m{}\x1B[0m@\x1B[1;38;5;75m{}\x1B[0m", user, hostname.trim()),
        format!("\x1B[1;38;5;75mOS:\x1B[0m redox-os"),
        format!("\x1B[1;38;5;75mKernel:\x1B[0m redox"),
        format!("\x1B[1;38;5;75mUptime:\x1B[0m {}", uptime_str),
        format!("\x1B[1;38;5;75mShell:\x1B[0m ion"),
        format!("\x1B[1;38;5;75mResolution:\x1B[0m {}x{}", width, height),
        format!("\x1B[1;38;5;75mDE:\x1B[0m orbital"),
        format!("\x1B[1;38;5;75mWM:\x1B[0m orbital"),
        format!("\x1B[1;38;5;75mFont:\x1B[0m unifont"),
        format!("\x1B[1;38;5;75mCPU:\x1B[0m {}", cpu),
        format!("\x1B[1;38;5;75mRAM:\x1B[0m {}", ram)
    ];

    let mut string = String::new();
    for i in 0..left.len() {
        string.push_str("\x1B[1;38;5;75m");
        string.push_str(left[i]);
        string.push_str("  \x1B[0m");
        if let Some(r) = right.get(i) {
            string.push_str(r);
        }
        string.push_str("\n");
    }

    io::stdout().write(string.as_bytes()).unwrap();
}
<commit_msg>screenfetch: Fix hostname, display the actual shell It displayed empty hostname when the scheme of the current working directory was not the file:/ scheme. Also the shell is not guaranteed to be Ion.<commit_after>#![deny(warnings)]

extern crate raw_cpuid;
extern crate syscall;

use std::env;
use std::fs::File;
use std::io::{self, Read, Write};
use std::path::Path;

// std::fmt::Write conflicts with std::io::Write, hence the alias
use std::fmt::Write as FmtWrite;

const SECONDS_PER_MINUTE: i64 = 60;
const SECONDS_PER_HOUR: i64 = 3600;
const SECONDS_PER_DAY: i64 = 86400;

fn main() {
    let user = env::var("USER").unwrap_or(String::new());
    let mut hostname = String::new();
    if let Ok(mut file) = File::open("file:/etc/hostname") {
        let _ = file.read_to_string(&mut hostname);
    }

    let mut uptime_str = String::new();

    let mut ts = syscall::TimeSpec::default();
    if syscall::clock_gettime(syscall::CLOCK_MONOTONIC, &mut ts).is_ok() {
        let uptime = ts.tv_sec;
        let uptime_secs = uptime % 60;
        let uptime_mins = (uptime / SECONDS_PER_MINUTE) % 60;
        let uptime_hours = (uptime / SECONDS_PER_HOUR) % 24;
        let uptime_days = (uptime / SECONDS_PER_DAY) % 365;

        let fmt_result;
        if uptime_days > 0 {
            fmt_result = write!(&mut uptime_str, "{}d {}h {}m {}s", uptime_days,
                                uptime_hours, uptime_mins, uptime_secs);
        } else if uptime_hours > 0 {
            fmt_result = write!(&mut uptime_str, "{}h {}m {}s", uptime_hours,
                                uptime_mins, uptime_secs);
        } else if uptime_mins > 0 {
            fmt_result = write!(&mut uptime_str, "{}m {}s", uptime_mins,
                                uptime_secs);
        } else {
            fmt_result = write!(&mut uptime_str, "{}s", uptime_secs);
        }

        if let Err(_) = fmt_result {
            // We probably don't want to panic! on this error
            println!("error: couldn't parse uptime");
        }
    }

    let mut shell = String::new();
    {
        if let Ok(shell_path) = env::var("SHELL") {
            if let Some(file_name) = Path::new(&shell_path).file_name() {
                shell = file_name.to_str().unwrap_or("").to_string();
            }
        }
    }

    let mut width = 0;
    let mut height = 0;
    if let Ok(display_name) = env::var("DISPLAY") {
        if let Ok(display) = syscall::open(&display_name, syscall::O_STAT) {
            let mut buf: [u8; 4096] = [0; 4096];
            if let Ok(count) = syscall::fpath(display, &mut buf) {
                let path = unsafe { String::from_utf8_unchecked(Vec::from(&buf[..count])) };
                let res = path.split(":").nth(1).unwrap_or("");
                width = res.split("/").nth(1).unwrap_or("").parse::<i32>().unwrap_or(0);
                height = res.split("/").nth(2).unwrap_or("").parse::<i32>().unwrap_or(0);
            }
            let _ = syscall::close(display);
        }
    }

    let mut cpu = String::new();
    {
        let cpuid = raw_cpuid::CpuId::new();
        if let Some(info) = cpuid.get_extended_function_info() {
            if let Some(brand) = info.processor_brand_string() {
                cpu = brand.trim().to_string();
            }
        }
    }

    let mut ram = String::new();
    {
        let mut stat = syscall::StatVfs::default();
        if let Ok(fd) = syscall::open("memory:", syscall::O_STAT) {
            if syscall::fstatvfs(fd, &mut stat).is_ok() {
                let size = stat.f_blocks * stat.f_bsize as u64;
                let used = (stat.f_blocks - stat.f_bfree) * stat.f_bsize as u64;

                ram = format!("{}MB / {}MB", (used + 1048575)/1048576, (size + 1048575)/1048576);
            }
            let _ = syscall::close(fd);
        }
    }

    let left = [
        "              :+yMMMMy+:              ",
        "         .+dddNMMMMMMMMNddd+.         ",
        "       sydMMMMo/sMMMMs/oMMMMdys       ",
        "    .oMMMdso              osdMMMo.    ",
        "   .+MMd/`   -:::::::`      `/dMM+.   ",
        "  +dMMN.     NMMNNNMMdyo`     .NMMd+  ",
        "  yMMN       NM+   oomMMN       NMMy  ",
        " hNMd.       NM+     `oMN       .dMNh ",
        " dMMh        NM+     `oMN        hMMd ",
        " dMMh        NM+-oooodMMN        hMMd ",
        " dMMh        NM+/MMMMdhs`        hMMd ",
        " hNMd.       NM+`/mMMm+         .dMNh ",
        "  yMMN       NM+   oNMd+        NMMy  ",
        "  +dMMN.     NM+    omMMm     .NMMd+  ",
        "   .+MMd/`   --.     .---   `/dMM+.   ",
        "    .oMMMdso              osdMMMo.    ",
        "       sydMMMMo////////oMMMMdys       ",
        "         .+dddNMMMMMMMMNddd+.         ",
        "              :++++++++:              ",
    ];

    let right = [
        format!("\x1B[1;38;5;75m{}\x1B[0m@\x1B[1;38;5;75m{}\x1B[0m", user, hostname.trim()),
        format!("\x1B[1;38;5;75mOS:\x1B[0m redox-os"),
        format!("\x1B[1;38;5;75mKernel:\x1B[0m redox"),
        format!("\x1B[1;38;5;75mUptime:\x1B[0m {}", uptime_str),
        format!("\x1B[1;38;5;75mShell:\x1B[0m {}", shell),
        format!("\x1B[1;38;5;75mResolution:\x1B[0m {}x{}", width, height),
        format!("\x1B[1;38;5;75mDE:\x1B[0m orbital"),
        format!("\x1B[1;38;5;75mWM:\x1B[0m orbital"),
        format!("\x1B[1;38;5;75mFont:\x1B[0m unifont"),
        format!("\x1B[1;38;5;75mCPU:\x1B[0m {}", cpu),
        format!("\x1B[1;38;5;75mRAM:\x1B[0m {}", ram)
    ];

    let mut string = String::new();
    for i in 0..left.len() {
        string.push_str("\x1B[1;38;5;75m");
        string.push_str(left[i]);
        string.push_str("  \x1B[0m");
        if let Some(r) = right.get(i) {
            string.push_str(r);
        }
        string.push_str("\n");
    }

    io::stdout().write(string.as_bytes()).unwrap();
}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate structopt_derive;
extern crate structopt;
extern crate csv;
extern crate mimir;
extern crate itertools;
extern crate mimirsbrunn;
#[macro_use]
extern crate log;

use std::rc::Rc;
use mimir::rubber::Rubber;
use mimirsbrunn::utils::{format_label, get_zip_codes_from_admins};
use mimir::MimirObject;
use mimirsbrunn::admin_geofinder::AdminGeoFinder;
use std::collections::HashMap;
use structopt::StructOpt;

const GLOBAL_STOP_INDEX_NAME: &'static str = "munin_global_stops";
const MAX_LAT: f64 = 90f64;
const MIN_LAT: f64 = -90f64;

const MAX_LON: f64 = 180f64;
const MIN_LON: f64 = -180f64;

#[derive(Debug, StructOpt)]
struct Args {
    /// NTFS stops.txt file.
    #[structopt(short = "i", long = "input")]
    input: String,
    /// Name of the dataset.
    #[structopt(short = "d", long = "dataset", default_value = "fr")]
    dataset: String,
    /// Elasticsearch parameters.
    #[structopt(short = "c", long = "connection-string",
                default_value = "http://localhost:9200/munin")]
    connection_string: String,
    /// City level to calculate weight.
    #[structopt(short = "C", long = "city-level", default_value = "8")]
    city_level: u32,
}

#[derive(Deserialize, Debug)]
enum StopConversionErr {
    ///StopArea is invisible in Autocomplete
    InvisibleStop,
    ///The stop in the line is not a StopArea
    NotStopArea,
    ///Values of one or more attributs are not valid
    InvalidStop(String),
}

#[derive(Debug, Deserialize)]
struct GtfsStop {
    stop_id: String,
    stop_lat: f64,
    stop_lon: f64,
    stop_name: String,
    location_type: Option<i32>,
    visible: Option<i32>,
    parent_station: Option<String>,
}

impl GtfsStop {
    fn incr_stop_point(&self, nb_stop_points: &mut HashMap<String, u32>) {
        match (self.location_type, &self.parent_station) {
            (Some(0), &Some(ref id)) |
            (None, &Some(ref id)) if !id.is_empty() => {
                *nb_stop_points
                    .entry(format!("stop_area:{}", id))
                    .or_insert(0) += 1
            }
            _ => (),
        }
    }
    // to be moved when TryInto is stablilized
    fn try_into(self) -> Result<mimir::Stop, StopConversionErr> {
        if self.location_type != Some(1) {
            Err(StopConversionErr::NotStopArea)
        } else if self.visible == Some(0) {
            Err(StopConversionErr::InvisibleStop)
        } else if self.stop_lat <= MIN_LAT || self.stop_lat >= MAX_LAT ||
                   self.stop_lon <= MIN_LON || self.stop_lon >= MAX_LON
        {
            //Here we return an error message
            Err(StopConversionErr::InvalidStop(
                format!(
                    "Invalid lon {:?} or lat {:?} for stop \
                                                        {:?}",
                    self.stop_lon,
                    self.stop_lat,
                    self.stop_name
                ).into(),
            ))
        } else {
            Ok(mimir::Stop {
                id: format!("stop_area:{}", self.stop_id), // prefix to match navitia's id
                coord: mimir::Coord::new(self.stop_lat, self.stop_lon),
                label: self.stop_name.clone(),
                weight: 0.,
                zip_codes: vec![],
                administrative_regions: vec![],
                name: self.stop_name,
                coverages: vec![],
            })
        }
    }
}

fn attach_stop(stop: &mut mimir::Stop, admins: Vec<Rc<mimir::Admin>>, city_level: u32) {
    stop.administrative_regions = admins;
    stop.label = format_label(&stop.administrative_regions, city_level, &stop.name);
    stop.zip_codes = get_zip_codes_from_admins(&stop.administrative_regions);
}

/// Attach the stops to administrative regions
///
/// The admins are loaded from Elasticsearch and stored in a quadtree
/// We attach a stop with all the admins that have a boundary containing
/// the coordinate of the stop
fn attach_stops_to_admins<'a, It: Iterator<Item = &'a mut mimir::Stop>>(
    stops: It,
    rubber: &mut Rubber,
    city_level: u32,
) {
    let admins = rubber.get_all_admins().unwrap_or_else(|_| {
        info!("Administratives regions not found in elasticsearch db");
        vec![]
    });

    info!("{} administrative regions loaded from mimir", admins.len());

    let admins_geofinder = admins.into_iter().collect::<AdminGeoFinder>();

    let mut nb_unmatched = 0u32;
    let mut nb_matched = 0u32;
    for mut stop in stops {
        let admins = admins_geofinder.get(&stop.coord);

        if admins.is_empty() {
            nb_unmatched += 1;
        } else {
            nb_matched += 1;
        }

        attach_stop(&mut stop, admins, city_level);
    }

    info!(
        "there is {}/{} stops without any admin",
        nb_unmatched,
        nb_matched + nb_unmatched
    );
}

// Update weight value for each stop_area from HashMap and stop's coverage
fn finalize_stop_area<'a, It: Iterator<Item = &'a mut mimir::Stop>>(
    stops: It,
    nb_stop_points: &HashMap<String, u32>,
    dataset: &str,
) {
    let max = *nb_stop_points.values().max().unwrap_or(&1) as f64;
    for stop in stops {
        if let Some(weight) = nb_stop_points.get(&stop.id) {
            stop.weight = *weight as f64 / max;
        }
        stop.coverages.push(dataset.to_string());
    }
}

/// merge the stops from all the different indexes
/// for the moment the merge is very simple and uses only the ID
/// (and we take the data from the first stop inserted)
fn merge_stops<It: IntoIterator<Item = mimir::Stop>>(
    stops: It,
) -> Box<Iterator<Item = mimir::Stop>> {
    let mut stops_by_id = HashMap::<String, mimir::Stop>::new();
    for mut stop in stops.into_iter() {
        let cov = std::mem::replace(&mut stop.coverages, vec![]);
        stops_by_id
            .entry(stop.id.clone())
            .or_insert(stop)
            .coverages
            .extend(cov.into_iter());
    }
    Box::new(stops_by_id.into_iter().map(|(_, v)| v))
}

fn get_all_stops(rubber: &mut Rubber, index: String) -> Result<Vec<mimir::Stop>, String> {
    rubber.get_all_objects_from_index(&index).map_err(
        |e| e.to_string(),
    )
}

fn update_global_stop_index<'a, It: Iterator<Item = &'a mimir::Stop>>(
    rubber: &mut Rubber,
    stops: It,
    dataset: &str,
) -> Result<String, String> {
    let dataset_index =
        mimir::rubber::get_main_type_and_dataset_index(mimir::Stop::doc_type(), dataset);
    let stops_indexes = rubber
        .get_all_aliased_index(&mimir::rubber::get_main_type_index(mimir::Stop::doc_type()))?
        .into_iter()
        .filter(|&(_, ref aliases)| !aliases.contains(&dataset_index))
        .map(|(index, _)| index);

    let all_other_es_stops: Vec<_> = stops_indexes
        .map(|index| get_all_stops(rubber, index).unwrap())
        .flat_map(|stops| stops.into_iter())
        .collect();

    let all_es_stops = all_other_es_stops.into_iter().chain(
        stops.into_iter().cloned(),
    );

    let all_merged_stops = merge_stops(all_es_stops);
    let es_index_name = mimir::rubber::get_date_index_name(GLOBAL_STOP_INDEX_NAME);

    rubber.create_index(&es_index_name)?;

    let nb_stops_added = rubber
        .bulk_index(&es_index_name, all_merged_stops)
        .map_err(|e| e.to_string())?;
    info!("{} stops added in the global index", nb_stops_added);
    // create global index
    // fill structure for each stop indexes
    Ok(es_index_name)
}

// publish the global stop index
// alias the new index to the global stop alias, and remove the old index
fn publish_global_index(rubber: &mut Rubber, new_global_index: &str) -> Result<(), String> {
    let last_global_indexes: Vec<_> = rubber
        .get_all_aliased_index(GLOBAL_STOP_INDEX_NAME)?
        .into_iter()
        .map(|(k, _)| k)
        .filter(|k| k != new_global_index)
        .collect();
    rubber.alias(
        GLOBAL_STOP_INDEX_NAME,
        &vec![new_global_index.to_string()],
        &last_global_indexes,
    )?;

    for index in last_global_indexes {
        rubber.delete_index(&index)?;
    }
    Ok(())
}

fn main() {
    mimir::logger_init().unwrap();
    info!("Launching stops2mimir...");

    let args = Args::from_args();

    info!("creation of indexes");
    let mut rubber = Rubber::new(&args.connection_string);
    let mut rdr = csv::Reader::from_path(args.input).unwrap();

    let mut nb_stop_points = HashMap::new();

    let mut stops: Vec<mimir::Stop> = rdr.deserialize()
        .filter_map(|rc| {
            rc.map_err(|e| warn!("skip csv line because: {}", e)).ok()
        })
        .filter_map(|stop: GtfsStop| {
            stop.incr_stop_point(&mut nb_stop_points);
            match stop.try_into() {
                Ok(s) => Some(s),
                Err(StopConversionErr::InvisibleStop) |
                Err(StopConversionErr::NotStopArea) => None,
                Err(StopConversionErr::InvalidStop(msg)) => {
                    warn!("skip csv line : {}", msg);
                    None
                }
            }
        })
        .collect();

    attach_stops_to_admins(stops.iter_mut(), &mut rubber, args.city_level);

    finalize_stop_area(stops.iter_mut(), &nb_stop_points, &args.dataset);

    let global_index = update_global_stop_index(&mut rubber, stops.iter(), &args.dataset).unwrap();

    info!("Importing {} stops into Mimir", stops.len());
    let nb_stops = rubber.index(&args.dataset, stops.iter()).unwrap();
    info!("Nb of indexed stops: {}", nb_stops);

    publish_global_index(&mut rubber, &global_index).unwrap();
}

#[test]
fn test_load_stops() {
    use itertools::Itertools;
    let mut rdr = csv::Reader::from_path("./tests/fixtures/stops.txt".to_string()).unwrap();

    let mut nb_stop_points = HashMap::new();
    let stops: Vec<mimir::Stop> = rdr.deserialize()
        .filter_map(Result::ok)
        .filter_map(|stop: GtfsStop| {
            stop.incr_stop_point(&mut nb_stop_points);
            match stop.try_into() {
                Ok(s) => Some(s),
                Err(StopConversionErr::InvisibleStop) => None,
                Err(StopConversionErr::NotStopArea) => None,
                Err(StopConversionErr::InvalidStop(msg)) => {
                    warn!("skip csv line because: {}", msg);
                    None
                }
            }
        })
        .collect();
    let ids: Vec<_> = stops.iter().map(|s| s.id.clone()).sorted();
    assert_eq!(
        ids,
        vec![
            "stop_area:SA:known_by_all_dataset",
            "stop_area:SA:main_station",
            "stop_area:SA:second_station",
            "stop_area:SA:station_no_city",
            "stop_area:SA:weight_1_station",
            "stop_area:SA:weight_3_station",
        ]
    );
    let weights: Vec<_> = ids.iter().map(|id| nb_stop_points.get(id)).collect();
    assert_eq!(
        weights,
        vec![None, Some(&1), Some(&1), None, Some(&1), Some(&3)]
    );
}
<commit_msg>stops2mimir: minor corrections<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate structopt_derive;
extern crate structopt;
extern crate csv;
extern crate mimir;
extern crate itertools;
extern crate mimirsbrunn;
#[macro_use]
extern crate log;

use std::rc::Rc;
use mimir::rubber::Rubber;
use mimirsbrunn::utils::{format_label, get_zip_codes_from_admins};
use mimir::MimirObject;
use mimirsbrunn::admin_geofinder::AdminGeoFinder;
use std::collections::HashMap;
use structopt::StructOpt;

const GLOBAL_STOP_INDEX_NAME: &'static str = "munin_global_stops";
const MAX_LAT: f64 = 90f64;
const MIN_LAT: f64 = -90f64;

const MAX_LON: f64 = 180f64;
const MIN_LON: f64 = -180f64;

#[derive(Debug, StructOpt)]
struct Args {
    /// NTFS stops.txt file.
    #[structopt(short = "i", long = "input")]
    input: String,
    /// Name of the dataset.
    #[structopt(short = "d", long = "dataset", default_value = "fr")]
    dataset: String,
    /// Elasticsearch parameters.
    #[structopt(short = "c", long = "connection-string",
                default_value = "http://localhost:9200/munin")]
    connection_string: String,
    /// City level to calculate weight.
    #[structopt(short = "C", long = "city-level", default_value = "8")]
    city_level: u32,
}

#[derive(Deserialize, Debug)]
enum StopConversionErr {
    ///StopArea is invisible in Autocomplete
    InvisibleStop,
    ///The stop in the line is not a StopArea
    NotStopArea,
    ///Values of one or more attributes are not valid
    InvalidStop(String),
}

#[derive(Debug, Deserialize)]
struct GtfsStop {
    stop_id: String,
    stop_lat: f64,
    stop_lon: f64,
    stop_name: String,
    location_type: Option<i32>,
    visible: Option<i32>,
    parent_station: Option<String>,
}

impl GtfsStop {
    fn incr_stop_point(&self, nb_stop_points: &mut HashMap<String, u32>) {
        match (self.location_type, &self.parent_station) {
            (Some(0), &Some(ref id)) |
            (None, &Some(ref id)) if !id.is_empty() => {
                *nb_stop_points
                    .entry(format!("stop_area:{}", id))
                    .or_insert(0) += 1
            }
            _ => (),
        }
    }
    // to be moved when TryInto is stablilized
    fn try_into(self) -> Result<mimir::Stop, StopConversionErr> {
        if self.location_type != Some(1) {
            Err(StopConversionErr::NotStopArea)
        } else if self.visible == Some(0) {
            Err(StopConversionErr::InvisibleStop)
        } else if self.stop_lat <= MIN_LAT || self.stop_lat >= MAX_LAT ||
                   self.stop_lon <= MIN_LON || self.stop_lon >= MAX_LON
        {
            //Here we return an error message
            Err(StopConversionErr::InvalidStop(
                format!(
                    "Invalid lon {:?} or lat {:?} for stop \
                                                        {:?}",
                    self.stop_lon,
                    self.stop_lat,
                    self.stop_name
                ).into(),
            ))
        } else {
            Ok(mimir::Stop {
                id: format!("stop_area:{}", self.stop_id), // prefix to match navitia's id
                coord: mimir::Coord::new(self.stop_lat, self.stop_lon),
                label: self.stop_name.clone(),
                weight: 0.,
                zip_codes: vec![],
                administrative_regions: vec![],
                name: self.stop_name,
                coverages: vec![],
            })
        }
    }
}

fn attach_stop(stop: &mut mimir::Stop, admins: Vec<Rc<mimir::Admin>>, city_level: u32) {
    stop.administrative_regions = admins;
    stop.label = format_label(&stop.administrative_regions, city_level, &stop.name);
    stop.zip_codes = get_zip_codes_from_admins(&stop.administrative_regions);
}

/// Attach the stops to administrative regions
///
/// The admins are loaded from Elasticsearch and stored in a quadtree
/// We attach a stop with all the admins that have a boundary containing
/// the coordinate of the stop
fn attach_stops_to_admins<'a, It: Iterator<Item = &'a mut mimir::Stop>>(
    stops: It,
    rubber: &mut Rubber,
    city_level: u32,
) {
    let admins = rubber.get_all_admins().unwrap_or_else(|_| {
        info!("Administratives regions not found in elasticsearch db");
        vec![]
    });

    info!("{} administrative regions loaded from mimir", admins.len());

    let admins_geofinder = admins.into_iter().collect::<AdminGeoFinder>();

    let mut nb_unmatched = 0u32;
    let mut nb_matched = 0u32;
    for mut stop in stops {
        let admins = admins_geofinder.get(&stop.coord);

        if admins.is_empty() {
            nb_unmatched += 1;
        } else {
            nb_matched += 1;
        }

        attach_stop(&mut stop, admins, city_level);
    }

    info!(
        "there are {}/{} stops without any admin",
        nb_unmatched,
        nb_matched + nb_unmatched
    );
}

// Update weight value for each stop_area from HashMap and stop's coverage
fn finalize_stop_area<'a, It: Iterator<Item = &'a mut mimir::Stop>>(
    stops: It,
    nb_stop_points: &HashMap<String, u32>,
    dataset: &str,
) {
    let max = *nb_stop_points.values().max().unwrap_or(&1) as f64;
    for stop in stops {
        if let Some(weight) = nb_stop_points.get(&stop.id) {
            stop.weight = *weight as f64 / max;
        }
        stop.coverages.push(dataset.to_string());
    }
}

/// merge the stops from all the different indexes
/// for the moment the merge is very simple and uses only the ID
/// (and we take the data from the first stop inserted)
fn merge_stops<It: IntoIterator<Item = mimir::Stop>>(
    stops: It,
) -> Box<Iterator<Item = mimir::Stop>> {
    let mut stops_by_id = HashMap::<String, mimir::Stop>::new();
    for mut stop in stops.into_iter() {
        let cov = std::mem::replace(&mut stop.coverages, vec![]);
        stops_by_id
            .entry(stop.id.clone())
            .or_insert(stop)
            .coverages
            .extend(cov.into_iter());
    }
    Box::new(stops_by_id.into_iter().map(|(_, v)| v))
}

fn get_all_stops(rubber: &mut Rubber, index: String) -> Result<Vec<mimir::Stop>, String> {
    rubber.get_all_objects_from_index(&index).map_err(
        |e| e.to_string(),
    )
}

fn update_global_stop_index<'a, It: Iterator<Item = &'a mimir::Stop>>(
    rubber: &mut Rubber,
    stops: It,
    dataset: &str,
) -> Result<String, String> {
    let dataset_index =
        mimir::rubber::get_main_type_and_dataset_index(mimir::Stop::doc_type(), dataset);
    let stops_indexes = rubber
        .get_all_aliased_index(&mimir::rubber::get_main_type_index(mimir::Stop::doc_type()))?
        .into_iter()
        .filter(|&(_, ref aliases)| !aliases.contains(&dataset_index))
        .map(|(index, _)| index);

    let all_other_es_stops: Vec<_> = stops_indexes
        .map(|index| get_all_stops(rubber, index).unwrap())
        .flat_map(|stops| stops.into_iter())
        .collect();

    let all_es_stops = all_other_es_stops.into_iter().chain(
        stops.into_iter().cloned(),
    );

    let all_merged_stops = merge_stops(all_es_stops);
    let es_index_name = mimir::rubber::get_date_index_name(GLOBAL_STOP_INDEX_NAME);

    rubber.create_index(&es_index_name)?;

    let nb_stops_added = rubber
        .bulk_index(&es_index_name, all_merged_stops)
        .map_err(|e| e.to_string())?;
    info!("{} stops added in the global index", nb_stops_added);
    // create global index
    // fill structure for each stop indexes
    Ok(es_index_name)
}

// publish the global stop index
// alias the new index to the global stop alias, and remove the old index
fn publish_global_index(rubber: &mut Rubber, new_global_index: &str) -> Result<(), String> {
    let last_global_indexes: Vec<_> = rubber
        .get_all_aliased_index(GLOBAL_STOP_INDEX_NAME)?
        .into_iter()
        .map(|(k, _)| k)
        .filter(|k| k != new_global_index)
        .collect();
    rubber.alias(
        GLOBAL_STOP_INDEX_NAME,
        &vec![new_global_index.to_string()],
        &last_global_indexes,
    )?;

    for index in last_global_indexes {
        rubber.delete_index(&index)?;
    }
    Ok(())
}

fn main() {
    mimir::logger_init().unwrap();
    info!("Launching stops2mimir...");

    let args = Args::from_args();

    info!("creation of indexes");
    let mut rubber = Rubber::new(&args.connection_string);
    let mut rdr = csv::Reader::from_path(args.input).unwrap();

    let mut nb_stop_points = HashMap::new();

    let mut stops: Vec<mimir::Stop> = rdr.deserialize()
        .filter_map(|rc| rc.map_err(|e| warn!("skip csv line: {}", e)).ok())
        .filter_map(|stop: GtfsStop| {
            stop.incr_stop_point(&mut nb_stop_points);
            match stop.try_into() {
                Ok(s) => Some(s),
                Err(StopConversionErr::InvisibleStop) |
                Err(StopConversionErr::NotStopArea) => None,
                Err(StopConversionErr::InvalidStop(msg)) => {
                    warn!("skip csv line : {}", msg);
                    None
                }
            }
        })
        .collect();

    attach_stops_to_admins(stops.iter_mut(), &mut rubber, args.city_level);

    finalize_stop_area(stops.iter_mut(), &nb_stop_points, &args.dataset);

    let global_index = update_global_stop_index(&mut rubber, stops.iter(), &args.dataset).unwrap();

    info!("Importing {} stops into Mimir", stops.len());
    let nb_stops = rubber.index(&args.dataset, stops.iter()).unwrap();
    info!("Nb of indexed stops: {}", nb_stops);

    publish_global_index(&mut rubber, &global_index).unwrap();
}

#[test]
fn test_load_stops() {
    use itertools::Itertools;
    let mut rdr = csv::Reader::from_path("./tests/fixtures/stops.txt".to_string()).unwrap();

    let mut nb_stop_points = HashMap::new();
    let stops: Vec<mimir::Stop> = rdr.deserialize()
        .filter_map(Result::ok)
        .filter_map(|stop: GtfsStop| {
            stop.incr_stop_point(&mut nb_stop_points);
            match stop.try_into() {
                Ok(s) => Some(s),
                Err(StopConversionErr::InvisibleStop) => None,
                Err(StopConversionErr::NotStopArea) => None,
                Err(StopConversionErr::InvalidStop(msg)) => {
                    warn!("skip csv line: {}", msg);
                    None
                }
            }
        })
        .collect();
    let ids: Vec<_> = stops.iter().map(|s| s.id.clone()).sorted();
    assert_eq!(
        ids,
        vec![
            "stop_area:SA:known_by_all_dataset",
            "stop_area:SA:main_station",
            "stop_area:SA:second_station",
            "stop_area:SA:station_no_city",
            "stop_area:SA:weight_1_station",
            "stop_area:SA:weight_3_station",
        ]
    );
    let weights: Vec<_> = ids.iter().map(|id| nb_stop_points.get(id)).collect();
    assert_eq!(
        weights,
        vec![None, Some(&1), Some(&1), None, Some(&1), Some(&3)]
    );
}
<|endoftext|>


<commit_before>use decoders::*;
use decoders::tiff::*;
use decoders::basics::*;
use std::mem::transmute;
extern crate itertools;
use self::itertools::Itertools;
use std::f32;
use std::cmp;

#[derive(Debug, Clone)]
pub struct ArwDecoder<'a> {
  buffer: &'a [u8],
  rawloader: &'a RawLoader,
  tiff: TiffIFD<'a>,
}

impl<'a> ArwDecoder<'a> {
  pub fn new(buf: &'a [u8], tiff: TiffIFD<'a>, rawloader: &'a RawLoader) -> ArwDecoder<'a> {
    ArwDecoder {
      buffer: buf,
      tiff: tiff,
      rawloader: rawloader,
    }
  }
}

impl<'a> Decoder for ArwDecoder<'a> {
  fn identify(&self) -> Result<&Camera, String> {
    let make = fetch_tag!(self.tiff, Tag::Make, "ARW: Couldn't find Make").get_str();
    let model = fetch_tag!(self.tiff, Tag::Model, "ARW: Couldn't find Model").get_str();
    self.rawloader.check_supported(make, model)
  }

  fn image(&self) -> Result<Image,String> {
    let camera = try!(self.identify());
    let data = self.tiff.find_ifds_with_tag(Tag::StripOffsets);
    if data.len() == 0 {
      return Err("ARW: Couldn't find the data IFD!".to_string())
    }
    let raw = data[0];
    let width = fetch_tag!(raw, Tag::ImageWidth, "ARW: Couldn't find width").get_u16(0) as u32;
    let height = fetch_tag!(raw, Tag::ImageLength, "ARW: Couldn't find height").get_u16(0) as u32;
    let offset = fetch_tag!(raw, Tag::StripOffsets, "ARW: Couldn't find offset").get_u32(0) as usize;
    let count = fetch_tag!(raw, Tag::StripByteCounts, "ARW: Couldn't find byte count").get_u32(0) as usize;
    let compression = fetch_tag!(raw, Tag::Compression, "ARW: Couldn't find Compression").get_u16(0);
    let bps = if camera.bps != 0 {
      camera.bps
    } else {
      fetch_tag!(raw, Tag::BitsPerSample, "ARW: Couldn't find bps").get_u16(0) as u32
    };
    let src = &self.buffer[offset .. self.buffer.len()];

    let image = match compression {
      1 => decode_16le(src, width as usize, height as usize),
      32767 => {
        if ((width*height*bps) as usize) != count*8 {
          ArwDecoder::decode_arw1(src, width, height)
        } else {
          match bps {
            8 => ArwDecoder::decode_arw2(src, width, height),
            12 => decode_12le(src, width as usize, height as usize),
            _ => return Err(format!("ARW2: Don't know how to decode images with {} bps", bps)),
          }
        }
      },
      _ => return Err(format!("ARW: Don't know how to decode type {}", compression).to_string()),
    };

    ok_image(camera, width, height, try!(self.get_wb()), image)
  }
}

impl<'a> ArwDecoder<'a> {
  fn decode_arw1(buf: &[u8], width: u32, height: u32) -> Vec<u16> {
    let mut buffer: Vec<u16> = vec![0; (width*height) as usize];

    buffer
  }

  fn decode_arw2(buf: &[u8], width: u32, height: u32) -> Vec<u16> {
    let mut buffer: Vec<u16> = vec![0; (width*height) as usize];
    let mut pump = BitPump::new(buf);

    for row in 0..height {
      // Process 16 pixels at a time in interleaved fashion
      let mut col = 0;
      while col < (width-30) {
        let max = pump.get_bits(11);
        let min = pump.get_bits(11);
        let imax = pump.get_bits(4);
        let imin = pump.get_bits(4);
        let mut sh = 0;
        while sh<4 && (0x80 << sh) <= (max - min) {sh = sh + 1;}
        for i in 0..16 {
          let val = if i == imax {
            max
          } else if i == imin {
            min
          } else {
            cmp::min(0x7ff,(pump.get_bits(7) << sh) + min)
          };
          buffer[(row*width+col+i*2) as usize] = val as u16;
        }
        col += if (col & 1) != 0 {31} else {1};  // Skip to next 16 pixels
      }
    }

    buffer
  }

  fn get_wb(&self) -> Result<[f32;4], String> {
    let priv_offset = fetch_tag!(self.tiff, Tag::DNGPrivateArea, "ARW: Couldn't find private offset").get_u32(0);
    let priv_tiff = TiffIFD::new(self.buffer, priv_offset as usize, 0, 0, LITTLE_ENDIAN);
    let sony_offset = fetch_tag!(priv_tiff, Tag::SonyOffset, "ARW: Couldn't find sony offset").get_u32(0) as usize;
    let sony_length = fetch_tag!(priv_tiff, Tag::SonyLength, "ARW: Couldn't find sony length").get_u32(0) as usize;
    let sony_key = fetch_tag!(priv_tiff, Tag::SonyKey, "ARW: Couldn't find sony key").get_u32(0);
    let decrypted_buf = ArwDecoder::sony_decrypt(self.buffer, sony_offset, sony_length, sony_key);
    let decrypted_tiff = TiffIFD::new(&decrypted_buf, 0, sony_offset, 0, LITTLE_ENDIAN);
    let grgb_levels = decrypted_tiff.find_entry(Tag::SonyGRBG);
    let rggb_levels = decrypted_tiff.find_entry(Tag::SonyRGGB);
    if grgb_levels.is_some() {
      let levels = grgb_levels.unwrap();
      Ok([levels.get_u16(1) as f32, levels.get_u16(0) as f32, levels.get_u16(2) as f32, f32::NAN])
    } else if rggb_levels.is_some() {
      let levels = rggb_levels.unwrap();
      Ok([levels.get_u16(0) as f32, levels.get_u16(1) as f32, levels.get_u16(3) as f32, f32::NAN])
    } else {
      Err("ARW: Couldn't find GRGB or RGGB levels".to_string())
    }
  }

  fn sony_decrypt(buf: &[u8], offset: usize, length: usize, key: u32) -> Vec<u8>{
    let mut pad: [u32; 128] = [0 as u32; 128];
    let mut mkey = key;
    // Make sure we always have space for the final bytes even if the buffer
    // isn't a multiple of 4
    let mut out = vec![0;length+4];
    // Initialize the decryption pad from the key
    for p in 0..4 {
      mkey = mkey.wrapping_mul(48828125).wrapping_add(1);
      pad[p] = mkey;
    }
    pad[3] = pad[3] << 1 | (pad[0]^pad[2]) >> 31;
    for p in 4..127 {
      pad[p] = (pad[p-4]^pad[p-2]) << 1 | (pad[p-3]^pad[p-1]) >> 31;
    }
    for p in 0..127 {
      pad[p] = u32::from_be(pad[p]);
    }

    for i in (0..length).step(4) {
      let p = i/4 + 127;
      pad[p & 127] = pad[(p+1) & 127] ^ pad[(p+1+64) & 127];
      let output = LEu32(buf, offset+i) ^ pad[p & 127];
      let bytes: [u8; 4] = unsafe { transmute(output.to_le()) };
      out[i]   = bytes[0];
      out[i+1] = bytes[1];
      out[i+2] = bytes[2];
      out[i+3] = bytes[3];
    }
    out
  }
}
<commit_msg>ARW: shut up warnings while ARW1 is not done<commit_after>use decoders::*;
use decoders::tiff::*;
use decoders::basics::*;
use std::mem::transmute;
extern crate itertools;
use self::itertools::Itertools;
use std::f32;
use std::cmp;

#[derive(Debug, Clone)]
pub struct ArwDecoder<'a> {
  buffer: &'a [u8],
  rawloader: &'a RawLoader,
  tiff: TiffIFD<'a>,
}

impl<'a> ArwDecoder<'a> {
  pub fn new(buf: &'a [u8], tiff: TiffIFD<'a>, rawloader: &'a RawLoader) -> ArwDecoder<'a> {
    ArwDecoder {
      buffer: buf,
      tiff: tiff,
      rawloader: rawloader,
    }
  }
}

impl<'a> Decoder for ArwDecoder<'a> {
  fn identify(&self) -> Result<&Camera, String> {
    let make = fetch_tag!(self.tiff, Tag::Make, "ARW: Couldn't find Make").get_str();
    let model = fetch_tag!(self.tiff, Tag::Model, "ARW: Couldn't find Model").get_str();
    self.rawloader.check_supported(make, model)
  }

  fn image(&self) -> Result<Image,String> {
    let camera = try!(self.identify());
    let data = self.tiff.find_ifds_with_tag(Tag::StripOffsets);
    if data.len() == 0 {
      return Err("ARW: Couldn't find the data IFD!".to_string())
    }
    let raw = data[0];
    let width = fetch_tag!(raw, Tag::ImageWidth, "ARW: Couldn't find width").get_u16(0) as u32;
    let height = fetch_tag!(raw, Tag::ImageLength, "ARW: Couldn't find height").get_u16(0) as u32;
    let offset = fetch_tag!(raw, Tag::StripOffsets, "ARW: Couldn't find offset").get_u32(0) as usize;
    let count = fetch_tag!(raw, Tag::StripByteCounts, "ARW: Couldn't find byte count").get_u32(0) as usize;
    let compression = fetch_tag!(raw, Tag::Compression, "ARW: Couldn't find Compression").get_u16(0);
    let bps = if camera.bps != 0 {
      camera.bps
    } else {
      fetch_tag!(raw, Tag::BitsPerSample, "ARW: Couldn't find bps").get_u16(0) as u32
    };
    let src = &self.buffer[offset .. self.buffer.len()];

    let image = match compression {
      1 => decode_16le(src, width as usize, height as usize),
      32767 => {
        if ((width*height*bps) as usize) != count*8 {
          ArwDecoder::decode_arw1(src, width, height)
        } else {
          match bps {
            8 => ArwDecoder::decode_arw2(src, width, height),
            12 => decode_12le(src, width as usize, height as usize),
            _ => return Err(format!("ARW2: Don't know how to decode images with {} bps", bps)),
          }
        }
      },
      _ => return Err(format!("ARW: Don't know how to decode type {}", compression).to_string()),
    };

    ok_image(camera, width, height, try!(self.get_wb()), image)
  }
}

impl<'a> ArwDecoder<'a> {
  fn decode_arw1(buf: &[u8], width: u32, height: u32) -> Vec<u16> {
    let mut buffer: Vec<u16> = vec![0; (width*height) as usize];

    buffer[0] = buf[0] as u16; // Shut up the warnings for now

    buffer
  }

  fn decode_arw2(buf: &[u8], width: u32, height: u32) -> Vec<u16> {
    let mut buffer: Vec<u16> = vec![0; (width*height) as usize];
    let mut pump = BitPump::new(buf);

    for row in 0..height {
      // Process 16 pixels at a time in interleaved fashion
      let mut col = 0;
      while col < (width-30) {
        let max = pump.get_bits(11);
        let min = pump.get_bits(11);
        let imax = pump.get_bits(4);
        let imin = pump.get_bits(4);
        let mut sh = 0;
        while sh<4 && (0x80 << sh) <= (max - min) {sh = sh + 1;}
        for i in 0..16 {
          let val = if i == imax {
            max
          } else if i == imin {
            min
          } else {
            cmp::min(0x7ff,(pump.get_bits(7) << sh) + min)
          };
          buffer[(row*width+col+i*2) as usize] = val as u16;
        }
        col += if (col & 1) != 0 {31} else {1};  // Skip to next 16 pixels
      }
    }

    buffer
  }

  fn get_wb(&self) -> Result<[f32;4], String> {
    let priv_offset = fetch_tag!(self.tiff, Tag::DNGPrivateArea, "ARW: Couldn't find private offset").get_u32(0);
    let priv_tiff = TiffIFD::new(self.buffer, priv_offset as usize, 0, 0, LITTLE_ENDIAN);
    let sony_offset = fetch_tag!(priv_tiff, Tag::SonyOffset, "ARW: Couldn't find sony offset").get_u32(0) as usize;
    let sony_length = fetch_tag!(priv_tiff, Tag::SonyLength, "ARW: Couldn't find sony length").get_u32(0) as usize;
    let sony_key = fetch_tag!(priv_tiff, Tag::SonyKey, "ARW: Couldn't find sony key").get_u32(0);
    let decrypted_buf = ArwDecoder::sony_decrypt(self.buffer, sony_offset, sony_length, sony_key);
    let decrypted_tiff = TiffIFD::new(&decrypted_buf, 0, sony_offset, 0, LITTLE_ENDIAN);
    let grgb_levels = decrypted_tiff.find_entry(Tag::SonyGRBG);
    let rggb_levels = decrypted_tiff.find_entry(Tag::SonyRGGB);
    if grgb_levels.is_some() {
      let levels = grgb_levels.unwrap();
      Ok([levels.get_u16(1) as f32, levels.get_u16(0) as f32, levels.get_u16(2) as f32, f32::NAN])
    } else if rggb_levels.is_some() {
      let levels = rggb_levels.unwrap();
      Ok([levels.get_u16(0) as f32, levels.get_u16(1) as f32, levels.get_u16(3) as f32, f32::NAN])
    } else {
      Err("ARW: Couldn't find GRGB or RGGB levels".to_string())
    }
  }

  fn sony_decrypt(buf: &[u8], offset: usize, length: usize, key: u32) -> Vec<u8>{
    let mut pad: [u32; 128] = [0 as u32; 128];
    let mut mkey = key;
    // Make sure we always have space for the final bytes even if the buffer
    // isn't a multiple of 4
    let mut out = vec![0;length+4];
    // Initialize the decryption pad from the key
    for p in 0..4 {
      mkey = mkey.wrapping_mul(48828125).wrapping_add(1);
      pad[p] = mkey;
    }
    pad[3] = pad[3] << 1 | (pad[0]^pad[2]) >> 31;
    for p in 4..127 {
      pad[p] = (pad[p-4]^pad[p-2]) << 1 | (pad[p-3]^pad[p-1]) >> 31;
    }
    for p in 0..127 {
      pad[p] = u32::from_be(pad[p]);
    }

    for i in (0..length).step(4) {
      let p = i/4 + 127;
      pad[p & 127] = pad[(p+1) & 127] ^ pad[(p+1+64) & 127];
      let output = LEu32(buf, offset+i) ^ pad[p & 127];
      let bytes: [u8; 4] = unsafe { transmute(output.to_le()) };
      out[i]   = bytes[0];
      out[i+1] = bytes[1];
      out[i+2] = bytes[2];
      out[i+3] = bytes[3];
    }
    out
  }
}
<|endoftext|>


<commit_before>use std::sync::{Barrier, RwLock, Arc, RwLockWriteGuard};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::atomic::Ordering as AtOrd;
extern crate rand;
use rand::{ThreadRng, Rng, SeedableRng, StdRng, XorShiftRng};
use rand::distributions::{IndependentSample, Range};

extern crate gr;
use gr::{GI, width_box, split_box, midpoint_box, eps_tol};

extern crate gelpia_utils;
use gelpia_utils::{Quple, INF, NINF, Flt, Parameters};

extern crate function;
use function::FuncObj;

type GARng = XorShiftRng;

#[derive(Clone)]
pub struct Individual {
    pub solution: Vec<GI>,
    pub fitness: Flt,
}

pub fn ea(x_e: Vec<GI>,
          param: Parameters,
          population: Arc<RwLock<Vec<Individual>>>,
          f_bestag: Arc<RwLock<Flt>>,
          x_bestbb: Arc<RwLock<Vec<GI>>>,
          b1: Arc<Barrier>,
          b2: Arc<Barrier>,
          stop: Arc<AtomicBool>,
          sync: Arc<AtomicBool>,
          fo_c: FuncObj) -> (Flt, Vec<GI>, bool) {
    // Constant function
    if x_e.len() == 0 {
        return (0.0, x_e, true);
    }
    let seed = param.seed;
    let input = ea_core(&x_e, &param, &stop, &sync, &b1, &b2, &f_bestag,
                        &x_bestbb, population, &fo_c, seed);
    let (ans_i, _) = fo_c.call(&input);
    let ans = ans_i.upper();

    (ans, input, true)
}

fn ea_core(x_e: &Vec<GI>, param: &Parameters, stop: &Arc<AtomicBool>,
           sync: &Arc<AtomicBool>, b1: &Arc<Barrier>, b2: &Arc<Barrier>,
           f_bestag: &Arc<RwLock<Flt>>,
           x_bestbb: &Arc<RwLock<Vec<GI>>>,
           population: Arc<RwLock<Vec<Individual>>>, fo_c: &FuncObj,
           seed: u32)
           -> (Vec<GI>) {
    let rng_seed: u32 =
        match seed {
            0 => 3735928579,
            1 => rand::thread_rng().next_u32(),
            _ => seed,
        };
    let seed_r: [u32; 4] = [(rng_seed & 0xFF000000) >> 24,
                            (rng_seed & 0xFF0000) >> 16,
                            (rng_seed & 0xFF00) >> 8 ,
                            rng_seed & 0xFF];
    let mut rng: GARng = GARng::from_seed(seed_r);

    let dimension = Range::new(0, x_e.len());
    let mut ranges = Vec::new();
    for g in x_e {
        ranges.push(Range::new(g.lower(), g.upper()));
    }

    while !stop.load(AtOrd::Acquire) {
        if sync.load(AtOrd::Acquire) {
            b1.wait();
            b2.wait();
        }
        let ref mut population = *population.write().unwrap();
        sample(param.population, population, fo_c, &ranges, &mut rng);

        population.sort_by(|a, b| b.fitness.partial_cmp(&a.fitness).unwrap());

        for iteration in 0..100 {
            population.truncate(param.elitism);

            for _ in 0..param.selection {
                population.push(rand_individual(fo_c, &ranges, &mut rng));
            }

            next_generation(param.population, population, fo_c, param.mutation,
                            param.crossover, &dimension, &ranges, &mut rng);
            population.sort_by(|a, b| b.fitness.partial_cmp(&a.fitness).unwrap());

            // Report fittest of the fit.
            {
                let mut fbest = f_bestag.write().unwrap();

                *fbest =
                    if *fbest < population[0].fitness { population[0].fitness }
                else { *fbest };
            }

            // Kill worst of the worst
            let mut ftg = Vec::new();
            {
                let bestbb = x_bestbb.read().unwrap();
                // From The Gods
                for i in 0..bestbb.len() {
                    ftg.push(Range::new(bestbb[i].lower(), bestbb[i].upper()));
                }
            }
            let worst_ind = population.len() - 1;
            population[worst_ind] = rand_individual(fo_c,
                                                    &ftg,
                                                    &mut rng);
            population.sort_by(|a, b| b.fitness.partial_cmp(&a.fitness).unwrap());
        }
    }

    let ref population = *population.read().unwrap();
    let result = if !population.is_empty() {
        population[0].solution.clone()
    } else {
        x_e.clone()
    };

    result
}

fn sample(population_size: usize, population: &mut Vec<Individual>,
          fo_c: &FuncObj, ranges: &Vec<Range<f64>>, rng: &mut GARng)
          -> () {
    for _ in 0..population_size-population.len() {
        population.push(rand_individual(fo_c, ranges, rng));
    }

    ()
}

fn rand_individual(fo_c: &FuncObj, ranges: &Vec<Range<f64>>, rng: &mut GARng)
                   -> (Individual) {
    let mut new_sol = Vec::new();
    for r in ranges {
        new_sol.push(GI::new_p(r.ind_sample(rng)));
    }
    let (fitness_i, _) = fo_c.call(&new_sol);
    let fitness = fitness_i.lower();

    Individual{solution:new_sol, fitness:fitness}
}

fn next_generation(population_size:usize, population: &mut Vec<Individual>,
                   fo_c: &FuncObj, mut_rate: f64, crossover: f64,
                   dimension: &Range<usize>, ranges: &Vec<Range<f64>>,
                   rng: &mut GARng)
                   -> () {

    let elites = population.clone();

    for _ in 0..population_size-population.len() {
        if rng.gen::<f64>() < crossover {
            population.push(breed((&mut *rng).choose(&elites).unwrap(),
                                  rng.choose(&elites).unwrap(),
                                  fo_c,
                                  dimension, rng));
        } else {
            population.push(mutate(rng.choose(&elites).unwrap(), fo_c, mut_rate,
                                   dimension, ranges, rng));
        }
    }

    ()
}

fn mutate(input: &Individual, fo_c: &FuncObj, mut_rate: f64,
          dimension: &Range<usize>, ranges: &Vec<Range<f64>>, rng: &mut GARng)
          -> (Individual) {
    let mut output_sol = Vec::new();

    for (r, &ind) in ranges.iter().zip(input.solution.iter()) {
        output_sol.push(
            if rng.gen::<f64>() < mut_rate {
                ind
            } else {
                GI::new_p(r.ind_sample(&mut *rng))
            });
    }

    let (fitness_i, _) = fo_c.call(&output_sol);
    let fitness = fitness_i.lower();

    Individual{solution: output_sol, fitness: fitness}
}

fn breed(parent1: &Individual, parent2: &Individual, fo_c: &FuncObj,
         dimension: &Range<usize>, rng: &mut GARng) -> (Individual) {
    let mut child = parent1.clone();
    let crossover_point = dimension.ind_sample(rng);
    child.solution.truncate(crossover_point);
    let mut rest = parent2.clone().solution.split_off(crossover_point);
    child.solution.append(&mut rest);
    let (fitness_i, _) = fo_c.call(&child.solution);
    child.fitness = fitness_i.lower();
    child
}
<commit_msg>Make ea check more often for stop condition<commit_after>use std::sync::{Barrier, RwLock, Arc, RwLockWriteGuard};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::atomic::Ordering as AtOrd;
extern crate rand;
use rand::{ThreadRng, Rng, SeedableRng, StdRng, XorShiftRng};
use rand::distributions::{IndependentSample, Range};

extern crate gr;
use gr::{GI, width_box, split_box, midpoint_box, eps_tol};

extern crate gelpia_utils;
use gelpia_utils::{Quple, INF, NINF, Flt, Parameters};

extern crate function;
use function::FuncObj;

type GARng = XorShiftRng;

#[derive(Clone)]
pub struct Individual {
    pub solution: Vec<GI>,
    pub fitness: Flt,
}

pub fn ea(x_e: Vec<GI>,
          param: Parameters,
          population: Arc<RwLock<Vec<Individual>>>,
          f_bestag: Arc<RwLock<Flt>>,
          x_bestbb: Arc<RwLock<Vec<GI>>>,
          b1: Arc<Barrier>,
          b2: Arc<Barrier>,
          stop: Arc<AtomicBool>,
          sync: Arc<AtomicBool>,
          fo_c: FuncObj) -> (Flt, Vec<GI>, bool) {
    // Constant function
    if x_e.len() == 0 {
        return (0.0, x_e, true);
    }
    let seed = param.seed;
    let input = ea_core(&x_e, &param, &stop, &sync, &b1, &b2, &f_bestag,
                        &x_bestbb, population, &fo_c, seed);
    let (ans_i, _) = fo_c.call(&input);
    let ans = ans_i.upper();

    (ans, input, true)
}

fn ea_core(x_e: &Vec<GI>, param: &Parameters, stop: &Arc<AtomicBool>,
           sync: &Arc<AtomicBool>, b1: &Arc<Barrier>, b2: &Arc<Barrier>,
           f_bestag: &Arc<RwLock<Flt>>,
           x_bestbb: &Arc<RwLock<Vec<GI>>>,
           population: Arc<RwLock<Vec<Individual>>>, fo_c: &FuncObj,
           seed: u32)
           -> (Vec<GI>) {
    let rng_seed: u32 =
        match seed {
            0 => 3735928579,
            1 => rand::thread_rng().next_u32(),
            _ => seed,
        };
    let seed_r: [u32; 4] = [(rng_seed & 0xFF000000) >> 24,
                            (rng_seed & 0xFF0000) >> 16,
                            (rng_seed & 0xFF00) >> 8 ,
                            rng_seed & 0xFF];
    let mut rng: GARng = GARng::from_seed(seed_r);

    let dimension = Range::new(0, x_e.len());
    let mut ranges = Vec::new();
    for g in x_e {
        ranges.push(Range::new(g.lower(), g.upper()));
    }

    while !stop.load(AtOrd::Acquire) {
        if sync.load(AtOrd::Acquire) {
            b1.wait();
            b2.wait();
        }
        let ref mut population = *population.write().unwrap();
        sample(param.population, population, fo_c, &ranges, &mut rng);

        population.sort_by(|a, b| b.fitness.partial_cmp(&a.fitness).unwrap());

        for iteration in 0..100 {
            if stop.load(AtOrd::Acquire) {
                let result = if !population.is_empty() {
                    population[0].solution.clone()
                } else {
                    x_e.clone()
                };
                return result;
                }
            population.truncate(param.elitism);

            for _ in 0..param.selection {
                population.push(rand_individual(fo_c, &ranges, &mut rng));
            }

            next_generation(param.population, population, fo_c, param.mutation,
                            param.crossover, &dimension, &ranges, &mut rng);
            population.sort_by(|a, b| b.fitness.partial_cmp(&a.fitness).unwrap());

            // Report fittest of the fit.
            {
                let mut fbest = f_bestag.write().unwrap();

                *fbest =
                    if *fbest < population[0].fitness { population[0].fitness }
                else { *fbest };
            }

            // Kill worst of the worst
            let mut ftg = Vec::new();
            {
                let bestbb = x_bestbb.read().unwrap();
                // From The Gods
                for i in 0..bestbb.len() {
                    ftg.push(Range::new(bestbb[i].lower(), bestbb[i].upper()));
                }
            }
            let worst_ind = population.len() - 1;
            population[worst_ind] = rand_individual(fo_c,
                                                    &ftg,
                                                    &mut rng);
            population.sort_by(|a, b| b.fitness.partial_cmp(&a.fitness).unwrap());
        }
    }

    let ref population = *population.read().unwrap();
    let result = if !population.is_empty() {
        population[0].solution.clone()
    } else {
        x_e.clone()
    };

    result
}

fn sample(population_size: usize, population: &mut Vec<Individual>,
          fo_c: &FuncObj, ranges: &Vec<Range<f64>>, rng: &mut GARng)
          -> () {
    for _ in 0..population_size-population.len() {
        population.push(rand_individual(fo_c, ranges, rng));
    }

    ()
}

fn rand_individual(fo_c: &FuncObj, ranges: &Vec<Range<f64>>, rng: &mut GARng)
                   -> (Individual) {
    let mut new_sol = Vec::new();
    for r in ranges {
        new_sol.push(GI::new_p(r.ind_sample(rng)));
    }
    let (fitness_i, _) = fo_c.call(&new_sol);
    let fitness = fitness_i.lower();

    Individual{solution:new_sol, fitness:fitness}
}

fn next_generation(population_size:usize, population: &mut Vec<Individual>,
                   fo_c: &FuncObj, mut_rate: f64, crossover: f64,
                   dimension: &Range<usize>, ranges: &Vec<Range<f64>>,
                   rng: &mut GARng)
                   -> () {

    let elites = population.clone();

    for _ in 0..population_size-population.len() {
        if rng.gen::<f64>() < crossover {
            population.push(breed((&mut *rng).choose(&elites).unwrap(),
                                  rng.choose(&elites).unwrap(),
                                  fo_c,
                                  dimension, rng));
        } else {
            population.push(mutate(rng.choose(&elites).unwrap(), fo_c, mut_rate,
                                   dimension, ranges, rng));
        }
    }

    ()
}

fn mutate(input: &Individual, fo_c: &FuncObj, mut_rate: f64,
          dimension: &Range<usize>, ranges: &Vec<Range<f64>>, rng: &mut GARng)
          -> (Individual) {
    let mut output_sol = Vec::new();

    for (r, &ind) in ranges.iter().zip(input.solution.iter()) {
        output_sol.push(
            if rng.gen::<f64>() < mut_rate {
                ind
            } else {
                GI::new_p(r.ind_sample(&mut *rng))
            });
    }

    let (fitness_i, _) = fo_c.call(&output_sol);
    let fitness = fitness_i.lower();

    Individual{solution: output_sol, fitness: fitness}
}

fn breed(parent1: &Individual, parent2: &Individual, fo_c: &FuncObj,
         dimension: &Range<usize>, rng: &mut GARng) -> (Individual) {
    let mut child = parent1.clone();
    let crossover_point = dimension.ind_sample(rng);
    child.solution.truncate(crossover_point);
    let mut rest = parent2.clone().solution.split_off(crossover_point);
    child.solution.append(&mut rest);
    let (fitness_i, _) = fo_c.call(&child.solution);
    child.fitness = fitness_i.lower();
    child
}
<|endoftext|>


<commit_before>// Copyright 2018 Bobby Powers. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

#![feature(alloc_system)]

extern crate alloc_system;

//extern crate libc;
//use libc::geteuid;

use std::cmp::{Eq, Ordering};
use std::fs::{self, File};
use std::io::{BufRead, BufReader, Error, ErrorKind, Read, Result};

const PROC_PATH: &'static str = "/proc";

const TY_PSS: &'static str = "Pss:";
const TY_SWAP: &'static str = "Swap:";
const TY_PRIVATE_CLEAN: &'static str = "Private_Clean:";
const TY_PRIVATE_DIRTY: &'static str = "Private_Dirty:";

struct CmdStat {
    name: String,
    pid: i32,
    pss: f32,
    shared: f32,
    swap: f32,
    count: i32, // used when summing up
}

impl CmdStat {
    fn new(pid: i32) -> Result<CmdStat> {
        let name = proc_name(pid)?;

        let mut stats = CmdStat {
            name: name,
            pid: pid,
            pss: 0.0,
            shared: 0.0,
            swap: 0.0,
            count: 1,
        };

        stats.collect_memory_usage()?;

        Ok(stats)
    }

    fn collect_memory_usage(&mut self) -> Result<()> {
        let path = format!("/proc/{}/smaps_rollup", self.pid);
        let file = File::open(path)?;

        let mut private: f32 = 0.0;

        for line in BufReader::new(file).lines() {
            let line = line.unwrap();
            if line.starts_with(TY_PSS) {
                if let Ok(n) = parse_line(&line) {
                    self.pss += n;
                }
            } else if line.starts_with(TY_SWAP) {
                if let Ok(n) = parse_line(&line) {
                    self.swap += n;
                }
            } else if line.starts_with(TY_PRIVATE_CLEAN) || line.starts_with(TY_PRIVATE_DIRTY) {
                if let Ok(n) = parse_line(&line) {
                    private += n;
                }
            }
        }

        self.shared = self.pss - private;

        Ok(())
    }
}

fn parse_line(line: &str) -> Result<f32> {
    let kbs = &line[16..24];
    let num: f32 = kbs
        .trim_start()
        .parse()
        .map_err(|_e| Error::new(ErrorKind::Other, "parse error"))?;
    Ok(num)
}

impl PartialEq for CmdStat {
    fn eq(&self, other: &CmdStat) -> bool {
        self.name == other.name
    }
}
impl Eq for CmdStat {}

impl PartialOrd for CmdStat {
    fn partial_cmp(&self, other: &CmdStat) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

impl Ord for CmdStat {
    fn cmp(&self, other: &CmdStat) -> Ordering {
        self.name.cmp(&other.name)
    }
}

#[inline(never)]
fn is_digit(d: char) -> bool {
    d >= '0' && d <= '9'
}

#[inline(never)]
fn first_char(name: &std::ffi::OsStr) -> char {
    use std::os::unix::ffi::OsStrExt;
    name.as_bytes()[0] as char
}

fn is_dir(md: fs::Metadata) -> Result<fs::Metadata> {
    if md.is_dir() {
        Ok(md)
    } else {
        Err(Error::new(ErrorKind::NotFound, "not a directory"))
    }
}

#[inline(never)]
fn get_pids() -> Result<Vec<i32>> {
    let dir = fs::read_dir(PROC_PATH)?;

    // TODO: look at filter_map
    let pids: Vec<i32> = dir
        .filter(|e| e.is_ok()) // ignore bad dirent results
        .map(|e| e.unwrap()) // make this a list of dirents
        .filter(|e| is_digit(first_char(&e.file_name())))
        .filter(|e| fs::metadata(e.path()).and_then(|md| is_dir(md)).is_ok())
        .map(|e| i32::from_str_radix(e.file_name().to_str().unwrap(), 10).unwrap())
        .collect();

    Ok(pids)
}

fn proc_cmdline(pid: i32) -> Result<String> {
    let mut buf = String::new();
    let path = format!("/proc/{}/cmdline", pid);

    // TODO: we don't really care about reading all of the
    // cmdline, just the first 1024 chars is enough.
    let mut f = File::open(path)?;

    f.read_to_string(&mut buf)?;

    Ok(buf)
}

fn proc_name(pid: i32) -> Result<String> {
    let path = format!("/proc/{}/exe", pid);

    let owned_path = fs::read_link(path)?;
    let full_path = owned_path.to_string_lossy();
    let cmdline = proc_cmdline(pid)?;

    let result = if cmdline.starts_with(&*full_path) {
        // the basename (last component) of the path
        owned_path
            .file_name()
            .unwrap()
            .to_string_lossy()
            .to_string()
    } else {
        cmdline
    };

    Ok(result)
}

fn cmdstats_for(pids: Vec<i32>) -> Vec<CmdStat> {
    pids.into_iter()
        .filter_map(|pid| CmdStat::new(pid).ok())
        .collect()
}

fn main() {
    // command line flag parsing

    // check euid
    // unsafe {
    // 	let uid = geteuid();
    // 	if uid != 0 {
    // 	    println!("we aren't root");
    // 	}
    // }

    let pids_r = get_pids();
    if let Err(err) = pids_r {
        println!("get_pids: {}", err);
        return;
    }

    let pids = pids_r.unwrap();

    let mut stats = cmdstats_for(pids);

    stats.sort_unstable();

    stats.dedup_by(|a, b| {
        if a.name != b.name {
            return false;
        }
        b.pss += a.pss;
        b.shared += a.shared;
        b.swap += a.swap;
        b.count += 1;
        true
    });

    stats.sort_unstable_by(|a, b| a.pss.partial_cmp(&b.pss).unwrap_or(Ordering::Equal));

    // TODO: this could be a single iteration
    let total_pss = stats.iter().fold(0.0, |sum, stat| sum + stat.pss);
    let total_shared = stats.iter().fold(0.0, |sum, stat| sum + stat.shared);
    let total_swap = stats.iter().fold(0.0, |sum, stat| sum + stat.swap);

    for cmd in &stats {
        let swap = "";
        println!("{:10.1}{:10.1}{:10}\t{} ({})", cmd.pss/1024.0, cmd.shared/1024.0, swap, cmd.name, cmd.count)
    }

    // print_results
    println!("#{:9.1}{:20.1}\tTOTAL USED BY PROCESSES", total_pss/1024.0, total_swap/1024.0);
}
<commit_msg>small readability cleanups<commit_after>// Copyright 2018 Bobby Powers. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

#![feature(alloc_system)]

extern crate alloc_system;

//extern crate libc;
//use libc::geteuid;

use std::cmp::{Eq, Ordering};
use std::fs::{self, File};
use std::io::{BufRead, BufReader, Error, ErrorKind, Read, Result};

const PROC_PATH: &'static str = "/proc";

struct CmdStat {
    name: String,
    pid: i32,
    pss: f32,
    shared: f32,
    swap: f32,
    count: i32, // used when summing up
}

impl CmdStat {
    fn new(pid: i32) -> Result<CmdStat> {
        let name = proc_name(pid)?;

        let mut stats = CmdStat {
            name: name,
            pid: pid,
            pss: 0.0,
            shared: 0.0,
            swap: 0.0,
            count: 1,
        };

        stats.collect_memory_usage()?;

        Ok(stats)
    }

    fn collect_memory_usage(&mut self) -> Result<()> {
        const TY_PSS: &'static str = "Pss:";
        const TY_SWAP: &'static str = "Swap:";
        const TY_PRIVATE_CLEAN: &'static str = "Private_Clean:";
        const TY_PRIVATE_DIRTY: &'static str = "Private_Dirty:";

        let path = format!("/proc/{}/smaps", self.pid);
        let file = File::open(path)?;

        let pss_adjust = 0.5;

        let mut private: f32 = 0.0;

        for line in BufReader::new(file).lines() {
            let line = line.unwrap();
            if line.starts_with(TY_PSS) {
                if let Ok(n) = parse_line(&line) {
                    self.pss += n + pss_adjust;
                }
            } else if line.starts_with(TY_SWAP) {
                if let Ok(n) = parse_line(&line) {
                    self.swap += n;
                }
            } else if line.starts_with(TY_PRIVATE_CLEAN) || line.starts_with(TY_PRIVATE_DIRTY) {
                if let Ok(n) = parse_line(&line) {
                    private += n;
                }
            }
        }

        self.shared = self.pss - private;

        Ok(())
    }
}

fn parse_line(line: &str) -> Result<f32> {
    let kbs = &line[16..24];
    let num: f32 = kbs
        .trim_start()
        .parse()
        .map_err(|_e| Error::new(ErrorKind::Other, "parse error"))?;
    Ok(num)
}

impl PartialEq for CmdStat {
    fn eq(&self, other: &CmdStat) -> bool {
        self.name == other.name
    }
}
impl Eq for CmdStat {}

impl PartialOrd for CmdStat {
    fn partial_cmp(&self, other: &CmdStat) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

impl Ord for CmdStat {
    fn cmp(&self, other: &CmdStat) -> Ordering {
        self.name.cmp(&other.name)
    }
}

#[inline(never)]
fn is_digit(d: char) -> bool {
    d >= '0' && d <= '9'
}

#[inline(never)]
fn first_char(name: &std::ffi::OsStr) -> char {
    use std::os::unix::ffi::OsStrExt;
    name.as_bytes()[0] as char
}

fn is_dir(md: fs::Metadata) -> Result<fs::Metadata> {
    if md.is_dir() {
        Ok(md)
    } else {
        Err(Error::new(ErrorKind::NotFound, "not a directory"))
    }
}

#[inline(never)]
fn get_pids() -> Result<Vec<i32>> {
    let dir = fs::read_dir(PROC_PATH)?;

    // TODO: look at filter_map
    let pids: Vec<i32> = dir
        .filter(|e| e.is_ok()) // ignore bad dirent results
        .map(|e| e.unwrap()) // make this a list of dirents
        .filter(|e| is_digit(first_char(&e.file_name())))
        .filter(|e| fs::metadata(e.path()).and_then(|md| is_dir(md)).is_ok())
        .map(|e| i32::from_str_radix(e.file_name().to_str().unwrap(), 10).unwrap())
        .collect();

    Ok(pids)
}

fn proc_cmdline(pid: i32) -> Result<String> {
    let mut buf = String::new();
    let path = format!("/proc/{}/cmdline", pid);

    // TODO: we don't really care about reading all of the
    // cmdline, just the first 1024 chars is enough.
    let mut f = File::open(path)?;

    f.read_to_string(&mut buf)?;

    Ok(buf)
}

fn proc_name(pid: i32) -> Result<String> {
    let path = format!("/proc/{}/exe", pid);

    let owned_path = fs::read_link(path)?;
    let full_path = owned_path.to_string_lossy();
    let cmdline = proc_cmdline(pid)?;

    let result = if cmdline.starts_with(&*full_path) {
        // the basename (last component) of the path
        owned_path
            .file_name()
            .unwrap()
            .to_string_lossy()
            .to_string()
    } else {
        cmdline
    };

    Ok(result)
}

fn cmdstats_for(pids: Vec<i32>) -> Vec<CmdStat> {
    pids.into_iter()
        .filter_map(|pid| CmdStat::new(pid).ok())
        .collect()
}

fn main() {
    // command line flag parsing

    // check euid
    // unsafe {
    // 	let uid = geteuid();
    // 	if uid != 0 {
    // 	    println!("we aren't root");
    // 	}
    // }

    let pids_r = get_pids();
    if let Err(err) = pids_r {
        println!("get_pids: {}", err);
        return;
    }

    let pids = pids_r.unwrap();

    let mut stats = cmdstats_for(pids);

    stats.sort_unstable();

    stats.dedup_by(|a, b| {
        if a.name != b.name {
            return false;
        }
        b.pss += a.pss;
        b.shared += a.shared;
        b.swap += a.swap;
        b.count += 1;
        true
    });

    stats.sort_unstable_by(|a, b| a.pss.partial_cmp(&b.pss).unwrap_or(Ordering::Equal));

    // TODO: this could be a single iteration
    let total_pss = stats.iter().fold(0.0, |sum, stat| sum + stat.pss);
    let total_swap = stats.iter().fold(0.0, |sum, stat| sum + stat.swap);

    for cmd in &stats {
        let swap = "";
        println!(
            "{:10.1}{:10.1}{:10}\t{} ({})",
            cmd.pss / 1024.0,
            cmd.shared / 1024.0,
            swap,
            cmd.name,
            cmd.count
        )
    }

    // print_results
    println!(
        "#{:9.1}{:20.1}\tTOTAL USED BY PROCESSES",
        total_pss / 1024.0,
        total_swap / 1024.0
    );
}
<|endoftext|>