<commit_before>use std::io::prelude::*;

use crate::core::{resolver, Resolve, ResolveVersion, Workspace};
use crate::util::errors::{CargoResult, CargoResultExt};
use crate::util::toml as cargo_toml;
use crate::util::Filesystem;

pub fn load_pkg_lockfile(ws: &Workspace<'_>) -> CargoResult<Option<Resolve>> {
    if !ws.root().join("Cargo.lock").exists() {
        return Ok(None);
    }

    let root = Filesystem::new(ws.root().to_path_buf());
    let mut f = root.open_ro("Cargo.lock", ws.config(), "Cargo.lock file")?;

    let mut s = String::new();
    f.read_to_string(&mut s)
        .chain_err(|| format!("failed to read file: {}", f.path().display()))?;

    let resolve = (|| -> CargoResult<Option<Resolve>> {
        let resolve: toml::Value = cargo_toml::parse(&s, f.path(), ws.config())?;
        let v: resolver::EncodableResolve = resolve.try_into()?;
        Ok(Some(v.into_resolve(&s, ws)?))
    })()
    .chain_err(|| format!("failed to parse lock file at: {}", f.path().display()))?;
    Ok(resolve)
}

/// Generate a toml String of Cargo.lock from a Resolve.
pub fn resolve_to_string(ws: &Workspace<'_>, resolve: &mut Resolve) -> CargoResult<String> {
    let (_orig, out, _ws_root) = resolve_to_string_orig(ws, resolve)?;
    Ok(out)
}

pub fn write_pkg_lockfile(ws: &Workspace<'_>, resolve: &mut Resolve) -> CargoResult<()> {
    let (orig, mut out, ws_root) = resolve_to_string_orig(ws, resolve)?;

    // If the lock file contents haven't changed so don't rewrite it. This is
    // helpful on read-only filesystems.
    if let Some(orig) = &orig {
        if are_equal_lockfiles(orig, &out, ws) {
            return Ok(());
        }
    }

    if !ws.config().lock_update_allowed() {
        if ws.config().offline() {
            anyhow::bail!("can't update in the offline mode");
        }

        let flag = if ws.config().network_allowed() {
            "--locked"
        } else {
            "--frozen"
        };
        anyhow::bail!(
            "the lock file {} needs to be updated but {} was passed to prevent this\n\
             If you want to try to generate the lock file without accessing the network, \
             use the --offline flag.",
            ws.root().to_path_buf().join("Cargo.lock").display(),
            flag
        );
    }

    // While we're updating the lock file anyway go ahead and update its
    // encoding to whatever the latest default is. That way we can slowly roll
    // out lock file updates as they're otherwise already updated, and changes
    // which don't touch dependencies won't seemingly spuriously update the lock
    // file.
    if resolve.version() < ResolveVersion::default() {
        resolve.set_version(ResolveVersion::default());
        out = serialize_resolve(resolve, orig.as_deref());
    }

    // Ok, if that didn't work just write it out
    ws_root
        .open_rw("Cargo.lock", ws.config(), "Cargo.lock file")
        .and_then(|mut f| {
            f.file().set_len(0)?;
            f.write_all(out.as_bytes())?;
            Ok(())
        })
        .chain_err(|| format!("failed to write {}", ws.root().join("Cargo.lock").display()))?;
    Ok(())
}

fn resolve_to_string_orig(
    ws: &Workspace<'_>,
    resolve: &mut Resolve,
) -> CargoResult<(Option<String>, String, Filesystem)> {
    // Load the original lock file if it exists.
    let ws_root = Filesystem::new(ws.root().to_path_buf());
    let orig = ws_root.open_ro("Cargo.lock", ws.config(), "Cargo.lock file");
    let orig = orig.and_then(|mut f| {
        let mut s = String::new();
        f.read_to_string(&mut s)?;
        Ok(s)
    });
    let out = serialize_resolve(resolve, orig.as_ref().ok().map(|s| &**s));
    Ok((orig.ok(), out, ws_root))
}

fn serialize_resolve(resolve: &Resolve, orig: Option<&str>) -> String {
    let toml = toml::Value::try_from(resolve).unwrap();

    let mut out = String::new();

    // At the start of the file we notify the reader that the file is generated.
    // Specifically Phabricator ignores files containing "@generated", so we use that.
    let marker_line = "# This file is automatically @generated by Cargo.";
    let extra_line = "# It is not intended for manual editing.";
    out.push_str(marker_line);
    out.push('\n');
    out.push_str(extra_line);
    out.push('\n');
    // and preserve any other top comments
    if let Some(orig) = orig {
        let mut comments = orig.lines().take_while(|line| line.starts_with('#'));
        if let Some(first) = comments.next() {
            if first != marker_line {
                out.push_str(first);
                out.push('\n');
            }
            if let Some(second) = comments.next() {
                if second != extra_line {
                    out.push_str(second);
                    out.push('\n');
                }
                for line in comments {
                    out.push_str(line);
                    out.push('\n');
                }
            }
        }
    }

    if let Some(version) = toml.get("version") {
        out.push_str(&format!("version = {}\n\n", version));
    }

    let deps = toml["package"].as_array().unwrap();
    for dep in deps {
        let dep = dep.as_table().unwrap();

        out.push_str("[[package]]\n");
        emit_package(dep, &mut out);
    }

    if let Some(patch) = toml.get("patch") {
        let list = patch["unused"].as_array().unwrap();
        for entry in list {
            out.push_str("[[patch.unused]]\n");
            emit_package(entry.as_table().unwrap(), &mut out);
            out.push_str("\n");
        }
    }

    if let Some(meta) = toml.get("metadata") {
        out.push_str("[metadata]\n");
        out.push_str(&meta.to_string());
    }

    // Historical versions of Cargo in the old format accidentally left trailing
    // blank newlines at the end of files, so we just leave that as-is. For all
    // encodings going forward, though, we want to be sure that our encoded lock
    // file doesn't contain any trailing newlines so trim out the extra if
    // necessary.
    if resolve.version() >= ResolveVersion::V2 {
        while out.ends_with("\n\n") {
            out.pop();
        }
    }
    out
}

fn are_equal_lockfiles(orig: &str, current: &str, ws: &Workspace<'_>) -> bool {
    let mut orig = orig.to_string();
    if has_crlf_line_endings(&orig) {
        orig = orig.replace("\r\n", "\n");
    }

    // If we want to try and avoid updating the lock file, parse both and
    // compare them; since this is somewhat expensive, don't do it in the
    // common case where we can update lock files.
    if !ws.config().lock_update_allowed() {
        let res: CargoResult<bool> = (|| {
            let old: resolver::EncodableResolve = toml::from_str(&orig)?;
            let new: resolver::EncodableResolve = toml::from_str(current)?;
            Ok(old.into_resolve(&orig, ws)? == new.into_resolve(current, ws)?)
        })();
        if let Ok(true) = res {
            return true;
        }
    }

    current == orig
}

fn has_crlf_line_endings(s: &str) -> bool {
    // Only check the first line.
    if let Some(lf) = s.find('\n') {
        s[..lf].ends_with('\r')
    } else {
        false
    }
}

fn emit_package(dep: &toml::value::Table, out: &mut String) {
    out.push_str(&format!("name = {}\n", &dep["name"]));
    out.push_str(&format!("version = {}\n", &dep["version"]));

    if dep.contains_key("source") {
        out.push_str(&format!("source = {}\n", &dep["source"]));
    }
    if dep.contains_key("checksum") {
        out.push_str(&format!("checksum = {}\n", &dep["checksum"]));
    }

    if let Some(s) = dep.get("dependencies") {
        let slice = s.as_array().unwrap();

        if !slice.is_empty() {
            out.push_str("dependencies = [\n");

            for child in slice.iter() {
                out.push_str(&format!(" {},\n", child));
            }

            out.push_str("]\n");
        }
        out.push_str("\n");
    } else if dep.contains_key("replace") {
        out.push_str(&format!("replace = {}\n\n", &dep["replace"]));
    }
}
<commit_msg>Auto merge of #8641 - weihanglo:fix/remove-alloc, r=Eh2406<commit_after>use std::io::prelude::*;

use crate::core::{resolver, Resolve, ResolveVersion, Workspace};
use crate::util::errors::{CargoResult, CargoResultExt};
use crate::util::toml as cargo_toml;
use crate::util::Filesystem;

pub fn load_pkg_lockfile(ws: &Workspace<'_>) -> CargoResult<Option<Resolve>> {
    if !ws.root().join("Cargo.lock").exists() {
        return Ok(None);
    }

    let root = Filesystem::new(ws.root().to_path_buf());
    let mut f = root.open_ro("Cargo.lock", ws.config(), "Cargo.lock file")?;

    let mut s = String::new();
    f.read_to_string(&mut s)
        .chain_err(|| format!("failed to read file: {}", f.path().display()))?;

    let resolve = (|| -> CargoResult<Option<Resolve>> {
        let resolve: toml::Value = cargo_toml::parse(&s, f.path(), ws.config())?;
        let v: resolver::EncodableResolve = resolve.try_into()?;
        Ok(Some(v.into_resolve(&s, ws)?))
    })()
    .chain_err(|| format!("failed to parse lock file at: {}", f.path().display()))?;
    Ok(resolve)
}

/// Generate a toml String of Cargo.lock from a Resolve.
pub fn resolve_to_string(ws: &Workspace<'_>, resolve: &mut Resolve) -> CargoResult<String> {
    let (_orig, out, _ws_root) = resolve_to_string_orig(ws, resolve)?;
    Ok(out)
}

pub fn write_pkg_lockfile(ws: &Workspace<'_>, resolve: &mut Resolve) -> CargoResult<()> {
    let (orig, mut out, ws_root) = resolve_to_string_orig(ws, resolve)?;

    // If the lock file contents haven't changed so don't rewrite it. This is
    // helpful on read-only filesystems.
    if let Some(orig) = &orig {
        if are_equal_lockfiles(orig, &out, ws) {
            return Ok(());
        }
    }

    if !ws.config().lock_update_allowed() {
        if ws.config().offline() {
            anyhow::bail!("can't update in the offline mode");
        }

        let flag = if ws.config().network_allowed() {
            "--locked"
        } else {
            "--frozen"
        };
        anyhow::bail!(
            "the lock file {} needs to be updated but {} was passed to prevent this\n\
             If you want to try to generate the lock file without accessing the network, \
             use the --offline flag.",
            ws.root().to_path_buf().join("Cargo.lock").display(),
            flag
        );
    }

    // While we're updating the lock file anyway go ahead and update its
    // encoding to whatever the latest default is. That way we can slowly roll
    // out lock file updates as they're otherwise already updated, and changes
    // which don't touch dependencies won't seemingly spuriously update the lock
    // file.
    if resolve.version() < ResolveVersion::default() {
        resolve.set_version(ResolveVersion::default());
        out = serialize_resolve(resolve, orig.as_deref());
    }

    // Ok, if that didn't work just write it out
    ws_root
        .open_rw("Cargo.lock", ws.config(), "Cargo.lock file")
        .and_then(|mut f| {
            f.file().set_len(0)?;
            f.write_all(out.as_bytes())?;
            Ok(())
        })
        .chain_err(|| format!("failed to write {}", ws.root().join("Cargo.lock").display()))?;
    Ok(())
}

fn resolve_to_string_orig(
    ws: &Workspace<'_>,
    resolve: &mut Resolve,
) -> CargoResult<(Option<String>, String, Filesystem)> {
    // Load the original lock file if it exists.
    let ws_root = Filesystem::new(ws.root().to_path_buf());
    let orig = ws_root.open_ro("Cargo.lock", ws.config(), "Cargo.lock file");
    let orig = orig.and_then(|mut f| {
        let mut s = String::new();
        f.read_to_string(&mut s)?;
        Ok(s)
    });
    let out = serialize_resolve(resolve, orig.as_ref().ok().map(|s| &**s));
    Ok((orig.ok(), out, ws_root))
}

fn serialize_resolve(resolve: &Resolve, orig: Option<&str>) -> String {
    let toml = toml::Value::try_from(resolve).unwrap();

    let mut out = String::new();

    // At the start of the file we notify the reader that the file is generated.
    // Specifically Phabricator ignores files containing "@generated", so we use that.
    let marker_line = "# This file is automatically @generated by Cargo.";
    let extra_line = "# It is not intended for manual editing.";
    out.push_str(marker_line);
    out.push('\n');
    out.push_str(extra_line);
    out.push('\n');
    // and preserve any other top comments
    if let Some(orig) = orig {
        let mut comments = orig.lines().take_while(|line| line.starts_with('#'));
        if let Some(first) = comments.next() {
            if first != marker_line {
                out.push_str(first);
                out.push('\n');
            }
            if let Some(second) = comments.next() {
                if second != extra_line {
                    out.push_str(second);
                    out.push('\n');
                }
                for line in comments {
                    out.push_str(line);
                    out.push('\n');
                }
            }
        }
    }

    if let Some(version) = toml.get("version") {
        out.push_str(&format!("version = {}\n\n", version));
    }

    let deps = toml["package"].as_array().unwrap();
    for dep in deps {
        let dep = dep.as_table().unwrap();

        out.push_str("[[package]]\n");
        emit_package(dep, &mut out);
    }

    if let Some(patch) = toml.get("patch") {
        let list = patch["unused"].as_array().unwrap();
        for entry in list {
            out.push_str("[[patch.unused]]\n");
            emit_package(entry.as_table().unwrap(), &mut out);
            out.push_str("\n");
        }
    }

    if let Some(meta) = toml.get("metadata") {
        out.push_str("[metadata]\n");
        out.push_str(&meta.to_string());
    }

    // Historical versions of Cargo in the old format accidentally left trailing
    // blank newlines at the end of files, so we just leave that as-is. For all
    // encodings going forward, though, we want to be sure that our encoded lock
    // file doesn't contain any trailing newlines so trim out the extra if
    // necessary.
    if resolve.version() >= ResolveVersion::V2 {
        while out.ends_with("\n\n") {
            out.pop();
        }
    }
    out
}

fn are_equal_lockfiles(orig: &str, current: &str, ws: &Workspace<'_>) -> bool {
    // If we want to try and avoid updating the lock file, parse both and
    // compare them; since this is somewhat expensive, don't do it in the
    // common case where we can update lock files.
    if !ws.config().lock_update_allowed() {
        let res: CargoResult<bool> = (|| {
            let old: resolver::EncodableResolve = toml::from_str(orig)?;
            let new: resolver::EncodableResolve = toml::from_str(current)?;
            Ok(old.into_resolve(orig, ws)? == new.into_resolve(current, ws)?)
        })();
        if let Ok(true) = res {
            return true;
        }
    }

    orig.lines().eq(current.lines())
}

fn emit_package(dep: &toml::value::Table, out: &mut String) {
    out.push_str(&format!("name = {}\n", &dep["name"]));
    out.push_str(&format!("version = {}\n", &dep["version"]));

    if dep.contains_key("source") {
        out.push_str(&format!("source = {}\n", &dep["source"]));
    }
    if dep.contains_key("checksum") {
        out.push_str(&format!("checksum = {}\n", &dep["checksum"]));
    }

    if let Some(s) = dep.get("dependencies") {
        let slice = s.as_array().unwrap();

        if !slice.is_empty() {
            out.push_str("dependencies = [\n");

            for child in slice.iter() {
                out.push_str(&format!(" {},\n", child));
            }

            out.push_str("]\n");
        }
        out.push_str("\n");
    } else if dep.contains_key("replace") {
        out.push_str(&format!("replace = {}\n\n", &dep["replace"]));
    }
}
<|endoftext|>


<commit_before>use super::gi_docgen;
use crate::{config::gobjects::GStatus, library::FunctionKind, nameutil, Env};
use once_cell::sync::Lazy;
use regex::{Captures, Match, Regex};

const LANGUAGE_SEP_BEGIN: &str = "<!-- language=\"";
const LANGUAGE_SEP_END: &str = "\" -->";
const LANGUAGE_BLOCK_BEGIN: &str = "|[";
const LANGUAGE_BLOCK_END: &str = "\n]|";

pub fn reformat_doc(input: &str, env: &Env, in_type: &str) -> String {
    code_blocks_transformation(input, env, in_type)
}

fn try_split<'a>(src: &'a str, needle: &str) -> (&'a str, Option<&'a str>) {
    match src.find(needle) {
        Some(pos) => (&src[..pos], Some(&src[pos + needle.len()..])),
        None => (src, None),
    }
}

fn code_blocks_transformation(mut input: &str, env: &Env, in_type: &str) -> String {
    let mut out = String::with_capacity(input.len());

    loop {
        input = match try_split(input, LANGUAGE_BLOCK_BEGIN) {
            (before, Some(after)) => {
                out.push_str(&format(before, env, in_type));
                if let (before, Some(after)) =
                    try_split(get_language(after, &mut out), LANGUAGE_BLOCK_END)
                {
                    out.push_str(before);
                    out.push_str("\n```");
                    after
                } else {
                    after
                }
            }
            (before, None) => {
                out.push_str(&format(before, env, in_type));
                return out;
            }
        };
    }
}

fn get_language<'a>(entry: &'a str, out: &mut String) -> &'a str {
    if let (_, Some(after)) = try_split(entry, LANGUAGE_SEP_BEGIN) {
        if let (before, Some(after)) = try_split(after, LANGUAGE_SEP_END) {
            out.push_str(&format!("\n```{}", before));
            return after;
        }
    }
    out.push_str("\n```text");
    entry
}

fn format(input: &str, env: &Env, in_type: &str) -> String {
    let mut ret = String::with_capacity(input.len());
    // We run gi_docgen first because it's super picky about the types it replaces
    let no_c_types_re = gi_docgen::replace_c_types(input, env, in_type);
    ret.push_str(&replace_c_types(&no_c_types_re, env, in_type));
    ret
}

static SYMBOL: Lazy<Regex> = Lazy::new(|| Regex::new(r"([#%])(\w+\b)([:.]+[\w-]+\b)?").unwrap());
static FUNCTION: Lazy<Regex> =
    Lazy::new(|| Regex::new(r"([@#%])?(\w+\b[:.]+)?(\b[a-z0-9_]+)\(\)").unwrap());
// **note**
// The optional . at the end is to make the regex more relaxed for some weird broken cases on gtk3's docs
// it doesn't hurt other docs so please don't drop it
static GDK_GTK: Lazy<Regex> =
    Lazy::new(|| Regex::new(r"`([^\(:])?((G[dts]k|Pango)\w+\b)(\.)?`").unwrap());
static TAGS: Lazy<Regex> = Lazy::new(|| Regex::new(r"<[\w/-]+>").unwrap());
static SPACES: Lazy<Regex> = Lazy::new(|| Regex::new(r"[ ]{2,}").unwrap());

fn replace_c_types(entry: &str, env: &Env, in_type: &str) -> String {
    let out = FUNCTION.replace_all(entry, |caps: &Captures<'_>| {
        let name = &caps[3];
        find_function(name, env)
    });

    let out = SYMBOL.replace_all(&out, |caps: &Captures<'_>| {
        match &caps[2] {
            "TRUE" => "[`true`]".to_string(),
            "FALSE" => "[`false`]".to_string(),
            "NULL" => "[`None`]".to_string(),
            symbol_name => {
                if &caps[1] == "%" {
                    find_constant_or_variant(symbol_name, env)
                } else {
                    let method_name = caps.get(3).map(|m| m.as_str().trim_start_matches('.'));
                    // would be #
                    find_type_by_name(symbol_name, method_name, env)
                }
            }
        }
    });
    let out = GDK_GTK.replace_all(&out, |caps: &Captures<'_>| find_struct(&caps[2], env));
    let out = TAGS.replace_all(&out, "`$0`");
    SPACES.replace_all(&out, " ").into_owned()
}

fn find_type_by_name(symbol: &str, method_name: Option<&str>, env: &Env) -> String {
    let symbols = env.symbols.borrow();
    if let Some(method) = method_name {
        let is_obj_func = env
            .analysis
            .objects
            .iter()
            .find(|(_, o)| o.full_name == symbol)
            .and_then(|(_, obj_info)| {
                obj_info
                    .functions
                    .iter()
                    .filter(|fn_info| fn_info.status != GStatus::Ignore)
                    .filter(|fn_info| !fn_info.is_special() && !fn_info.is_async_finish(env))
                    .find(|fn_info| fn_info.name == method)
                    .map(|fn_info| (obj_info, fn_info))
            });
        let is_record_func = env
            .analysis
            .records
            .iter()
            .find(|(_, r)| r.type_(&env.library).c_type == symbol)
            .and_then(|(_, record_info)| {
                record_info
                    .functions
                    .iter()
                    .filter(|fn_info| fn_info.status != GStatus::Ignore)
                    .filter(|fn_info| !fn_info.is_special() && !fn_info.is_async_finish(env))
                    .find(|fn_info| fn_info.glib_name == method)
                    .map(|fn_info| (record_info, fn_info))
            });
        if let Some((obj_info, fn_info)) = is_obj_func {
            let sym = symbols.by_tid(obj_info.type_id).unwrap(); // we are sure the object exists
            let (type_name, visible_type_name) = if obj_info.final_type
                || fn_info.kind == FunctionKind::Constructor
                || fn_info.kind == FunctionKind::Function
            {
                (obj_info.name.clone(), obj_info.name.clone())
            } else {
                let type_name = if fn_info.status == GStatus::Generate {
                    obj_info.trait_name.clone()
                } else {
                    format!("{}Manual", obj_info.trait_name)
                };
                (format!("prelude::{}", type_name), type_name)
            };
            let name = sym.full_rust_name().replace(&obj_info.name, &type_name);
            format!(
                "[{visible_type_name}::{fn_name}](crate::{name}::{fn_name})",
                name = name,
                visible_type_name = visible_type_name,
                fn_name = fn_info.codegen_name()
            )
        } else if let Some((record_info, fn_info)) = is_record_func {
            let sym = symbols.by_tid(record_info.type_id).unwrap(); // we are sure the object exists
            format!(
                "[{name}::{fn_name}](crate::{name}::{fn_name})",
                name = sym.full_rust_name(),
                fn_name = fn_info.codegen_name()
            )
        } else {
            format!("`{}::{}`", symbol, method)
        }
    } else {
        find_struct(symbol, env)
    }
}

fn find_constant_or_variant(symbol: &str, env: &Env) -> String {
    let symbols = env.symbols.borrow();
    if let Some(const_info) = env
        .analysis
        .constants
        .iter()
        .find(|c| c.glib_name == symbol)
    {
        // for whatever reason constants are not part of the symbols list
        format!("[{name}](crate::{name})", name = const_info.name)
    } else if let Some((flag_info, member_info)) = env.analysis.flags.iter().find_map(|f| {
        f.type_(&env.library)
            .members
            .iter()
            .find(|m| m.c_identifier == symbol)
            .map(|m| (f, m))
    }) {
        let sym = symbols.by_tid(flag_info.type_id).unwrap();
        format!(
            "[{flag_name}::{member_name}](crate::{parent}{member_name})",
            member_name = nameutil::bitfield_member_name(&member_info.name),
            flag_name = flag_info.name,
            parent = sym.parent()
        )
    } else if let Some((enum_info, member_info)) = env.analysis.enumerations.iter().find_map(|e| {
        e.type_(&env.library)
            .members
            .iter()
            .find(|m| m.c_identifier == symbol)
            .map(|m| (e, m))
    }) {
        let sym = symbols.by_tid(enum_info.type_id).unwrap();
        format!(
            "[{enum_name}::{member}](crate::{parent}{member})",
            enum_name = enum_info.name,
            member = nameutil::enum_member_name(&member_info.name),
            parent = sym.parent()
        )
    } else {
        format!("`{}`", symbol)
    }
}

fn find_struct(name: &str, env: &Env) -> String {
    let symbols = env.symbols.borrow();

    let symbol = if let Some(obj) = env
        .analysis
        .objects
        .iter()
        .find(|(_, o)| o.c_type == name)
        .map(|(_, o)| o)
    {
        symbols.by_tid(obj.type_id)
    } else if let Some(record) = env
        .analysis
        .records
        .iter()
        .find(|(_, r)| r.type_(&env.library).c_type == name)
        .map(|(_, r)| r)
    {
        symbols.by_tid(record.type_id)
    } else {
        None
    };
    symbol
        .map(|sym| format!("[{name}](crate::{name})", name = sym.full_rust_name()))
        .unwrap_or_else(|| name.to_string())
}

/// Find a function in all the possible items, if not found return the original name surrounded with backsticks
/// A function can either be a struct/interface/record method, a global function or maybe a virtual function
fn find_function(name: &str, env: &Env) -> String {
    let symbols = env.symbols.borrow();
    let is_obj_func = env.analysis.objects.iter().find_map(|(_, obj_info)| {
        obj_info
            .functions
            .iter()
            .filter(|fn_info| fn_info.status != GStatus::Ignore)
            .filter(|fn_info| !fn_info.is_special() && !fn_info.is_async_finish(env))
            .find(|fn_info| fn_info.glib_name == name)
            .map(|fn_info| (obj_info, fn_info))
    });
    let is_record_func = env.analysis.records.iter().find_map(|(_, record_info)| {
        record_info
            .functions
            .iter()
            .filter(|fn_info| fn_info.status != GStatus::Ignore)
            .filter(|fn_info| !fn_info.is_special() && !fn_info.is_async_finish(env))
            .find(|fn_info| fn_info.glib_name == name)
            .map(|fn_info| (record_info, fn_info))
    });
    let is_globa_func = env.analysis.global_functions.as_ref().and_then(|info| {
        info.functions
            .iter()
            .filter(|fn_info| fn_info.status != GStatus::Ignore)
            .filter(|fn_info| !fn_info.is_special() && !fn_info.is_async_finish(env))
            .find(|fn_info| fn_info.glib_name == name)
    });

    if let Some((obj_info, fn_info)) = is_obj_func {
        let sym = symbols.by_tid(obj_info.type_id).unwrap(); // we are sure the object exists
        let (type_name, visible_type_name) = if obj_info.final_type
            || fn_info.kind == FunctionKind::Constructor
            || fn_info.kind == FunctionKind::Function
        {
            (obj_info.name.clone(), obj_info.name.clone())
        } else {
            let type_name = if fn_info.status == GStatus::Generate {
                obj_info.trait_name.clone()
            } else {
                format!("{}Manual", obj_info.trait_name)
            };
            (format!("prelude::{}", type_name), type_name)
        };
        let name = sym.full_rust_name().replace(&obj_info.name, &type_name);
        format!(
            "[{visible_type_name}::{fn_name}](crate::{name}::{fn_name})",
            name = name,
            visible_type_name = visible_type_name,
            fn_name = fn_info.codegen_name()
        )
    } else if let Some((record_info, fn_info)) = is_record_func {
        let sym = symbols.by_tid(record_info.type_id).unwrap(); // we are sure the object exists
        format!(
            "[{name}::{fn_name}](crate::{name}::{fn_name})",
            name = sym.full_rust_name(),
            fn_name = fn_info.codegen_name()
        )
    } else if let Some(fn_info) = is_globa_func {
        format!(
            "[{fn_name}()](crate::{fn_name})",
            fn_name = fn_info.codegen_name()
        )
    } else {
        format!("`{}`", name)
    }
}
<commit_msg>docs: re-add @param parsing back<commit_after>use super::gi_docgen;
use crate::{config::gobjects::GStatus, library::FunctionKind, nameutil, Env};
use once_cell::sync::Lazy;
use regex::{Captures, Match, Regex};

const LANGUAGE_SEP_BEGIN: &str = "<!-- language=\"";
const LANGUAGE_SEP_END: &str = "\" -->";
const LANGUAGE_BLOCK_BEGIN: &str = "|[";
const LANGUAGE_BLOCK_END: &str = "\n]|";

pub fn reformat_doc(input: &str, env: &Env, in_type: &str) -> String {
    code_blocks_transformation(input, env, in_type)
}

fn try_split<'a>(src: &'a str, needle: &str) -> (&'a str, Option<&'a str>) {
    match src.find(needle) {
        Some(pos) => (&src[..pos], Some(&src[pos + needle.len()..])),
        None => (src, None),
    }
}

fn code_blocks_transformation(mut input: &str, env: &Env, in_type: &str) -> String {
    let mut out = String::with_capacity(input.len());

    loop {
        input = match try_split(input, LANGUAGE_BLOCK_BEGIN) {
            (before, Some(after)) => {
                out.push_str(&format(before, env, in_type));
                if let (before, Some(after)) =
                    try_split(get_language(after, &mut out), LANGUAGE_BLOCK_END)
                {
                    out.push_str(before);
                    out.push_str("\n```");
                    after
                } else {
                    after
                }
            }
            (before, None) => {
                out.push_str(&format(before, env, in_type));
                return out;
            }
        };
    }
}

fn get_language<'a>(entry: &'a str, out: &mut String) -> &'a str {
    if let (_, Some(after)) = try_split(entry, LANGUAGE_SEP_BEGIN) {
        if let (before, Some(after)) = try_split(after, LANGUAGE_SEP_END) {
            out.push_str(&format!("\n```{}", before));
            return after;
        }
    }
    out.push_str("\n```text");
    entry
}

fn format(input: &str, env: &Env, in_type: &str) -> String {
    let mut ret = String::with_capacity(input.len());
    // We run gi_docgen first because it's super picky about the types it replaces
    let no_c_types_re = gi_docgen::replace_c_types(input, env, in_type);
    ret.push_str(&replace_c_types(&no_c_types_re, env, in_type));
    ret
}

static SYMBOL: Lazy<Regex> = Lazy::new(|| Regex::new(r"([#%])(\w+\b)([:.]+[\w-]+\b)?").unwrap());
static PARAM_SYMBOL: Lazy<Regex> =
    Lazy::new(|| Regex::new(r"([@])(\w+\b)([:.]+[\w-]+\b)?").unwrap());
static FUNCTION: Lazy<Regex> =
    Lazy::new(|| Regex::new(r"([@#%])?(\w+\b[:.]+)?(\b[a-z0-9_]+)\(\)").unwrap());
// **note**
// The optional . at the end is to make the regex more relaxed for some weird broken cases on gtk3's docs
// it doesn't hurt other docs so please don't drop it
static GDK_GTK: Lazy<Regex> =
    Lazy::new(|| Regex::new(r"`([^\(:])?((G[dts]k|Pango)\w+\b)(\.)?`").unwrap());
static TAGS: Lazy<Regex> = Lazy::new(|| Regex::new(r"<[\w/-]+>").unwrap());
static SPACES: Lazy<Regex> = Lazy::new(|| Regex::new(r"[ ]{2,}").unwrap());

fn replace_c_types(entry: &str, env: &Env, in_type: &str) -> String {
    let out = FUNCTION.replace_all(entry, |caps: &Captures<'_>| {
        let name = &caps[3];
        find_function(name, env)
    });

    let out = SYMBOL.replace_all(&out, |caps: &Captures<'_>| {
        match &caps[2] {
            "TRUE" => "[`true`]".to_string(),
            "FALSE" => "[`false`]".to_string(),
            "NULL" => "[`None`]".to_string(),
            symbol_name => {
                if &caps[1] == "%" {
                    find_constant_or_variant(symbol_name, env)
                } else {
                    let method_name = caps.get(3).map(|m| m.as_str().trim_start_matches('.'));
                    // would be #
                    find_type_by_name(symbol_name, method_name, env)
                }
            }
        }
    });
    let out = GDK_GTK.replace_all(&out, |caps: &Captures<'_>| find_struct(&caps[2], env));
    let out = PARAM_SYMBOL.replace_all(&out, |caps: &Captures<'_>| format!("`{}`", &caps[2]));
    let out = TAGS.replace_all(&out, "`$0`");
    SPACES.replace_all(&out, " ").into_owned()
}

fn find_type_by_name(symbol: &str, method_name: Option<&str>, env: &Env) -> String {
    let symbols = env.symbols.borrow();
    if let Some(method) = method_name {
        let is_obj_func = env
            .analysis
            .objects
            .iter()
            .find(|(_, o)| o.full_name == symbol)
            .and_then(|(_, obj_info)| {
                obj_info
                    .functions
                    .iter()
                    .filter(|fn_info| fn_info.status != GStatus::Ignore)
                    .filter(|fn_info| !fn_info.is_special() && !fn_info.is_async_finish(env))
                    .find(|fn_info| fn_info.name == method)
                    .map(|fn_info| (obj_info, fn_info))
            });
        let is_record_func = env
            .analysis
            .records
            .iter()
            .find(|(_, r)| r.type_(&env.library).c_type == symbol)
            .and_then(|(_, record_info)| {
                record_info
                    .functions
                    .iter()
                    .filter(|fn_info| fn_info.status != GStatus::Ignore)
                    .filter(|fn_info| !fn_info.is_special() && !fn_info.is_async_finish(env))
                    .find(|fn_info| fn_info.glib_name == method)
                    .map(|fn_info| (record_info, fn_info))
            });
        if let Some((obj_info, fn_info)) = is_obj_func {
            let sym = symbols.by_tid(obj_info.type_id).unwrap(); // we are sure the object exists
            let (type_name, visible_type_name) = if obj_info.final_type
                || fn_info.kind == FunctionKind::Constructor
                || fn_info.kind == FunctionKind::Function
            {
                (obj_info.name.clone(), obj_info.name.clone())
            } else {
                let type_name = if fn_info.status == GStatus::Generate {
                    obj_info.trait_name.clone()
                } else {
                    format!("{}Manual", obj_info.trait_name)
                };
                (format!("prelude::{}", type_name), type_name)
            };
            let name = sym.full_rust_name().replace(&obj_info.name, &type_name);
            format!(
                "[{visible_type_name}::{fn_name}](crate::{name}::{fn_name})",
                name = name,
                visible_type_name = visible_type_name,
                fn_name = fn_info.codegen_name()
            )
        } else if let Some((record_info, fn_info)) = is_record_func {
            let sym = symbols.by_tid(record_info.type_id).unwrap(); // we are sure the object exists
            format!(
                "[{name}::{fn_name}](crate::{name}::{fn_name})",
                name = sym.full_rust_name(),
                fn_name = fn_info.codegen_name()
            )
        } else {
            format!("`{}::{}`", symbol, method)
        }
    } else {
        find_struct(symbol, env)
    }
}

fn find_constant_or_variant(symbol: &str, env: &Env) -> String {
    let symbols = env.symbols.borrow();
    if let Some(const_info) = env
        .analysis
        .constants
        .iter()
        .find(|c| c.glib_name == symbol)
    {
        // for whatever reason constants are not part of the symbols list
        format!("[{name}](crate::{name})", name = const_info.name)
    } else if let Some((flag_info, member_info)) = env.analysis.flags.iter().find_map(|f| {
        f.type_(&env.library)
            .members
            .iter()
            .find(|m| m.c_identifier == symbol)
            .map(|m| (f, m))
    }) {
        let sym = symbols.by_tid(flag_info.type_id).unwrap();
        format!(
            "[{flag_name}::{member_name}](crate::{parent}{member_name})",
            member_name = nameutil::bitfield_member_name(&member_info.name),
            flag_name = flag_info.name,
            parent = sym.parent()
        )
    } else if let Some((enum_info, member_info)) = env.analysis.enumerations.iter().find_map(|e| {
        e.type_(&env.library)
            .members
            .iter()
            .find(|m| m.c_identifier == symbol)
            .map(|m| (e, m))
    }) {
        let sym = symbols.by_tid(enum_info.type_id).unwrap();
        format!(
            "[{enum_name}::{member}](crate::{parent}{member})",
            enum_name = enum_info.name,
            member = nameutil::enum_member_name(&member_info.name),
            parent = sym.parent()
        )
    } else {
        format!("`{}`", symbol)
    }
}

fn find_struct(name: &str, env: &Env) -> String {
    let symbols = env.symbols.borrow();

    let symbol = if let Some(obj) = env
        .analysis
        .objects
        .iter()
        .find(|(_, o)| o.c_type == name)
        .map(|(_, o)| o)
    {
        symbols.by_tid(obj.type_id)
    } else if let Some(record) = env
        .analysis
        .records
        .iter()
        .find(|(_, r)| r.type_(&env.library).c_type == name)
        .map(|(_, r)| r)
    {
        symbols.by_tid(record.type_id)
    } else {
        None
    };
    symbol
        .map(|sym| format!("[{name}](crate::{name})", name = sym.full_rust_name()))
        .unwrap_or_else(|| name.to_string())
}

/// Find a function in all the possible items, if not found return the original name surrounded with backsticks
/// A function can either be a struct/interface/record method, a global function or maybe a virtual function
fn find_function(name: &str, env: &Env) -> String {
    let symbols = env.symbols.borrow();
    let is_obj_func = env.analysis.objects.iter().find_map(|(_, obj_info)| {
        obj_info
            .functions
            .iter()
            .filter(|fn_info| fn_info.status != GStatus::Ignore)
            .filter(|fn_info| !fn_info.is_special() && !fn_info.is_async_finish(env))
            .find(|fn_info| fn_info.glib_name == name)
            .map(|fn_info| (obj_info, fn_info))
    });
    let is_record_func = env.analysis.records.iter().find_map(|(_, record_info)| {
        record_info
            .functions
            .iter()
            .filter(|fn_info| fn_info.status != GStatus::Ignore)
            .filter(|fn_info| !fn_info.is_special() && !fn_info.is_async_finish(env))
            .find(|fn_info| fn_info.glib_name == name)
            .map(|fn_info| (record_info, fn_info))
    });
    let is_globa_func = env.analysis.global_functions.as_ref().and_then(|info| {
        info.functions
            .iter()
            .filter(|fn_info| fn_info.status != GStatus::Ignore)
            .filter(|fn_info| !fn_info.is_special() && !fn_info.is_async_finish(env))
            .find(|fn_info| fn_info.glib_name == name)
    });

    if let Some((obj_info, fn_info)) = is_obj_func {
        let sym = symbols.by_tid(obj_info.type_id).unwrap(); // we are sure the object exists
        let (type_name, visible_type_name) = if obj_info.final_type
            || fn_info.kind == FunctionKind::Constructor
            || fn_info.kind == FunctionKind::Function
        {
            (obj_info.name.clone(), obj_info.name.clone())
        } else {
            let type_name = if fn_info.status == GStatus::Generate {
                obj_info.trait_name.clone()
            } else {
                format!("{}Manual", obj_info.trait_name)
            };
            (format!("prelude::{}", type_name), type_name)
        };
        let name = sym.full_rust_name().replace(&obj_info.name, &type_name);
        format!(
            "[{visible_type_name}::{fn_name}](crate::{name}::{fn_name})",
            name = name,
            visible_type_name = visible_type_name,
            fn_name = fn_info.codegen_name()
        )
    } else if let Some((record_info, fn_info)) = is_record_func {
        let sym = symbols.by_tid(record_info.type_id).unwrap(); // we are sure the object exists
        format!(
            "[{name}::{fn_name}](crate::{name}::{fn_name})",
            name = sym.full_rust_name(),
            fn_name = fn_info.codegen_name()
        )
    } else if let Some(fn_info) = is_globa_func {
        format!(
            "[{fn_name}()](crate::{fn_name})",
            fn_name = fn_info.codegen_name()
        )
    } else {
        format!("`{}`", name)
    }
}
<|endoftext|>


<commit_before>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// Test that an `&mut self` method, when invoked on an lvalue whose
// type is `&mut [u8]`, passes in a pointer to the lvalue and not a
// temporary. Issue #19147.

#![feature(core)]

use std::slice;

pub type IoResult<T> = Result<T, ()>;

trait MyWriter {
    fn my_write(&mut self, buf: &[u8]) -> IoResult<()>;
}

impl<'a> MyWriter for &'a mut [u8] {
    fn my_write(&mut self, buf: &[u8]) -> IoResult<()> {
        slice::bytes::copy_memory(buf, *self);

        let write_len = buf.len();
        unsafe {
            *self = slice::from_raw_parts_mut(
                self.as_mut_ptr().offset(write_len as isize),
                self.len() - write_len
            );
        }

        Ok(())
    }
}

fn main() {
    let mut buf = [0; 6];

    {
        let mut writer: &mut [_] = &mut buf;
        writer.my_write(&[0, 1, 2]).unwrap();
        writer.my_write(&[3, 4, 5]).unwrap();
    }

    // If `my_write` is not modifying `buf` in place, then we will
    // wind up with `[3, 4, 5, 0, 0, 0]` because the first call to
    // `my_write()` doesn't update the starting point for the write.

    assert_eq!(buf, [0, 1, 2, 3, 4, 5]);
}
<commit_msg>remove the last mention of IoResult<commit_after>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// Test that an `&mut self` method, when invoked on an lvalue whose
// type is `&mut [u8]`, passes in a pointer to the lvalue and not a
// temporary. Issue #19147.

#![feature(core)]

use std::slice;

trait MyWriter {
    fn my_write(&mut self, buf: &[u8]) -> Result<(), ()>;
}

impl<'a> MyWriter for &'a mut [u8] {
    fn my_write(&mut self, buf: &[u8]) -> Result<(), ()> {
        slice::bytes::copy_memory(buf, *self);

        let write_len = buf.len();
        unsafe {
            *self = slice::from_raw_parts_mut(
                self.as_mut_ptr().offset(write_len as isize),
                self.len() - write_len
            );
        }

        Ok(())
    }
}

fn main() {
    let mut buf = [0; 6];

    {
        let mut writer: &mut [_] = &mut buf;
        writer.my_write(&[0, 1, 2]).unwrap();
        writer.my_write(&[3, 4, 5]).unwrap();
    }

    // If `my_write` is not modifying `buf` in place, then we will
    // wind up with `[3, 4, 5, 0, 0, 0]` because the first call to
    // `my_write()` doesn't update the starting point for the write.

    assert_eq!(buf, [0, 1, 2, 3, 4, 5]);
}
<|endoftext|>


<commit_before>pub trait Property {
    fn inherits_automatically() -> bool;
    fn inherit_from(&self, &Self) -> Self;
}

/// Generates a property definition that simply parses strings to enum variants
/// or to a tuple struct of the given type.
///
/// For example, the SVG spec defines the `stroke-linejoin` property
/// to have possible values `miter | round | bevel | inherit`, with a default
/// of `miter`.  We can define the property like this:
///
/// ```
/// make_property!(
/// StrokeLinejoin,
/// default: Miter,
///
/// "miter" => Miter,
/// "round" => Round,
/// "bevel" => Bevel,
/// );
/// ```
///
/// The macro will generate a `StrokeLinejoin` enum with the provided
/// variants.  It will generate an `impl Default for StrokeLinejoin`
/// with the provided `default:` value.  Finally, it will generate an
/// `impl Parse for StrokeLinejoin`, from `parsers::Parse`, where
/// `type Data = ()` and `type Err = AttributeError`.
#[macro_export]
macro_rules! make_property {
    ($name: ident,
     default: $default: ident,
     inherits_automatically: $inherits_automatically: expr,
     identifiers:
     $($str_prop: expr => $variant: ident,)+
    ) => {
        #[derive(Debug, Copy, Clone, PartialEq)]
        pub enum $name {
            $($variant),+
        }

        impl_default!($name, $name::$default);
        impl_property!($name, $inherits_automatically);

        impl ::parsers::Parse for $name {
            type Data = ();
            type Err = ::error::AttributeError;

            fn parse(s: &str, _: Self::Data) -> Result<$name, ::error::AttributeError> {
                match s.trim() {
                    $($str_prop => Ok($name::$variant),)+

                    _ => Err(::error::AttributeError::from(::parsers::ParseError::new("invalid value"))),
                }
            }
        }
    };

    ($name: ident,
     default: $default: expr,
     inherits_automatically: $inherits_automatically: expr,
     newtype_from_str: $type: ty
    ) => {
        #[derive(Debug, Clone, PartialEq)]
        pub struct $name(pub $type);

        impl_default!($name, $name($default));
        impl_property!($name, $inherits_automatically);

        impl ::parsers::Parse for $name {
            type Data = ();
            type Err = ::error::AttributeError;

            fn parse(s: &str, _: Self::Data) -> Result<$name, ::error::AttributeError> {
                s.trim()
                    .parse()
                    .map(|val| $name(val))
                    .map_err(|_| ::error::AttributeError::from(
                        ::parsers::ParseError::new("parse error")
                    ))
            }
        }
    };

    ($name: ident,
     default: $default: expr,
     inherits_automatically: $inherits_automatically: expr,
     newtype_parse: $type: ty,
     parse_data_type: $parse_data_type: ty
    ) => {
        #[derive(Debug, Clone, PartialEq)]
        pub struct $name(pub $type);

        impl_default!($name, $name($default));
        impl_property!($name, $inherits_automatically);

        impl ::parsers::Parse for $name {
            type Data = $parse_data_type;
            type Err = ::error::AttributeError;

            fn parse(s: &str, d: Self::Data) -> Result<$name, ::error::AttributeError> {
                Ok($name(<$type as ::parsers::Parse>::parse(s, d)?))
            }
        }
    };

    ($name: ident,
     default: $default: expr,
     inherits_automatically: $inherits_automatically: expr,
     newtype: $type: ty
    ) => {
        #[derive(Debug, Clone, PartialEq)]
        pub struct $name(pub $type);

        impl_default!($name, $name($default));
        impl_property!($name, $inherits_automatically);
    };

    ($name: ident,
     inherits_automatically: $inherits_automatically: expr,
     fields:
     $($field_name: ident : $field_type: ty, default: $field_default : expr,)+
    ) => {
        #[derive(Debug, Clone, PartialEq)]
        pub struct $name {
            $(pub $field_name: $field_type),+
        }

        impl_default!($name, $name { $($field_name: $field_default),+ });
        impl_property!($name, $inherits_automatically);
    };
}

macro_rules! impl_default {
    ($name:ident, $default:expr) => {
        impl Default for $name {
            fn default() -> $name {
                $default
            }
        }
    };
}

macro_rules! impl_property {
    ($name:ident, $inherits_automatically:expr) => {
        impl ::property_macros::Property for $name {
            fn inherits_automatically() -> bool {
                $inherits_automatically
            }

            fn inherit_from(&self, v: &Self) -> Self {
                self.clone()
            }
        }
    };
}

#[cfg(test)]
mod tests {
    use super::*;

    use parsers::Parse;

    #[test]
    fn check_generated_property() {
        make_property! {
            Foo,
            default: Def,
            inherits_automatically: true,

            identifiers:
            "def" => Def,
            "bar" => Bar,
            "baz" => Baz,
        }

        assert_eq!(<Foo as Default>::default(), Foo::Def);
        assert_eq!(<Foo as Property>::inherits_automatically(), true);
        assert!(<Foo as Parse>::parse("blargh", ()).is_err());
        assert_eq!(<Foo as Parse>::parse("bar", ()), Ok(Foo::Bar));

        make_property! {
            Bar,
            default: "bar".to_string(),
            inherits_automatically: true,
            newtype_from_str: String
        }

        assert_eq!(<Bar as Default>::default(), Bar("bar".to_string()));
        assert_eq!(<Bar as Property>::inherits_automatically(), true);
        assert_eq!(
            <Bar as Parse>::parse("test", ()),
            Ok(Bar("test".to_string()))
        );

        make_property! {
            Baz,
            default: 42f64,
            inherits_automatically: true,
            newtype_from_str: f64
        }

        assert_eq!(<Baz as Default>::default(), Baz(42f64));
        assert_eq!(<Baz as Property>::inherits_automatically(), true);
        assert_eq!(<Baz as Parse>::parse("42", ()), Ok(Baz(42f64)));
    }
}
<commit_msg>make_property!() - Let the caller fully implement Property for at least one of our variants<commit_after>pub trait Property {
    fn inherits_automatically() -> bool;
    fn inherit_from(&self, &Self) -> Self;
}

/// Generates a property definition that simply parses strings to enum variants
/// or to a tuple struct of the given type.
///
/// For example, the SVG spec defines the `stroke-linejoin` property
/// to have possible values `miter | round | bevel | inherit`, with a default
/// of `miter`.  We can define the property like this:
///
/// ```
/// make_property!(
/// StrokeLinejoin,
/// default: Miter,
///
/// "miter" => Miter,
/// "round" => Round,
/// "bevel" => Bevel,
/// );
/// ```
///
/// The macro will generate a `StrokeLinejoin` enum with the provided
/// variants.  It will generate an `impl Default for StrokeLinejoin`
/// with the provided `default:` value.  Finally, it will generate an
/// `impl Parse for StrokeLinejoin`, from `parsers::Parse`, where
/// `type Data = ()` and `type Err = AttributeError`.
#[macro_export]
macro_rules! make_property {
    ($name: ident,
     default: $default: ident,
     inherits_automatically: $inherits_automatically: expr,
     identifiers:
     $($str_prop: expr => $variant: ident,)+
    ) => {
        #[derive(Debug, Copy, Clone, PartialEq)]
        pub enum $name {
            $($variant),+
        }

        impl_default!($name, $name::$default);
        impl_property!($name, $inherits_automatically);

        impl ::parsers::Parse for $name {
            type Data = ();
            type Err = ::error::AttributeError;

            fn parse(s: &str, _: Self::Data) -> Result<$name, ::error::AttributeError> {
                match s.trim() {
                    $($str_prop => Ok($name::$variant),)+

                    _ => Err(::error::AttributeError::from(::parsers::ParseError::new("invalid value"))),
                }
            }
        }
    };

    ($name: ident,
     default: $default: expr,
     inherits_automatically: $inherits_automatically: expr,
     newtype_from_str: $type: ty
    ) => {
        #[derive(Debug, Clone, PartialEq)]
        pub struct $name(pub $type);

        impl_default!($name, $name($default));
        impl_property!($name, $inherits_automatically);

        impl ::parsers::Parse for $name {
            type Data = ();
            type Err = ::error::AttributeError;

            fn parse(s: &str, _: Self::Data) -> Result<$name, ::error::AttributeError> {
                s.trim()
                    .parse()
                    .map(|val| $name(val))
                    .map_err(|_| ::error::AttributeError::from(
                        ::parsers::ParseError::new("parse error")
                    ))
            }
        }
    };

    ($name: ident,
     default: $default: expr,
     inherits_automatically: $inherits_automatically: expr,
     newtype_parse: $type: ty,
     parse_data_type: $parse_data_type: ty
    ) => {
        #[derive(Debug, Clone, PartialEq)]
        pub struct $name(pub $type);

        impl_default!($name, $name($default));

        impl ::property_macros::Property for $name {
            fn inherits_automatically() -> bool {
                $inherits_automatically
            }

            fn inherit_from(&self, _v: &Self) -> Self {
                self.clone()
            }
        }

        impl ::parsers::Parse for $name {
            type Data = $parse_data_type;
            type Err = ::error::AttributeError;

            fn parse(s: &str, d: Self::Data) -> Result<$name, ::error::AttributeError> {
                Ok($name(<$type as ::parsers::Parse>::parse(s, d)?))
            }
        }
    };

    ($name: ident,
     default: $default: expr,
     newtype_parse: $type: ty,
     parse_data_type: $parse_data_type: ty,
     property_impl { $prop: item }
    ) => {
        #[derive(Debug, Clone, PartialEq)]
        pub struct $name(pub $type);

        impl_default!($name, $name($default));

        $prop

        impl ::parsers::Parse for $name {
            type Data = $parse_data_type;
            type Err = ::error::AttributeError;

            fn parse(s: &str, d: Self::Data) -> Result<$name, ::error::AttributeError> {
                Ok($name(<$type as ::parsers::Parse>::parse(s, d)?))
            }
        }
    };

    ($name: ident,
     default: $default: expr,
     inherits_automatically: $inherits_automatically: expr,
     newtype: $type: ty
    ) => {
        #[derive(Debug, Clone, PartialEq)]
        pub struct $name(pub $type);

        impl_default!($name, $name($default));
        impl_property!($name, $inherits_automatically);
    };

    ($name: ident,
     inherits_automatically: $inherits_automatically: expr,
     fields:
     $($field_name: ident : $field_type: ty, default: $field_default : expr,)+
    ) => {
        #[derive(Debug, Clone, PartialEq)]
        pub struct $name {
            $(pub $field_name: $field_type),+
        }

        impl_default!($name, $name { $($field_name: $field_default),+ });
        impl_property!($name, $inherits_automatically);
    };
}

macro_rules! impl_default {
    ($name:ident, $default:expr) => {
        impl Default for $name {
            fn default() -> $name {
                $default
            }
        }
    };
}

macro_rules! impl_property {
    ($name:ident, $inherits_automatically:expr) => {
        impl ::property_macros::Property for $name {
            fn inherits_automatically() -> bool {
                $inherits_automatically
            }

            fn inherit_from(&self, _v: &Self) -> Self {
                self.clone()
            }
        }
    };
}

#[cfg(test)]
mod tests {
    use super::*;

    use cssparser::RGBA;
    use parsers::Parse;

    #[test]
    fn check_generated_property() {
        make_property! {
            Foo,
            default: Def,
            inherits_automatically: true,

            identifiers:
            "def" => Def,
            "bar" => Bar,
            "baz" => Baz,
        }

        assert_eq!(<Foo as Default>::default(), Foo::Def);
        assert_eq!(<Foo as Property>::inherits_automatically(), true);
        assert!(<Foo as Parse>::parse("blargh", ()).is_err());
        assert_eq!(<Foo as Parse>::parse("bar", ()), Ok(Foo::Bar));

        make_property! {
            Bar,
            default: "bar".to_string(),
            inherits_automatically: true,
            newtype_from_str: String
        }

        assert_eq!(<Bar as Default>::default(), Bar("bar".to_string()));
        assert_eq!(<Bar as Property>::inherits_automatically(), true);
        assert_eq!(
            <Bar as Parse>::parse("test", ()),
            Ok(Bar("test".to_string()))
        );

        make_property! {
            Baz,
            default: 42f64,
            inherits_automatically: true,
            newtype_from_str: f64
        }

        assert_eq!(<Baz as Default>::default(), Baz(42f64));
        assert_eq!(<Baz as Property>::inherits_automatically(), true);
        assert_eq!(<Baz as Parse>::parse("42", ()), Ok(Baz(42f64)));
    }

    #[test]
    fn check_inherit_from() {
        make_property! {
            AddColor,
            default: RGBA::new(1, 1, 1, 1),
            newtype_parse: RGBA,
            parse_data_type: (),
            property_impl {
                impl Property for AddColor {
                    fn inherits_automatically() -> bool {
                        true
                    }

                    fn inherit_from(&self, v: &Self) -> Self {
                        AddColor(RGBA::new(
                            self.0.red + v.0.red,
                            self.0.green + v.0.green,
                            self.0.blue + v.0.blue,
                            self.0.alpha + v.0.alpha
                        ))
                    }
                }
            }
        }

        let a = <AddColor as Parse>::parse("#01010101", ()).unwrap();
        let b = <AddColor as Parse>::parse("#02030405", ()).unwrap();

        let c = b.inherit_from(&a);

        assert_eq!(c, AddColor(RGBA::new(3, 4, 5, 6)));
    }
}
<|endoftext|>


<commit_before>//  Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::*;
use dubp::{
    block::parser::parse_json_block_from_serde_value, block::parser::ParseJsonBlockError,
    block::prelude::DubpBlockTrait, block::DubpBlock, common::prelude::BlockNumber,
};
use duniter_core::dbs::{databases::bc_v1::BcV1DbReadable, FileBackend};
use fast_threadpool::{ThreadPool, ThreadPoolConfig};
use std::{ops::Deref, path::PathBuf};

const CHUNK_SIZE: usize = 250;

pub(crate) fn migrate(profile_path: PathBuf) -> anyhow::Result<()> {
    let start_time = Instant::now();

    // Remove bc_db and gva_db
    std::fs::remove_dir_all(profile_path.join("data/bc_v2_sled"))?;
    std::fs::remove_dir_all(profile_path.join("data/gva_v1_sled"))?;

    // Open bc_db and gva_db
    let (bc_db, shared_dbs) = duniter_core::dbs::open_dbs(Some(profile_path.as_path()))?;
    let gva_db = duniter_gva_indexer::get_gva_db_rw(Some(profile_path.as_path()));

    if let Err(e) = migrate_inner(&bc_db, gva_db, profile_path, shared_dbs, start_time) {
        // Clear bc_db and gva_db
        bc_db.clear()?;
        gva_db.clear()?;

        Err(e)
    } else {
        Ok(())
    }
}

fn migrate_inner(
    bc_db: &BcV2Db<FileBackend>,
    gva_db: &'static GvaV1Db<FileBackend>,
    profile_path: PathBuf,
    shared_dbs: SharedDbs<FileBackend>,
    start_time: Instant,
) -> anyhow::Result<()> {
    let data_path = profile_path.join(crate::DATA_DIR);
    let duniter_js_db = BcV1Db::<LevelDb>::open(LevelDbConf {
        db_path: data_path.as_path().join("leveldb"),
        ..Default::default()
    })?;

    let dbs_pool = ThreadPool::start(ThreadPoolConfig::default(), shared_dbs).into_sync_handler();

    if let Some(target) = get_target_block_number(&duniter_js_db)? {
        println!("target block: #{}", target.0);

        let (s, r) = flume::unbounded();
        let reader_handle = std::thread::spawn(move || {
            duniter_js_db.main_blocks().iter(.., |it| {
                it.values().try_for_each(|block_res| {
                    s.send(block_res).map_err(|_| anyhow!("fail to send"))
                })
            })
        });
        let (s2, r2) = flume::unbounded();
        let parser_handle = std::thread::spawn(move || {
            let target_u64 = target.0 as u64;
            let mut db_blocks = Vec::with_capacity(CHUNK_SIZE);
            while let Ok(db_block_res) = r.recv() {
                let db_block = db_block_res?;
                let db_block_number = db_block.number;
                db_blocks.push(db_block);
                if db_blocks.len() == CHUNK_SIZE || db_block_number == target_u64 {
                    let blocks = std::mem::take(&mut db_blocks)
                        .into_par_iter()
                        .map(|db_block| match serde_json::to_value(&db_block) {
                            Ok(json_block) => {
                                match parse_json_block_from_serde_value(&json_block) {
                                    Ok(block) => match block {
                                        DubpBlock::V10(block_v10) => Ok(block_v10),
                                    },
                                    Err(e) => Err(anyhow::Error::new::<ParseJsonBlockError>(e)),
                                }
                            }
                            Err(e) => Err(anyhow::Error::new::<serde_json::Error>(e)),
                        })
                        .collect::<anyhow::Result<Vec<_>>>()?;
                    s2.send(blocks).map_err(|_| anyhow!("fail to send"))?;
                    db_blocks.reserve_exact(CHUNK_SIZE);
                }
            }
            Ok::<(), anyhow::Error>(())
        });

        let mut current = None;
        let mut currency_params = Default::default();
        while let Ok(chunk) = r2.recv() {
            if !chunk.is_empty() {
                println!(
                    "Apply chunk #{}-#{} ..",
                    chunk[0].number(),
                    chunk[chunk.len() - 1].number()
                );
                if let Some(currency_parameters) = chunk[0].currency_parameters() {
                    currency_params = currency_parameters;
                }
                let chunk = Arc::from(chunk);
                let chunk_arc_clone = Arc::clone(&chunk);
                let gva_handle = dbs_pool
                    .launch(move |_| {
                        for block in chunk_arc_clone.deref() {
                            duniter_gva_indexer::apply_block(block, currency_params, gva_db)?;
                        }
                        Ok::<_, KvError>(())
                    })
                    .expect("gva:apply_chunk: dbs pool disconnected");
                current = Some(duniter_core::dbs_write_ops::apply_block::apply_chunk(
                    bc_db, current, &dbs_pool, chunk, None,
                )?);
                gva_handle
                    .join()
                    .expect("gva:apply_chunk: dbs pool disconnected")?;
            }
        }

        reader_handle.join().expect("reader thread panic")?;
        parser_handle.join().expect("parser thread panic")?;

        println!("Flush DBs caches on disk...");
        bc_db.save()?;
        gva_db.save()?;

        if let Some(current) = current {
            if current.number != target.0 {
                Err(anyhow::anyhow!("Migration fail: current != target"))
            } else {
                let duration = start_time.elapsed();
                println!(
                    "Migration successfully completed on {} seconds.",
                    duration.as_secs()
                );
                Ok(())
            }
        } else {
            Err(anyhow::anyhow!("Migration fail: rust dbs are empty"))
        }
    } else {
        Err(anyhow::anyhow!("Empty blockchain"))
    }
}

fn get_target_block_number(duniter_js_db: &BcV1Db<LevelDb>) -> KvResult<Option<BlockNumber>> {
    duniter_js_db.main_blocks().iter_rev(.., |it| {
        it.keys().map(|k_res| k_res.map(|bn| bn.0)).next_res()
    })
}

/*fn get_chunk(duniter_js_db: &BcV1Db<LevelDb>, i: u32) -> anyhow::Result<Vec<DubpBlockV10>> {
    let start = BlockNumberKeyV1(BlockNumber(i * CHUNK_SIZE));
    let end = BlockNumberKeyV1(BlockNumber(((i + 1) * CHUNK_SIZE) - 1));
    println!("get_chunk({}): range {}..{}", i, start.0, end.0);
    let db_blocks = duniter_js_db
        .main_blocks()
        .iter(start..=end, |it| it.values().collect::<KvResult<Vec<_>>>())?;

    db_blocks
        .into_par_iter()
        .map(|db_block| match serde_json::to_value(&db_block) {
            Ok(json_block) => match parse_json_block_from_serde_value(&json_block) {
                Ok(block) => match block {
                    DubpBlock::V10(block_v10) => Ok(block_v10),
                },
                Err(e) => Err(anyhow::Error::new::<ParseJsonBlockError>(e)),
            },
            Err(e) => Err(anyhow::Error::new::<serde_json::Error>(e)),
        })
        .collect()
}
*/
<commit_msg>fix(dex):migrate: add gva blocks chunks phase<commit_after>//  Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::*;
use dubp::{
    block::parser::parse_json_block_from_serde_value, block::parser::ParseJsonBlockError,
    block::prelude::DubpBlockTrait, block::DubpBlock, common::prelude::BlockNumber,
};
use duniter_core::dbs::{databases::bc_v1::BcV1DbReadable, FileBackend};
use fast_threadpool::{ThreadPool, ThreadPoolConfig};
use std::{ops::Deref, path::PathBuf};

const CHUNK_SIZE: usize = 250;

pub(crate) fn migrate(profile_path: PathBuf) -> anyhow::Result<()> {
    let start_time = Instant::now();

    // Remove bc_db and gva_db
    std::fs::remove_dir_all(profile_path.join("data/bc_v2_sled"))?;
    std::fs::remove_dir_all(profile_path.join("data/gva_v1_sled"))?;

    // Open bc_db and gva_db
    let (bc_db, shared_dbs) = duniter_core::dbs::open_dbs(Some(profile_path.as_path()))?;
    let gva_db = duniter_gva_indexer::get_gva_db_rw(Some(profile_path.as_path()));

    if let Err(e) = migrate_inner(&bc_db, gva_db, profile_path, shared_dbs, start_time) {
        // Clear bc_db and gva_db
        bc_db.clear()?;
        gva_db.clear()?;

        Err(e)
    } else {
        Ok(())
    }
}

fn migrate_inner(
    bc_db: &BcV2Db<FileBackend>,
    gva_db: &'static GvaV1Db<FileBackend>,
    profile_path: PathBuf,
    shared_dbs: SharedDbs<FileBackend>,
    start_time: Instant,
) -> anyhow::Result<()> {
    let data_path = profile_path.join(crate::DATA_DIR);
    let duniter_js_db = BcV1Db::<LevelDb>::open(LevelDbConf {
        db_path: data_path.as_path().join("leveldb"),
        ..Default::default()
    })?;

    let dbs_pool = ThreadPool::start(ThreadPoolConfig::default(), shared_dbs).into_sync_handler();

    if let Some(target) = get_target_block_number(&duniter_js_db)? {
        println!("target block: #{}", target.0);

        let (s, r) = flume::unbounded();
        let reader_handle = std::thread::spawn(move || {
            duniter_js_db.main_blocks().iter(.., |it| {
                it.values().try_for_each(|block_res| {
                    s.send(block_res).map_err(|_| anyhow!("fail to send"))
                })
            })
        });
        let (s2, r2) = flume::unbounded();
        let parser_handle = std::thread::spawn(move || {
            let target_u64 = target.0 as u64;
            let mut db_blocks = Vec::with_capacity(CHUNK_SIZE);
            while let Ok(db_block_res) = r.recv() {
                let db_block = db_block_res?;
                let db_block_number = db_block.number;
                db_blocks.push(db_block);
                if db_blocks.len() == CHUNK_SIZE || db_block_number == target_u64 {
                    let blocks = std::mem::take(&mut db_blocks)
                        .into_par_iter()
                        .map(|db_block| match serde_json::to_value(&db_block) {
                            Ok(json_block) => {
                                match parse_json_block_from_serde_value(&json_block) {
                                    Ok(block) => match block {
                                        DubpBlock::V10(block_v10) => Ok(block_v10),
                                    },
                                    Err(e) => Err(anyhow::Error::new::<ParseJsonBlockError>(e)),
                                }
                            }
                            Err(e) => Err(anyhow::Error::new::<serde_json::Error>(e)),
                        })
                        .collect::<anyhow::Result<Vec<_>>>()?;
                    s2.send(blocks).map_err(|_| anyhow!("fail to send"))?;
                    db_blocks.reserve_exact(CHUNK_SIZE);
                }
            }
            Ok::<(), anyhow::Error>(())
        });

        let mut current = None;
        let mut currency_params = Default::default();
        while let Ok(chunk) = r2.recv() {
            if !chunk.is_empty() {
                println!(
                    "Apply chunk #{}-#{} ..",
                    chunk[0].number(),
                    chunk[chunk.len() - 1].number()
                );
                if let Some(currency_parameters) = chunk[0].currency_parameters() {
                    currency_params = currency_parameters;
                }
                let chunk = Arc::from(chunk);
                let chunk_arc_clone = Arc::clone(&chunk);
                let profile_path_clone = profile_path.clone();
                let gva_chunks_handle = dbs_pool
                    .launch(move |_| {
                        for block in chunk_arc_clone.deref() {
                            duniter_gva_indexer::apply_block_blocks_chunk(
                                block,
                                gva_db,
                                &profile_path_clone,
                            )?;
                        }
                        Ok::<_, KvError>(())
                    })
                    .expect("gva:apply_chunk: dbs pool disconnected");
                let chunk_arc_clone = Arc::clone(&chunk);
                let gva_handle = dbs_pool
                    .launch(move |_| {
                        for block in chunk_arc_clone.deref() {
                            duniter_gva_indexer::apply_block(block, currency_params, gva_db)?;
                        }
                        Ok::<_, KvError>(())
                    })
                    .expect("gva:apply_chunk: dbs pool disconnected");
                current = Some(duniter_core::dbs_write_ops::apply_block::apply_chunk(
                    bc_db, current, &dbs_pool, chunk, None,
                )?);
                gva_chunks_handle
                    .join()
                    .expect("gva:apply_block_blocks_chunk: dbs pool disconnected")?;
                gva_handle
                    .join()
                    .expect("gva:apply_chunk: dbs pool disconnected")?;
            }
        }

        reader_handle.join().expect("reader thread panic")?;
        parser_handle.join().expect("parser thread panic")?;

        println!("Flush DBs caches on disk...");
        bc_db.save()?;
        gva_db.save()?;

        if let Some(current) = current {
            if current.number != target.0 {
                Err(anyhow::anyhow!("Migration fail: current != target"))
            } else {
                let duration = start_time.elapsed();
                println!(
                    "Migration successfully completed on {} seconds.",
                    duration.as_secs()
                );
                Ok(())
            }
        } else {
            Err(anyhow::anyhow!("Migration fail: rust dbs are empty"))
        }
    } else {
        Err(anyhow::anyhow!("Empty blockchain"))
    }
}

fn get_target_block_number(duniter_js_db: &BcV1Db<LevelDb>) -> KvResult<Option<BlockNumber>> {
    duniter_js_db.main_blocks().iter_rev(.., |it| {
        it.keys().map(|k_res| k_res.map(|bn| bn.0)).next_res()
    })
}

/*fn get_chunk(duniter_js_db: &BcV1Db<LevelDb>, i: u32) -> anyhow::Result<Vec<DubpBlockV10>> {
    let start = BlockNumberKeyV1(BlockNumber(i * CHUNK_SIZE));
    let end = BlockNumberKeyV1(BlockNumber(((i + 1) * CHUNK_SIZE) - 1));
    println!("get_chunk({}): range {}..{}", i, start.0, end.0);
    let db_blocks = duniter_js_db
        .main_blocks()
        .iter(start..=end, |it| it.values().collect::<KvResult<Vec<_>>>())?;

    db_blocks
        .into_par_iter()
        .map(|db_block| match serde_json::to_value(&db_block) {
            Ok(json_block) => match parse_json_block_from_serde_value(&json_block) {
                Ok(block) => match block {
                    DubpBlock::V10(block_v10) => Ok(block_v10),
                },
                Err(e) => Err(anyhow::Error::new::<ParseJsonBlockError>(e)),
            },
            Err(e) => Err(anyhow::Error::new::<serde_json::Error>(e)),
        })
        .collect()
}
*/
<|endoftext|>


<commit_before>//  Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

#![deny(
    clippy::unwrap_used,
    missing_copy_implementations,
    trivial_casts,
    trivial_numeric_casts,
    unstable_features,
    unused_import_braces
)]

use std::borrow::Cow;

use dubp::common::crypto::keys::ed25519::PublicKey;
use dubp::documents::prelude::*;
use dubp::documents::transaction::TransactionDocumentV10;
use duniter_dbs::kv_typed::prelude::*;
use duniter_dbs::{GvaV1DbReadable, TxsMpV2Db, TxsMpV2DbReadable};
use thiserror::Error;

#[derive(Debug, Error)]
pub enum TxMpError {
    #[error("{0}")]
    Db(KvError),
    #[error("Mempool full")]
    Full,
    #[error("Transaction already written in blockchain")]
    TxAlreadyWritten,
}

impl From<KvError> for TxMpError {
    fn from(e: KvError) -> Self {
        TxMpError::Db(e)
    }
}

#[derive(Clone, Copy, Debug)]
pub struct TxsMempool {
    max_size: usize,
}

impl TxsMempool {
    pub fn new(max_size: usize) -> Self {
        TxsMempool { max_size }
    }
    pub fn accept_new_tx<GvaDb: GvaV1DbReadable, TxsMpDb: TxsMpV2DbReadable>(
        &self,
        gva_db_ro: &GvaDb,
        server_pubkey: PublicKey,
        tx: TransactionDocumentV10,
        txs_mp_db_ro: &TxsMpDb,
    ) -> Result<(), TxMpError> {
        if duniter_dbs_read_ops::txs_history::tx_exist(gva_db_ro, tx.get_hash())? {
            Err(TxMpError::TxAlreadyWritten)
        } else if tx.issuers().contains(&server_pubkey)
            || txs_mp_db_ro.txs().count()? < self.max_size
        {
            Ok(())
        } else {
            Err(TxMpError::Full)
        }
    }

    pub fn add_pending_tx<B: Backend, GvaDb: GvaV1DbReadable>(
        &self,
        gva_db_ro: &GvaDb,
        server_pubkey: PublicKey,
        txs_mp_db: &TxsMpV2Db<B>,
        tx: &TransactionDocumentV10,
    ) -> Result<(), TxMpError> {
        if duniter_dbs_read_ops::txs_history::tx_exist(gva_db_ro, tx.get_hash())? {
            Err(TxMpError::TxAlreadyWritten)
        } else if tx.issuers().contains(&server_pubkey) {
            duniter_dbs_write_ops::txs_mp::add_pending_tx(
                |_, _| Ok(()),
                txs_mp_db,
                Cow::Borrowed(tx),
            )?;
            Ok(())
        } else {
            duniter_dbs_write_ops::txs_mp::add_pending_tx(
                |_tx, txs| {
                    if txs.count()? < self.max_size {
                        Err(KvError::Custom(TxMpError::Full.into()))
                    } else {
                        Ok(())
                    }
                },
                txs_mp_db,
                Cow::Borrowed(tx),
            )?;
            Ok(())
        }
    }

    #[doc(hidden)]
    pub fn add_pending_tx_force<B: Backend>(
        &self,
        txs_mp_db: &TxsMpV2Db<B>,
        tx: &TransactionDocumentV10,
    ) -> KvResult<()> {
        duniter_dbs_write_ops::txs_mp::add_pending_tx(|_, _| Ok(()), txs_mp_db, Cow::Borrowed(tx))?;
        Ok(())
    }

    pub fn get_free_rooms<TxsMpDb: TxsMpV2DbReadable>(
        &self,
        txs_mp_db_ro: &TxsMpDb,
    ) -> KvResult<usize> {
        Ok(self.max_size - txs_mp_db_ro.txs().count()?)
    }
}
<commit_msg>[fix] mempools: add_pending_tx: error in control condition<commit_after>//  Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

#![deny(
    clippy::unwrap_used,
    missing_copy_implementations,
    trivial_casts,
    trivial_numeric_casts,
    unstable_features,
    unused_import_braces
)]

use std::borrow::Cow;

use dubp::common::crypto::keys::ed25519::PublicKey;
use dubp::documents::prelude::*;
use dubp::documents::transaction::TransactionDocumentV10;
use duniter_dbs::kv_typed::prelude::*;
use duniter_dbs::{GvaV1DbReadable, TxsMpV2Db, TxsMpV2DbReadable};
use thiserror::Error;

#[derive(Debug, Error)]
pub enum TxMpError {
    #[error("{0}")]
    Db(KvError),
    #[error("Mempool full")]
    Full,
    #[error("Transaction already written in blockchain")]
    TxAlreadyWritten,
}

impl From<KvError> for TxMpError {
    fn from(e: KvError) -> Self {
        TxMpError::Db(e)
    }
}

#[derive(Clone, Copy, Debug)]
pub struct TxsMempool {
    max_size: usize,
}

impl TxsMempool {
    pub fn new(max_size: usize) -> Self {
        TxsMempool { max_size }
    }
    pub fn accept_new_tx<GvaDb: GvaV1DbReadable, TxsMpDb: TxsMpV2DbReadable>(
        &self,
        gva_db_ro: &GvaDb,
        server_pubkey: PublicKey,
        tx: TransactionDocumentV10,
        txs_mp_db_ro: &TxsMpDb,
    ) -> Result<(), TxMpError> {
        if duniter_dbs_read_ops::txs_history::tx_exist(gva_db_ro, tx.get_hash())? {
            Err(TxMpError::TxAlreadyWritten)
        } else if tx.issuers().contains(&server_pubkey)
            || txs_mp_db_ro.txs().count()? < self.max_size
        {
            Ok(())
        } else {
            Err(TxMpError::Full)
        }
    }

    pub fn add_pending_tx<B: Backend, GvaDb: GvaV1DbReadable>(
        &self,
        gva_db_ro: &GvaDb,
        server_pubkey: PublicKey,
        txs_mp_db: &TxsMpV2Db<B>,
        tx: &TransactionDocumentV10,
    ) -> Result<(), TxMpError> {
        if duniter_dbs_read_ops::txs_history::tx_exist(gva_db_ro, tx.get_hash())? {
            Err(TxMpError::TxAlreadyWritten)
        } else if tx.issuers().contains(&server_pubkey) {
            duniter_dbs_write_ops::txs_mp::add_pending_tx(
                |_, _| Ok(()),
                txs_mp_db,
                Cow::Borrowed(tx),
            )?;
            Ok(())
        } else {
            duniter_dbs_write_ops::txs_mp::add_pending_tx(
                |_tx, txs| {
                    if txs.count()? >= self.max_size {
                        Err(KvError::Custom(TxMpError::Full.into()))
                    } else {
                        Ok(())
                    }
                },
                txs_mp_db,
                Cow::Borrowed(tx),
            )?;
            Ok(())
        }
    }

    #[doc(hidden)]
    pub fn add_pending_tx_force<B: Backend>(
        &self,
        txs_mp_db: &TxsMpV2Db<B>,
        tx: &TransactionDocumentV10,
    ) -> KvResult<()> {
        duniter_dbs_write_ops::txs_mp::add_pending_tx(|_, _| Ok(()), txs_mp_db, Cow::Borrowed(tx))?;
        Ok(())
    }

    pub fn get_free_rooms<TxsMpDb: TxsMpV2DbReadable>(
        &self,
        txs_mp_db_ro: &TxsMpDb,
    ) -> KvResult<usize> {
        Ok(self.max_size - txs_mp_db_ro.txs().count()?)
    }
}
<|endoftext|>


<commit_before>use std::cell::RefCell;
use std::io;
use std::net::{Ipv4Addr, SocketAddr};
use std::rc::{Rc, Weak};
use mio::{Event, PollOpt, Ready, Token};
use mio::net::TcpStream;

use super::client::Client;
use super::connection::{self, Connection, ConnectionId};
use super::ipv4_header::IPv4Header;
use super::ipv4_packet::{IPv4Packet, MAX_PACKET_LENGTH};
use super::packetizer::Packetizer;
use super::selector::Selector;
use super::stream_buffer::StreamBuffer;
use super::tcp_header::TCPHeader;
use super::transport_header::TransportHeader;

const TAG: &'static str = "TCPConnection";

enum TCPState {
    SynSent,
    SynReceived,
    Established,
    CloseWait,
    LastAck,
}

struct TCPConnection {
    id: ConnectionId,
    client: Weak<RefCell<Client>>,
    stream: TcpStream,
    token: Token,
    client_to_network: StreamBuffer,
    network_to_client: Packetizer,
    closed: bool,
    state: TCPState,
    syn_sequence_number: u32,
    sequence_number: u32,
    acknowledgement_number: u32,
    their_acknowledgement_numbe: u32,
    client_window: u16,
    remote_closed: bool,
}

impl TCPConnection {
    pub fn new(selector: &mut Selector, id: ConnectionId, client: Weak<RefCell<Client>>, ipv4_header: &IPv4Header, transport_header: &TransportHeader) -> io::Result<Rc<RefCell<Self>>> {
        let stream = TCPConnection::create_stream(&id)?;

        if let TransportHeader::TCP(ref tcp_header) = *transport_header {
            // shrink the TCP options to pass a minimal refrence header to the packetizer
            let mut shrinked_tcp_header_raw = [0u8; 20];
            shrinked_tcp_header_raw.copy_from_slice(&tcp_header.raw()[..20]);
            let mut shrinked_tcp_header_data = tcp_header.data().clone();
            let mut shrinked_tcp_header = shrinked_tcp_header_data.bind_mut(&mut shrinked_tcp_header_raw);
            shrinked_tcp_header.shrink_options();
            assert_eq!(20, shrinked_tcp_header.header_length());
            let shrinked_transport_header = TCPHeader::from(shrinked_tcp_header).into();

            let packetizer = Packetizer::new(&ipv4_header, &shrinked_transport_header);
        } else {
            panic!("Not a TCP header");
        }
    }

    fn create_stream(id: &ConnectionId) -> io::Result<TcpStream> {
        let rewritten_destination = connection::rewritten_destination(id.destination_ip(), id.destination_port()).into();
        TcpStream::connect(&rewritten_destination)
    }
}
<commit_msg>Implement TCPConnection skeleton<commit_after>use std::cell::RefCell;
use std::io;
use std::net::{Ipv4Addr, SocketAddr};
use std::rc::{Rc, Weak};
use mio::{Event, PollOpt, Ready, Token};
use mio::net::TcpStream;

use super::client::Client;
use super::connection::{self, Connection, ConnectionId};
use super::ipv4_header::IPv4Header;
use super::ipv4_packet::{IPv4Packet, MAX_PACKET_LENGTH};
use super::packetizer::Packetizer;
use super::selector::Selector;
use super::stream_buffer::StreamBuffer;
use super::tcp_header::TCPHeader;
use super::transport_header::TransportHeader;

const TAG: &'static str = "TCPConnection";

enum TCPState {
    Init,
    SynSent,
    SynReceived,
    Established,
    CloseWait,
    LastAck,
}

struct TCPConnection {
    id: ConnectionId,
    client: Weak<RefCell<Client>>,
    stream: TcpStream,
    token: Token,
    client_to_network: StreamBuffer,
    network_to_client: Packetizer,
    closed: bool,
    state: TCPState,
    syn_sequence_number: u32,
    sequence_number: u32,
    acknowledgement_number: u32,
    their_acknowledgement_number: u32,
    client_window: u16,
    remote_closed: bool,
}

impl TCPConnection {
    pub fn new(selector: &mut Selector, id: ConnectionId, client: Weak<RefCell<Client>>, ipv4_header: &IPv4Header, transport_header: &TransportHeader) -> io::Result<Rc<RefCell<Self>>> {
        let stream = TCPConnection::create_stream(&id)?;

        if let TransportHeader::TCP(ref tcp_header) = *transport_header {
            // shrink the TCP options to pass a minimal refrence header to the packetizer
            let mut shrinked_tcp_header_raw = [0u8; 20];
            shrinked_tcp_header_raw.copy_from_slice(&tcp_header.raw()[..20]);
            let mut shrinked_tcp_header_data = tcp_header.data().clone();
            let mut shrinked_tcp_header = shrinked_tcp_header_data.bind_mut(&mut shrinked_tcp_header_raw);
            shrinked_tcp_header.shrink_options();
            assert_eq!(20, shrinked_tcp_header.header_length());
            let shrinked_transport_header = TCPHeader::from(shrinked_tcp_header).into();

            let packetizer = Packetizer::new(&ipv4_header, &shrinked_transport_header);

            let rc = Rc::new(RefCell::new(Self {
                id: id,
                client: client,
                stream: stream,
                token: Token(0), // default value, will be set afterwards
                client_to_network: StreamBuffer::new(4 * MAX_PACKET_LENGTH),
                network_to_client: packetizer,
                closed: false,
                state: TCPState::Init,
                syn_sequence_number: 0,
                sequence_number: 0,
                acknowledgement_number: 0,
                their_acknowledgement_number: 0,
                client_window: 0,
                remote_closed: false,
            }));

            {
                let rc_clone = rc.clone();
                let handler = move |selector: &mut Selector, ready| {
                    let mut self_ref = rc_clone.borrow_mut();
                    self_ref.on_ready(selector, ready);
                };
                let mut self_ref = rc.borrow_mut();
                let token = selector.register(&self_ref.stream, handler, Ready::readable(), PollOpt::level())?;
                self_ref.token = token;
            }
            Ok(rc)
        } else {
            panic!("Not a TCP header");
        }
    }

    fn create_stream(id: &ConnectionId) -> io::Result<TcpStream> {
        let rewritten_destination = connection::rewritten_destination(id.destination_ip(), id.destination_port()).into();
        TcpStream::connect(&rewritten_destination)
    }

    fn process_send(&mut self, selector: &mut Selector) {
        // TODO
    }

    fn process_receive(&mut self, selector: &mut Selector) {
        // TODO
    }

    fn update_interests(&mut self, selector: &mut Selector) {
        // TODO
    }

    fn on_ready(&mut self, selector: &mut Selector, event: Event) {
        assert!(!self.closed);
        let ready = event.readiness();
        if ready.is_writable() {
            self.process_send(selector);
        }
        if !self.closed && ready.is_readable() {
            self.process_receive(selector);
        }
        if !self.closed {
            self.update_interests(selector);
        }
    }
}
<|endoftext|>


<commit_before>//! Linear Regression module
//!
//! Contains implemention of linear regression using
//! OLS and gradient descent optimization.
//!
//! The regressor will automatically add the intercept term
//! so you do not need to format the input matrices yourself.
//!
//! # Usage
//!
//! ```
//! use rusty_machine::learning::lin_reg::LinRegressor;
//! use rusty_machine::learning::SupModel;
//! use rusty_machine::linalg::matrix::Matrix;
//! use rusty_machine::linalg::vector::Vector;
//!
//! let inputs = Matrix::new(4,1,vec![1.0,3.0,5.0,7.0]);
//! let targets = Vector::new(vec![1.,5.,9.,13.]);
//!
//! let mut lin_mod = LinRegressor::default();
//!
//! // Train the model
//! lin_mod.train(&inputs, &targets);
//!
//! // Now we'll predict a new point
//! let new_point = Matrix::new(1,1,vec![10.]);
//! let output = lin_mod.predict(&new_point);
//!
//! // Hopefully we classified our new point correctly!
//! assert!(output[0] > 17f64, "Our regressor isn't very good!");
//! ```

use learning::SupModel;
use linalg::matrix::Matrix;
use linalg::vector::Vector;
use learning::toolkit::cost_fn::CostFunc;
use learning::toolkit::cost_fn::MeanSqError;
use learning::optim::grad_desc::GradientDesc;
use learning::optim::OptimAlgorithm;
use learning::optim::Optimizable;

/// Linear Regression Model.
///
/// Contains option for optimized parameter.
pub struct LinRegressor {
    /// The parameters for the regression model.
    parameters: Option<Vector<f64>>,
}

impl Default for LinRegressor {
    fn default() -> LinRegressor {
        LinRegressor { parameters: None }
    }
}

impl LinRegressor {

    /// Get the parameters from the model.
    ///
    /// Returns an option that is None if the model has not been trained.
    pub fn parameters(&self) -> Option<Vector<f64>> {
        match self.parameters {
            None => None,
            Some(ref x) => Some(x.clone()),
        }
    }
}

impl SupModel<Matrix<f64>, Vector<f64>> for LinRegressor {
    /// Train the linear regression model.
    ///
    /// Takes training data and output values as input.
    ///
    /// # Examples
    ///
    /// ```
    /// use rusty_machine::learning::lin_reg::LinRegressor;
    /// use rusty_machine::linalg::matrix::Matrix;
    /// use rusty_machine::linalg::vector::Vector;
    /// use rusty_machine::learning::SupModel;
    ///
    /// let mut lin_mod = LinRegressor::default();
    /// let inputs = Matrix::new(3,1, vec![2.0, 3.0, 4.0]);
    /// let targets = Vector::new(vec![5.0, 6.0, 7.0]);
    ///
    /// lin_mod.train(&inputs, &targets);
    /// ```
    fn train(&mut self, inputs: &Matrix<f64>, targets: &Vector<f64>) {
        let ones = Matrix::<f64>::ones(inputs.rows(), 1);
        let full_inputs = ones.hcat(inputs);

        let xt = full_inputs.transpose();

        self.parameters = Some(((&xt * full_inputs).inverse() * &xt) * targets);
    }

    /// Predict output value from input data.
    ///
    /// Model must be trained before prediction can be made.
    fn predict(&self, inputs: &Matrix<f64>) -> Vector<f64> {
        if let Some(ref v) = self.parameters {
            let ones = Matrix::<f64>::ones(inputs.rows(), 1);
            let full_inputs = ones.hcat(inputs);
            full_inputs * v
        }
        else {
            panic!("Model has not been trained.");
        }
    }
}

impl Optimizable for LinRegressor {
    type Inputs = Matrix<f64>;
    type Targets = Vector<f64>;

    fn compute_grad(&self, params: &[f64], inputs: &Matrix<f64>, targets: &Vector<f64>) -> (f64, Vec<f64>) {

        let beta_vec = Vector::new(params.to_vec());
        let outputs = inputs * beta_vec;

        let cost = MeanSqError::cost(&outputs, targets);
        let grad = (inputs.transpose() * (outputs-targets)) / (inputs.rows() as f64);

        (cost, grad.into_vec())
    }
}

impl LinRegressor {

    /// Train the linear regressor using Gradient Descent.
    pub fn train_with_optimization(&mut self, inputs: &Matrix<f64>, targets: &Vector<f64>) {
        let ones = Matrix::<f64>::ones(inputs.rows(), 1);
        let full_inputs = ones.hcat(inputs);

        let initial_params = vec![0.; full_inputs.cols()];

        let gd = GradientDesc::default();
        let optimal_w = gd.optimize(self, &initial_params[..], &full_inputs, targets);
        self.parameters = Some(Vector::new(optimal_w));
    }
}<commit_msg>Adding test for optimized lin reg training.<commit_after>//! Linear Regression module
//!
//! Contains implemention of linear regression using
//! OLS and gradient descent optimization.
//!
//! The regressor will automatically add the intercept term
//! so you do not need to format the input matrices yourself.
//!
//! # Usage
//!
//! ```
//! use rusty_machine::learning::lin_reg::LinRegressor;
//! use rusty_machine::learning::SupModel;
//! use rusty_machine::linalg::matrix::Matrix;
//! use rusty_machine::linalg::vector::Vector;
//!
//! let inputs = Matrix::new(4,1,vec![1.0,3.0,5.0,7.0]);
//! let targets = Vector::new(vec![1.,5.,9.,13.]);
//!
//! let mut lin_mod = LinRegressor::default();
//!
//! // Train the model
//! lin_mod.train(&inputs, &targets);
//!
//! // Now we'll predict a new point
//! let new_point = Matrix::new(1,1,vec![10.]);
//! let output = lin_mod.predict(&new_point);
//!
//! // Hopefully we classified our new point correctly!
//! assert!(output[0] > 17f64, "Our regressor isn't very good!");
//! ```

use learning::SupModel;
use linalg::matrix::Matrix;
use linalg::vector::Vector;
use learning::toolkit::cost_fn::CostFunc;
use learning::toolkit::cost_fn::MeanSqError;
use learning::optim::grad_desc::GradientDesc;
use learning::optim::OptimAlgorithm;
use learning::optim::Optimizable;

/// Linear Regression Model.
///
/// Contains option for optimized parameter.
pub struct LinRegressor {
    /// The parameters for the regression model.
    parameters: Option<Vector<f64>>,
}

impl Default for LinRegressor {
    fn default() -> LinRegressor {
        LinRegressor { parameters: None }
    }
}

impl LinRegressor {

    /// Get the parameters from the model.
    ///
    /// Returns an option that is None if the model has not been trained.
    pub fn parameters(&self) -> Option<Vector<f64>> {
        match self.parameters {
            None => None,
            Some(ref x) => Some(x.clone()),
        }
    }
}

impl SupModel<Matrix<f64>, Vector<f64>> for LinRegressor {
    /// Train the linear regression model.
    ///
    /// Takes training data and output values as input.
    ///
    /// # Examples
    ///
    /// ```
    /// use rusty_machine::learning::lin_reg::LinRegressor;
    /// use rusty_machine::linalg::matrix::Matrix;
    /// use rusty_machine::linalg::vector::Vector;
    /// use rusty_machine::learning::SupModel;
    ///
    /// let mut lin_mod = LinRegressor::default();
    /// let inputs = Matrix::new(3,1, vec![2.0, 3.0, 4.0]);
    /// let targets = Vector::new(vec![5.0, 6.0, 7.0]);
    ///
    /// lin_mod.train(&inputs, &targets);
    /// ```
    fn train(&mut self, inputs: &Matrix<f64>, targets: &Vector<f64>) {
        let ones = Matrix::<f64>::ones(inputs.rows(), 1);
        let full_inputs = ones.hcat(inputs);

        let xt = full_inputs.transpose();

        self.parameters = Some(((&xt * full_inputs).inverse() * &xt) * targets);
    }

    /// Predict output value from input data.
    ///
    /// Model must be trained before prediction can be made.
    fn predict(&self, inputs: &Matrix<f64>) -> Vector<f64> {
        if let Some(ref v) = self.parameters {
            let ones = Matrix::<f64>::ones(inputs.rows(), 1);
            let full_inputs = ones.hcat(inputs);
            full_inputs * v
        }
        else {
            panic!("Model has not been trained.");
        }
    }
}

impl Optimizable for LinRegressor {
    type Inputs = Matrix<f64>;
    type Targets = Vector<f64>;

    fn compute_grad(&self, params: &[f64], inputs: &Matrix<f64>, targets: &Vector<f64>) -> (f64, Vec<f64>) {

        let beta_vec = Vector::new(params.to_vec());
        let outputs = inputs * beta_vec;

        let cost = MeanSqError::cost(&outputs, targets);
        let grad = (inputs.transpose() * (outputs-targets)) / (inputs.rows() as f64);

        (cost, grad.into_vec())
    }
}

impl LinRegressor {

    /// Train the linear regressor using Gradient Descent.
    ///
    /// # Examples
    ///
    /// ```
    /// use rusty_machine::learning::lin_reg::LinRegressor;
    /// use rusty_machine::learning::SupModel;
    /// use rusty_machine::linalg::matrix::Matrix;
    /// use rusty_machine::linalg::vector::Vector;
    ///
    /// let inputs = Matrix::new(4,1,vec![1.0,3.0,5.0,7.0]);
    /// let targets = Vector::new(vec![1.,5.,9.,13.]);
    ///
    /// let mut lin_mod = LinRegressor::default();
    ///
    /// // Train the model
    /// lin_mod.train_with_optimization(&inputs, &targets);
    ///
    /// // Now we'll predict a new point
    /// let new_point = Matrix::new(1,1,vec![10.]);
    /// let _ = lin_mod.predict(&new_point);
    /// ```
    pub fn train_with_optimization(&mut self, inputs: &Matrix<f64>, targets: &Vector<f64>) {
        let ones = Matrix::<f64>::ones(inputs.rows(), 1);
        let full_inputs = ones.hcat(inputs);

        let initial_params = vec![0.; full_inputs.cols()];

        let gd = GradientDesc::default();
        let optimal_w = gd.optimize(self, &initial_params[..], &full_inputs, targets);
        self.parameters = Some(Vector::new(optimal_w));
    }
}<|endoftext|>