<commit_before>use crate::config::Config;
use crate::error::{Error, Result};
use crate::github_api::Client;
use crate::page::{DiffOp, Line, Page};
use crate::pull_request;
use std::borrow::Cow;
use std::mem;
use url::Url;

#[cfg(target_os = "windows")]
fn to_slash<S: AsRef<str>>(s: &S) -> String {
    use path_slash::PathExt;
    use std::path::Path;
    Path::new(s.as_ref()).to_slash_lossy()
}

// Do nothing on Windows
#[cfg(not(target_os = "windows"))]
fn to_slash<S: AsRef<str>>(s: &S) -> &str {
    s.as_ref()
}

// TODO: Omit fallback and return Result<String>
fn first_available_url<T: AsRef<str>>(
    candidates: &mut [String],
    fallback: String,
    https_proxy: &Option<T>,
) -> String {
    let mut builder = reqwest::Client::builder();
    if let Some(ref p) = https_proxy {
        if let Ok(p) = reqwest::Proxy::https(p.as_ref()) {
            builder = builder.proxy(p);
        } else {
            return fallback;
        }
    }
    if let Ok(client) = builder.build() {
        for mut candidate in candidates.iter_mut() {
            let req = client.head(candidate.as_str());
            if let Ok(res) = req.send() {
                let status = res.status();
                if status == reqwest::StatusCode::OK {
                    return mem::replace(&mut candidate, String::new());
                }
            }
        }
    }
    fallback
}

fn build_github_like_url<S: AsRef<str>>(
    host: &str,
    user: &str,
    repo: &str,
    api_endpoint: Option<S>,
    cfg: &Config,
    page: &Page,
) -> Result<String> {
    match page {
        Page::Open { website: true, .. } => {
            match host {
                "github.com" => {
                    if let Some(endpoint) = api_endpoint {
                        if let Ok(client) = Client::build(
                            endpoint.as_ref(),
                            cfg.env.github_token.as_ref(),
                            &cfg.env.https_proxy,
                        ) {
                            if let Ok(Some(homepage)) = client.repo_homepage(user, repo) {
                                return Ok(homepage);
                            }
                        }
                    }
                    let host = &host[0..host.len() - 4];
                    Ok(format!("https://{}.{}.io/{}", user, host, repo))
                }
                "gitlab.com" => Ok(format!("https://{}.gitlab.io/{}", user, repo)),
                host if host.starts_with("gitlab.") => {
                    Ok(format!("https://{}.{}/{}", user, host, repo))
                }
                // For GitHub Enterprise:
                //   https://help.github.com/enterprise/user/articles/user-organization-and-project-pages/
                host => {
                    // Token is always required for GitHub Enterprise
                    if let (Some(ref endpoint), Some(ref token)) =
                        (api_endpoint, &cfg.env.ghe_token)
                    {
                        if let Ok(client) =
                            Client::build(endpoint.as_ref(), Some(token), &cfg.env.https_proxy)
                        {
                            if let Ok(Some(homepage)) = client.repo_homepage(user, repo) {
                                return Ok(homepage);
                            }
                        }
                    }
                    let with_subdomain = format!("https://pages.{}/{}/{}", host, user, repo);
                    let without_subdomain = format!("https://{}/pages/{}/{}", host, user, repo);
                    Ok(first_available_url(
                        &mut [with_subdomain],
                        without_subdomain,
                        &cfg.env.https_proxy,
                    ))
                }
            }
        }
        Page::Open {
            pull_request: true, ..
        } => {
            if let Some(endpoint) = api_endpoint {
                match pull_request::find_page(endpoint.as_ref(), user, repo, cfg)? {
                    pull_request::Page::Existing { url } => Ok(url),
                    pull_request::Page::New {
                        author,
                        repo,
                        default_branch,
                        branch,
                    } => Ok(format!(
                        "https://{}/{}/{}/compare/{}...{}",
                        host, author, repo, default_branch, branch,
                    )),
                    pull_request::Page::NewAtParent {
                        author,
                        repo,
                        fork_author,
                        default_branch,
                        branch,
                    } => Ok(format!(
                        "https://{}/{}/{}/compare/{}...{}:{}",
                        host, author, repo, default_branch, fork_author, branch,
                    )),
                }
            } else {
                Err(Error::PullReqNotSupported {
                    service: host.to_string(),
                })
            }
        }
        Page::Open { .. } => {
            if let Some(ref b) = cfg.branch {
                Ok(format!("https://{}/{}/{}/tree/{}", host, user, repo, b))
            } else {
                Ok(format!("https://{}/{}/{}", host, user, repo))
            }
        }
        Page::Tag { ref tagname, .. } => Ok(format!(
            "https://{}/{}/{}/tree/{}",
            host, user, repo, tagname,
        )),
        Page::Diff {
            ref lhs,
            ref rhs,
            ref op,
        } => Ok(format!(
            "https://{}/{}/{}/compare/{}{}{}",
            host, user, repo, lhs, op, rhs,
        )),
        Page::Commit { ref hash } => Ok(format!(
            "https://{}/{}/{}/commit/{}",
            host, user, repo, hash
        )),
        Page::FileOrDir {
            ref relative_path,
            ref hash,
            line,
            blame,
        } => Ok(format!(
            "https://{host}/{user}/{repo}/{feat}/{hash}/{path}{anchor}",
            host = host,
            user = user,
            repo = repo,
            feat = if *blame { "blame" } else { "blob" },
            hash = hash,
            path = to_slash(relative_path),
            anchor = match line {
                None => "".to_string(),
                Some(Line::At(line)) => format!("#L{}", line),
                Some(Line::Range(start, end)) => format!("#L{}-L{}", start, end),
            },
        )),
        Page::Issue { number } => Ok(format!(
            "https://{}/{}/{}/issues/{}",
            host, user, repo, number
        )),
    }
}

fn build_gitlab_url(
    host: &str,
    user: &str,
    repo: &str,
    cfg: &Config,
    page: &Page,
) -> Result<String> {
    if let Page::Diff { op, .. } = page {
        if *op == DiffOp::TwoDots {
            return Err(Error::GitLabDiffNotSupported);
        }
    }
    build_github_like_url::<&str>(host, user, repo, None, cfg, page)
}

fn build_bitbucket_url(user: &str, repo: &str, cfg: &Config, page: &Page) -> Result<String> {
    match page {
        Page::Open { website: true, .. } => {
            // Build bitbucket cloud URL:
            //   https://confluence.atlassian.com/bitbucket/publishing-a-website-on-bitbucket-cloud-221449776.html
            let with_user = format!("https://{}.bitbucket.io/{}", user, repo);
            let without_user = format!("https://{}.bitbucket.io", user);
            Ok(first_available_url(
                &mut [with_user],
                without_user,
                &cfg.env.https_proxy,
            ))
        }
        Page::Open {
            pull_request: true, ..
        } => Err(Error::PullReqNotSupported {
            service: "bitbucket.org".to_string(),
        }),
        Page::Open { .. } => {
            if let Some(ref b) = cfg.branch {
                Ok(format!(
                    "https://bitbucket.org/{}/{}/branch/{}",
                    user, repo, b,
                ))
            } else {
                Ok(format!("https://bitbucket.org/{}/{}", user, repo))
            }
        }
        Page::Diff { .. } => Err(Error::BitbucketDiffNotSupported),
        Page::Commit { ref hash } => Ok(format!(
            "https://bitbucket.org/{}/{}/commits/{}",
            user, repo, hash,
        )),
        // On Bitbucket, there is no tag-specific page. However, unlike GitHub, bitbucket supports
        // tag commit. Open the tag commit page instead.
        Page::Tag { ref commit, .. } => Ok(format!(
            "https://bitbucket.org/{}/{}/commits/{}",
            user, repo, commit,
        )),
        Page::FileOrDir {
            ref relative_path,
            ref hash,
            line,
            blame,
        } => Ok(format!(
            "https://bitbucket.org/{user}/{repo}/{feat}/{hash}/{path}{anchor}",
            user = user,
            repo = repo,
            feat = if *blame { "annotate" } else { "src" },
            hash = hash,
            path = to_slash(relative_path),
            anchor = match line {
                None => "".to_string(),
                Some(Line::At(line)) => format!("#lines-{}", line),
                Some(Line::Range(start, end)) => format!("#lines-{}:{}", start, end),
            },
        )),
        Page::Issue { number } => Ok(format!(
            "https://bitbucket.org/{}/{}/issues/{}",
            user, repo, number,
        )),
    }
}

fn build_azdevops_url(team: &str, repo: &str, cfg: &Config, page: &Page) -> Result<String> {
    match page {
        Page::Open {
            pull_request: true, ..
        } => {
            if let Some(ref b) = cfg.branch {
                Ok(format!("https://dev.azure.com/{}/_git/{}/pullrequestcreate?sourceRef={}&targetRef=master", team, repo, b))
            } else {
                Err(Error::NoLocalRepoFound {
                    operation: "opening a pull request without specifying branch".to_string(),
                })
            }
        }

        Page::Open { .. } => {
            if let Some(ref b) = cfg.branch {
                Ok(format!(
                    "https://dev.azure.com/{}/_git/{}?version=GB{}",
                    team, repo, b
                ))
            } else {
                Ok(format!("https://dev.azure.com/{}/{}", team, repo))
            }
        }

        Page::Commit { ref hash } => Ok(format!(
            "https://dev.azure.com/{}/_git/{}/commit/{}",
            team, repo, hash
        )),

        Page::Tag { ref tagname, .. } => Ok(format!(
            "https://dev.azure.com/{}/_git/{}?version=GT{}",
            team, repo, tagname
        )),

        Page::FileOrDir {
            ref relative_path,
            ref hash,
            line: None,
            blame,
        } => Ok(format!(
            "https://dev.azure.com/{}/_git/{}/commit/{}?path={}{}",
            team,
            repo,
            hash,
            to_slash(relative_path),
            if *blame { "?_a=annotate" } else { "" },
        )),

        Page::Issue { number } => Ok(format!(
            "https://dev.azure.com/{}/{}/_workitems/edit/{}",
            team, repo, number
        )),

        _ => Err(Error::AzureDevOpsNotSupported),
    }
}

// Note: Parse '/user/repo.git' or '/user/repo' or 'user/repo' into 'user' and 'repo'
pub fn slug_from_path<'a>(path: &'a str) -> Result<(&'a str, &'a str)> {
    let mut split = path.split('/').skip_while(|s| s.is_empty());
    let mut user = split.next().ok_or_else(|| Error::NoUserInPath {
        path: path.to_string(),
    })?;

    if user == "v3-special-az-devops-case" {
        // Skip v3 in azure devops ssh urls.
        user = split.next().ok_or_else(|| Error::NoUserInPath {
            path: path.to_string(),
        })?;
    }

    let mut repo = split.next().ok_or_else(|| Error::NoRepoInPath {
        path: path.to_string(),
    })?;
    if repo.ends_with(".git") {
        // Slice '.git' from 'repo.git'
        repo = &repo[0..repo.len() - 4];
    } else if repo.ends_with("_git") {
        // Handle special case for weird azure urls.
        repo = split.next().ok_or_else(|| Error::NoRepoInPath {
            path: path.to_string(),
        })?;
    }
    Ok((user, repo))
}

fn preprocess_repo_to_url(repo: &str) -> Result<Url> {
    // Workaround Url::parse not being able to parse the SSH urls for AzureDevOps,
    // it seems like the URL's don't adhere to the RFC?
    let processed_repo = if repo.contains("visualstudio.com:v3") || repo.contains("azure.com:v3") {
        // Hack: It should be nice to find a cleaner solution than having
        // this special marker that slug_from_path knows about.
        repo.replace(":v3/", ":22/v3-special-az-devops-case/")
    } else {
        repo.to_string()
    };

    Url::parse(&processed_repo).map_err(|e| Error::BrokenUrl {
        url: processed_repo,
        msg: format!("{}", e),
    })
}

// Known URL formats
//  1. https://hosting_service.com/user/repo.git
//  2. git@hosting_service.com:user/repo.git (-> ssh://git@hosting_service.com:22/user/repo.git)
pub fn build_page_url(page: &Page, cfg: &Config) -> Result<String> {
    let repo_url = &cfg.repo;
    let url = preprocess_repo_to_url(&repo_url)?;
    let env = &cfg.env;

    let path = url.path();
    let (user, repo_name) = slug_from_path(path)?;
    let host = url.host_str().ok_or_else(|| Error::BrokenUrl {
        url: repo_url.to_string(),
        msg: "No host in URL".to_string(),
    })?;

    match host {
        "github.com" => {
            build_github_like_url(host, user, repo_name, Some("api.github.com"), cfg, page)
        }
        "gitlab.com" => build_gitlab_url(host, user, repo_name, cfg, page),
        "bitbucket.org" => build_bitbucket_url(user, repo_name, cfg, page),
        "visualstudio.com" => build_azdevops_url(user, repo_name, cfg, page),
        "vs-ssh.visualstudio.com" => build_azdevops_url(user, repo_name, cfg, page),
        "dev.azure.com" => build_azdevops_url(user, repo_name, cfg, page),
        "ssh.dev.azure.com" => build_azdevops_url(user, repo_name, cfg, page),
        _ => {
            let is_gitlab = host.starts_with("gitlab.");
            let port = if host.starts_with("github.") {
                env.ghe_ssh_port
            } else if is_gitlab {
                env.gitlab_ssh_port
            } else {
                match env.ghe_url_host {
                    Some(ref v) if v == host => env.ghe_ssh_port,
                    _ => {
                        return Err(Error::UnknownHostingService {
                            url: repo_url.to_string(),
                        });
                    }
                }
            };

            let host = match port {
                Some(port) => Cow::Owned(format!("{}:{}", host, port)),
                None => Cow::Borrowed(host),
            };

            if is_gitlab {
                build_gitlab_url(&host, user, repo_name, cfg, page)
            } else {
                build_github_like_url(
                    &host,
                    user,
                    repo_name,
                    Some(format!("{}/api/v3", host)),
                    cfg,
                    page,
                )
            }
        }
    }
}
<commit_msg>pull request feedback<commit_after>use crate::config::Config;
use crate::error::{Error, Result};
use crate::github_api::Client;
use crate::page::{DiffOp, Line, Page};
use crate::pull_request;
use std::borrow::Cow;
use std::mem;
use url::Url;

#[cfg(target_os = "windows")]
fn to_slash<S: AsRef<str>>(s: &S) -> String {
    use path_slash::PathExt;
    use std::path::Path;
    Path::new(s.as_ref()).to_slash_lossy()
}

// Do nothing on Windows
#[cfg(not(target_os = "windows"))]
fn to_slash<S: AsRef<str>>(s: &S) -> &str {
    s.as_ref()
}

// TODO: Omit fallback and return Result<String>
fn first_available_url<T: AsRef<str>>(
    candidates: &mut [String],
    fallback: String,
    https_proxy: &Option<T>,
) -> String {
    let mut builder = reqwest::Client::builder();
    if let Some(ref p) = https_proxy {
        if let Ok(p) = reqwest::Proxy::https(p.as_ref()) {
            builder = builder.proxy(p);
        } else {
            return fallback;
        }
    }
    if let Ok(client) = builder.build() {
        for mut candidate in candidates.iter_mut() {
            let req = client.head(candidate.as_str());
            if let Ok(res) = req.send() {
                let status = res.status();
                if status == reqwest::StatusCode::OK {
                    return mem::replace(&mut candidate, String::new());
                }
            }
        }
    }
    fallback
}

fn build_github_like_url<S: AsRef<str>>(
    host: &str,
    user: &str,
    repo: &str,
    api_endpoint: Option<S>,
    cfg: &Config,
    page: &Page,
) -> Result<String> {
    match page {
        Page::Open { website: true, .. } => {
            match host {
                "github.com" => {
                    if let Some(endpoint) = api_endpoint {
                        if let Ok(client) = Client::build(
                            endpoint.as_ref(),
                            cfg.env.github_token.as_ref(),
                            &cfg.env.https_proxy,
                        ) {
                            if let Ok(Some(homepage)) = client.repo_homepage(user, repo) {
                                return Ok(homepage);
                            }
                        }
                    }
                    let host = &host[0..host.len() - 4];
                    Ok(format!("https://{}.{}.io/{}", user, host, repo))
                }
                "gitlab.com" => Ok(format!("https://{}.gitlab.io/{}", user, repo)),
                host if host.starts_with("gitlab.") => {
                    Ok(format!("https://{}.{}/{}", user, host, repo))
                }
                // For GitHub Enterprise:
                //   https://help.github.com/enterprise/user/articles/user-organization-and-project-pages/
                host => {
                    // Token is always required for GitHub Enterprise
                    if let (Some(ref endpoint), Some(ref token)) =
                        (api_endpoint, &cfg.env.ghe_token)
                    {
                        if let Ok(client) =
                            Client::build(endpoint.as_ref(), Some(token), &cfg.env.https_proxy)
                        {
                            if let Ok(Some(homepage)) = client.repo_homepage(user, repo) {
                                return Ok(homepage);
                            }
                        }
                    }
                    let with_subdomain = format!("https://pages.{}/{}/{}", host, user, repo);
                    let without_subdomain = format!("https://{}/pages/{}/{}", host, user, repo);
                    Ok(first_available_url(
                        &mut [with_subdomain],
                        without_subdomain,
                        &cfg.env.https_proxy,
                    ))
                }
            }
        }
        Page::Open {
            pull_request: true, ..
        } => {
            if let Some(endpoint) = api_endpoint {
                match pull_request::find_page(endpoint.as_ref(), user, repo, cfg)? {
                    pull_request::Page::Existing { url } => Ok(url),
                    pull_request::Page::New {
                        author,
                        repo,
                        default_branch,
                        branch,
                    } => Ok(format!(
                        "https://{}/{}/{}/compare/{}...{}",
                        host, author, repo, default_branch, branch,
                    )),
                    pull_request::Page::NewAtParent {
                        author,
                        repo,
                        fork_author,
                        default_branch,
                        branch,
                    } => Ok(format!(
                        "https://{}/{}/{}/compare/{}...{}:{}",
                        host, author, repo, default_branch, fork_author, branch,
                    )),
                }
            } else {
                Err(Error::PullReqNotSupported {
                    service: host.to_string(),
                })
            }
        }
        Page::Open { .. } => {
            if let Some(ref b) = cfg.branch {
                Ok(format!("https://{}/{}/{}/tree/{}", host, user, repo, b))
            } else {
                Ok(format!("https://{}/{}/{}", host, user, repo))
            }
        }
        Page::Tag { ref tagname, .. } => Ok(format!(
            "https://{}/{}/{}/tree/{}",
            host, user, repo, tagname,
        )),
        Page::Diff {
            ref lhs,
            ref rhs,
            ref op,
        } => Ok(format!(
            "https://{}/{}/{}/compare/{}{}{}",
            host, user, repo, lhs, op, rhs,
        )),
        Page::Commit { ref hash } => Ok(format!(
            "https://{}/{}/{}/commit/{}",
            host, user, repo, hash
        )),
        Page::FileOrDir {
            ref relative_path,
            ref hash,
            line,
            blame,
        } => Ok(format!(
            "https://{host}/{user}/{repo}/{feat}/{hash}/{path}{anchor}",
            host = host,
            user = user,
            repo = repo,
            feat = if *blame { "blame" } else { "blob" },
            hash = hash,
            path = to_slash(relative_path),
            anchor = match line {
                None => "".to_string(),
                Some(Line::At(line)) => format!("#L{}", line),
                Some(Line::Range(start, end)) => format!("#L{}-L{}", start, end),
            },
        )),
        Page::Issue { number } => Ok(format!(
            "https://{}/{}/{}/issues/{}",
            host, user, repo, number
        )),
    }
}

fn build_gitlab_url(
    host: &str,
    user: &str,
    repo: &str,
    cfg: &Config,
    page: &Page,
) -> Result<String> {
    if let Page::Diff { op, .. } = page {
        if *op == DiffOp::TwoDots {
            return Err(Error::GitLabDiffNotSupported);
        }
    }
    build_github_like_url::<&str>(host, user, repo, None, cfg, page)
}

fn build_bitbucket_url(user: &str, repo: &str, cfg: &Config, page: &Page) -> Result<String> {
    match page {
        Page::Open { website: true, .. } => {
            // Build bitbucket cloud URL:
            //   https://confluence.atlassian.com/bitbucket/publishing-a-website-on-bitbucket-cloud-221449776.html
            let with_user = format!("https://{}.bitbucket.io/{}", user, repo);
            let without_user = format!("https://{}.bitbucket.io", user);
            Ok(first_available_url(
                &mut [with_user],
                without_user,
                &cfg.env.https_proxy,
            ))
        }
        Page::Open {
            pull_request: true, ..
        } => Err(Error::PullReqNotSupported {
            service: "bitbucket.org".to_string(),
        }),
        Page::Open { .. } => {
            if let Some(ref b) = cfg.branch {
                Ok(format!(
                    "https://bitbucket.org/{}/{}/branch/{}",
                    user, repo, b,
                ))
            } else {
                Ok(format!("https://bitbucket.org/{}/{}", user, repo))
            }
        }
        Page::Diff { .. } => Err(Error::BitbucketDiffNotSupported),
        Page::Commit { ref hash } => Ok(format!(
            "https://bitbucket.org/{}/{}/commits/{}",
            user, repo, hash,
        )),
        // On Bitbucket, there is no tag-specific page. However, unlike GitHub, bitbucket supports
        // tag commit. Open the tag commit page instead.
        Page::Tag { ref commit, .. } => Ok(format!(
            "https://bitbucket.org/{}/{}/commits/{}",
            user, repo, commit,
        )),
        Page::FileOrDir {
            ref relative_path,
            ref hash,
            line,
            blame,
        } => Ok(format!(
            "https://bitbucket.org/{user}/{repo}/{feat}/{hash}/{path}{anchor}",
            user = user,
            repo = repo,
            feat = if *blame { "annotate" } else { "src" },
            hash = hash,
            path = to_slash(relative_path),
            anchor = match line {
                None => "".to_string(),
                Some(Line::At(line)) => format!("#lines-{}", line),
                Some(Line::Range(start, end)) => format!("#lines-{}:{}", start, end),
            },
        )),
        Page::Issue { number } => Ok(format!(
            "https://bitbucket.org/{}/{}/issues/{}",
            user, repo, number,
        )),
    }
}

fn build_azdevops_url(team: &str, repo: &str, cfg: &Config, page: &Page) -> Result<String> {
    match page {
        Page::Open {
            pull_request: true, ..
        } => {
            if let Some(ref b) = cfg.branch {
                Ok(format!("https://dev.azure.com/{}/_git/{}/pullrequestcreate?sourceRef={}&targetRef=master", team, repo, b))
            } else {
                Err(Error::NoLocalRepoFound {
                    operation: "opening a pull request without specifying branch".to_string(),
                })
            }
        }

        Page::Open { .. } => {
            if let Some(ref b) = cfg.branch {
                Ok(format!(
                    "https://dev.azure.com/{}/_git/{}?version=GB{}",
                    team, repo, b
                ))
            } else {
                Ok(format!("https://dev.azure.com/{}/{}", team, repo))
            }
        }

        Page::Commit { ref hash } => Ok(format!(
            "https://dev.azure.com/{}/_git/{}/commit/{}",
            team, repo, hash
        )),

        Page::Tag { ref tagname, .. } => Ok(format!(
            "https://dev.azure.com/{}/_git/{}?version=GT{}",
            team, repo, tagname
        )),

        Page::FileOrDir {
            ref relative_path,
            ref hash,
            line: None,
            blame,
        } => Ok(format!(
            "https://dev.azure.com/{}/_git/{}/commit/{}?path={}{}",
            team,
            repo,
            hash,
            to_slash(relative_path),
            if *blame { "?_a=annotate" } else { "" },
        )),

        Page::Issue { number } => Ok(format!(
            "https://dev.azure.com/{}/{}/_workitems/edit/{}",
            team, repo, number
        )),

        _ => Err(Error::AzureDevOpsNotSupported),
    }
}

fn is_azdevops(host: &str) -> bool {
    match host {
        "visualstudio.com" => true,
        "vs-ssh.visualstudio.com" => true,
        "dev.azure.com" => true,
        "ssh.dev.azure.com" => true,
        _ => false,
    }
}

// Note: Parse '/team/_git/repo' or '/team/repo' into 'team' and 'repo'
pub fn azdevops_slug_from_path<'a>(path: &'a str) -> Result<(&'a str, &'a str)> {
    let mut split = path.split('/').skip_while(|s| s.is_empty());

    let mut team = split.next().ok_or_else(|| Error::NoUserInPath {
        path: path.to_string(),
    })?;

    // Strip off v3 from Azure DevOps ssh:// paths.
    // See: preprocess_repo_to_url
    //
    // Example: ssh://git@ssh.dev.azure.com:v3/team/repo/repo
    //
    if team == "v3" {
        team = split.next().ok_or_else(|| Error::NoRepoInPath {
            path: path.to_string(),
        })?;
    }

    let mut repo = split.next().ok_or_else(|| Error::NoRepoInPath {
        path: path.to_string(),
    })?;

    if repo.ends_with("_git") {
        repo = split.next().ok_or_else(|| Error::NoRepoInPath {
            path: path.to_string(),
        })?;
    }
    Ok((team, repo))
}

// Note: Parse '/user/repo.git' or '/user/repo' or 'user/repo' into 'user' and 'repo'
pub fn slug_from_path<'a>(path: &'a str) -> Result<(&'a str, &'a str)> {
    let mut split = path.split('/').skip_while(|s| s.is_empty());
    let user = split.next().ok_or_else(|| Error::NoUserInPath {
        path: path.to_string(),
    })?;

    let mut repo = split.next().ok_or_else(|| Error::NoRepoInPath {
        path: path.to_string(),
    })?;

    if repo.ends_with(".git") {
        // Slice '.git' from 'repo.git'
        repo = &repo[0..repo.len() - 4];
    }

    Ok((user, repo))
}

fn preprocess_repo_to_url(repo: &str) -> Result<Url> {
    // Workaround Url::parse not being able to parse the SSH urls for AzureDevOps
    // as they don't specify a port number, but use the colon syntax. It seems like
    // the URL's don't adhere to the RFC? So we force a port number to the default
    // SSH port so the Url will parse correctly.
    //
    // Example: ssh://git@ssh.dev.azure.com:v3/team/repo/repo
    //
    let processed_repo = if repo.contains("visualstudio.com:v3") || repo.contains("azure.com:v3") {
        repo.replace(":v3/", ":22/v3/")
    } else {
        repo.to_string()
    };

    Url::parse(&processed_repo).map_err(|e| Error::BrokenUrl {
        url: processed_repo,
        msg: format!("{}", e),
    })
}

// Known URL formats
//  1. https://hosting_service.com/user/repo.git
//  2. git@hosting_service.com:user/repo.git (-> ssh://git@hosting_service.com:22/user/repo.git)
pub fn build_page_url(page: &Page, cfg: &Config) -> Result<String> {
    let repo_url = &cfg.repo;
    let url = preprocess_repo_to_url(&repo_url)?;
    let env = &cfg.env;

    let path = url.path();
    let host = url.host_str().ok_or_else(|| Error::BrokenUrl {
        url: repo_url.to_string(),
        msg: "No host in URL".to_string(),
    })?;

    let (user, repo_name) = if is_azdevops(host) {
        azdevops_slug_from_path(path)?
    } else {
        slug_from_path(path)?
    };

    match host {
        "github.com" => {
            build_github_like_url(host, user, repo_name, Some("api.github.com"), cfg, page)
        }
        "gitlab.com" => build_gitlab_url(host, user, repo_name, cfg, page),
        "bitbucket.org" => build_bitbucket_url(user, repo_name, cfg, page),
        "visualstudio.com" => build_azdevops_url(user, repo_name, cfg, page),
        "vs-ssh.visualstudio.com" => build_azdevops_url(user, repo_name, cfg, page),
        "dev.azure.com" => build_azdevops_url(user, repo_name, cfg, page),
        "ssh.dev.azure.com" => build_azdevops_url(user, repo_name, cfg, page),
        _ => {
            let is_gitlab = host.starts_with("gitlab.");
            let port = if host.starts_with("github.") {
                env.ghe_ssh_port
            } else if is_gitlab {
                env.gitlab_ssh_port
            } else {
                match env.ghe_url_host {
                    Some(ref v) if v == host => env.ghe_ssh_port,
                    _ => {
                        return Err(Error::UnknownHostingService {
                            url: repo_url.to_string(),
                        });
                    }
                }
            };

            let host = match port {
                Some(port) => Cow::Owned(format!("{}:{}", host, port)),
                None => Cow::Borrowed(host),
            };

            if is_gitlab {
                build_gitlab_url(&host, user, repo_name, cfg, page)
            } else {
                build_github_like_url(
                    &host,
                    user,
                    repo_name,
                    Some(format!("{}/api/v3", host)),
                    cfg,
                    page,
                )
            }
        }
    }
}
<|endoftext|>


<commit_before>//! A trait that can provide the `Span` of the complete contents of a syntax
//! tree node.
//!
//! *This module is available if Syn is built with both the `"parsing"` and
//! `"printing"` features.*
//!
//! # Example
//!
//! Suppose in a procedural macro we have a [`Type`] that we want to assert
//! implements the [`Sync`] trait. Maybe this is the type of one of the fields
//! of a struct for which we are deriving a trait implementation, and we need to
//! be able to pass a reference to one of those fields across threads.
//!
//! [`Type`]: ../enum.Type.html
//! [`Sync`]: https://doc.rust-lang.org/std/marker/trait.Sync.html
//!
//! If the field type does *not* implement `Sync` as required, we want the
//! compiler to report an error pointing out exactly which type it was.
//!
//! The following macro code takes a variable `ty` of type `Type` and produces a
//! static assertion that `Sync` is implemented for that type.
//!
//! ```edition2018
//! # extern crate proc_macro;
//! #
//! use proc_macro::TokenStream;
//! use proc_macro2::Span;
//! use quote::quote_spanned;
//! use syn::Type;
//! use syn::spanned::Spanned;
//!
//! # const IGNORE_TOKENS: &str = stringify! {
//! #[proc_macro_derive(MyMacro)]
//! # };
//! pub fn my_macro(input: TokenStream) -> TokenStream {
//!     # let ty = get_a_type();
//!     /* ... */
//!
//!     let assert_sync = quote_spanned! {ty.span()=>
//!         struct _AssertSync where #ty: Sync;
//!     };
//!
//!     /* ... */
//!     # input
//! }
//! #
//! # fn get_a_type() -> Type {
//! #     unimplemented!()
//! # }
//! ```
//!
//! By inserting this `assert_sync` fragment into the output code generated by
//! our macro, the user's code will fail to compile if `ty` does not implement
//! `Sync`. The errors they would see look like the following.
//!
//! ```text
//! error[E0277]: the trait bound `*const i32: std::marker::Sync` is not satisfied
//!   --> src/main.rs:10:21
//!    |
//! 10 |     bad_field: *const i32,
//!    |                ^^^^^^^^^^ `*const i32` cannot be shared between threads safely
//! ```
//!
//! In this technique, using the `Type`'s span for the error message makes the
//! error appear in the correct place underlining the right type.

use proc_macro2::{Span, TokenStream};
use quote::ToTokens;

/// A trait that can provide the `Span` of the complete contents of a syntax
/// tree node.
///
/// This trait is automatically implemented for all types that implement
/// [`ToTokens`] from the `quote` crate. It is sealed and cannot be implemented
/// outside of the Syn crate other than by implementing `ToTokens`.
///
/// [`ToTokens`]: quote::ToTokens
///
/// See the [module documentation] for an example.
///
/// [module documentation]: self
///
/// *This trait is available if Syn is built with both the `"parsing"` and
/// `"printing"` features.*
pub trait Spanned: private::Sealed {
    /// Returns a `Span` covering the complete contents of this syntax tree
    /// node, or [`Span::call_site()`] if this node is empty.
    ///
    /// [`Span::call_site()`]: proc_macro2::Span::call_site
    fn span(&self) -> Span;
}

mod private {
    use quote::ToTokens;
    pub trait Sealed {}
    impl<T: ToTokens> Sealed for T {}
}

impl<T> Spanned for T
where
    T: ToTokens,
{
    fn span(&self) -> Span {
        join_spans(self.into_token_stream())
    }
}

fn join_spans(tokens: TokenStream) -> Span {
    let mut iter = tokens.into_iter().filter_map(|tt| {
        // FIXME: This shouldn't be required, since optimally spans should
        // never be invalid. This filter_map can probably be removed when
        // https://github.com/rust-lang/rust/issues/43081 is resolved.
        let span = tt.span();
        let debug = format!("{:?}", span);
        if debug.ends_with("bytes(0..0)") {
            None
        } else {
            Some(span)
        }
    });

    let mut joined = match iter.next() {
        Some(span) => span,
        None => return Span::call_site(),
    };

    #[cfg(procmacro2_semver_exempt)]
    {
        for next in iter {
            if let Some(span) = joined.join(next) {
                joined = span;
            }
        }
    }

    #[cfg(not(procmacro2_semver_exempt))]
    {
        // We can't join spans without procmacro2_semver_exempt so just grab the
        // first one.
        joined = joined;
    }

    joined
}
<commit_msg>Use quote's Spanned to provide impl for Span<commit_after>//! A trait that can provide the `Span` of the complete contents of a syntax
//! tree node.
//!
//! *This module is available if Syn is built with both the `"parsing"` and
//! `"printing"` features.*
//!
//! # Example
//!
//! Suppose in a procedural macro we have a [`Type`] that we want to assert
//! implements the [`Sync`] trait. Maybe this is the type of one of the fields
//! of a struct for which we are deriving a trait implementation, and we need to
//! be able to pass a reference to one of those fields across threads.
//!
//! [`Type`]: ../enum.Type.html
//! [`Sync`]: https://doc.rust-lang.org/std/marker/trait.Sync.html
//!
//! If the field type does *not* implement `Sync` as required, we want the
//! compiler to report an error pointing out exactly which type it was.
//!
//! The following macro code takes a variable `ty` of type `Type` and produces a
//! static assertion that `Sync` is implemented for that type.
//!
//! ```edition2018
//! # extern crate proc_macro;
//! #
//! use proc_macro::TokenStream;
//! use proc_macro2::Span;
//! use quote::quote_spanned;
//! use syn::Type;
//! use syn::spanned::Spanned;
//!
//! # const IGNORE_TOKENS: &str = stringify! {
//! #[proc_macro_derive(MyMacro)]
//! # };
//! pub fn my_macro(input: TokenStream) -> TokenStream {
//!     # let ty = get_a_type();
//!     /* ... */
//!
//!     let assert_sync = quote_spanned! {ty.span()=>
//!         struct _AssertSync where #ty: Sync;
//!     };
//!
//!     /* ... */
//!     # input
//! }
//! #
//! # fn get_a_type() -> Type {
//! #     unimplemented!()
//! # }
//! ```
//!
//! By inserting this `assert_sync` fragment into the output code generated by
//! our macro, the user's code will fail to compile if `ty` does not implement
//! `Sync`. The errors they would see look like the following.
//!
//! ```text
//! error[E0277]: the trait bound `*const i32: std::marker::Sync` is not satisfied
//!   --> src/main.rs:10:21
//!    |
//! 10 |     bad_field: *const i32,
//!    |                ^^^^^^^^^^ `*const i32` cannot be shared between threads safely
//! ```
//!
//! In this technique, using the `Type`'s span for the error message makes the
//! error appear in the correct place underlining the right type.

use proc_macro2::Span;
use quote::spanned::Spanned as ToTokens;

/// A trait that can provide the `Span` of the complete contents of a syntax
/// tree node.
///
/// This trait is automatically implemented for all types that implement
/// [`ToTokens`] from the `quote` crate, as well as for `Span` itself.
///
/// [`ToTokens`]: quote::ToTokens
///
/// See the [module documentation] for an example.
///
/// [module documentation]: self
///
/// *This trait is available if Syn is built with both the `"parsing"` and
/// `"printing"` features.*
pub trait Spanned {
    /// Returns a `Span` covering the complete contents of this syntax tree
    /// node, or [`Span::call_site()`] if this node is empty.
    ///
    /// [`Span::call_site()`]: proc_macro2::Span::call_site
    fn span(&self) -> Span;
}

impl<T: ?Sized + ToTokens> Spanned for T {
    fn span(&self) -> Span {
        self.__span()
    }
}
<|endoftext|>


<commit_before>use gfx;

use rustfft;
use rustfft::num_complex::Complex;

use std::f32;
use std::sync::Arc;

pub struct Spectro {
    bufsize: usize,
    window: Vec<f32>,
    fft: Arc<rustfft::FFT<f32>>
}

impl Spectro {
    pub fn new(bufsize: usize) -> Spectro {
        let window = (0..bufsize)
            .map(|i| ((f32::consts::PI * i as f32) / (bufsize - 1) as f32).sin().powi(2))
            .collect();
        let mut planner = rustfft::FFTplanner::new(false);

        Spectro {
            bufsize: bufsize,
            window: window,
            fft: planner.plan_fft(bufsize)
        }
    }

    pub fn draw(&self, signal: &[f32], surface: &mut gfx::Surface) {
        if signal.len() != self.bufsize {
            panic!()
        }

        // Multiply signal with the window function
        let mut input: Vec<Complex<f32>> = signal.iter().zip(&self.window)
            .map(|(sample, window_n)| Complex { re: sample * window_n, im: 0.0 })
            .collect();
        let mut output = vec![Complex { re: 0.0, im: 0.0 }; self.bufsize];

        self.fft.process(&mut input, &mut output);

        let dbs: Vec<f32> = output.iter()
            .map(|val| 10f32 * (val.re.powi(2) + val.im.powi(2)).log10() / 10f32.log10())
            .collect();

        let min_db = dbs.iter().cloned().fold(f32::NAN, f32::min);
        let max_db = dbs.iter().cloned().fold(f32::NAN, f32::max);

        let min_db_limit = 0f32;
        let max_db_limit = 100f32;

        surface.clear(gfx::Color { r: 0, g: 0, b: 0, a: 255 });

        println!("min, max db = {}, {}", min_db, max_db);

        let graph_color = gfx::Color { r: 255, g: 255, b: 255, a: 255 };
        let mut prev_point = gfx::Point { x: 0, y: surface.height - 1 };

        for x in 0..surface.width {
            let mut db = dbs[((x as f64 / surface.width as f64) * output.len() as f64) as usize];
            if db < min_db_limit {
                db = min_db_limit;
            } else if db > max_db_limit {
                db = max_db_limit;
            }

            let scaled = (db - min_db_limit) / (max_db_limit - min_db_limit);
            let y = surface.height - (scaled * surface.height as f32) as isize - 1;

            if x < 3 {
                println!("x={} index={} db={} scaled={} y={}",
                         x,
                         ((x as f64 / surface.width as f64) * output.len() as f64) as usize,
                         db,
                         scaled,
                         y);
            }

            let next_point = gfx::Point { x: x, y: y };

            surface.bresenham(prev_point.clone(), next_point.clone(), graph_color.clone());
            prev_point = next_point.clone();
        }
    }
}
<commit_msg>Switch to logarithmic frequency scale<commit_after>use gfx;

use rustfft;
use rustfft::num_complex::Complex;

use std::f32;
use std::sync::Arc;

pub struct Spectro {
    bufsize: usize,
    sample_rate: f32,
    window: Vec<f32>,
    fft: Arc<rustfft::FFT<f32>>
}

impl Spectro {
    pub fn new(bufsize: usize) -> Spectro {
        let window = (0..bufsize)
            .map(|i| ((f32::consts::PI * i as f32) / (bufsize - 1) as f32).sin().powi(2))
            .collect();
        let mut planner = rustfft::FFTplanner::new(false);

        Spectro {
            bufsize: bufsize,
            sample_rate: 44100.0,
            window: window,
            fft: planner.plan_fft(bufsize)
        }
    }

    pub fn draw(&self, signal: &[f32], surface: &mut gfx::Surface) {
        if signal.len() != self.bufsize {
            panic!()
        }

        // Multiply signal with the window function
        let mut input: Vec<Complex<f32>> = signal.iter().zip(&self.window)
            .map(|(sample, window_n)| Complex { re: sample * window_n, im: 0.0 })
            .collect();
        let mut output = vec![Complex { re: 0.0, im: 0.0 }; self.bufsize];

        self.fft.process(&mut input, &mut output);

        let energies: Vec<f32> = output.iter()
            .map(|val| val.re.powi(2) + val.im.powi(2))
            .collect();

        surface.clear(gfx::Color { r: 0, g: 0, b: 0, a: 255 });

        let graph_color = gfx::Color { r: 255, g: 255, b: 255, a: 255 };
        let mut prev_point = gfx::Point { x: 0, y: surface.height - 1 };

        let min_freq: f32 = self.sample_rate / self.bufsize as f32;
        let max_freq: f32  = self.sample_rate / 2f32;

        let mut prev_freq = min_freq;

        for x in 0..surface.width {
            let step = x as f32 / surface.width as f32;
            let freq = min_freq * (max_freq / min_freq).powf(step);

            let bin_index0 = (prev_freq / min_freq) as usize;
            let bin_index1 = (freq / min_freq) as usize;

            let bin_energy = (&energies[bin_index0 .. bin_index1 + 1])
                .iter()
                .fold(0.0, |sum, e| sum + e);
            let db = 10.0 * bin_energy.log10();

            println!("energy for x={}: {}, db: {}", x, bin_energy, db);

            let next_point = gfx::Point {
                x: x,
                y: surface.height - ((f32::max(db, 0.0) / 70.0) * surface.height as f32) as isize - 1
            };

            surface.bresenham(prev_point.clone(), next_point.clone(), graph_color.clone());

            prev_freq = freq;
            prev_point = next_point.clone();
        }
    }
}
<|endoftext|>


<commit_before>//  Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::*;
use dubp::{
    block::parser::parse_json_block_from_serde_value, block::parser::ParseJsonBlockError,
    block::prelude::DubpBlockTrait, block::DubpBlock, common::prelude::BlockNumber,
};
use duniter_core::dbs::{databases::bc_v1::BcV1DbReadable, FileBackend};
use fast_threadpool::{ThreadPool, ThreadPoolConfig};
use std::{ops::Deref, path::PathBuf};

const CHUNK_SIZE: usize = 250;

pub(crate) fn migrate(profile_path: PathBuf) -> anyhow::Result<()> {
    let start_time = Instant::now();
    let (bc_db, shared_dbs) = duniter_core::dbs::open_dbs(Some(profile_path.as_path()))?;
    let gva_db = duniter_gva_indexer::get_gva_db_rw(Some(profile_path.as_path()));

    // Clear bc_db and gva_db
    bc_db.clear()?;
    gva_db.clear()?;

    if let Err(e) = migrate_inner(&bc_db, gva_db, profile_path, shared_dbs, start_time) {
        // Clear bc_db and gva_db
        bc_db.clear()?;
        gva_db.clear()?;

        Err(e)
    } else {
        Ok(())
    }
}

fn migrate_inner(
    bc_db: &BcV2Db<FileBackend>,
    gva_db: &'static GvaV1Db<FileBackend>,
    profile_path: PathBuf,
    shared_dbs: SharedDbs<FileBackend>,
    start_time: Instant,
) -> anyhow::Result<()> {
    let data_path = profile_path.join(crate::DATA_DIR);
    let duniter_js_db = BcV1Db::<LevelDb>::open(LevelDbConf {
        db_path: data_path.as_path().join("leveldb"),
        ..Default::default()
    })?;

    let dbs_pool = ThreadPool::start(ThreadPoolConfig::default(), shared_dbs).into_sync_handler();

    if let Some(target) = get_target_block_number(&duniter_js_db)? {
        println!("target block: #{}", target.0);

        let (s, r) = flume::unbounded();
        let reader_handle = std::thread::spawn(move || {
            duniter_js_db.main_blocks().iter(.., |it| {
                it.values().try_for_each(|block_res| {
                    s.send(block_res).map_err(|_| anyhow!("fail to send"))
                })
            })
        });
        let (s2, r2) = flume::unbounded();
        let parser_handle = std::thread::spawn(move || {
            let target_u64 = target.0 as u64;
            let mut db_blocks = Vec::with_capacity(CHUNK_SIZE);
            while let Ok(db_block_res) = r.recv() {
                let db_block = db_block_res?;
                let db_block_number = db_block.number;
                db_blocks.push(db_block);
                if db_blocks.len() == CHUNK_SIZE || db_block_number == target_u64 {
                    let blocks = std::mem::take(&mut db_blocks)
                        .into_par_iter()
                        .map(|db_block| match serde_json::to_value(&db_block) {
                            Ok(json_block) => {
                                match parse_json_block_from_serde_value(&json_block) {
                                    Ok(block) => match block {
                                        DubpBlock::V10(block_v10) => Ok(block_v10),
                                    },
                                    Err(e) => Err(anyhow::Error::new::<ParseJsonBlockError>(e)),
                                }
                            }
                            Err(e) => Err(anyhow::Error::new::<serde_json::Error>(e)),
                        })
                        .collect::<anyhow::Result<Vec<_>>>()?;
                    s2.send(blocks).map_err(|_| anyhow!("fail to send"))?;
                    db_blocks.reserve_exact(CHUNK_SIZE);
                }
            }
            Ok::<(), anyhow::Error>(())
        });

        let mut current = None;
        let mut currency_params = Default::default();
        while let Ok(chunk) = r2.recv() {
            if !chunk.is_empty() {
                println!(
                    "Apply chunk #{}-#{} ..",
                    chunk[0].number(),
                    chunk[chunk.len() - 1].number()
                );
                if let Some(currency_parameters) = chunk[0].currency_parameters() {
                    currency_params = currency_parameters;
                }
                let chunk = Arc::from(chunk);
                let chunk_arc_clone = Arc::clone(&chunk);
                let gva_handle = dbs_pool
                    .launch(move |_| {
                        for block in chunk_arc_clone.deref() {
                            duniter_gva_indexer::apply_block(block, currency_params, gva_db)?;
                        }
                        Ok::<_, KvError>(())
                    })
                    .expect("gva:apply_chunk: dbs pool disconnected");
                current = Some(duniter_core::dbs_write_ops::apply_block::apply_chunk(
                    bc_db, current, &dbs_pool, chunk, None,
                )?);
                gva_handle
                    .join()
                    .expect("gva:apply_chunk: dbs pool disconnected")?;
            }
        }

        reader_handle.join().expect("reader thread panic")?;
        parser_handle.join().expect("parser thread panic")?;

        println!("Flush DBs caches on disk...");
        bc_db.save()?;
        gva_db.save()?;

        if let Some(current) = current {
            if current.number != target.0 {
                Err(anyhow::anyhow!("Migration fail: current != target"))
            } else {
                let duration = start_time.elapsed();
                println!(
                    "Migration successfully completed on {} seconds.",
                    duration.as_secs()
                );
                Ok(())
            }
        } else {
            Err(anyhow::anyhow!("Migration fail: rust dbs are empty"))
        }
    } else {
        Err(anyhow::anyhow!("Empty blockchain"))
    }
}

fn get_target_block_number(duniter_js_db: &BcV1Db<LevelDb>) -> KvResult<Option<BlockNumber>> {
    duniter_js_db.main_blocks().iter_rev(.., |it| {
        it.keys().map(|k_res| k_res.map(|bn| bn.0)).next_res()
    })
}

/*fn get_chunk(duniter_js_db: &BcV1Db<LevelDb>, i: u32) -> anyhow::Result<Vec<DubpBlockV10>> {
    let start = BlockNumberKeyV1(BlockNumber(i * CHUNK_SIZE));
    let end = BlockNumberKeyV1(BlockNumber(((i + 1) * CHUNK_SIZE) - 1));
    println!("get_chunk({}): range {}..{}", i, start.0, end.0);
    let db_blocks = duniter_js_db
        .main_blocks()
        .iter(start..=end, |it| it.values().collect::<KvResult<Vec<_>>>())?;

    db_blocks
        .into_par_iter()
        .map(|db_block| match serde_json::to_value(&db_block) {
            Ok(json_block) => match parse_json_block_from_serde_value(&json_block) {
                Ok(block) => match block {
                    DubpBlock::V10(block_v10) => Ok(block_v10),
                },
                Err(e) => Err(anyhow::Error::new::<ParseJsonBlockError>(e)),
            },
            Err(e) => Err(anyhow::Error::new::<serde_json::Error>(e)),
        })
        .collect()
}
*/
<commit_msg>perf(dex):migrate: remove dbs instead of clear their content<commit_after>//  Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::*;
use dubp::{
    block::parser::parse_json_block_from_serde_value, block::parser::ParseJsonBlockError,
    block::prelude::DubpBlockTrait, block::DubpBlock, common::prelude::BlockNumber,
};
use duniter_core::dbs::{databases::bc_v1::BcV1DbReadable, FileBackend};
use fast_threadpool::{ThreadPool, ThreadPoolConfig};
use std::{ops::Deref, path::PathBuf};

const CHUNK_SIZE: usize = 250;

pub(crate) fn migrate(profile_path: PathBuf) -> anyhow::Result<()> {
    let start_time = Instant::now();

    // Remove bc_db and gva_db
    std::fs::remove_dir_all(profile_path.join("data/bc_v2_sled"))?;
    std::fs::remove_dir_all(profile_path.join("data/gva_v1_sled"))?;

    // Open bc_db and gva_db
    let (bc_db, shared_dbs) = duniter_core::dbs::open_dbs(Some(profile_path.as_path()))?;
    let gva_db = duniter_gva_indexer::get_gva_db_rw(Some(profile_path.as_path()));

    if let Err(e) = migrate_inner(&bc_db, gva_db, profile_path, shared_dbs, start_time) {
        // Clear bc_db and gva_db
        bc_db.clear()?;
        gva_db.clear()?;

        Err(e)
    } else {
        Ok(())
    }
}

fn migrate_inner(
    bc_db: &BcV2Db<FileBackend>,
    gva_db: &'static GvaV1Db<FileBackend>,
    profile_path: PathBuf,
    shared_dbs: SharedDbs<FileBackend>,
    start_time: Instant,
) -> anyhow::Result<()> {
    let data_path = profile_path.join(crate::DATA_DIR);
    let duniter_js_db = BcV1Db::<LevelDb>::open(LevelDbConf {
        db_path: data_path.as_path().join("leveldb"),
        ..Default::default()
    })?;

    let dbs_pool = ThreadPool::start(ThreadPoolConfig::default(), shared_dbs).into_sync_handler();

    if let Some(target) = get_target_block_number(&duniter_js_db)? {
        println!("target block: #{}", target.0);

        let (s, r) = flume::unbounded();
        let reader_handle = std::thread::spawn(move || {
            duniter_js_db.main_blocks().iter(.., |it| {
                it.values().try_for_each(|block_res| {
                    s.send(block_res).map_err(|_| anyhow!("fail to send"))
                })
            })
        });
        let (s2, r2) = flume::unbounded();
        let parser_handle = std::thread::spawn(move || {
            let target_u64 = target.0 as u64;
            let mut db_blocks = Vec::with_capacity(CHUNK_SIZE);
            while let Ok(db_block_res) = r.recv() {
                let db_block = db_block_res?;
                let db_block_number = db_block.number;
                db_blocks.push(db_block);
                if db_blocks.len() == CHUNK_SIZE || db_block_number == target_u64 {
                    let blocks = std::mem::take(&mut db_blocks)
                        .into_par_iter()
                        .map(|db_block| match serde_json::to_value(&db_block) {
                            Ok(json_block) => {
                                match parse_json_block_from_serde_value(&json_block) {
                                    Ok(block) => match block {
                                        DubpBlock::V10(block_v10) => Ok(block_v10),
                                    },
                                    Err(e) => Err(anyhow::Error::new::<ParseJsonBlockError>(e)),
                                }
                            }
                            Err(e) => Err(anyhow::Error::new::<serde_json::Error>(e)),
                        })
                        .collect::<anyhow::Result<Vec<_>>>()?;
                    s2.send(blocks).map_err(|_| anyhow!("fail to send"))?;
                    db_blocks.reserve_exact(CHUNK_SIZE);
                }
            }
            Ok::<(), anyhow::Error>(())
        });

        let mut current = None;
        let mut currency_params = Default::default();
        while let Ok(chunk) = r2.recv() {
            if !chunk.is_empty() {
                println!(
                    "Apply chunk #{}-#{} ..",
                    chunk[0].number(),
                    chunk[chunk.len() - 1].number()
                );
                if let Some(currency_parameters) = chunk[0].currency_parameters() {
                    currency_params = currency_parameters;
                }
                let chunk = Arc::from(chunk);
                let chunk_arc_clone = Arc::clone(&chunk);
                let gva_handle = dbs_pool
                    .launch(move |_| {
                        for block in chunk_arc_clone.deref() {
                            duniter_gva_indexer::apply_block(block, currency_params, gva_db)?;
                        }
                        Ok::<_, KvError>(())
                    })
                    .expect("gva:apply_chunk: dbs pool disconnected");
                current = Some(duniter_core::dbs_write_ops::apply_block::apply_chunk(
                    bc_db, current, &dbs_pool, chunk, None,
                )?);
                gva_handle
                    .join()
                    .expect("gva:apply_chunk: dbs pool disconnected")?;
            }
        }

        reader_handle.join().expect("reader thread panic")?;
        parser_handle.join().expect("parser thread panic")?;

        println!("Flush DBs caches on disk...");
        bc_db.save()?;
        gva_db.save()?;

        if let Some(current) = current {
            if current.number != target.0 {
                Err(anyhow::anyhow!("Migration fail: current != target"))
            } else {
                let duration = start_time.elapsed();
                println!(
                    "Migration successfully completed on {} seconds.",
                    duration.as_secs()
                );
                Ok(())
            }
        } else {
            Err(anyhow::anyhow!("Migration fail: rust dbs are empty"))
        }
    } else {
        Err(anyhow::anyhow!("Empty blockchain"))
    }
}

fn get_target_block_number(duniter_js_db: &BcV1Db<LevelDb>) -> KvResult<Option<BlockNumber>> {
    duniter_js_db.main_blocks().iter_rev(.., |it| {
        it.keys().map(|k_res| k_res.map(|bn| bn.0)).next_res()
    })
}

/*fn get_chunk(duniter_js_db: &BcV1Db<LevelDb>, i: u32) -> anyhow::Result<Vec<DubpBlockV10>> {
    let start = BlockNumberKeyV1(BlockNumber(i * CHUNK_SIZE));
    let end = BlockNumberKeyV1(BlockNumber(((i + 1) * CHUNK_SIZE) - 1));
    println!("get_chunk({}): range {}..{}", i, start.0, end.0);
    let db_blocks = duniter_js_db
        .main_blocks()
        .iter(start..=end, |it| it.values().collect::<KvResult<Vec<_>>>())?;

    db_blocks
        .into_par_iter()
        .map(|db_block| match serde_json::to_value(&db_block) {
            Ok(json_block) => match parse_json_block_from_serde_value(&json_block) {
                Ok(block) => match block {
                    DubpBlock::V10(block_v10) => Ok(block_v10),
                },
                Err(e) => Err(anyhow::Error::new::<ParseJsonBlockError>(e)),
            },
            Err(e) => Err(anyhow::Error::new::<serde_json::Error>(e)),
        })
        .collect()
}
*/
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! A caching layer for xi plugins. Will be split out into its own crate once it's a bit more stable.

use serde_json::Value;

use plugin_base;
use plugin_base::PluginRequest;

pub use plugin_base::{Error, Spans, SpansBuilder};

const CHUNK_SIZE: usize = 1024 * 1024;

/// A handler that the plugin needs to instantiate.
pub trait Handler {
    fn init_buf(&mut self, ctx: PluginCtx, buf_size: usize);
    fn update(&mut self, ctx: PluginCtx);
    #[allow(unused_variables)]
    fn idle(&mut self, ctx: PluginCtx, token: usize) {}
}

/// The caching state
#[derive(Default)]
struct State {
    buf_size: usize,
    rev: usize,
    cache: Option<String>,
    cache_offset: usize,

    // line iteration state
    line_num: usize,
    offset_of_line: usize,
}

pub struct PluginCtx<'a> {
    state: &'a mut State,
    peer: plugin_base::PluginCtx<'a>,
}

struct MyHandler<'a, H: 'a> {
    handler: &'a mut H,
    state: State,
}

impl<'a, H: Handler> plugin_base::Handler for MyHandler<'a, H> {
    fn call(&mut self, req: &PluginRequest, peer: plugin_base::PluginCtx) -> Option<Value> {
        let ctx = PluginCtx {
            state: &mut self.state,
            peer: peer,
        };
        match *req {
            PluginRequest::Ping => {
                print_err!("got ping");
                None
            }
            PluginRequest::InitBuf { buf_size, rev } => {
                print_err!("got init_buf buf_size = {}, rev = {}", buf_size, rev);
                ctx.state.buf_size = buf_size;
                ctx.state.rev = rev;
                self.handler.init_buf(ctx, buf_size);
                None
            }
            PluginRequest::Update { start, end, new_len, rev, edit_type, author, text } => {
                print_err!("got update notification {:?}", edit_type);
                ctx.state.buf_size = ctx.state.buf_size - (end - start) + new_len;
                ctx.state.rev = rev;
                if let (Some(text), Some(mut cache)) = (text, ctx.state.cache.take()) {
                    let off = ctx.state.cache_offset;
                    if start >= off && start <= off + cache.len() {
                        let tail = if end < off + cache.len() {
                            Some(cache[end - off ..].to_string())
                        } else {
                            None
                        };
                        cache.truncate(start - off);
                        cache.push_str(text);
                        if let Some(tail) = tail {
                            cache.push_str(&tail);
                        }
                        ctx.state.cache = Some(cache);
                    }
                }
                ctx.state.line_num = 0;
                ctx.state.offset_of_line = 0;
                self.handler.update(ctx);
                Some(Value::Null)
            }
        }
    }

    fn idle(&mut self, peer: plugin_base::PluginCtx, token: usize) {
        let ctx = PluginCtx {
            state: &mut self.state,
            peer: peer,
        };
        self.handler.idle(ctx, token);
    }
}

pub fn mainloop<H: Handler>(handler: &mut H) {
    let mut my_handler = MyHandler {
        handler: handler,
        state: State::default(),
    };
    plugin_base::mainloop(&mut my_handler);
}

impl<'a> PluginCtx<'a> {
    pub fn get_line(&mut self, line_num: usize) -> Result<Option<String>, Error> {
        if line_num != self.state.line_num {
            print_err!("can't handle non-sequential line numbers yet");
            return Ok(None);
        }
        let offset_of_line = self.state.offset_of_line;
        if offset_of_line == self.state.buf_size {
            return Ok(None);
        }
        if self.state.cache.is_none() || offset_of_line < self.state.cache_offset ||
                offset_of_line >= self.state.cache_offset +
                    self.state.cache.as_ref().unwrap().len() {
            self.state.cache = Some(self.peer.get_data(offset_of_line, CHUNK_SIZE,
                self.state.rev)?);
            self.state.cache_offset = offset_of_line;
        }
        loop {
            let offset_in_cache = offset_of_line - self.state.cache_offset;
            match memchr(b'\n', &self.state.cache.as_ref().unwrap().as_bytes()[offset_in_cache..]) {
                None => {
                    let cache_len = self.state.cache.as_ref().unwrap().len();
                    if self.state.cache_offset + cache_len == self.state.buf_size {
                        // incomplete last line
                        let cache = self.state.cache.as_ref().unwrap();
                        let result = String::from(&cache[offset_in_cache..]);
                        self.state.offset_of_line += result.len();
                        self.state.line_num += 1;
                        return Ok(Some(result));
                    }
                    // fetch next chunk
                    let next_offset = self.state.cache_offset + cache_len;
                    let next_chunk = self.peer.get_data(next_offset, CHUNK_SIZE,
                            self.state.rev)?;
                    self.state.cache_offset = offset_of_line;
                    let mut new_cache = String::with_capacity(cache_len - offset_in_cache +
                            next_chunk.len());
                    new_cache.push_str(&self.state.cache.as_ref().unwrap()[offset_in_cache..]);
                    new_cache.push_str(&next_chunk);
                    self.state.cache = Some(new_cache);
                }
                Some(pos) => {
                    let cache = self.state.cache.as_ref().unwrap();
                    let result = String::from(&cache[offset_in_cache .. offset_in_cache + pos + 1]);
                    self.state.offset_of_line += pos + 1;
                    self.state.line_num += 1;
                    return Ok(Some(result));
                }
            }
        }
    }

    /// Send style spans to the core. Note: these are based on the revision as of the
    /// current state of the cache (the revision is sent to the core).
    pub fn set_fg_spans(&self, start: usize, len: usize, spans: Spans) {
        self.peer.set_fg_spans(start, len, spans, self.state.rev);
    }

    /// Determines whether an incoming request (or notification) is pending. This
    /// is intended to reduce latency for bulk operations done in the background.
    pub fn request_is_pending(&self) -> bool {
        self.peer.request_is_pending()
    }

    /// Schedule the idle handler to be run when there are no requests pending.
    pub fn schedule_idle(&mut self, token: usize) {
        self.peer.schedule_idle(token);
    }
}

// TODO: use burntsushi memchr, or import this from xi_rope
fn memchr(needle: u8, haystack: &[u8]) -> Option<usize> {
    haystack.iter().position(|&b| b == needle)
}
<commit_msg>Return number instead of Null on plugin update<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! A caching layer for xi plugins. Will be split out into its own crate once it's a bit more stable.

use serde_json::Value;

use plugin_base;
use plugin_base::PluginRequest;

pub use plugin_base::{Error, Spans, SpansBuilder};

const CHUNK_SIZE: usize = 1024 * 1024;

/// A handler that the plugin needs to instantiate.
pub trait Handler {
    fn init_buf(&mut self, ctx: PluginCtx, buf_size: usize);
    fn update(&mut self, ctx: PluginCtx);
    #[allow(unused_variables)]
    fn idle(&mut self, ctx: PluginCtx, token: usize) {}
}

/// The caching state
#[derive(Default)]
struct State {
    buf_size: usize,
    rev: usize,
    cache: Option<String>,
    cache_offset: usize,

    // line iteration state
    line_num: usize,
    offset_of_line: usize,
}

pub struct PluginCtx<'a> {
    state: &'a mut State,
    peer: plugin_base::PluginCtx<'a>,
}

struct MyHandler<'a, H: 'a> {
    handler: &'a mut H,
    state: State,
}

impl<'a, H: Handler> plugin_base::Handler for MyHandler<'a, H> {
    fn call(&mut self, req: &PluginRequest, peer: plugin_base::PluginCtx) -> Option<Value> {
        let ctx = PluginCtx {
            state: &mut self.state,
            peer: peer,
        };
        match *req {
            PluginRequest::Ping => {
                print_err!("got ping");
                None
            }
            PluginRequest::InitBuf { buf_size, rev } => {
                print_err!("got init_buf buf_size = {}, rev = {}", buf_size, rev);
                ctx.state.buf_size = buf_size;
                ctx.state.rev = rev;
                self.handler.init_buf(ctx, buf_size);
                None
            }
            PluginRequest::Update { start, end, new_len, rev, edit_type, author, text } => {
                //print_err!("got update notification {:?}", edit_type);
                ctx.state.buf_size = ctx.state.buf_size - (end - start) + new_len;
                ctx.state.rev = rev;
                if let (Some(text), Some(mut cache)) = (text, ctx.state.cache.take()) {
                    let off = ctx.state.cache_offset;
                    if start >= off && start <= off + cache.len() {
                        let tail = if end < off + cache.len() {
                            Some(cache[end - off ..].to_string())
                        } else {
                            None
                        };
                        cache.truncate(start - off);
                        cache.push_str(text);
                        if let Some(tail) = tail {
                            cache.push_str(&tail);
                        }
                        ctx.state.cache = Some(cache);
                    }
                }
                ctx.state.line_num = 0;
                ctx.state.offset_of_line = 0;
                self.handler.update(ctx);
                Some(Value::from(0i32))
            }
        }
    }

    fn idle(&mut self, peer: plugin_base::PluginCtx, token: usize) {
        let ctx = PluginCtx {
            state: &mut self.state,
            peer: peer,
        };
        self.handler.idle(ctx, token);
    }
}

pub fn mainloop<H: Handler>(handler: &mut H) {
    let mut my_handler = MyHandler {
        handler: handler,
        state: State::default(),
    };
    plugin_base::mainloop(&mut my_handler);
}

impl<'a> PluginCtx<'a> {
    pub fn get_line(&mut self, line_num: usize) -> Result<Option<String>, Error> {
        if line_num != self.state.line_num {
            print_err!("can't handle non-sequential line numbers yet");
            return Ok(None);
        }
        let offset_of_line = self.state.offset_of_line;
        if offset_of_line == self.state.buf_size {
            return Ok(None);
        }
        if self.state.cache.is_none() || offset_of_line < self.state.cache_offset ||
                offset_of_line >= self.state.cache_offset +
                    self.state.cache.as_ref().unwrap().len() {
            self.state.cache = Some(self.peer.get_data(offset_of_line, CHUNK_SIZE,
                self.state.rev)?);
            self.state.cache_offset = offset_of_line;
        }
        loop {
            let offset_in_cache = offset_of_line - self.state.cache_offset;
            match memchr(b'\n', &self.state.cache.as_ref().unwrap().as_bytes()[offset_in_cache..]) {
                None => {
                    let cache_len = self.state.cache.as_ref().unwrap().len();
                    if self.state.cache_offset + cache_len == self.state.buf_size {
                        // incomplete last line
                        let cache = self.state.cache.as_ref().unwrap();
                        let result = String::from(&cache[offset_in_cache..]);
                        self.state.offset_of_line += result.len();
                        self.state.line_num += 1;
                        return Ok(Some(result));
                    }
                    // fetch next chunk
                    let next_offset = self.state.cache_offset + cache_len;
                    let next_chunk = self.peer.get_data(next_offset, CHUNK_SIZE,
                            self.state.rev)?;
                    self.state.cache_offset = offset_of_line;
                    let mut new_cache = String::with_capacity(cache_len - offset_in_cache +
                            next_chunk.len());
                    new_cache.push_str(&self.state.cache.as_ref().unwrap()[offset_in_cache..]);
                    new_cache.push_str(&next_chunk);
                    self.state.cache = Some(new_cache);
                }
                Some(pos) => {
                    let cache = self.state.cache.as_ref().unwrap();
                    let result = String::from(&cache[offset_in_cache .. offset_in_cache + pos + 1]);
                    self.state.offset_of_line += pos + 1;
                    self.state.line_num += 1;
                    return Ok(Some(result));
                }
            }
        }
    }

    /// Send style spans to the core. Note: these are based on the revision as of the
    /// current state of the cache (the revision is sent to the core).
    pub fn set_fg_spans(&self, start: usize, len: usize, spans: Spans) {
        self.peer.set_fg_spans(start, len, spans, self.state.rev);
    }

    /// Determines whether an incoming request (or notification) is pending. This
    /// is intended to reduce latency for bulk operations done in the background.
    pub fn request_is_pending(&self) -> bool {
        self.peer.request_is_pending()
    }

    /// Schedule the idle handler to be run when there are no requests pending.
    pub fn schedule_idle(&mut self, token: usize) {
        self.peer.schedule_idle(token);
    }
}

// TODO: use burntsushi memchr, or import this from xi_rope
fn memchr(needle: u8, haystack: &[u8]) -> Option<usize> {
    haystack.iter().position(|&b| b == needle)
}
<|endoftext|>