<commit_before>use pombase_rc_string::RcString;

use crate::web::config::Config;
use crate::web::data::{GeneDetails, ReferenceDetails, TermDetails,
                       ContainerType, GenotypeShort,
                       OntAnnotationId, AnnotationContainer, OrthologAnnotationContainer};

fn format_page(header: &str, body: &str) -> String {
    format! ("
<!DOCTYPE html>

    {}

    {}

",
             header, body)
}

fn make_gene_title(config: &Config, gene_details: &GeneDetails) -> String {
    let name_and_uniquename =
        if let Some(ref name) = gene_details.name {
            format!("{} ({})", name, gene_details.uniquename)
        } else {
            format!("{}", gene_details.uniquename)
        };

    let feature_type =
        if gene_details.feature_type.ends_with("gene") {
            String::from(gene_details.feature_type.as_str())
        } else {
            format!("{} gene", gene_details.feature_type)
        };

    if let Some(ref product) = gene_details.product {
        format!("{} - {} {} - {}", config.database_name, feature_type,
                name_and_uniquename, product)
    } else {
        format!("{} - {} {}", config.database_name, feature_type, name_and_uniquename)
    }
}

fn header(config: &Config, title: &str) -> String  {
    format!(r##"
  <meta charset="utf-8">
  <title>{}</title>
  <base href="/">
  <meta http-equiv="X-UA-Compatible" content="IE=edge">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <meta name="title" content="{}">
  <meta name="author" content="{}">
  <meta name="description" content="{}">
  <meta name="google-site-verification" content="H8vhCdG7XsSbh-8f20VSipvu2PnZ22YSGZm81jE0-Pk" />
  <link rel="shortcut icon" href="/assets/favicon.ico">
  <meta name="apple-mobile-web-app-title" content="{}">
  <meta name="application-name" content="{}">
"##,
            title, title, config.database_name, title, config.database_name,
            config.database_name
    )
}

fn gene_summary(config: &Config, gene_details: &GeneDetails) -> String {
    let mut summ = String::new();

    summ += "
\n";
    if let Some(ref name) = gene_details.name {
        summ += &format!("
Gene standard name

{}
\n",  name);
    }
    summ += &format!("
Systematic ID

{}
\n", gene_details.uniquename);

    if gene_details.synonyms.len() > 0 {
        let synonyms: Vec<RcString> =
            gene_details.synonyms.iter().map(|s| s.name.clone()).collect();
        summ += &format!("
Synonyms

{}
\n", synonyms.join(", "));
    }

    if let Some(ref uniprot_identifier) = gene_details.uniprot_identifier {
        summ += &format!("
UniProt ID

{}
\n", uniprot_identifier);
    }

    if let Some(ref orfeome_identifier) = gene_details.orfeome_identifier {
        summ += &format!("
ORFeome ID

{}
\n", orfeome_identifier);
    }

    if let Some(gene_organism) = config.organism_by_taxonid(gene_details.taxonid) {
        summ += &format!("
Organism

{}\n", gene_organism.scientific_name());

        if gene_organism.alternative_names.len() > 0 {
            summ += &format!(" ({})", gene_organism.alternative_names.join(", "));
        }
        summ += "
\n";
    }

    if let Some(ref characterisation_status) = gene_details.characterisation_status {
        summ += &format!("
Characterisation status

{}\n",
                         characterisation_status);
    }

    summ += "
\n";

    summ
}

fn genotype_display_name(genotype_short: &GenotypeShort) -> String {
    let mut bits: Vec<_> = genotype_short.display_uniquename.split("-").collect();
    bits.dedup();
    bits.join(" ")
}

fn get_annotations(ont_annotation_ids: &Vec<OntAnnotationId>,
                   container: &dyn AnnotationContainer) -> String {
    let mut ret = String::new();
    let mut references = vec![];
    let mut genes = vec![];
    let mut genotypes = vec![];

    for ont_annotation_id in ont_annotation_ids {
        if let Some(annotation_details) = container.annotation_details().get(ont_annotation_id) {
            if let Some(ref reference) = annotation_details.reference {
                references.push(reference);
            }

            if let Some(ref genotype) = annotation_details.genotype {
                genotypes.push(genotype);
            }

            for gene_uniquename in &annotation_details.genes {
                if let Some(maybe_gene_short) =
                    container.genes_by_uniquename().get(gene_uniquename.as_str()) {
                        if let Some(gene_short) = maybe_gene_short {
                            genes.push(gene_short)
                        }
                    }
            }
        }
    }

    if container.container_type() != ContainerType::Reference && references.len() > 0 {
        references.sort();
        references.dedup();
        ret += "
References:
\n
";
        for reference in references {
            ret += &format!("
{}
\n", reference, reference);
        }
        ret += "
\n";
    }

    if container.container_type() != ContainerType::Gene && genes.len() > 0 {
        genes.sort();
        genes.dedup();
        ret += "
Genes:
\n
";
        for gene in genes {
            ret += &format!("
{}
\n", gene.uniquename,
                            gene.display_name());
        }
        ret += "
\n";
    }

    if container.container_type() != ContainerType::Genotype && genotypes.len() > 0 {
        genotypes.sort();
        genotypes.dedup();
        if let Some(genotypes_by_uniquename) = container.genotypes_by_uniquename() {
            ret += "
Genotypes:
\n
";
            for uniquename in genotypes {
                if let Some(genotype_short) = genotypes_by_uniquename.get(uniquename.as_str()) {
                    ret += &format!("
{}
\n",
                                    uniquename,
                                    genotype_display_name(genotype_short));
                }
            }
            ret += "
\n";
        }
    }

    ret
}

fn annotation_section(config: &Config, container: &dyn AnnotationContainer) -> String {
    let mut annotation_html = String::new();

    let mut cv_names: Vec<RcString> = vec![];

    for cv_name in container.cv_annotations().keys() {
        cv_names.push(cv_name.clone());
    }

    let cmp_display_names =
        |cv_name_a: &RcString, cv_name_b: &RcString| {
            let cv_config_a = config.cv_config_by_name(&cv_name_a);
            let cv_display_name_a = cv_config_a.display_name;
            let cv_config_b = config.cv_config_by_name(&cv_name_b);
            let cv_display_name_b = cv_config_b.display_name;

            cv_display_name_a.cmp(&cv_display_name_b)
        };

    cv_names.sort_by(cmp_display_names);

    for cv_name in cv_names {
        let term_annotations = container.cv_annotations().get(&cv_name).unwrap();
        let cv_config = config.cv_config_by_name(&cv_name);
        let cv_display_name = cv_config.display_name;
        annotation_html += &format!("<sect>\n
{}
\n", cv_display_name);
        for term_annotation in term_annotations {
            let term_name =
                if let Some(term_short_opt) =
                    container.terms_by_termid().get(&term_annotation.term) {
                        if let Some(term_short) = term_short_opt {
                            String::from(term_short.name.as_str())
                        } else {
                            String::from("(unknown term name)")
                        }
                    } else {
                        String::from("(unknown term name)")
                    };
            annotation_html += &format!("<sect>
{} - \
{}
\n{}</sect>\n",
                                        term_annotation.term,
                                        term_annotation.term,
                                        term_annotation.term,
                                        term_name,
                                        get_annotations(&term_annotation.annotations,
                                                        container));
        }
        annotation_html += "</sect>\n";
    }

    annotation_html
}

fn orthologs(config: &Config, container: &dyn OrthologAnnotationContainer) -> String {
    let mut orth_html = String::new();

    if container.ortholog_annotations().is_empty() {
        return orth_html;
    }

    orth_html += "<sect>
Orthologs
\n
\n";

    for orth_annotation in container.ortholog_annotations().iter() {
        let maybe_orth_org = config.organism_by_taxonid(orth_annotation.ortholog_taxonid);
        let orth_org_scientific_name =
            if let Some(orth_org) = maybe_orth_org {
                orth_org.scientific_name()
            } else {
                String::from("Unknown organism")
            };
        if let Some(maybe_orth_gene_short) =
            container.genes_by_uniquename().get(&orth_annotation.ortholog_uniquename) {
                if let Some(orth_gene_short) = maybe_orth_gene_short {
                    orth_html += &format!("
{} {}
\n", orth_gene_short.display_name(),
                                          orth_org_scientific_name);
                }
            }
    }

    orth_html += "
</sect>\n";

    orth_html
}

fn gene_body(config: &Config, title: &str, gene_details: &GeneDetails) -> String {
    let mut body = String::new();

    body += &format!("
{}
\n", title);

    body += &format!("<sect>
Welcome to {}
\n
{}
</sect>\n",
                     config.database_name, config.site_description);

    body += &format!("<sect>
Gene summary
\n{}</sect>\n",
                     gene_summary(config, gene_details));

    body += &format!("<sect>
Annotation
\n{}</sect>\n",
                     annotation_section(config, gene_details));

    body += &orthologs(config, gene_details);

    body
}

pub fn render_simple_gene_page(config: &Config, gene_details: &GeneDetails) -> String  {
    let title = make_gene_title(config, gene_details);

    format_page(&header(config, &title), &gene_body(config, &title, gene_details))
}

fn make_reference_title(config: &Config, reference_details: &ReferenceDetails) -> String {
    if let Some(ref title) = reference_details.title {
        format!("{} - {} - {}", config.database_name, reference_details.uniquename, title)
    } else {
        format!("{} - {}", config.database_name, reference_details.uniquename)
    }
}

fn reference_summary(reference_details: &ReferenceDetails) -> String {
    let mut summ = String::new();

    summ += "
\n";

    summ += &format!("
PubMed ID

{}
\n", reference_details.uniquename);

    if let Some(ref title) = reference_details.title {
        summ += &format!("
Title

{}
\n", title);
    }

    if let Some(ref authors) = reference_details.authors {
        summ += &format!("
Authors

{}
\n", authors);
    }

    if let Some(ref citation) = reference_details.citation {
        summ += &format!("
Citation

{}
\n", citation);
    }

    if let Some(ref publication_year) = reference_details.publication_year {
        summ += &format!("
Publication year

{}
\n", publication_year);
    }

    summ += "
\n";

    summ
}

fn reference_body(config: &Config, title: &str, reference_details: &ReferenceDetails) -> String {
    let mut body = String::new();

    body += &format!("
{}
\n", title);

    body += &format!("<sect>
Welcome to {}
\n
{}
</sect>\n",
                     config.database_name, config.site_description);

    body += &format!("<sect>
Reference summary
\n{}</sect>\n",
                     reference_summary(reference_details));

    body += &format!("<sect>
Annotation
\n{}</sect>\n",
                     annotation_section(config, reference_details));

    body += &orthologs(config, reference_details);

    body
}

pub fn render_simple_reference_page(config: &Config, reference_details: &ReferenceDetails) -> String  {
    let title = make_reference_title(config, reference_details);

    format_page(&header(config, &title), &reference_body(config, &title, reference_details))
}

fn make_term_title(config: &Config, term_details: &TermDetails) -> String {
    let cv_config = config.cv_config_by_name(&term_details.cv_name);
    let cv_display_name = cv_config.display_name;

    format!("{} - {} - {} - {}", config.database_name, term_details.termid,
            term_details.name, cv_display_name)
}

fn term_summary(config: &Config, term_details: &TermDetails) -> String {
    let mut summ = String::new();

    summ += "
\n";

    summ += &format!("
Term ID

{}
\n", term_details.termid);
    summ += &format!("
Term name

{}
\n", term_details.name);

    let cv_config = config.cv_config_by_name(&term_details.cv_name);
    let cv_display_name = cv_config.display_name;

    summ += &format!("
CV name

{}
\n", cv_display_name);

    summ += "
\n";

    summ
}

fn term_body(config: &Config, title: &str, term_details: &TermDetails) -> String {
    let mut body = String::new();

    body += &format!("
{}
\n", title);

    body += &format!("<sect>
Welcome to {}
\n
{}
</sect>\n",
                     config.database_name, config.site_description);

    body += &format!("<sect>
Term summary
\n{}</sect>\n",
                     term_summary(config, term_details));

    body += &format!("<sect>
Annotation
\n{}</sect>\n",
                     annotation_section(config, term_details));

    body
}

pub fn render_simple_term_page(config: &Config, term_details: &TermDetails) -> String  {
    let title = make_term_title(config, term_details);

    format_page(&header(config, &title), &term_body(config, &title, term_details))
}
<commit_msg>Move organism closer to top of simple gene pages<commit_after>use pombase_rc_string::RcString;

use crate::web::config::Config;
use crate::web::data::{GeneDetails, ReferenceDetails, TermDetails,
                       ContainerType, GenotypeShort,
                       OntAnnotationId, AnnotationContainer, OrthologAnnotationContainer};

fn format_page(header: &str, body: &str) -> String {
    format! ("
<!DOCTYPE html>

    {}

    {}

",
             header, body)
}

fn make_gene_title(config: &Config, gene_details: &GeneDetails) -> String {
    let name_and_uniquename =
        if let Some(ref name) = gene_details.name {
            format!("{} ({})", name, gene_details.uniquename)
        } else {
            format!("{}", gene_details.uniquename)
        };

    let feature_type =
        if gene_details.feature_type.ends_with("gene") {
            String::from(gene_details.feature_type.as_str())
        } else {
            format!("{} gene", gene_details.feature_type)
        };

    if let Some(ref product) = gene_details.product {
        format!("{} - {} {} - {}", config.database_name, feature_type,
                name_and_uniquename, product)
    } else {
        format!("{} - {} {}", config.database_name, feature_type, name_and_uniquename)
    }
}

fn header(config: &Config, title: &str) -> String  {
    format!(r##"
  <meta charset="utf-8">
  <title>{}</title>
  <base href="/">
  <meta http-equiv="X-UA-Compatible" content="IE=edge">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <meta name="title" content="{}">
  <meta name="author" content="{}">
  <meta name="description" content="{}">
  <meta name="google-site-verification" content="H8vhCdG7XsSbh-8f20VSipvu2PnZ22YSGZm81jE0-Pk" />
  <link rel="shortcut icon" href="/assets/favicon.ico">
  <meta name="apple-mobile-web-app-title" content="{}">
  <meta name="application-name" content="{}">
"##,
            title, title, config.database_name, title, config.database_name,
            config.database_name
    )
}

fn gene_summary(config: &Config, gene_details: &GeneDetails) -> String {
    let mut summ = String::new();

    summ += "
\n";
    if let Some(ref name) = gene_details.name {
        summ += &format!("
Gene standard name

{}
\n",  name);
    }
    summ += &format!("
Systematic ID

{}
\n", gene_details.uniquename);

    if let Some(gene_organism) = config.organism_by_taxonid(gene_details.taxonid) {
        summ += &format!("
Organism

{}\n", gene_organism.scientific_name());

        if gene_organism.alternative_names.len() > 0 {
            summ += &format!(" ({})", gene_organism.alternative_names.join(", "));
        }
        summ += "
\n";
    }

    if gene_details.synonyms.len() > 0 {
        let synonyms: Vec<RcString> =
            gene_details.synonyms.iter().map(|s| s.name.clone()).collect();
        summ += &format!("
Synonyms

{}
\n", synonyms.join(", "));
    }

    if let Some(ref uniprot_identifier) = gene_details.uniprot_identifier {
        summ += &format!("
UniProt ID

{}
\n", uniprot_identifier);
    }

    if let Some(ref orfeome_identifier) = gene_details.orfeome_identifier {
        summ += &format!("
ORFeome ID

{}
\n", orfeome_identifier);
    }

    if let Some(ref characterisation_status) = gene_details.characterisation_status {
        summ += &format!("
Characterisation status

{}\n",
                         characterisation_status);
    }

    summ += "
\n";

    summ
}

fn genotype_display_name(genotype_short: &GenotypeShort) -> String {
    let mut bits: Vec<_> = genotype_short.display_uniquename.split("-").collect();
    bits.dedup();
    bits.join(" ")
}

fn get_annotations(ont_annotation_ids: &Vec<OntAnnotationId>,
                   container: &dyn AnnotationContainer) -> String {
    let mut ret = String::new();
    let mut references = vec![];
    let mut genes = vec![];
    let mut genotypes = vec![];

    for ont_annotation_id in ont_annotation_ids {
        if let Some(annotation_details) = container.annotation_details().get(ont_annotation_id) {
            if let Some(ref reference) = annotation_details.reference {
                references.push(reference);
            }

            if let Some(ref genotype) = annotation_details.genotype {
                genotypes.push(genotype);
            }

            for gene_uniquename in &annotation_details.genes {
                if let Some(maybe_gene_short) =
                    container.genes_by_uniquename().get(gene_uniquename.as_str()) {
                        if let Some(gene_short) = maybe_gene_short {
                            genes.push(gene_short)
                        }
                    }
            }
        }
    }

    if container.container_type() != ContainerType::Reference && references.len() > 0 {
        references.sort();
        references.dedup();
        ret += "
References:
\n
";
        for reference in references {
            ret += &format!("
{}
\n", reference, reference);
        }
        ret += "
\n";
    }

    if container.container_type() != ContainerType::Gene && genes.len() > 0 {
        genes.sort();
        genes.dedup();
        ret += "
Genes:
\n
";
        for gene in genes {
            ret += &format!("
{}
\n", gene.uniquename,
                            gene.display_name());
        }
        ret += "
\n";
    }

    if container.container_type() != ContainerType::Genotype && genotypes.len() > 0 {
        genotypes.sort();
        genotypes.dedup();
        if let Some(genotypes_by_uniquename) = container.genotypes_by_uniquename() {
            ret += "
Genotypes:
\n
";
            for uniquename in genotypes {
                if let Some(genotype_short) = genotypes_by_uniquename.get(uniquename.as_str()) {
                    ret += &format!("
{}
\n",
                                    uniquename,
                                    genotype_display_name(genotype_short));
                }
            }
            ret += "
\n";
        }
    }

    ret
}

fn annotation_section(config: &Config, container: &dyn AnnotationContainer) -> String {
    let mut annotation_html = String::new();

    let mut cv_names: Vec<RcString> = vec![];

    for cv_name in container.cv_annotations().keys() {
        cv_names.push(cv_name.clone());
    }

    let cmp_display_names =
        |cv_name_a: &RcString, cv_name_b: &RcString| {
            let cv_config_a = config.cv_config_by_name(&cv_name_a);
            let cv_display_name_a = cv_config_a.display_name;
            let cv_config_b = config.cv_config_by_name(&cv_name_b);
            let cv_display_name_b = cv_config_b.display_name;

            cv_display_name_a.cmp(&cv_display_name_b)
        };

    cv_names.sort_by(cmp_display_names);

    for cv_name in cv_names {
        let term_annotations = container.cv_annotations().get(&cv_name).unwrap();
        let cv_config = config.cv_config_by_name(&cv_name);
        let cv_display_name = cv_config.display_name;
        annotation_html += &format!("<sect>\n
{}
\n", cv_display_name);
        for term_annotation in term_annotations {
            let term_name =
                if let Some(term_short_opt) =
                    container.terms_by_termid().get(&term_annotation.term) {
                        if let Some(term_short) = term_short_opt {
                            String::from(term_short.name.as_str())
                        } else {
                            String::from("(unknown term name)")
                        }
                    } else {
                        String::from("(unknown term name)")
                    };
            annotation_html += &format!("<sect>
{} - \
{}
\n{}</sect>\n",
                                        term_annotation.term,
                                        term_annotation.term,
                                        term_annotation.term,
                                        term_name,
                                        get_annotations(&term_annotation.annotations,
                                                        container));
        }
        annotation_html += "</sect>\n";
    }

    annotation_html
}

fn orthologs(config: &Config, container: &dyn OrthologAnnotationContainer) -> String {
    let mut orth_html = String::new();

    if container.ortholog_annotations().is_empty() {
        return orth_html;
    }

    orth_html += "<sect>
Orthologs
\n
\n";

    for orth_annotation in container.ortholog_annotations().iter() {
        let maybe_orth_org = config.organism_by_taxonid(orth_annotation.ortholog_taxonid);
        let orth_org_scientific_name =
            if let Some(orth_org) = maybe_orth_org {
                orth_org.scientific_name()
            } else {
                String::from("Unknown organism")
            };
        if let Some(maybe_orth_gene_short) =
            container.genes_by_uniquename().get(&orth_annotation.ortholog_uniquename) {
                if let Some(orth_gene_short) = maybe_orth_gene_short {
                    orth_html += &format!("
{} {}
\n", orth_gene_short.display_name(),
                                          orth_org_scientific_name);
                }
            }
    }

    orth_html += "
</sect>\n";

    orth_html
}

fn gene_body(config: &Config, title: &str, gene_details: &GeneDetails) -> String {
    let mut body = String::new();

    body += &format!("
{}
\n", title);

    body += &format!("<sect>
Welcome to {}
\n
{}
</sect>\n",
                     config.database_name, config.site_description);

    body += &format!("<sect>
Gene summary
\n{}</sect>\n",
                     gene_summary(config, gene_details));

    body += &format!("<sect>
Annotation
\n{}</sect>\n",
                     annotation_section(config, gene_details));

    body += &orthologs(config, gene_details);

    body
}

pub fn render_simple_gene_page(config: &Config, gene_details: &GeneDetails) -> String  {
    let title = make_gene_title(config, gene_details);

    format_page(&header(config, &title), &gene_body(config, &title, gene_details))
}

fn make_reference_title(config: &Config, reference_details: &ReferenceDetails) -> String {
    if let Some(ref title) = reference_details.title {
        format!("{} - {} - {}", config.database_name, reference_details.uniquename, title)
    } else {
        format!("{} - {}", config.database_name, reference_details.uniquename)
    }
}

fn reference_summary(reference_details: &ReferenceDetails) -> String {
    let mut summ = String::new();

    summ += "
\n";

    summ += &format!("
PubMed ID

{}
\n", reference_details.uniquename);

    if let Some(ref title) = reference_details.title {
        summ += &format!("
Title

{}
\n", title);
    }

    if let Some(ref authors) = reference_details.authors {
        summ += &format!("
Authors

{}
\n", authors);
    }

    if let Some(ref citation) = reference_details.citation {
        summ += &format!("
Citation

{}
\n", citation);
    }

    if let Some(ref publication_year) = reference_details.publication_year {
        summ += &format!("
Publication year

{}
\n", publication_year);
    }

    summ += "
\n";

    summ
}

fn reference_body(config: &Config, title: &str, reference_details: &ReferenceDetails) -> String {
    let mut body = String::new();

    body += &format!("
{}
\n", title);

    body += &format!("<sect>
Welcome to {}
\n
{}
</sect>\n",
                     config.database_name, config.site_description);

    body += &format!("<sect>
Reference summary
\n{}</sect>\n",
                     reference_summary(reference_details));

    body += &format!("<sect>
Annotation
\n{}</sect>\n",
                     annotation_section(config, reference_details));

    body += &orthologs(config, reference_details);

    body
}

pub fn render_simple_reference_page(config: &Config, reference_details: &ReferenceDetails) -> String  {
    let title = make_reference_title(config, reference_details);

    format_page(&header(config, &title), &reference_body(config, &title, reference_details))
}

fn make_term_title(config: &Config, term_details: &TermDetails) -> String {
    let cv_config = config.cv_config_by_name(&term_details.cv_name);
    let cv_display_name = cv_config.display_name;

    format!("{} - {} - {} - {}", config.database_name, term_details.termid,
            term_details.name, cv_display_name)
}

fn term_summary(config: &Config, term_details: &TermDetails) -> String {
    let mut summ = String::new();

    summ += "
\n";

    summ += &format!("
Term ID

{}
\n", term_details.termid);
    summ += &format!("
Term name

{}
\n", term_details.name);

    let cv_config = config.cv_config_by_name(&term_details.cv_name);
    let cv_display_name = cv_config.display_name;

    summ += &format!("
CV name

{}
\n", cv_display_name);

    summ += "
\n";

    summ
}

fn term_body(config: &Config, title: &str, term_details: &TermDetails) -> String {
    let mut body = String::new();

    body += &format!("
{}
\n", title);

    body += &format!("<sect>
Welcome to {}
\n
{}
</sect>\n",
                     config.database_name, config.site_description);

    body += &format!("<sect>
Term summary
\n{}</sect>\n",
                     term_summary(config, term_details));

    body += &format!("<sect>
Annotation
\n{}</sect>\n",
                     annotation_section(config, term_details));

    body
}

pub fn render_simple_term_page(config: &Config, term_details: &TermDetails) -> String  {
    let title = make_term_title(config, term_details);

    format_page(&header(config, &title), &term_body(config, &title, term_details))
}
<|endoftext|>


<commit_before>use std::io::{Result, Write};

use env::Env;
use library;
use nameutil;
use super::ffi_type::*;
use super::super::general::version_condition;
use traits::*;

//used as glib:get-type in GLib-2.0.gir
const INTERN: &'static str= "intern";

pub fn generate_records_funcs<W: Write>(w: &mut W, env: &Env, records: &[&library::Record]) -> Result<()> {
    let intern_str = INTERN.to_string();
    for record in records {
        let glib_get_type = record.glib_get_type.as_ref().unwrap_or(&intern_str);
        try!(generate_object_funcs(w, env, &record.c_type,
            glib_get_type, &record.functions));
    }

    Ok(())
}

pub fn generate_classes_funcs<W: Write>(w: &mut W, env: &Env, classes: &[&library::Class]) -> Result<()> {
    for klass in classes {
        try!(generate_object_funcs(w, env, &klass.c_type,
            &klass.glib_get_type, &klass.functions));
    }

    Ok(())
}

pub fn generate_interfaces_funcs<W: Write>(w: &mut W, env: &Env, interfaces: &[&library::Interface]) -> Result<()> {
    for interface in interfaces {
        try!(generate_object_funcs(w, env,  &interface.c_type,
            &interface.glib_get_type, &interface.functions));
    }

    Ok(())
}

pub fn generate_other_funcs<W: Write>(w: &mut W, env: &Env, functions: &[library::Function]) -> Result<()> {
    generate_object_funcs(w, env,  "Other functions", INTERN, functions)
}

fn generate_object_funcs<W: Write>(w: &mut W, env: &Env, c_type: &str,
    glib_get_type: &str, functions: &[library::Function]) -> Result<()> {
    let write_get_type = glib_get_type != INTERN;
    if write_get_type || !functions.is_empty() {
        try!(writeln!(w, ""));
        try!(writeln!(w, "    //========================================================================="));
        try!(writeln!(w, "    // {}", c_type));
        try!(writeln!(w, "    //========================================================================="));
    }
    if write_get_type {
        try!(writeln!(w, "    pub fn {:<36}() -> GType;", glib_get_type));
    }

    for func in functions {
        let (commented, sig) = function_signature(env, func, false);
        let comment = if commented { "//" } else { "" };
        try!(version_condition(w, &env.config.library_name,
            env.config.min_cfg_version, func.version, commented, 1));
        try!(writeln!(w, "    {}pub fn {:<36}{};",
                      comment, func.c_identifier.as_ref().unwrap(), sig));
    }

    Ok(())
}

pub fn generate_callbacks<W: Write>(w: &mut W, env: &Env, callbacks: &[&library::Function]) -> Result<()> {
    for func in callbacks {
        let (commented, sig) = function_signature(env, func, true);
        let comment = if commented { "//" } else { "" };
        try!(writeln!(w, "{}pub type {} = Option<unsafe extern \"C\" fn{}>;",
                      comment, func.c_identifier.as_ref().unwrap(), sig));
    }

    Ok(())
}

pub fn function_signature(env: &Env, func: &library::Function, bare: bool) -> (bool, String) {
    let (mut commented, ret_str) = function_return_value(env, func);

    let mut parameter_strs: Vec<String> = Vec::new();
    for par in &func.parameters {
        let (c, par_str) = function_parameter(env, par, bare);
        parameter_strs.push(par_str);
        if c { commented = true; }
    }

    (commented, format!("({}){}", parameter_strs.connect(", "), ret_str))
}

fn function_return_value(env: &Env, func: &library::Function) -> (bool, String) {
    if func.ret.typ == Default::default() { return (false, String::new()) }
    let ffi_type = ffi_type(env, func.ret.typ, &func.ret.c_type);
    let commented = ffi_type.is_err();
    (commented, format!(" -> {}", ffi_type.as_str()))
}

fn function_parameter(env: &Env, par: &library::Parameter, bare: bool) -> (bool, String) {
    if let &library::Type::Fundamental(library::Fundamental::VarArgs) = env.library.type_(par.typ) {
        return (false, "...".into());
    }
    let ffi_type = ffi_type(env, par.typ, &par.c_type);
    let commented = ffi_type.is_err();
    let res = if bare {
        ffi_type.as_str().into()
    }
    else {
        format!("{}: {}", nameutil::mangle_keywords(&*par.name), ffi_type.as_str())
    };
    (commented, res)
}
<commit_msg>sys: don't align function arguments opening braces<commit_after>use std::io::{Result, Write};

use env::Env;
use library;
use nameutil;
use super::ffi_type::*;
use super::super::general::version_condition;
use traits::*;

//used as glib:get-type in GLib-2.0.gir
const INTERN: &'static str= "intern";

pub fn generate_records_funcs<W: Write>(w: &mut W, env: &Env, records: &[&library::Record]) -> Result<()> {
    let intern_str = INTERN.to_string();
    for record in records {
        let glib_get_type = record.glib_get_type.as_ref().unwrap_or(&intern_str);
        try!(generate_object_funcs(w, env, &record.c_type,
            glib_get_type, &record.functions));
    }

    Ok(())
}

pub fn generate_classes_funcs<W: Write>(w: &mut W, env: &Env, classes: &[&library::Class]) -> Result<()> {
    for klass in classes {
        try!(generate_object_funcs(w, env, &klass.c_type,
            &klass.glib_get_type, &klass.functions));
    }

    Ok(())
}

pub fn generate_interfaces_funcs<W: Write>(w: &mut W, env: &Env, interfaces: &[&library::Interface]) -> Result<()> {
    for interface in interfaces {
        try!(generate_object_funcs(w, env,  &interface.c_type,
            &interface.glib_get_type, &interface.functions));
    }

    Ok(())
}

pub fn generate_other_funcs<W: Write>(w: &mut W, env: &Env, functions: &[library::Function]) -> Result<()> {
    generate_object_funcs(w, env,  "Other functions", INTERN, functions)
}

fn generate_object_funcs<W: Write>(w: &mut W, env: &Env, c_type: &str,
    glib_get_type: &str, functions: &[library::Function]) -> Result<()> {
    let write_get_type = glib_get_type != INTERN;
    if write_get_type || !functions.is_empty() {
        try!(writeln!(w, ""));
        try!(writeln!(w, "    //========================================================================="));
        try!(writeln!(w, "    // {}", c_type));
        try!(writeln!(w, "    //========================================================================="));
    }
    if write_get_type {
        try!(writeln!(w, "    pub fn {}() -> GType;", glib_get_type));
    }

    for func in functions {
        let (commented, sig) = function_signature(env, func, false);
        let comment = if commented { "//" } else { "" };
        try!(version_condition(w, &env.config.library_name,
            env.config.min_cfg_version, func.version, commented, 1));
        try!(writeln!(w, "    {}pub fn {}{};",
                      comment, func.c_identifier.as_ref().unwrap(), sig));
    }

    Ok(())
}

pub fn generate_callbacks<W: Write>(w: &mut W, env: &Env, callbacks: &[&library::Function]) -> Result<()> {
    for func in callbacks {
        let (commented, sig) = function_signature(env, func, true);
        let comment = if commented { "//" } else { "" };
        try!(writeln!(w, "{}pub type {} = Option<unsafe extern \"C\" fn{}>;",
                      comment, func.c_identifier.as_ref().unwrap(), sig));
    }

    Ok(())
}

pub fn function_signature(env: &Env, func: &library::Function, bare: bool) -> (bool, String) {
    let (mut commented, ret_str) = function_return_value(env, func);

    let mut parameter_strs: Vec<String> = Vec::new();
    for par in &func.parameters {
        let (c, par_str) = function_parameter(env, par, bare);
        parameter_strs.push(par_str);
        if c { commented = true; }
    }

    (commented, format!("({}){}", parameter_strs.connect(", "), ret_str))
}

fn function_return_value(env: &Env, func: &library::Function) -> (bool, String) {
    if func.ret.typ == Default::default() { return (false, String::new()) }
    let ffi_type = ffi_type(env, func.ret.typ, &func.ret.c_type);
    let commented = ffi_type.is_err();
    (commented, format!(" -> {}", ffi_type.as_str()))
}

fn function_parameter(env: &Env, par: &library::Parameter, bare: bool) -> (bool, String) {
    if let &library::Type::Fundamental(library::Fundamental::VarArgs) = env.library.type_(par.typ) {
        return (false, "...".into());
    }
    let ffi_type = ffi_type(env, par.typ, &par.c_type);
    let commented = ffi_type.is_err();
    let res = if bare {
        ffi_type.as_str().into()
    }
    else {
        format!("{}: {}", nameutil::mangle_keywords(&*par.name), ffi_type.as_str())
    };
    (commented, res)
}
<|endoftext|>


<commit_before>import driver::session;
import syntax::ast;
import middle::ty;
import std::io;

fn get_symbol(session::session sess, ast::def_id def) -> str {
    auto cnum = def._0;
    auto node_id = def._1;
    auto cstore = sess.get_cstore();
    auto cdata = cstore::get_crate_data(cstore, cnum).data;
    ret decoder::get_symbol(cdata, node_id);
}

fn get_tag_variants(ty::ctxt tcx, ast::def_id def) -> ty::variant_info[] {
    decoder::get_tag_variants(tcx, def)
}

fn get_type(ty::ctxt tcx, ast::def_id def) -> ty::ty_param_count_and_ty {
    decoder::get_type(tcx, def)
}

fn get_type_param_count(ty::ctxt tcx, &ast::def_id def) -> uint {
    auto cnum = def._0;
    auto node_id = def._1;
    auto cstore = tcx.sess.get_cstore();
    auto cdata = cstore::get_crate_data(cstore, cnum).data;
    ret decoder::get_type_param_count(cdata, node_id);
}

fn lookup_defs(session::session sess, ast::crate_num cnum,
               vec[ast::ident] path) -> vec[ast::def] {
    auto cstore = sess.get_cstore();
    auto cdata = cstore::get_crate_data(cstore, cnum).data;
    ret decoder::lookup_defs(cdata, cnum, path);
}

fn get_crate_attributes(&vec[u8] data) -> ast::attribute[] {
    decoder::get_crate_attributes(data)
}

fn list_crate_metadata(vec[u8] data, io::writer out) {
    decoder::list_crate_metadata(data, out)
}

// Local Variables:
// mode: rust
// fill-column: 78;
// indent-tabs-mode: nil
// c-basic-offset: 4
// buffer-file-coding-system: utf-8-unix
// compile-command: "make -k -C $RBUILD 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
// End:
<commit_msg>Remove unused functions from metadata::csearch<commit_after>import driver::session;
import syntax::ast;
import middle::ty;
import std::io;

fn get_symbol(session::session sess, ast::def_id def) -> str {
    auto cnum = def._0;
    auto node_id = def._1;
    auto cstore = sess.get_cstore();
    auto cdata = cstore::get_crate_data(cstore, cnum).data;
    ret decoder::get_symbol(cdata, node_id);
}

fn get_tag_variants(ty::ctxt tcx, ast::def_id def) -> ty::variant_info[] {
    decoder::get_tag_variants(tcx, def)
}

fn get_type(ty::ctxt tcx, ast::def_id def) -> ty::ty_param_count_and_ty {
    decoder::get_type(tcx, def)
}

fn get_type_param_count(ty::ctxt tcx, &ast::def_id def) -> uint {
    auto cnum = def._0;
    auto node_id = def._1;
    auto cstore = tcx.sess.get_cstore();
    auto cdata = cstore::get_crate_data(cstore, cnum).data;
    ret decoder::get_type_param_count(cdata, node_id);
}

fn lookup_defs(session::session sess, ast::crate_num cnum,
               vec[ast::ident] path) -> vec[ast::def] {
    auto cstore = sess.get_cstore();
    auto cdata = cstore::get_crate_data(cstore, cnum).data;
    ret decoder::lookup_defs(cdata, cnum, path);
}

// Local Variables:
// mode: rust
// fill-column: 78;
// indent-tabs-mode: nil
// c-basic-offset: 4
// buffer-file-coding-system: utf-8-unix
// compile-command: "make -k -C $RBUILD 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
// End:
<|endoftext|>


<commit_before>import front::ast;
import front::ast::method;
import front::ast::ann;
import front::ast::item;
import front::ast::item_fn;
import front::ast::_fn;
import front::ast::obj_field;
import front::ast::_obj;
import front::ast::stmt;
import front::ast::ident;
import front::ast::def_id;
import front::ast::ty_param;
import front::ast::crate;

import front::ast::expr;
import middle::ty::type_is_nil;
import middle::ty::ret_ty_of_fn;
import util::common::span;
import tstate::ann::ts_ann;
import tstate::ann::empty_poststate;
import tstate::ann::true_precond;
import tstate::ann::true_postcond;
import tstate::ann::false_postcond;
import tstate::ann::precond;
import tstate::ann::postcond;
import tstate::ann::poststate;
import tstate::ann::prestate;
import tstate::ann::implies;
import tstate::ann::ann_precond;
import tstate::ann::ann_prestate;
import std::vec::map;
import std::vec;
import std::vec::slice;
import std::vec::unzip;
import std::vec::plus_option;
import std::vec::cat_options;

import std::option;
import std::option::t;
import std::option::some;
import std::option::none;

import aux::fn_ctxt;
import aux::crate_ctxt;
import aux::new_crate_ctxt;
import aux::expr_precond;
import aux::expr_prestate;
import aux::expr_poststate;
import aux::stmt_poststate;
import aux::stmt_to_ann;
import aux::num_locals;
import aux::fixed_point_states;
import aux::bitv_to_str;

import util::common::ty_to_str;
import util::common::log_stmt_err;
import aux::log_bitv_err;
import bitvectors::promises;

import annotate::annotate_crate;
import collect_locals::mk_f_to_fn_info;
import pre_post_conditions::fn_pre_post;
import states::find_pre_post_state_fn;

fn check_states_expr(&fn_ctxt fcx, @expr e) -> () {
    let precond prec  = expr_precond(fcx.ccx, e);
    let prestate pres = expr_prestate(fcx.ccx, e);

    if (!implies(pres, prec)) {
        auto s = "";
        s += ("Unsatisfied precondition constraint for expression:\n");
        s += util::common::expr_to_str(e);
        s += ("Precondition: ");
        s += bitv_to_str(fcx.enclosing, prec);
        s += ("Prestate: ");
        s += bitv_to_str(fcx.enclosing, pres);
        fcx.ccx.tcx.sess.span_err(e.span, s);
    }
}

fn check_states_stmt(&fn_ctxt fcx, &stmt s) -> () {
    auto a = stmt_to_ann(fcx.ccx, s);
    let precond prec    = ann_precond(a);
    let prestate pres   = ann_prestate(a);

    /*
      log_err("check_states_stmt:");
      log_stmt_err(s);
      log_err("prec = ");
      log_bitv_err(fcx.enclosing, prec);
      log_err("pres = ");
      log_bitv_err(fcx.enclosing, pres);
    */

    if (!implies(pres, prec)) {
        auto ss = "";
        ss += ("Unsatisfied precondition constraint for statement:\n");
        ss += util::common::stmt_to_str(s);
        ss += ("Precondition: ");
        ss += bitv_to_str(fcx.enclosing, prec);
        ss += ("Prestate: ");
        ss += bitv_to_str(fcx.enclosing, pres);
        fcx.ccx.tcx.sess.span_err(s.span, ss);
    }
}

fn check_states_against_conditions(&fn_ctxt fcx, &_fn f, &ann a) -> () {
    auto enclosing = fcx.enclosing;
    auto nv   = num_locals(enclosing);
    auto post = @empty_poststate(nv);

    fn do_one_(fn_ctxt fcx, &@stmt s, @poststate post) -> () {
        check_states_stmt(fcx, *s);
        *post = stmt_poststate(fcx.ccx, *s);
    }

    auto do_one = bind do_one_(fcx, _, post);

    vec::map[@stmt, ()](do_one, f.body.node.stmts);
    fn do_inner_(fn_ctxt fcx, &@expr e, @poststate post) -> () {
        check_states_expr(fcx, e);
        *post = expr_poststate(fcx.ccx, e);
    }
    auto do_inner = bind do_inner_(fcx, _, post);
    option::map[@expr, ()](do_inner, f.body.node.expr);

    /* Finally, check that the return value is initialized */
    if (f.proto == ast::proto_fn
        && ! promises(*post, fcx.id, enclosing)
        && ! type_is_nil(fcx.ccx.tcx,
                         ret_ty_of_fn(fcx.ccx.tcx, a)) ) {
        /* FIXME: call span_err, not span_warn, once I finish implementing
           ! annotations */
        fcx.ccx.tcx.sess.span_warn(f.body.span, "In function " + fcx.name +
          ", not all control paths return a value");
        fcx.ccx.tcx.sess.span_note(f.decl.output.span,
            "see declared return type of '" + ty_to_str(*f.decl.output) +
            "'");
    }

}

fn check_fn_states(&fn_ctxt fcx, &_fn f, &ann a) -> () {
    /* Compute the pre- and post-states for this function */
    auto g = find_pre_post_state_fn;
    fixed_point_states(fcx, g, f);

    /* Now compare each expr's pre-state to its precondition
       and post-state to its postcondition */
    check_states_against_conditions(fcx, f, a);
}

fn fn_states(&crate_ctxt ccx, &_fn f, &ident i, &def_id id, &ann a) -> () {
    /* Look up the var-to-bit-num map for this function */
    assert (ccx.fm.contains_key(id));
    auto f_info = ccx.fm.get(id);

    auto fcx = rec(enclosing=f_info, id=id, name=i, ccx=ccx);
    check_fn_states(fcx, f, a);
}

fn check_crate(ty::ctxt cx, @crate crate) -> () {
    let crate_ctxt ccx = new_crate_ctxt(cx);

    /* Build the global map from function id to var-to-bit-num-map */
    mk_f_to_fn_info(ccx, crate);

    /* Add a blank ts_ann for every statement (and expression) */
    annotate_crate(ccx, *crate);

    /* Compute the pre and postcondition for every subexpression */
    auto do_pre_post = walk::default_visitor();
    do_pre_post = rec(visit_fn_pre = bind fn_pre_post(ccx,_,_,_,_)
                      with do_pre_post);
    walk::walk_crate(do_pre_post, *crate);

    /* Check the pre- and postcondition against the pre- and poststate
       for every expression */
    auto do_states = walk::default_visitor();
    do_states = rec(visit_fn_pre = bind fn_states(ccx,_,_,_,_)
                    with do_states);
    walk::walk_crate(do_states, *crate);
}

//
// Local Variables:
// mode: rust
// fill-column: 78;
// indent-tabs-mode: nil
// c-basic-offset: 4
// buffer-file-coding-system: utf-8-unix
// compile-command: "make -k -C $RBUILD 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
// End:
//
<commit_msg>Error instead of warning for missing returns. Huzzah!<commit_after>import front::ast;
import front::ast::method;
import front::ast::ann;
import front::ast::item;
import front::ast::item_fn;
import front::ast::_fn;
import front::ast::obj_field;
import front::ast::_obj;
import front::ast::stmt;
import front::ast::ident;
import front::ast::def_id;
import front::ast::ty_param;
import front::ast::crate;

import front::ast::expr;
import middle::ty::type_is_nil;
import middle::ty::ret_ty_of_fn;
import util::common::span;
import tstate::ann::ts_ann;
import tstate::ann::empty_poststate;
import tstate::ann::true_precond;
import tstate::ann::true_postcond;
import tstate::ann::false_postcond;
import tstate::ann::precond;
import tstate::ann::postcond;
import tstate::ann::poststate;
import tstate::ann::prestate;
import tstate::ann::implies;
import tstate::ann::ann_precond;
import tstate::ann::ann_prestate;
import std::vec::map;
import std::vec;
import std::vec::slice;
import std::vec::unzip;
import std::vec::plus_option;
import std::vec::cat_options;

import std::option;
import std::option::t;
import std::option::some;
import std::option::none;

import aux::fn_ctxt;
import aux::crate_ctxt;
import aux::new_crate_ctxt;
import aux::expr_precond;
import aux::expr_prestate;
import aux::expr_poststate;
import aux::stmt_poststate;
import aux::stmt_to_ann;
import aux::num_locals;
import aux::fixed_point_states;
import aux::bitv_to_str;

import util::common::ty_to_str;
import util::common::log_stmt_err;
import aux::log_bitv_err;
import bitvectors::promises;

import annotate::annotate_crate;
import collect_locals::mk_f_to_fn_info;
import pre_post_conditions::fn_pre_post;
import states::find_pre_post_state_fn;

fn check_states_expr(&fn_ctxt fcx, @expr e) -> () {
    let precond prec  = expr_precond(fcx.ccx, e);
    let prestate pres = expr_prestate(fcx.ccx, e);

    if (!implies(pres, prec)) {
        auto s = "";
        s += ("Unsatisfied precondition constraint for expression:\n");
        s += util::common::expr_to_str(e);
        s += ("Precondition: ");
        s += bitv_to_str(fcx.enclosing, prec);
        s += ("Prestate: ");
        s += bitv_to_str(fcx.enclosing, pres);
        fcx.ccx.tcx.sess.span_err(e.span, s);
    }
}

fn check_states_stmt(&fn_ctxt fcx, &stmt s) -> () {
    auto a = stmt_to_ann(fcx.ccx, s);
    let precond prec    = ann_precond(a);
    let prestate pres   = ann_prestate(a);

    /*
      log_err("check_states_stmt:");
      log_stmt_err(s);
      log_err("prec = ");
      log_bitv_err(fcx.enclosing, prec);
      log_err("pres = ");
      log_bitv_err(fcx.enclosing, pres);
    */

    if (!implies(pres, prec)) {
        auto ss = "";
        ss += ("Unsatisfied precondition constraint for statement:\n");
        ss += util::common::stmt_to_str(s);
        ss += ("Precondition: ");
        ss += bitv_to_str(fcx.enclosing, prec);
        ss += ("Prestate: ");
        ss += bitv_to_str(fcx.enclosing, pres);
        fcx.ccx.tcx.sess.span_err(s.span, ss);
    }
}

fn check_states_against_conditions(&fn_ctxt fcx, &_fn f, &ann a) -> () {
    auto enclosing = fcx.enclosing;
    auto nv   = num_locals(enclosing);
    auto post = @empty_poststate(nv);

    fn do_one_(fn_ctxt fcx, &@stmt s, @poststate post) -> () {
        check_states_stmt(fcx, *s);
        *post = stmt_poststate(fcx.ccx, *s);
    }

    auto do_one = bind do_one_(fcx, _, post);

    vec::map[@stmt, ()](do_one, f.body.node.stmts);
    fn do_inner_(fn_ctxt fcx, &@expr e, @poststate post) -> () {
        check_states_expr(fcx, e);
        *post = expr_poststate(fcx.ccx, e);
    }
    auto do_inner = bind do_inner_(fcx, _, post);
    option::map[@expr, ()](do_inner, f.body.node.expr);

    /* Finally, check that the return value is initialized */
    if (f.proto == ast::proto_fn
        && ! promises(*post, fcx.id, enclosing)
        && ! type_is_nil(fcx.ccx.tcx,
                         ret_ty_of_fn(fcx.ccx.tcx, a)) ) {
        fcx.ccx.tcx.sess.span_note(f.body.span, "In function " + fcx.name +
          ", not all control paths return a value");
        fcx.ccx.tcx.sess.span_err(f.decl.output.span,
            "see declared return type of '" + ty_to_str(*f.decl.output) +
            "'");
    }

}

fn check_fn_states(&fn_ctxt fcx, &_fn f, &ann a) -> () {
    /* Compute the pre- and post-states for this function */
    auto g = find_pre_post_state_fn;
    fixed_point_states(fcx, g, f);

    /* Now compare each expr's pre-state to its precondition
       and post-state to its postcondition */
    check_states_against_conditions(fcx, f, a);
}

fn fn_states(&crate_ctxt ccx, &_fn f, &ident i, &def_id id, &ann a) -> () {
    /* Look up the var-to-bit-num map for this function */
    assert (ccx.fm.contains_key(id));
    auto f_info = ccx.fm.get(id);

    auto fcx = rec(enclosing=f_info, id=id, name=i, ccx=ccx);
    check_fn_states(fcx, f, a);
}

fn check_crate(ty::ctxt cx, @crate crate) -> () {
    let crate_ctxt ccx = new_crate_ctxt(cx);

    /* Build the global map from function id to var-to-bit-num-map */
    mk_f_to_fn_info(ccx, crate);

    /* Add a blank ts_ann for every statement (and expression) */
    annotate_crate(ccx, *crate);

    /* Compute the pre and postcondition for every subexpression */
    auto do_pre_post = walk::default_visitor();
    do_pre_post = rec(visit_fn_pre = bind fn_pre_post(ccx,_,_,_,_)
                      with do_pre_post);
    walk::walk_crate(do_pre_post, *crate);

    /* Check the pre- and postcondition against the pre- and poststate
       for every expression */
    auto do_states = walk::default_visitor();
    do_states = rec(visit_fn_pre = bind fn_states(ccx,_,_,_,_)
                    with do_states);
    walk::walk_crate(do_states, *crate);
}

//
// Local Variables:
// mode: rust
// fill-column: 78;
// indent-tabs-mode: nil
// c-basic-offset: 4
// buffer-file-coding-system: utf-8-unix
// compile-command: "make -k -C $RBUILD 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
// End:
//
<|endoftext|>


<commit_before>#![feature(test)]
extern crate test;
extern crate unidiff;

use std::io::prelude::*;
use std::fs::File;

use test::Bencher;
use unidiff::PatchSet;

#[bench]
fn bench_parse_diff_simple(b: &mut Bencher) {
    let mut buf = String::new();
    File::open("tests/fixtures/sample0.diff").and_then(|mut r| r.read_to_string(&mut buf)).unwrap();
    let mut patch = PatchSet::new();

    b.iter(|| {
        patch.parse(&buf).unwrap();
    });
}
<commit_msg>Fix simple benchmark<commit_after>#![feature(test)]
extern crate test;
extern crate unidiff;

use std::io::prelude::*;
use std::fs::File;

use test::Bencher;
use unidiff::PatchSet;

#[bench]
fn bench_parse_diff_simple(b: &mut Bencher) {
    let mut buf = String::new();
    File::open("tests/fixtures/sample0.diff").and_then(|mut r| r.read_to_string(&mut buf)).unwrap();

    b.iter(|| {
        let mut patch = PatchSet::new();
        patch.parse(&buf).unwrap();
    });
}
<|endoftext|>


<commit_before>use perlin::index::{Index, PersistentIndex};
use perlin::index::boolean_index::{FsPostingProvider, BooleanIndex};
use std::fmt::Debug;
use std::env;
use std::path::Path;
use std::fs::File;
use rand;

pub fn prepare_index(documents: usize,
                                                                     document_size: usize)
                                                                             -> BooleanIndex<usize> {

  //  println!("Preparing Index with {} documents and {} terms per document", documents, document_size);
    let mut tmp_dir = env::temp_dir();
    tmp_dir.push(&format!("bench_index_{}_{}.bin", documents, document_size));

    if tmp_dir.exists() {
        let result = BooleanIndex::read_from(&mut File::open(tmp_dir.as_path()).unwrap()).unwrap();
        result
    } else {
        let rng = ZipfGenerator::new(voc_size(20, 0.5, documents * document_size));
        let mut index = BooleanIndex::new(Box::new(FsPostingProvider::new(Path::new(&format!("/tmp/fs_{}{}.bin", documents, document_size)))));
        let mut docs = Vec::with_capacity(documents);
        for i in 0..documents {
            if i % 1000 == 0
            {
                println!("{}/{} documents", i, documents);
            }
            docs.push(rng.take(document_size));
        }
        index.index_documents(docs);
        index.write_to(&mut File::create(tmp_dir.as_path()).unwrap()).unwrap();
        index
    }
}

// Implementation of Heaps' Law
fn voc_size(k: usize, b: f64, tokens: usize) -> usize {
    ((k as f64) * (tokens as f64).powf(b)) as usize
}

#[derive(Clone)]
struct ZipfGenerator {
    voc_size: usize,
    factor: f64,
    acc_probs: Box<Vec<f64>>,
}

impl ZipfGenerator {
    fn new(voc_size: usize) -> Self {
        let mut res = ZipfGenerator {
            voc_size: voc_size,
            factor: (1.78 * voc_size as f64).ln(),
            acc_probs: Box::new(Vec::with_capacity(voc_size)),
        };
        let mut acc = 0.0;
        for i in 1..voc_size {
            acc += 1.0 / (i as f64 * res.factor);
            res.acc_probs.push(acc);
        }
        res
    }
}

impl<'a> Iterator for &'a ZipfGenerator {
    type Item = usize;

    fn next(&mut self) -> Option<Self::Item> {
        let dice = rand::random::<f64>();
        let mut c = 0;
        loop {
            if dice < self.acc_probs[c] {
                return Some(c);
            }
            c += 1;
        }
    }
}
<commit_msg>fixed Benches<commit_after>use perlin::index::{Index, PersistentIndex};
use perlin::index::boolean_index::BooleanIndex;
use perlin::index::storage::fs_storage::FsStorage;
use std::fmt::Debug;
use std::env;
use std::path::Path;
use std::fs::{create_dir_all, File};
use rand;

pub fn prepare_index(documents: usize, document_size: usize) -> BooleanIndex<usize> {
    let mut tmp_dir = env::temp_dir();
    tmp_dir.push(&format!("bench_index_{}_{}.bin", documents, document_size));

    if tmp_dir.exists() {
        let result = BooleanIndex::read_from(&mut File::open(tmp_dir.as_path()).unwrap()).unwrap();
        result
    } else {
        let rng = ZipfGenerator::new(voc_size(20, 0.5, documents * document_size));
        assert!(create_dir_all(Path::new(&format!("/tmp/fs_{}{}", documents, document_size)))
            .is_ok());
        let mut index =
            BooleanIndex::new(Box::new(FsStorage::new(Path::new(&format!("/tmp/fs_{}{}",
                                                                          documents,
                                                                          document_size)))));
        let mut docs = Vec::with_capacity(documents);
        for i in 0..documents {
            if i % 1000 == 0 {
                println!("{}/{} documents", i, documents);
            }
            docs.push(rng.take(document_size));
        }
        index.index_documents(docs);
        index.write_to(&mut File::create(tmp_dir.as_path()).unwrap()).unwrap();
        index
    }
}

// Implementation of Heaps' Law
fn voc_size(k: usize, b: f64, tokens: usize) -> usize {
    ((k as f64) * (tokens as f64).powf(b)) as usize
}

#[derive(Clone)]
struct ZipfGenerator {
    voc_size: usize,
    factor: f64,
    acc_probs: Box<Vec<f64>>,
}

impl ZipfGenerator {
    fn new(voc_size: usize) -> Self {
        let mut res = ZipfGenerator {
            voc_size: voc_size,
            factor: (1.78 * voc_size as f64).ln(),
            acc_probs: Box::new(Vec::with_capacity(voc_size)),
        };
        let mut acc = 0.0;
        for i in 1..voc_size {
            acc += 1.0 / (i as f64 * res.factor);
            res.acc_probs.push(acc);
        }
        res
    }
}

impl<'a> Iterator for &'a ZipfGenerator {
    type Item = usize;

    fn next(&mut self) -> Option<Self::Item> {
        let dice = rand::random::<f64>();
        let mut c = 0;
        loop {
            if dice < self.acc_probs[c] {
                return Some(c);
            }
            c += 1;
        }
    }
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013, David Renshaw (dwrenshaw@gmail.com)
 *
 * See the LICENSE file in the capnproto-rust root directory.
 */

extern mod extra;

use std;
use std::rand::*;
use capnp;
use common::*;
use catrank_capnp::*;

pub type RequestBuilder = SearchResultList::Builder;
pub type ResponseBuilder = SearchResultList::Builder;
pub type Expectation = int;

pub fn newRequestReader<'a>(sr : capnp::layout::StructReader<'a>) -> SearchResultList::Reader<'a> {
    SearchResultList::Reader::new(sr)
}

pub fn newResponseReader<'a>(sr : capnp::layout::StructReader<'a>) -> SearchResultList::Reader<'a> {
    SearchResultList::Reader::new(sr)
}

pub struct ScoredResult<'self> {
    score : f64,
    result : SearchResult::Reader<'self>
}

static URL_PREFIX : &'static str = "http://example.com";

pub fn setupRequest(rng : &mut FastRand, request : SearchResultList::Builder) -> int {
    let count = rng.nextLessThan(1000) as uint;
    let mut goodCount : int = 0;

    let list = request.initResults(count);

    for i in range(0, count) {
        let result = list[i];
        result.setScore(1000.0 - i as f64);
        let urlSize = rng.nextLessThan(100) as uint;

        // TODO: modify string field in place with Text::Builder?
        let mut url = ~"http://example.com/";

        for _ in range(0, urlSize) {
            url.push_char(std::char::from_u32(97 + rng.nextLessThan(26)).unwrap());
        }

        // Add this just to match capnproto-c++, which inits url to be
        // one byte longer than it needs to be.
        url.push_char('\x00');

        result.setUrl(url);

        let isCat = rng.nextLessThan(8) == 0;
        let isDog = rng.nextLessThan(8) == 0;
        if (isCat && !isDog) {
            goodCount += 1;
        }

        let mut snippet = ~" ";

        let prefix = rng.nextLessThan(20) as uint;
        for _ in range(0, prefix) {
            snippet.push_str(WORDS[rng.nextLessThan(WORDS.len() as u32)]);
        }
        if (isCat) { snippet.push_str("cat ") }
        if (isDog) { snippet.push_str("dog ") }

        let suffix = rng.nextLessThan(20) as uint;
        for _ in range(0, suffix) {
            snippet.push_str(WORDS[rng.nextLessThan(WORDS.len() as u32)]);
        }

        result.setSnippet(snippet);
    }

    goodCount
}

pub fn handleRequest(request : SearchResultList::Reader,
                     response : SearchResultList::Builder) {
    let mut scoredResults : ~[ScoredResult] = ~[];

    let results = request.getResults();
    for i in range(0, results.size()) {
        let result = results[i];
        let mut score = result.getScore();
        if (result.getSnippet().contains(" cat ")) {
            score *= 10000.0;
        }
        if (result.getSnippet().contains(" dog ")) {
            score /= 10000.0;
        }
        scoredResults.push(ScoredResult {score : score, result : result});
    }

    do extra::sort::quick_sort(scoredResults) |v1, v2| {v1.score <= v2.score };

    let list = response.initResults(scoredResults.len());
    for i in range(0, list.size()) {
        let item = list[i];
        let result = scoredResults[i];
        item.setScore(result.score);
        item.setUrl(result.result.getUrl());
        item.setSnippet(result.result.getSnippet());
    }
}

pub fn checkResponse(response : SearchResultList::Reader, expectedGoodCount : int) -> bool {
    let mut goodCount : int = 0;
    let results = response.getResults();
    for i in range(0, results.size()) {
        let result = results[i];
        if (result.getScore() > 1001.0) {
            goodCount += 1;
        } else {
            break;
        }
    }
    return goodCount == expectedGoodCount;
}
<commit_msg>remove extra null terminator<commit_after>/*
 * Copyright (c) 2013, David Renshaw (dwrenshaw@gmail.com)
 *
 * See the LICENSE file in the capnproto-rust root directory.
 */

extern mod extra;

use std;
use std::rand::*;
use capnp;
use common::*;
use catrank_capnp::*;

pub type RequestBuilder = SearchResultList::Builder;
pub type ResponseBuilder = SearchResultList::Builder;
pub type Expectation = int;

pub fn newRequestReader<'a>(sr : capnp::layout::StructReader<'a>) -> SearchResultList::Reader<'a> {
    SearchResultList::Reader::new(sr)
}

pub fn newResponseReader<'a>(sr : capnp::layout::StructReader<'a>) -> SearchResultList::Reader<'a> {
    SearchResultList::Reader::new(sr)
}

pub struct ScoredResult<'self> {
    score : f64,
    result : SearchResult::Reader<'self>
}

static URL_PREFIX : &'static str = "http://example.com";

pub fn setupRequest(rng : &mut FastRand, request : SearchResultList::Builder) -> int {
    let count = rng.nextLessThan(1000) as uint;
    let mut goodCount : int = 0;

    let list = request.initResults(count);

    for i in range(0, count) {
        let result = list[i];
        result.setScore(1000.0 - i as f64);
        let urlSize = rng.nextLessThan(100) as uint;

        // TODO: modify string field in place with Text::Builder?
        let mut url = ~"http://example.com/";

        for _ in range(0, urlSize) {
            url.push_char(std::char::from_u32(97 + rng.nextLessThan(26)).unwrap());
        }

        result.setUrl(url);

        let isCat = rng.nextLessThan(8) == 0;
        let isDog = rng.nextLessThan(8) == 0;
        if (isCat && !isDog) {
            goodCount += 1;
        }

        let mut snippet = ~" ";

        let prefix = rng.nextLessThan(20) as uint;
        for _ in range(0, prefix) {
            snippet.push_str(WORDS[rng.nextLessThan(WORDS.len() as u32)]);
        }
        if (isCat) { snippet.push_str("cat ") }
        if (isDog) { snippet.push_str("dog ") }

        let suffix = rng.nextLessThan(20) as uint;
        for _ in range(0, suffix) {
            snippet.push_str(WORDS[rng.nextLessThan(WORDS.len() as u32)]);
        }

        result.setSnippet(snippet);
    }

    goodCount
}

pub fn handleRequest(request : SearchResultList::Reader,
                     response : SearchResultList::Builder) {
    let mut scoredResults : ~[ScoredResult] = ~[];

    let results = request.getResults();
    for i in range(0, results.size()) {
        let result = results[i];
        let mut score = result.getScore();
        if (result.getSnippet().contains(" cat ")) {
            score *= 10000.0;
        }
        if (result.getSnippet().contains(" dog ")) {
            score /= 10000.0;
        }
        scoredResults.push(ScoredResult {score : score, result : result});
    }

    do extra::sort::quick_sort(scoredResults) |v1, v2| {v1.score <= v2.score };

    let list = response.initResults(scoredResults.len());
    for i in range(0, list.size()) {
        let item = list[i];
        let result = scoredResults[i];
        item.setScore(result.score);
        item.setUrl(result.result.getUrl());
        item.setSnippet(result.result.getSnippet());
    }
}

pub fn checkResponse(response : SearchResultList::Reader, expectedGoodCount : int) -> bool {
    let mut goodCount : int = 0;
    let results = response.getResults();
    for i in range(0, results.size()) {
        let result = results[i];
        if (result.getScore() > 1001.0) {
            goodCount += 1;
        } else {
            break;
        }
    }
    return goodCount == expectedGoodCount;
}
<|endoftext|>


<commit_before>use std::borrow::Cow;
use std::cmp::{max, min, Ordering, Reverse};
use std::collections::VecDeque;
use std::future::Future;
use std::sync::{atomic, Arc};
use std::time::{Duration, Instant};

use async_trait::async_trait;
use lazy_static::lazy_static;
use log::Level;
use parking_lot::Mutex;
use regex::Regex;
use task_executor::Executor;
use tokio::sync::{Notify, Semaphore, SemaphorePermit};
use tokio::time::sleep;
use workunit_store::{in_workunit, RunningWorkunit, WorkunitMetadata};

use crate::{Context, FallibleProcessResultWithPlatform, Process};

lazy_static! {
  // TODO: Runtime formatting is unstable in Rust, so we imitate it.
  static ref CONCURRENCY_TEMPLATE_RE: Regex = Regex::new(r"\{pants_concurrency\}").unwrap();
}

///
/// A CommandRunner wrapper which limits the number of concurrent requests and which provides
/// concurrency information to the process being executed.
///
/// If a Process sets a non-zero `concurrency_available` value, it may be preempted (i.e. canceled
/// and restarted) with a new concurrency value for a short period after starting.
///
#[derive(Clone)]
pub struct CommandRunner {
  inner: Arc<dyn crate::CommandRunner>,
  sema: AsyncSemaphore,
}

impl CommandRunner {
  pub fn new(
    executor: &Executor,
    inner: Box<dyn crate::CommandRunner>,
    bound: usize,
  ) -> CommandRunner {
    CommandRunner {
      inner: inner.into(),
      sema: AsyncSemaphore::new(
        executor,
        bound,
        // TODO: Make configurable.
        Duration::from_millis(200),
      ),
    }
  }
}

#[async_trait]
impl crate::CommandRunner for CommandRunner {
  async fn run(
    &self,
    context: Context,
    workunit: &mut RunningWorkunit,
    process: Process,
  ) -> Result<FallibleProcessResultWithPlatform, String> {
    let semaphore_acquisition = self.sema.acquire(process.concurrency_available);
    let permit = in_workunit!(
      context.workunit_store.clone(),
      "acquire_command_runner_slot".to_owned(),
      WorkunitMetadata {
        level: Level::Trace,
        ..WorkunitMetadata::default()
      },
      |workunit| async move {
        let _blocking_token = workunit.blocking();
        semaphore_acquisition.await
      }
    )
    .await;

    loop {
      let mut process = process.clone();
      let concurrency_available = permit.concurrency();
      log::debug!(
        "Running {} under semaphore with concurrency id: {}, and concurrency: {}",
        process.description,
        permit.concurrency_slot(),
        concurrency_available,
      );

      // TODO: Both of these templating cases should be implemented at the lowest possible level:
      // they might currently be applied above a cache.
      if let Some(ref execution_slot_env_var) = process.execution_slot_variable {
        process.env.insert(
          execution_slot_env_var.clone(),
          format!("{}", permit.concurrency_slot()),
        );
      }
      if process.concurrency_available > 0 {
        let concurrency = format!("{}", permit.concurrency());
        let mut matched = false;
        process.argv = std::mem::take(&mut process.argv)
          .into_iter()
          .map(
            |arg| match CONCURRENCY_TEMPLATE_RE.replace_all(&arg, &concurrency) {
              Cow::Owned(altered) => {
                matched = true;
                altered
              }
              Cow::Borrowed(_original) => arg,
            },
          )
          .collect();
        if !matched {
          return Err(format!(
            "Process {} set `concurrency_available={}`, but did not include \
                             the `{}` template variable in its arguments.",
            process.description, process.concurrency_available, *CONCURRENCY_TEMPLATE_RE
          ));
        }
      }

      let running_process = self.inner.run(context.clone(), workunit, process.clone());
      tokio::select! {
        _ = permit.notified_concurrency_changed() => {
          log::debug!(
            "Process {} was preempted, and went from concurrency {} to concurrency {}",
            process.description,
            concurrency_available,
            permit.concurrency(),
          );
          continue;
        },
        res = running_process => {
          // The process completed.
          return res;
        }
      }
    }
  }
}

/// A wrapped Semaphore which adds concurrency metadata which supports overcommit.
#[derive(Clone)]
pub(crate) struct AsyncSemaphore {
  sema: Arc<Semaphore>,
  state: Arc<Mutex<State>>,
  preemptible_duration: Duration,
}

pub(crate) struct State {
  total_concurrency: usize,
  available_ids: VecDeque<usize>,
  tasks: Vec<Arc<Task>>,
}

impl State {
  #[cfg(test)]
  pub(crate) fn new_for_tests(total_concurrency: usize, tasks: Vec<Arc<Task>>) -> Self {
    Self {
      total_concurrency,
      available_ids: VecDeque::new(),
      tasks,
    }
  }
}

impl AsyncSemaphore {
  pub fn new(
    executor: &Executor,
    permits: usize,
    preemptible_duration: Duration,
  ) -> AsyncSemaphore {
    let mut available_ids = VecDeque::new();
    for id in 1..=permits {
      available_ids.push_back(id);
    }

    let state = Arc::new(Mutex::new(State {
      total_concurrency: permits,
      available_ids,
      tasks: Vec::new(),
    }));

    // Spawn a task which will periodically balance Tasks.
    let _balancer_task = {
      let state = Arc::downgrade(&state);
      executor.spawn(async move {
        loop {
          sleep(preemptible_duration / 4).await;
          if let Some(state) = state.upgrade() {
            // Balance tasks.
            let mut state = state.lock();
            balance(Instant::now(), &mut state);
          } else {
            // The AsyncSemaphore was torn down.
            break;
          }
        }
      })
    };

    AsyncSemaphore {
      sema: Arc::new(Semaphore::new(permits)),
      state,
      preemptible_duration,
    }
  }

  #[cfg(test)]
  pub(crate) fn available_permits(&self) -> usize {
    self.sema.available_permits()
  }

  ///
  /// Runs the given Future-creating function (and the Future it returns) under the semaphore.
  ///
  /// NB: This method does not support preemption, or controlling concurrency.
  ///
  // TODO: https://github.com/rust-lang/rust/issues/46379
  #[allow(dead_code)]
  pub(crate) async fn with_acquired<F, B, O>(self, f: F) -> O
  where
    F: FnOnce(usize) -> B,
    B: Future<Output = O>,
  {
    let permit = self.acquire(1).await;
    let res = f(permit.task.id).await;
    drop(permit);
    res
  }

  ///
  /// Acquire a slot on the semaphore when it becomes available. Additionally, attempt to acquire
  /// the given amount of concurrency. The amount actually acquired will be reported on the
  /// returned Permit.
  ///
  pub async fn acquire(&self, concurrency_desired: usize) -> Permit<'_> {
    let permit = self.sema.acquire().await.expect("semaphore closed");
    let task = {
      let mut state = self.state.lock();
      let id = state
        .available_ids
        .pop_front()
        .expect("More permits were distributed than ids exist.");

      // A Task is initially given its fair share of the available concurrency: i.e., the first
      // arriving task gets all of the slots, and the second arriving gets half, even though that
      // means that we overcommit. Balancing will adjust concurrency later, to the extent that it
      // can given preemption timeouts.
      //
      // This is because we cannot anticipate the number of inbound processes, and we never want to
      // delay a process from starting.
      let concurrency_desired = max(concurrency_desired, 1);
      let concurrency_actual = min(
        concurrency_desired,
        state.total_concurrency / (state.tasks.len() + 1),
      );
      let task = Arc::new(Task::new(
        id,
        concurrency_desired,
        concurrency_actual,
        Instant::now() + self.preemptible_duration,
      ));
      state.tasks.push(task.clone());
      task
    };
    Permit {
      state: self.state.clone(),
      _permit: permit,
      task,
    }
  }
}

pub struct Permit<'a> {
  state: Arc<Mutex<State>>,
  // NB: Kept for its `Drop` impl.
  _permit: SemaphorePermit<'a>,
  task: Arc<Task>,
}

impl Permit<'_> {
  pub fn concurrency_slot(&self) -> usize {
    self.task.id
  }

  pub fn concurrency(&self) -> usize {
    self.task.concurrency()
  }

  pub async fn notified_concurrency_changed(&self) {
    self.task.notify_concurrency_changed.notified().await
  }
}

impl<'a> Drop for Permit<'a> {
  fn drop(&mut self) {
    let mut state = self.state.lock();
    state.available_ids.push_back(self.task.id);
    let tasks_position = state
      .tasks
      .iter()
      .position(|t| t.id == self.task.id)
      .unwrap();
    state.tasks.swap_remove(tasks_position);
  }
}

pub(crate) struct Task {
  id: usize,
  concurrency_desired: usize,
  pub(crate) concurrency_actual: atomic::AtomicUsize,
  notify_concurrency_changed: Notify,
  preemptible_until: Instant,
}

impl Task {
  pub(crate) fn new(
    id: usize,
    concurrency_desired: usize,
    concurrency_actual: usize,
    preemptible_until: Instant,
  ) -> Self {
    assert!(concurrency_actual <= concurrency_desired);
    Self {
      id,
      concurrency_desired,
      concurrency_actual: atomic::AtomicUsize::new(concurrency_actual),
      notify_concurrency_changed: Notify::new(),
      preemptible_until,
    }
  }

  pub(crate) fn concurrency(&self) -> usize {
    self.concurrency_actual.load(atomic::Ordering::SeqCst)
  }

  fn preemptible(&self, now: Instant) -> bool {
    self.preemptible_until > now
  }
}

/// Given a set of Tasks with their desired and actual concurrency, balance the concurrency levels
/// of any preemptible tasks, and notify them of the changes. Returns the number of Tasks that were
/// preempted.
///
/// This method only internally mutates tasks (by adjusting their concurrency levels and notifying
/// them), but takes State as mutable in order to guarantee that it gets an atomic view of the
/// tasks.
pub(crate) fn balance(now: Instant, state: &mut State) -> usize {
  let concurrency_used: usize = state.tasks.iter().map(|t| t.concurrency()).sum();
  let mut desired_change_in_commitment =
    state.total_concurrency as isize - concurrency_used as isize;
  let mut prempted = 0;

  // To reduce the number of tasks that we preempty, we preempt them in order by the amount of
  // concurrency that they desire or can relinquish.
  match desired_change_in_commitment.cmp(&0) {
    Ordering::Equal => {
      // Nothing to do! Although some tasks might not have their desired concurrency levels, it's
      // probably not worth preempting any tasks to fix that.
    }
    Ordering::Less => {
      // We're overcommitted: order by the amount that they can relinquish.
      let mut preemptible_tasks = state
        .tasks
        .iter()
        .filter_map(|t| {
          // A task may never have less than one slot.
          let relinquishable = t.concurrency() - 1;
          if relinquishable > 0 && t.preemptible(now) {
            Some((relinquishable, t))
          } else {
            None
          }
        })
        .collect::<Vec<_>>();
      preemptible_tasks.sort_by_key(|(relinquishable, _)| Reverse(*relinquishable));

      for (relinquishable, task) in preemptible_tasks {
        if desired_change_in_commitment == 0 {
          break;
        }

        let relinquish = min(relinquishable, (-desired_change_in_commitment) as usize);
        desired_change_in_commitment += relinquish as isize;
        task
          .concurrency_actual
          .fetch_sub(relinquish, atomic::Ordering::SeqCst);
        task.notify_concurrency_changed.notify_one();
        prempted += 1;
      }
    }
    Ordering::Greater => {
      // We're undercommitted: order by the amount that they are owed.
      let mut preemptible_tasks = state
        .tasks
        .iter()
        .filter_map(|t| {
          let desired = t.concurrency_desired - t.concurrency();
          if desired > 0 && t.preemptible(now) {
            Some((desired, t))
          } else {
            None
          }
        })
        .collect::<Vec<_>>();
      preemptible_tasks.sort_by_key(|(desired, _)| Reverse(*desired));

      for (desired, task) in preemptible_tasks {
        if desired_change_in_commitment == 0 {
          break;
        }

        let acquire = min(desired, desired_change_in_commitment as usize);
        desired_change_in_commitment -= acquire as isize;
        task
          .concurrency_actual
          .fetch_add(acquire, atomic::Ordering::SeqCst);
        task.notify_concurrency_changed.notify_one();
        prempted += 1;
      }
    }
  }

  prempted
}
<commit_msg>Fix rendering of "Scheduling: ..." workunits introduced by workunit filtering. (#14681)<commit_after>use std::borrow::Cow;
use std::cmp::{max, min, Ordering, Reverse};
use std::collections::VecDeque;
use std::future::Future;
use std::sync::{atomic, Arc};
use std::time::{Duration, Instant};

use async_trait::async_trait;
use lazy_static::lazy_static;
use log::Level;
use parking_lot::Mutex;
use regex::Regex;
use task_executor::Executor;
use tokio::sync::{Notify, Semaphore, SemaphorePermit};
use tokio::time::sleep;
use workunit_store::{in_workunit, RunningWorkunit, WorkunitMetadata};

use crate::{Context, FallibleProcessResultWithPlatform, Process};

lazy_static! {
  // TODO: Runtime formatting is unstable in Rust, so we imitate it.
  static ref CONCURRENCY_TEMPLATE_RE: Regex = Regex::new(r"\{pants_concurrency\}").unwrap();
}

///
/// A CommandRunner wrapper which limits the number of concurrent requests and which provides
/// concurrency information to the process being executed.
///
/// If a Process sets a non-zero `concurrency_available` value, it may be preempted (i.e. canceled
/// and restarted) with a new concurrency value for a short period after starting.
///
#[derive(Clone)]
pub struct CommandRunner {
  inner: Arc<dyn crate::CommandRunner>,
  sema: AsyncSemaphore,
}

impl CommandRunner {
  pub fn new(
    executor: &Executor,
    inner: Box<dyn crate::CommandRunner>,
    bound: usize,
  ) -> CommandRunner {
    CommandRunner {
      inner: inner.into(),
      sema: AsyncSemaphore::new(
        executor,
        bound,
        // TODO: Make configurable.
        Duration::from_millis(200),
      ),
    }
  }
}

#[async_trait]
impl crate::CommandRunner for CommandRunner {
  async fn run(
    &self,
    context: Context,
    workunit: &mut RunningWorkunit,
    process: Process,
  ) -> Result<FallibleProcessResultWithPlatform, String> {
    let semaphore_acquisition = self.sema.acquire(process.concurrency_available);
    let permit = in_workunit!(
      context.workunit_store.clone(),
      "acquire_command_runner_slot".to_owned(),
      WorkunitMetadata {
        // TODO: The UI uses the presence of a blocked workunit below a parent as an indication that
        // the parent is blocked. If this workunit is filtered out, parents nodes which are waiting
        // for the semaphore will render, even though they are effectively idle.
        //
        // https://github.com/pantsbuild/pants/issues/14680 will likely allow for a more principled
        // solution to this problem, such as removing the mutable `blocking` flag, and then never
        // filtering blocked workunits at creation time, regardless of level.
        level: Level::Debug,
        ..WorkunitMetadata::default()
      },
      |workunit| async move {
        let _blocking_token = workunit.blocking();
        semaphore_acquisition.await
      }
    )
    .await;

    loop {
      let mut process = process.clone();
      let concurrency_available = permit.concurrency();
      log::debug!(
        "Running {} under semaphore with concurrency id: {}, and concurrency: {}",
        process.description,
        permit.concurrency_slot(),
        concurrency_available,
      );

      // TODO: Both of these templating cases should be implemented at the lowest possible level:
      // they might currently be applied above a cache.
      if let Some(ref execution_slot_env_var) = process.execution_slot_variable {
        process.env.insert(
          execution_slot_env_var.clone(),
          format!("{}", permit.concurrency_slot()),
        );
      }
      if process.concurrency_available > 0 {
        let concurrency = format!("{}", permit.concurrency());
        let mut matched = false;
        process.argv = std::mem::take(&mut process.argv)
          .into_iter()
          .map(
            |arg| match CONCURRENCY_TEMPLATE_RE.replace_all(&arg, &concurrency) {
              Cow::Owned(altered) => {
                matched = true;
                altered
              }
              Cow::Borrowed(_original) => arg,
            },
          )
          .collect();
        if !matched {
          return Err(format!(
            "Process {} set `concurrency_available={}`, but did not include \
                             the `{}` template variable in its arguments.",
            process.description, process.concurrency_available, *CONCURRENCY_TEMPLATE_RE
          ));
        }
      }

      let running_process = self.inner.run(context.clone(), workunit, process.clone());
      tokio::select! {
        _ = permit.notified_concurrency_changed() => {
          log::debug!(
            "Process {} was preempted, and went from concurrency {} to concurrency {}",
            process.description,
            concurrency_available,
            permit.concurrency(),
          );
          continue;
        },
        res = running_process => {
          // The process completed.
          return res;
        }
      }
    }
  }
}

/// A wrapped Semaphore which adds concurrency metadata which supports overcommit.
#[derive(Clone)]
pub(crate) struct AsyncSemaphore {
  sema: Arc<Semaphore>,
  state: Arc<Mutex<State>>,
  preemptible_duration: Duration,
}

pub(crate) struct State {
  total_concurrency: usize,
  available_ids: VecDeque<usize>,
  tasks: Vec<Arc<Task>>,
}

impl State {
  #[cfg(test)]
  pub(crate) fn new_for_tests(total_concurrency: usize, tasks: Vec<Arc<Task>>) -> Self {
    Self {
      total_concurrency,
      available_ids: VecDeque::new(),
      tasks,
    }
  }
}

impl AsyncSemaphore {
  pub fn new(
    executor: &Executor,
    permits: usize,
    preemptible_duration: Duration,
  ) -> AsyncSemaphore {
    let mut available_ids = VecDeque::new();
    for id in 1..=permits {
      available_ids.push_back(id);
    }

    let state = Arc::new(Mutex::new(State {
      total_concurrency: permits,
      available_ids,
      tasks: Vec::new(),
    }));

    // Spawn a task which will periodically balance Tasks.
    let _balancer_task = {
      let state = Arc::downgrade(&state);
      executor.spawn(async move {
        loop {
          sleep(preemptible_duration / 4).await;
          if let Some(state) = state.upgrade() {
            // Balance tasks.
            let mut state = state.lock();
            balance(Instant::now(), &mut state);
          } else {
            // The AsyncSemaphore was torn down.
            break;
          }
        }
      })
    };

    AsyncSemaphore {
      sema: Arc::new(Semaphore::new(permits)),
      state,
      preemptible_duration,
    }
  }

  #[cfg(test)]
  pub(crate) fn available_permits(&self) -> usize {
    self.sema.available_permits()
  }

  ///
  /// Runs the given Future-creating function (and the Future it returns) under the semaphore.
  ///
  /// NB: This method does not support preemption, or controlling concurrency.
  ///
  // TODO: https://github.com/rust-lang/rust/issues/46379
  #[allow(dead_code)]
  pub(crate) async fn with_acquired<F, B, O>(self, f: F) -> O
  where
    F: FnOnce(usize) -> B,
    B: Future<Output = O>,
  {
    let permit = self.acquire(1).await;
    let res = f(permit.task.id).await;
    drop(permit);
    res
  }

  ///
  /// Acquire a slot on the semaphore when it becomes available. Additionally, attempt to acquire
  /// the given amount of concurrency. The amount actually acquired will be reported on the
  /// returned Permit.
  ///
  pub async fn acquire(&self, concurrency_desired: usize) -> Permit<'_> {
    let permit = self.sema.acquire().await.expect("semaphore closed");
    let task = {
      let mut state = self.state.lock();
      let id = state
        .available_ids
        .pop_front()
        .expect("More permits were distributed than ids exist.");

      // A Task is initially given its fair share of the available concurrency: i.e., the first
      // arriving task gets all of the slots, and the second arriving gets half, even though that
      // means that we overcommit. Balancing will adjust concurrency later, to the extent that it
      // can given preemption timeouts.
      //
      // This is because we cannot anticipate the number of inbound processes, and we never want to
      // delay a process from starting.
      let concurrency_desired = max(concurrency_desired, 1);
      let concurrency_actual = min(
        concurrency_desired,
        state.total_concurrency / (state.tasks.len() + 1),
      );
      let task = Arc::new(Task::new(
        id,
        concurrency_desired,
        concurrency_actual,
        Instant::now() + self.preemptible_duration,
      ));
      state.tasks.push(task.clone());
      task
    };
    Permit {
      state: self.state.clone(),
      _permit: permit,
      task,
    }
  }
}

pub struct Permit<'a> {
  state: Arc<Mutex<State>>,
  // NB: Kept for its `Drop` impl.
  _permit: SemaphorePermit<'a>,
  task: Arc<Task>,
}

impl Permit<'_> {
  pub fn concurrency_slot(&self) -> usize {
    self.task.id
  }

  pub fn concurrency(&self) -> usize {
    self.task.concurrency()
  }

  pub async fn notified_concurrency_changed(&self) {
    self.task.notify_concurrency_changed.notified().await
  }
}

impl<'a> Drop for Permit<'a> {
  fn drop(&mut self) {
    let mut state = self.state.lock();
    state.available_ids.push_back(self.task.id);
    let tasks_position = state
      .tasks
      .iter()
      .position(|t| t.id == self.task.id)
      .unwrap();
    state.tasks.swap_remove(tasks_position);
  }
}

pub(crate) struct Task {
  id: usize,
  concurrency_desired: usize,
  pub(crate) concurrency_actual: atomic::AtomicUsize,
  notify_concurrency_changed: Notify,
  preemptible_until: Instant,
}

impl Task {
  pub(crate) fn new(
    id: usize,
    concurrency_desired: usize,
    concurrency_actual: usize,
    preemptible_until: Instant,
  ) -> Self {
    assert!(concurrency_actual <= concurrency_desired);
    Self {
      id,
      concurrency_desired,
      concurrency_actual: atomic::AtomicUsize::new(concurrency_actual),
      notify_concurrency_changed: Notify::new(),
      preemptible_until,
    }
  }

  pub(crate) fn concurrency(&self) -> usize {
    self.concurrency_actual.load(atomic::Ordering::SeqCst)
  }

  fn preemptible(&self, now: Instant) -> bool {
    self.preemptible_until > now
  }
}

/// Given a set of Tasks with their desired and actual concurrency, balance the concurrency levels
/// of any preemptible tasks, and notify them of the changes. Returns the number of Tasks that were
/// preempted.
///
/// This method only internally mutates tasks (by adjusting their concurrency levels and notifying
/// them), but takes State as mutable in order to guarantee that it gets an atomic view of the
/// tasks.
pub(crate) fn balance(now: Instant, state: &mut State) -> usize {
  let concurrency_used: usize = state.tasks.iter().map(|t| t.concurrency()).sum();
  let mut desired_change_in_commitment =
    state.total_concurrency as isize - concurrency_used as isize;
  let mut prempted = 0;

  // To reduce the number of tasks that we preempty, we preempt them in order by the amount of
  // concurrency that they desire or can relinquish.
  match desired_change_in_commitment.cmp(&0) {
    Ordering::Equal => {
      // Nothing to do! Although some tasks might not have their desired concurrency levels, it's
      // probably not worth preempting any tasks to fix that.
    }
    Ordering::Less => {
      // We're overcommitted: order by the amount that they can relinquish.
      let mut preemptible_tasks = state
        .tasks
        .iter()
        .filter_map(|t| {
          // A task may never have less than one slot.
          let relinquishable = t.concurrency() - 1;
          if relinquishable > 0 && t.preemptible(now) {
            Some((relinquishable, t))
          } else {
            None
          }
        })
        .collect::<Vec<_>>();
      preemptible_tasks.sort_by_key(|(relinquishable, _)| Reverse(*relinquishable));

      for (relinquishable, task) in preemptible_tasks {
        if desired_change_in_commitment == 0 {
          break;
        }

        let relinquish = min(relinquishable, (-desired_change_in_commitment) as usize);
        desired_change_in_commitment += relinquish as isize;
        task
          .concurrency_actual
          .fetch_sub(relinquish, atomic::Ordering::SeqCst);
        task.notify_concurrency_changed.notify_one();
        prempted += 1;
      }
    }
    Ordering::Greater => {
      // We're undercommitted: order by the amount that they are owed.
      let mut preemptible_tasks = state
        .tasks
        .iter()
        .filter_map(|t| {
          let desired = t.concurrency_desired - t.concurrency();
          if desired > 0 && t.preemptible(now) {
            Some((desired, t))
          } else {
            None
          }
        })
        .collect::<Vec<_>>();
      preemptible_tasks.sort_by_key(|(desired, _)| Reverse(*desired));

      for (desired, task) in preemptible_tasks {
        if desired_change_in_commitment == 0 {
          break;
        }

        let acquire = min(desired, desired_change_in_commitment as usize);
        desired_change_in_commitment -= acquire as isize;
        task
          .concurrency_actual
          .fetch_add(acquire, atomic::Ordering::SeqCst);
        task.notify_concurrency_changed.notify_one();
        prempted += 1;
      }
    }
  }

  prempted
}
<|endoftext|>


<commit_before>// This file is part of rgtk.
//
// rgtk is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// rgtk is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with rgtk.  If not, see <http://www.gnu.org/licenses/>.

//! GtkSizeGroup — Grouping widgets so they request the same size

use gtk::{mod, ffi, traits};

pub struct SizeGroup {
    pointer: *mut ffi::C_GtkSizeGroup
}

impl SizeGroup {
    pub fn new(mode: gtk::SizeGroupMode) -> Option<SizeGroup> {
        let tmp_pointer = unsafe { ffi::gtk_size_group_new(mode) };

        if tmp_pointer.is_null() {
            None
        } else {
            Some(SizeGroup {
                pointer: tmp_pointer
            })
        }
    }

    pub fn set_mode(&self, mode: gtk::SizeGroupMode) {
        unsafe { ffi::gtk_size_group_set_mode(self.pointer, mode) }
    }

    pub fn get_mode(&self) -> gtk::SizeGroupMode {
        unsafe { ffi::gtk_size_group_get_mode(self.pointer) }
    }

    pub fn set_ignore_hidden(&self, ignore_hidden: bool) {
        unsafe { ffi::gtk_size_group_set_ignore_hidden(self.pointer, ffi::to_gboolean(ignore_hidden)) }
    }

    pub fn get_ignore_hidden(&self) -> bool {
        unsafe { ffi::to_bool(ffi::gtk_size_group_get_ignore_hidden(self.pointer)) }
    }

    pub fn add_widget<T: traits::WidgetTrait>(&self, widget: &T) {
        unsafe { ffi::gtk_size_group_add_widget(self.pointer, widget.get_widget()) }
    }

    pub fn remove_widget<T: traits::WidgetTrait>(&self, widget: &T) {
        unsafe { ffi::gtk_size_group_remove_widget(self.pointer, widget.get_widget()) }
    }
}

impl_GObjectFunctions!(SizeGroup, C_GtkSizeGroup)<commit_msg>Replace trait import by gtk use<commit_after>// This file is part of rgtk.
//
// rgtk is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// rgtk is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with rgtk.  If not, see <http://www.gnu.org/licenses/>.

//! GtkSizeGroup — Grouping widgets so they request the same size

use gtk::{mod, ffi};

pub struct SizeGroup {
    pointer: *mut ffi::C_GtkSizeGroup
}

impl SizeGroup {
    pub fn new(mode: gtk::SizeGroupMode) -> Option<SizeGroup> {
        let tmp_pointer = unsafe { ffi::gtk_size_group_new(mode) };

        if tmp_pointer.is_null() {
            None
        } else {
            Some(SizeGroup {
                pointer: tmp_pointer
            })
        }
    }

    pub fn set_mode(&self, mode: gtk::SizeGroupMode) {
        unsafe { ffi::gtk_size_group_set_mode(self.pointer, mode) }
    }

    pub fn get_mode(&self) -> gtk::SizeGroupMode {
        unsafe { ffi::gtk_size_group_get_mode(self.pointer) }
    }

    pub fn set_ignore_hidden(&self, ignore_hidden: bool) {
        unsafe { ffi::gtk_size_group_set_ignore_hidden(self.pointer, ffi::to_gboolean(ignore_hidden)) }
    }

    pub fn get_ignore_hidden(&self) -> bool {
        unsafe { ffi::to_bool(ffi::gtk_size_group_get_ignore_hidden(self.pointer)) }
    }

    pub fn add_widget<T: gtk::WidgetTrait>(&self, widget: &T) {
        unsafe { ffi::gtk_size_group_add_widget(self.pointer, widget.get_widget()) }
    }

    pub fn remove_widget<T: gtk::WidgetTrait>(&self, widget: &T) {
        unsafe { ffi::gtk_size_group_remove_widget(self.pointer, widget.get_widget()) }
    }
}

impl_GObjectFunctions!(SizeGroup, C_GtkSizeGroup)<|endoftext|>