<commit_before>///# Parse ascii(1) `nametable` files
///
/// Nametables have an almost deceptively simple format:
///
///```text
///Mnemonics: "\"",
///ISO names: "Quotation Mark",
///Synonyms:  "Double Quote", "Quote", "String Quote",
///            "Dirk", "Literal Mark", "Double Glitch",
///XML name: """,
///Comment:   "# See ' and ` for matching names.",
///%%
///```
///
/// Any C Strings that are shorter than 4 elements count as "called"
/// abbreviations (think `TAB`, `NL`).  All control chars (in C, also
/// `0x7F`; it counts as a control char in rust) are printed in their
/// ^-notation, as `&`ed with `0x1F` (except again `0x7F`, this is
/// printed as ^?).
///
/// The first non-"called" name is the "Official name", and after is
/// an (optional) C escape and other names, with a Note if any other
/// name starts with a "#".
use std::fmt;
use std::fs;
use std::fs::File;
use std::io;
use std::io::Write;
use std::io::{BufRead, BufWriter, Cursor};
use std::path::Path;

use regex::Regex;

use fst_generator;

#[derive(Debug, Clone)]
struct ASCIIEntry {
    value: char,
    mnemonics: Vec<String>,
    synonyms: Vec<String>,
    note: Option<String>,
}

#[derive(Debug)]
struct ASCIIForDisplay<'a> {
    val: &'a ASCIIEntry,
}

impl ASCIIEntry {
    fn new(code: u8) -> ASCIIEntry {
        ASCIIEntry {
            value: code as char,
            mnemonics: vec![],
            synonyms: vec![],
            note: None,
        }
    }

    fn for_display(&self) -> ASCIIForDisplay {
        ASCIIForDisplay { val: self }
    }
}

fn split_name_line(line: &str) -> Vec<String> {
    lazy_static! {
        static ref QUOTES: Regex = Regex::new("\",\\s*\"").unwrap();
        static ref RIGHT_QUOTE: Regex = Regex::new("\"\\s*,\\s*$").unwrap();
        static ref BACKSLASH_SOMETHING: Regex = Regex::new(r"\\(.)").unwrap();
    }
    let line = line.trim_start();
    let line = RIGHT_QUOTE.replace_all(line, "$1");
    let line = line.trim_start_matches('"');
    QUOTES
        .split(line)
        .map(|s| BACKSLASH_SOMETHING.replace_all(s, "$1"))
        .collect()
}

#[test]
fn test_split_name_line() {
    assert_eq!(
        split_name_line(r#" "Shift In", "Locking Shift 0","#),
        vec!["Shift In", "Locking Shift 0"]
    );
    assert_eq!(split_name_line(r#""\\v","#), vec![r"\v"]);
    assert_eq!(split_name_line(r#""\"","#), vec![r#"""#]);
}

const NAMETABLE: &[u8] = include_bytes!("../data/ascii/nametable");

fn process_ascii_nametable() -> Result<Vec<ASCIIEntry>, io::Error> {
    let mut char_code: u8 = 0;

    let mut entry = ASCIIEntry::new(char_code);
    let mut entries: Vec<ASCIIEntry> = vec![];

    let reader = Cursor::new(NAMETABLE);
    for line in reader.lines() {
        let line = line?;
        let line = line.as_str();
        match line.chars().next() {
            Some('#') | None => {}
            Some(_) => {
                if line == "%%" {
                    entries.push(entry);
                    char_code += 1;
                    entry = ASCIIEntry::new(char_code);
                    continue;
                }
                // Otherwise, not at a spec boundary. Let's add names:
                lazy_static! {
                    static ref BEGINNING: Regex = Regex::new(r"^[A-Za-z ]*:\s*").unwrap();
                }
                let line = BEGINNING.replace_all(line, "");
                let line = line.trim_start();
                if entry.mnemonics.is_empty() {
                    entry.mnemonics = split_name_line(line);
                } else {
                    for element in split_name_line(line) {
                        if element.starts_with('#') {
                            entry.note = Some(element[2..element.len()].to_owned());
                        } else {
                            entry.synonyms.push(element);
                        }
                    }
                }
            }
        }
    }
    entries.push(entry);
    Ok(entries)
}

impl<'a> fmt::Display for ASCIIForDisplay<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        let val = self.val.clone();
        write!(
            f,
            "Information{{value:{:?}, mnemonics:&{:?}, synonyms:&{:?}, note:{:?}}},",
            val.value, val.mnemonics, val.synonyms, val.note
        )?;
        Ok(())
    }
}

const PREAMBLE: &str = r#"/// Generated with `make names`

#[derive(Clone)]
pub struct Information {
    pub value: char,
    pub mnemonics: &'static [&'static str],
    pub synonyms: &'static [&'static str],
    pub note: Option<&'static str>,
}

#[rustfmt::skip]
"#;

pub fn write_ascii_name_data(output_dir: &Path, sorted_names: &mut fst_generator::Names) {
    fs::create_dir_all(output_dir).unwrap();
    let table = process_ascii_nametable().unwrap();

    for entry in table.clone() {
        sorted_names.insert(entry.mnemonics, entry.value);
        sorted_names.insert(entry.synonyms, entry.value);
    }

    let mut out = BufWriter::new(File::create(output_dir.join("names.rs")).unwrap());

    write!(&mut out, "{}", PREAMBLE).unwrap();
    writeln!(
        &mut out,
        "static PRINTABLE_CHARS: &[Information; {}] = &[",
        table.len()
    )
    .unwrap();
    for entry in table.clone() {
        writeln!(&mut out, "    {}", entry.for_display()).unwrap();
    }
    writeln!(&mut out, "];").unwrap();
}
<commit_msg>Force the kine into giving us an owned string<commit_after>///# Parse ascii(1) `nametable` files
///
/// Nametables have an almost deceptively simple format:
///
///```text
///Mnemonics: "\"",
///ISO names: "Quotation Mark",
///Synonyms:  "Double Quote", "Quote", "String Quote",
///            "Dirk", "Literal Mark", "Double Glitch",
///XML name: """,
///Comment:   "# See ' and ` for matching names.",
///%%
///```
///
/// Any C Strings that are shorter than 4 elements count as "called"
/// abbreviations (think `TAB`, `NL`).  All control chars (in C, also
/// `0x7F`; it counts as a control char in rust) are printed in their
/// ^-notation, as `&`ed with `0x1F` (except again `0x7F`, this is
/// printed as ^?).
///
/// The first non-"called" name is the "Official name", and after is
/// an (optional) C escape and other names, with a Note if any other
/// name starts with a "#".
use std::fmt;
use std::fs;
use std::fs::File;
use std::io;
use std::io::Write;
use std::io::{BufRead, BufWriter, Cursor};
use std::path::Path;

use regex::Regex;

use fst_generator;

#[derive(Debug, Clone)]
struct ASCIIEntry {
    value: char,
    mnemonics: Vec<String>,
    synonyms: Vec<String>,
    note: Option<String>,
}

#[derive(Debug)]
struct ASCIIForDisplay<'a> {
    val: &'a ASCIIEntry,
}

impl ASCIIEntry {
    fn new(code: u8) -> ASCIIEntry {
        ASCIIEntry {
            value: code as char,
            mnemonics: vec![],
            synonyms: vec![],
            note: None,
        }
    }

    fn for_display(&self) -> ASCIIForDisplay {
        ASCIIForDisplay { val: self }
    }
}

fn split_name_line(line: &str) -> Vec<String> {
    lazy_static! {
        static ref QUOTES: Regex = Regex::new("\",\\s*\"").unwrap();
        static ref RIGHT_QUOTE: Regex = Regex::new("\"\\s*,\\s*$").unwrap();
        static ref BACKSLASH_SOMETHING: Regex = Regex::new(r"\\(.)").unwrap();
    }
    let line = line.trim_start();
    let line = RIGHT_QUOTE.replace_all(line, "$1");
    let line = line.trim_start_matches('"');
    QUOTES
        .split(line)
        .map(|s| BACKSLASH_SOMETHING.replace_all(s, "$1").into_owned())
        .collect()
}

#[test]
fn test_split_name_line() {
    assert_eq!(
        split_name_line(r#" "Shift In", "Locking Shift 0","#),
        vec!["Shift In", "Locking Shift 0"]
    );
    assert_eq!(split_name_line(r#""\\v","#), vec![r"\v"]);
    assert_eq!(split_name_line(r#""\"","#), vec![r#"""#]);
}

const NAMETABLE: &[u8] = include_bytes!("../data/ascii/nametable");

fn process_ascii_nametable() -> Result<Vec<ASCIIEntry>, io::Error> {
    let mut char_code: u8 = 0;

    let mut entry = ASCIIEntry::new(char_code);
    let mut entries: Vec<ASCIIEntry> = vec![];

    let reader = Cursor::new(NAMETABLE);
    for line in reader.lines() {
        let line = line?;
        let line = line.as_str();
        match line.chars().next() {
            Some('#') | None => {}
            Some(_) => {
                if line == "%%" {
                    entries.push(entry);
                    char_code += 1;
                    entry = ASCIIEntry::new(char_code);
                    continue;
                }
                // Otherwise, not at a spec boundary. Let's add names:
                lazy_static! {
                    static ref BEGINNING: Regex = Regex::new(r"^[A-Za-z ]*:\s*").unwrap();
                }
                let line = BEGINNING.replace_all(line, "");
                let line = line.trim_start();
                if entry.mnemonics.is_empty() {
                    entry.mnemonics = split_name_line(line);
                } else {
                    for element in split_name_line(line) {
                        if element.starts_with('#') {
                            entry.note = Some(element[2..element.len()].to_owned());
                        } else {
                            entry.synonyms.push(element);
                        }
                    }
                }
            }
        }
    }
    entries.push(entry);
    Ok(entries)
}

impl<'a> fmt::Display for ASCIIForDisplay<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        let val = self.val.clone();
        write!(
            f,
            "Information{{value:{:?}, mnemonics:&{:?}, synonyms:&{:?}, note:{:?}}},",
            val.value, val.mnemonics, val.synonyms, val.note
        )?;
        Ok(())
    }
}

const PREAMBLE: &str = r#"/// Generated with `make names`

#[derive(Clone)]
pub struct Information {
    pub value: char,
    pub mnemonics: &'static [&'static str],
    pub synonyms: &'static [&'static str],
    pub note: Option<&'static str>,
}

#[rustfmt::skip]
"#;

pub fn write_ascii_name_data(output_dir: &Path, sorted_names: &mut fst_generator::Names) {
    fs::create_dir_all(output_dir).unwrap();
    let table = process_ascii_nametable().unwrap();

    for entry in table.clone() {
        sorted_names.insert(entry.mnemonics, entry.value);
        sorted_names.insert(entry.synonyms, entry.value);
    }

    let mut out = BufWriter::new(File::create(output_dir.join("names.rs")).unwrap());

    write!(&mut out, "{}", PREAMBLE).unwrap();
    writeln!(
        &mut out,
        "static PRINTABLE_CHARS: &[Information; {}] = &[",
        table.len()
    )
    .unwrap();
    for entry in table.clone() {
        writeln!(&mut out, "    {}", entry.for_display()).unwrap();
    }
    writeln!(&mut out, "];").unwrap();
}
<|endoftext|>


<commit_before>//! Multi-way equijoin expression plan.
//!
//! This plan provides us the opportunity to map out a non-trivial differential
//! implementation for a complex join query. In particular, we are able to invoke
//! delta-query and worst-case optimal join plans, which avoid any intermediate
//! materialization.
//!
//! Each `MultiwayJoin` indicates several source collections, equality constraints
//! among their attributes, and then the set of attributes to produce as results.
//!
//! One naive implementation would take each input collection in order, and develop
//! the join restricted to the prefix of relations so far. Ideally the order would
//! be such that joined collections have equality constraints and prevent Cartesian
//! explosion. At each step, a new collection picks out some of the attributes and
//! instantiates a primitive binary join between the accumulated collection and the
//! next collection.
//!
//! A more sophisticated implementation establishes delta queries for each input
//! collection, which responds to changes in that input collection against the
//! current other input collections. For each input collection we may choose very
//! different join orders, as the order must follow equality constraints.
//!
//! A further implementation could develop the results attribute-by-attribute, as
//! opposed to collection-by-collection, which gives us the ability to use column
//! indices rather than whole-collection indices.

use std::hash::Hash;

use timely::dataflow::Scope;

use differential_dataflow::operators::arrange::{ArrangeBySelf, ArrangeByKey};

use differential_dataflow::{Collection, ExchangeData};
use plan::{Plan, Render};
use {TraceManager, Time, Diff};

/// A multiway join of muliple relations.
///
/// By expressing multiple relations and required equivalances between their attributes,
/// we can more efficiently design incremental update strategies without materializing
/// and indexing intermediate relations.
#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct MultiwayJoin<Value> {
    /// A list of (attribute index, input) pairs to extract.
    pub results: Vec<(usize, usize)>,
    /// A list of source collections.
    pub sources: Vec<Box<Plan<Value>>>,
    /// Equality constraints.
    ///
    /// Equality constraints are presented as lists of `(attr, input)` equivalence classes.
    /// This means that each `(attr, input)` pair can exist in at most one list; if it would
    /// appear in more than one list, those two lists should be merged.
    pub equalities: Vec<Vec<(usize, usize)>>,
}

impl<V: ExchangeData+Hash> Render for MultiwayJoin<V> {

    type Value = V;

    fn render<S: Scope<Timestamp = Time>>(
        &self,
        scope: &mut S,
        arrangements: &mut TraceManager<Self::Value>) -> Collection<S, Vec<Self::Value>, Diff>
    {
        // The plan here is that each stream of changes comes in, and has a delta query.
        // For each each stream, we need to work through each other relation ensuring that
        // each new relation we add has some attributes in common with the developing set.

        // Attributes we may need from any and all relations.
        let mut relevant_attributes = Vec::new();
        relevant_attributes.extend(self.results.iter().cloned());
        relevant_attributes.extend(self.equalities.iter().flat_map(|list| list.iter().cloned()));
        relevant_attributes.sort();
        relevant_attributes.dedup();

        // Into which we accumulate change streams.
        let mut accumulated_changes = Vec::new();

        // For each participating relation, we build a delta query dataflow.
        for (index, plan) in self.sources.iter().enumerate() {

            // Restrict down to relevant attributes.
            let mut attributes: Vec<(usize, usize)> =
            relevant_attributes
                .iter()
                .filter(|(_attr, input)| input == &index)
                .cloned()
                .collect::<Vec<_>>();

            let attributes_init = attributes.clone();

            // Ensure the plan is rendered and cached.
            if arrangements.get_unkeyed(&plan).is_none() {
                let collection = plan.render(scope, arrangements);
                arrangements.set_unkeyed(plan, &collection.arrange_by_self().trace);
            }
            let changes =
            arrangements
                .get_unkeyed(&plan)
                .expect("Surely we just ensured this")
                .import(scope)
                .as_collection(|val,&()| val.clone())
                .map(move |tuple| attributes_init.iter().map(|&(attr,_)|
                        tuple[attr].clone()).collect::<Vec<_>>()
                );

            // Before constructing the dataflow, which takes a borrow on `scope`,
            // we'll want to ensure that we have all of the necessary data assets
            // in place. This requires a bit of planning first, then the building.

            // Acquire a sane sequence in which to join the relations:
            //
            // This is a sequence of relation identifiers, starting with `index`,
            // such that each has at least one attribute in common with a prior
            // relation, and so can be effectively joined.
            let join_order = plan_join_order(index, &self.equalities);
            let mut join_plan = Vec::new();

            // Skipping `index`, join in each relation in sequence.
            for join_idx in join_order.into_iter().skip(1) {

                // To join a relation, we need to determine any constraints on
                // attributes in common with prior relations. Any other values
                // should be appended to tuples in `changes` with care taken to
                // update `attributes`.
                let (keys, priors) = determine_keys_priors(join_idx, &self.equalities, &attributes[..]);

                // The fields in `sources[join_idx]` that should be values are those
                // that are required output or participate in an equality constraint,
                // but *WHICH ARE NOT* in `keys`.
                let vals =
                relevant_attributes
                    .iter()
                    .filter(|&(attr,index)| index == &join_idx && !keys.contains(&attr))
                    .cloned()
                    .collect::<Vec<_>>();

                let mut projection = Vec::new();
                for &attr in keys.iter() {
                    projection.push(attr);
                }
                for &(attr, _index) in vals.iter() {
                    projection.push(attr);
                }
                // TODO: Sort, to improve chances of re-use opportunities.
                //       Requires understanding how attributes move to get the right
                //       key selectors out though.
                // projection.sort();
                // projection.dedup(); // Should already be deduplicated, probably?

                // Get a plan for the projection on to these few attributes.
                let plan = self.sources[join_idx].clone().project(projection);

                if arrangements.get_keyed(&plan, &keys[..]).is_none() {
                    let keys_clone = keys.clone();
                    let arrangement =
                    plan.render(scope, arrangements)
                        .map(move |tuple| (keys_clone.iter().map(|&i| tuple[i].clone()).collect::<Vec<_>>(), tuple))
                        .arrange_by_key();

                    arrangements.set_keyed(&plan, &keys[..], &arrangement.trace);
                }

                let arrangement =
                arrangements
                    .get_keyed(&plan, &keys[..])
                    .expect("Surely we just ensured this");

                let key_selector = std::rc::Rc::new(move |change: &Vec<V>|
                    priors.iter().map(|&p| change[p].clone()).collect::<Vec<_>>()
                );

                join_plan.push((join_idx, key_selector, arrangement));

                attributes.extend(vals.into_iter());
            }

            // Build the dataflow.
            use dogsdogsdogs::altneu::AltNeu;

            let scope_name = format!("DeltaRule: {}/{}", index, self.sources.len());
            let changes = scope.clone().scoped::<AltNeu<_>,_,_>(&scope_name, |inner| {

                // This should default to an `AltNeu::Alt` timestamp.
                let mut changes = changes.enter(inner);

                for (join_idx, key_selector, mut trace) in join_plan.into_iter() {

                    // Use alt or neu timestamps based on relative indices.
                    // Must have an `if` statement here as the two arrangement have different
                    // types, and we would to determine `alt` v `neu` once, rather than per
                    // tuple in the cursor.
                    changes =
                    if join_idx < index {
                        let arrangement = trace.import(scope).enter_at(inner, |_,_,t| AltNeu::alt(t.clone()));
                        dogsdogsdogs::operators::propose(&changes, arrangement, key_selector)
                    }
                    else {
                        let arrangement = trace.import(scope).enter_at(inner, |_,_,t| AltNeu::neu(t.clone()));
                        dogsdogsdogs::operators::propose(&changes, arrangement, key_selector)
                    }
                    .map(|(mut prefix, extensions)| { prefix.extend(extensions.into_iter()); prefix });
                }

                // Extract `self.results` in order, using `attributes`.
                let extract_map =
                self.results
                    .iter()
                    .map(move |x| attributes.iter().position(|i| i == x).expect("Output attribute not found!"))
                    .collect::<Vec<_>>();

                changes
                    .map(move |tuple| extract_map.iter().map(|&i| tuple[i].clone()).collect::<Vec<_>>())
                    .leave()
            });

            accumulated_changes.push(changes);
        }

        differential_dataflow::collection::concatenate(scope, accumulated_changes.into_iter())
    }
}

/// Sequences relations in `constraints`.
///
/// Relations become available for sequencing as soon as they share a constraint with
/// either `source` or another sequenced relation.
fn plan_join_order(source: usize, constraints: &[Vec<(usize, usize)>]) -> Vec<usize> {

    let mut result = vec![source];
    let mut active = true;
    while active {
        active = false;
        for constraint in constraints.iter() {
            // Check to see if the constraint contains a sequenced relation.
            if constraint.iter().any(|(_,index)| result.contains(index)) {
                // If so, sequence any unsequenced relations.
                for (_, index) in constraint.iter() {
                    if !result.contains(index) {
                        result.push(*index);
                        active = true;
                    }
                }
            }
        }
    }

    result
}

/// Identifies keys and values for a join.
///
/// The result is a sequence, for each
fn determine_keys_priors(
    relation: usize,
    constraints: &[Vec<(usize, usize)>],
    current_attributes: &[(usize, usize)],
)
-> (Vec<usize>, Vec<usize>)
{
    // The fields in `sources[join_idx]` that should be keys are those
    // that share an equality constraint with an element of `attributes`.
    // For each key, we should capture the associated `attributes` entry
    // so that we can easily prepare the keys of the `delta` stream.
    let mut keys = Vec::new();
    let mut priors = Vec::new();
    for constraint in constraints.iter() {

        // If there is an intersection between `constraint` and `current_attributes`,
        // we should capture the position in `current_attributes` and emit all of the
        // attributes for `relation`.
        if let Some(prior) = current_attributes.iter().position(|x| constraint.contains(x)) {
            for &(attr, index) in constraint.iter() {
                if index == relation {
                    keys.push(attr);
                    priors.push(prior);
                }
            }
        }
    }

    (keys, priors)
}
<commit_msg>more documentation<commit_after>//! Multi-way equijoin expression plan.
//!
//! This plan provides us the opportunity to map out a non-trivial differential
//! implementation for a complex join query. In particular, we are able to invoke
//! delta-query and worst-case optimal join plans, which avoid any intermediate
//! materialization.
//!
//! Each `MultiwayJoin` indicates several source collections, equality constraints
//! among their attributes, and then the set of attributes to produce as results.
//!
//! One naive implementation would take each input collection in order, and develop
//! the join restricted to the prefix of relations so far. Ideally the order would
//! be such that joined collections have equality constraints and prevent Cartesian
//! explosion. At each step, a new collection picks out some of the attributes and
//! instantiates a primitive binary join between the accumulated collection and the
//! next collection.
//!
//! A more sophisticated implementation establishes delta queries for each input
//! collection, which responds to changes in that input collection against the
//! current other input collections. For each input collection we may choose very
//! different join orders, as the order must follow equality constraints.
//!
//! A further implementation could develop the results attribute-by-attribute, as
//! opposed to collection-by-collection, which gives us the ability to use column
//! indices rather than whole-collection indices.

use std::hash::Hash;

use timely::dataflow::Scope;

use differential_dataflow::operators::arrange::{ArrangeBySelf, ArrangeByKey};

use differential_dataflow::{Collection, ExchangeData};
use plan::{Plan, Render};
use {TraceManager, Time, Diff};

/// A multiway join of muliple relations.
///
/// By expressing multiple relations and required equivalances between their attributes,
/// we can more efficiently design incremental update strategies without materializing
/// and indexing intermediate relations.
#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct MultiwayJoin<Value> {
    /// A list of (attribute index, input) pairs to extract.
    pub results: Vec<(usize, usize)>,
    /// A list of source collections.
    pub sources: Vec<Box<Plan<Value>>>,
    /// Equality constraints.
    ///
    /// Equality constraints are presented as lists of `(attr, input)` equivalence classes.
    /// This means that each `(attr, input)` pair can exist in at most one list; if it would
    /// appear in more than one list, those two lists should be merged.
    pub equalities: Vec<Vec<(usize, usize)>>,
}

impl<V: ExchangeData+Hash> Render for MultiwayJoin<V> {

    type Value = V;

    fn render<S: Scope<Timestamp = Time>>(
        &self,
        scope: &mut S,
        arrangements: &mut TraceManager<Self::Value>) -> Collection<S, Vec<Self::Value>, Diff>
    {
        // The idea here is the following:
        //
        // For each stream, we will determine a streaming delta query, in which changes
        // are joined against indexed forms of the other relations using `dogsdogsdogs`
        // stateless `propose` operators.
        //
        // For a query Q(x,y,z) := A(x,y), B(y,z), C(x,z) we might write dataflows like:
        //
        //   dQdA := dA(x,y), B(y,z), C(x,z)
        //   dQdB := dB(y,z), A(x,y), C(x,z)
        //   dQdC := dC(x,y), A(x,y), B(y,z)
        //
        // where each line is read from left to right as a sequence of `propose` joins,
        // which respond to timestamped delta changes by joining with the maintained
        // relation at that corresponding time.
        //
        // We take some care to make sure that when these joins are performed, a delta
        // interacts with relations as if we updated A, then B, then C, as if in sequence.
        // That is, when a dB delta joins A it observes all updates to A at times less or
        // equal to the delta's timestamp, but when a dB delta joins C it observes only
        // updates to C at times strictly less than the delta's timestamp.
        //
        // This is done to avoid double counting updates; any concurrent changes will be
        // accounted for by the last relation for which there is a concurrent update.

        // Attributes we may need from any and all relations.
        let mut relevant_attributes = Vec::new();
        relevant_attributes.extend(self.results.iter().cloned());
        relevant_attributes.extend(self.equalities.iter().flat_map(|list| list.iter().cloned()));
        relevant_attributes.sort();
        relevant_attributes.dedup();

        // Into which we accumulate change streams.
        let mut accumulated_changes = Vec::new();

        // For each participating relation, we build a delta query dataflow.
        for (index, plan) in self.sources.iter().enumerate() {

            // Restrict down to relevant attributes.
            let mut attributes: Vec<(usize, usize)> =
            relevant_attributes
                .iter()
                .filter(|(_attr, input)| input == &index)
                .cloned()
                .collect::<Vec<_>>();

            let attributes_init = attributes.clone();

            // Ensure the plan is rendered and cached.
            if arrangements.get_unkeyed(&plan).is_none() {
                let collection = plan.render(scope, arrangements);
                arrangements.set_unkeyed(plan, &collection.arrange_by_self().trace);
            }
            let changes =
            arrangements
                .get_unkeyed(&plan)
                .expect("Surely we just ensured this")
                .import(scope)
                .as_collection(|val,&()| val.clone())
                .map(move |tuple| attributes_init.iter().map(|&(attr,_)|
                        tuple[attr].clone()).collect::<Vec<_>>()
                );

            // Before constructing the dataflow, which takes a borrow on `scope`,
            // we'll want to ensure that we have all of the necessary data assets
            // in place. This requires a bit of planning first, then the building.

            // Acquire a sane sequence in which to join the relations:
            //
            // This is a sequence of relation identifiers, starting with `index`,
            // such that each has at least one attribute in common with a prior
            // relation, and so can be effectively joined.
            let join_order = plan_join_order(index, &self.equalities);
            let mut join_plan = Vec::new();

            // Skipping `index`, join in each relation in sequence.
            for join_idx in join_order.into_iter().skip(1) {

                // To join a relation, we need to determine any constraints on
                // attributes in common with prior relations. Any other values
                // should be appended to tuples in `changes` with care taken to
                // update `attributes`.
                let (keys, priors) = determine_keys_priors(join_idx, &self.equalities, &attributes[..]);

                // The fields in `sources[join_idx]` that should be values are those
                // that are required output or participate in an equality constraint,
                // but *WHICH ARE NOT* in `keys`.
                let vals =
                relevant_attributes
                    .iter()
                    .filter(|&(attr,index)| index == &join_idx && !keys.contains(&attr))
                    .cloned()
                    .collect::<Vec<_>>();

                let mut projection = Vec::new();
                for &attr in keys.iter() {
                    projection.push(attr);
                }
                for &(attr, _index) in vals.iter() {
                    projection.push(attr);
                }
                // TODO: Sort, to improve chances of re-use opportunities.
                //       Requires understanding how attributes move to get the right
                //       key selectors out though.
                // projection.sort();
                // projection.dedup(); // Should already be deduplicated, probably?

                // Get a plan for the projection on to these few attributes.
                let plan = self.sources[join_idx].clone().project(projection);

                if arrangements.get_keyed(&plan, &keys[..]).is_none() {
                    let keys_clone = keys.clone();
                    let arrangement =
                    plan.render(scope, arrangements)
                        .map(move |tuple| (keys_clone.iter().map(|&i| tuple[i].clone()).collect::<Vec<_>>(), tuple))
                        .arrange_by_key();

                    arrangements.set_keyed(&plan, &keys[..], &arrangement.trace);
                }

                let arrangement =
                arrangements
                    .get_keyed(&plan, &keys[..])
                    .expect("Surely we just ensured this");

                let key_selector = std::rc::Rc::new(move |change: &Vec<V>|
                    priors.iter().map(|&p| change[p].clone()).collect::<Vec<_>>()
                );

                join_plan.push((join_idx, key_selector, arrangement));

                attributes.extend(vals.into_iter());
            }

            // Build the dataflow.
            use dogsdogsdogs::altneu::AltNeu;

            let scope_name = format!("DeltaRule: {}/{}", index, self.sources.len());
            let changes = scope.clone().scoped::<AltNeu<_>,_,_>(&scope_name, |inner| {

                // This should default to an `AltNeu::Alt` timestamp.
                let mut changes = changes.enter(inner);

                for (join_idx, key_selector, mut trace) in join_plan.into_iter() {

                    // Use alt or neu timestamps based on relative indices.
                    // Must have an `if` statement here as the two arrangement have different
                    // types, and we would to determine `alt` v `neu` once, rather than per
                    // tuple in the cursor.
                    changes =
                    if join_idx < index {
                        let arrangement = trace.import(scope).enter_at(inner, |_,_,t| AltNeu::alt(t.clone()));
                        dogsdogsdogs::operators::propose(&changes, arrangement, key_selector)
                    }
                    else {
                        let arrangement = trace.import(scope).enter_at(inner, |_,_,t| AltNeu::neu(t.clone()));
                        dogsdogsdogs::operators::propose(&changes, arrangement, key_selector)
                    }
                    .map(|(mut prefix, extensions)| { prefix.extend(extensions.into_iter()); prefix });

                    // TODO: Equality constraints strictly within a relation have the effect
                    //       of "filtering" data, but they are ignored at the moment. We should
                    //       check for these and do something about it.
                }

                // Extract `self.results` in order, using `attributes`.
                let extract_map =
                self.results
                    .iter()
                    .map(move |x| attributes.iter().position(|i| i == x).expect("Output attribute not found!"))
                    .collect::<Vec<_>>();

                changes
                    .map(move |tuple| extract_map.iter().map(|&i| tuple[i].clone()).collect::<Vec<_>>())
                    .leave()
            });

            accumulated_changes.push(changes);
        }

        differential_dataflow::collection::concatenate(scope, accumulated_changes.into_iter())
    }
}

/// Sequences relations in `constraints`.
///
/// Relations become available for sequencing as soon as they share a constraint with
/// either `source` or another sequenced relation.
fn plan_join_order(source: usize, constraints: &[Vec<(usize, usize)>]) -> Vec<usize> {

    let mut result = vec![source];
    let mut active = true;
    while active {
        active = false;
        for constraint in constraints.iter() {
            // Check to see if the constraint contains a sequenced relation.
            if constraint.iter().any(|(_,index)| result.contains(index)) {
                // If so, sequence any unsequenced relations.
                for (_, index) in constraint.iter() {
                    if !result.contains(index) {
                        result.push(*index);
                        active = true;
                    }
                }
            }
        }
    }

    result
}

/// Identifies keys and values for a join.
///
/// The result is a sequence, for each
fn determine_keys_priors(
    relation: usize,
    constraints: &[Vec<(usize, usize)>],
    current_attributes: &[(usize, usize)],
)
-> (Vec<usize>, Vec<usize>)
{
    // The fields in `sources[join_idx]` that should be keys are those
    // that share an equality constraint with an element of `attributes`.
    // For each key, we should capture the associated `attributes` entry
    // so that we can easily prepare the keys of the `delta` stream.
    let mut keys = Vec::new();
    let mut priors = Vec::new();
    for constraint in constraints.iter() {

        // If there is an intersection between `constraint` and `current_attributes`,
        // we should capture the position in `current_attributes` and emit all of the
        // attributes for `relation`.
        if let Some(prior) = current_attributes.iter().position(|x| constraint.contains(x)) {
            for &(attr, index) in constraint.iter() {
                if index == relation {
                    keys.push(attr);
                    priors.push(prior);
                }
            }
        }
    }

    (keys, priors)
}
<|endoftext|>


<commit_before>// Copyright 2020 The Evcxr Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use crate::{code_block::CodeBlock, eval_context::ContextState};
use crate::{
    errors::{bail, CompilationError, Error},
    eval_context::Config,
};
use once_cell::sync::OnceCell;
use regex::Regex;
use std::fs;
use std::path::{Path, PathBuf};

fn shared_object_name_from_crate_name(crate_name: &str) -> String {
    if cfg!(target_os = "macos") {
        format!("lib{}.dylib", crate_name)
    } else if cfg!(target_os = "windows") {
        format!("{}.dll", crate_name)
    } else {
        format!("lib{}.so", crate_name)
    }
}

fn create_dir(dir: &Path) -> Result<(), Error> {
    if let Err(err) = fs::create_dir_all(dir) {
        bail!("Error creating directory '{:?}': {}", dir, err);
    }
    Ok(())
}

fn write_file(dir: &Path, basename: &str, contents: &str) -> Result<(), Error> {
    create_dir(dir)?;
    let filename = dir.join(basename);
    if let Err(err) = fs::write(&filename, contents) {
        bail!("Error writing '{:?}': {}", filename, err);
    }
    Ok(())
}

/// On Mac, if we copy the dylib, we get intermittent failures where we end up
/// with the previous version of the file when we shouldn't. On windows, if
/// rename the file, we get errors subsequently when something (perhaps the
/// Windows linker) tries to delete the file that it expects to still be there.
/// On Linux either renaming or copying works, but renaming should be more
/// efficient, so we do that.
#[cfg(windows)]
fn rename_or_copy_so_file(src: &Path, dest: &Path) -> Result<(), Error> {
    // Copy file by reading and writing instead of using std::fs::copy. The src
    // is a hard-linked file and we want to make extra sure that we end up with
    // a completely independent copy.
    fn alt_copy(src: &Path, dest: &Path) -> Result<(), std::io::Error> {
        use std::fs::File;
        std::io::copy(&mut File::open(src)?, &mut File::create(dest)?)?;
        Ok(())
    }
    if let Err(err) = alt_copy(src, dest) {
        bail!("Error copying '{:?}' to '{:?}': {}", src, dest, err);
    }
    Ok(())
}

#[cfg(not(windows))]
fn rename_or_copy_so_file(src: &Path, dest: &Path) -> Result<(), Error> {
    if let Err(err) = fs::rename(src, dest) {
        bail!("Error renaming '{:?}' to '{:?}': {}", src, dest, err);
    }
    Ok(())
}

pub(crate) struct Module {
    pub(crate) tmpdir: PathBuf,
    build_num: i32,
}

const CRATE_NAME: &str = "ctx";

impl Module {
    pub(crate) fn new(tmpdir: PathBuf) -> Result<Module, Error> {
        let module = Module {
            tmpdir,
            build_num: 0,
        };
        Ok(module)
    }

    pub(crate) fn deps_dir(&self) -> PathBuf {
        self.target_dir().join("debug").join("deps")
    }

    fn target_dir(&self) -> PathBuf {
        self.tmpdir.join("target")
    }

    fn so_path(&self) -> PathBuf {
        self.deps_dir()
            .join(shared_object_name_from_crate_name(CRATE_NAME))
    }

    fn src_dir(&self) -> PathBuf {
        self.tmpdir.join("src")
    }

    pub(crate) fn crate_dir(&self) -> &Path {
        &self.tmpdir
    }

    pub fn last_source(&self) -> Result<String, std::io::Error> {
        std::fs::read_to_string(self.src_dir().join("lib.rs"))
    }

    // Writes Cargo.toml. Should be called before compile.
    pub(crate) fn write_cargo_toml(&self, state: &ContextState) -> Result<(), Error> {
        write_file(
            self.crate_dir(),
            "Cargo.toml",
            &self.get_cargo_toml_contents(state),
        )
    }

    pub(crate) fn check(
        &mut self,
        code_block: &CodeBlock,
        config: &Config,
    ) -> Result<Vec<CompilationError>, Error> {
        self.write_code(code_block)?;
        let output = config
            .cargo_command("check")
            .arg("--message-format=json")
            .output();

        let cargo_output = match output {
            Ok(out) => out,
            Err(err) => bail!("Error running 'cargo check': {}", err),
        };
        let (errors, _non_json_error) = errors_from_cargo_output(&cargo_output, code_block);
        Ok(errors)
    }

    pub(crate) fn compile(
        &mut self,
        code_block: &CodeBlock,
        config: &Config,
    ) -> Result<SoFile, Error> {
        let mut command = config.cargo_command("rustc");
        if config.time_passes && config.toolchain != "nightly" {
            bail!("time_passes option requires nightly compiler");
        }

        command
            .arg("--message-format=json")
            .arg("--")
            .arg("-C")
            .arg("prefer-dynamic")
            .env("CARGO_TARGET_DIR", "target");
        if config.linker == "lld" {
            command
                .arg("-C")
                .arg(format!("link-arg=-fuse-ld={}", config.linker));
        }
        if let Some(sccache) = &config.sccache {
            command.env("RUSTC_WRAPPER", sccache);
        }
        if config.time_passes {
            command.arg("-Ztime-passes");
        }
        self.write_code(code_block)?;
        let cargo_output = run_cargo(command, code_block)?;
        if config.time_passes {
            let stdout = String::from_utf8_lossy(&cargo_output.stdout);
            eprintln!("{}", stdout);
        }
        self.build_num += 1;
        let copied_so_file = self
            .deps_dir()
            .join(shared_object_name_from_crate_name(&format!(
                "code_{}",
                self.build_num
            )));
        // Every time we compile, the output file is the same. We need to
        // renamed it so that we have a unique filename, otherwise we wouldn't
        // be able to load the result of the next compilation. Also, on Windows,
        // a loaded dll gets locked, so we couldn't even compile a second time
        // if we didn't load a different file.
        rename_or_copy_so_file(&self.so_path(), &copied_so_file)?;
        Ok(SoFile {
            path: copied_so_file,
        })
    }

    fn write_code(&self, code_block: &CodeBlock) -> Result<(), Error> {
        write_file(&self.src_dir(), "lib.rs", &code_block.code_string())?;
        self.maybe_bump_lib_mtime();
        Ok(())
    }

    #[cfg(not(target_os = "macos"))]
    fn maybe_bump_lib_mtime(&self) {}

    #[cfg(target_os = "macos")]
    fn maybe_bump_lib_mtime(&self) {
        // Some Macs use a filesystem that only has 1 second precision on file modification
        // timestamps. Cargo uses these timestamps to see if it needs to recompile things, otherwise
        // it just reuses the previous output. We set the modification timestamp on our source file
        // to be 10 seconds in the future. That way it's guaranteed to be newer than any outputs
        // produced by previous runs. In the event that setting the mtime fails, we just ignore it,
        // as this mostly affects tests and we don't want inability to set mtime to break things for
        // users.
        let _ = filetime::set_file_mtime(
            self.src_dir().join("lib.rs"),
            filetime::FileTime::from_unix_time(filetime::FileTime::now().unix_seconds() + 10, 0),
        );
    }

    fn get_cargo_toml_contents(&self, state: &ContextState) -> String {
        let crate_imports = state.format_cargo_deps();
        format!(
            r#"
[package]
name = "{}"
version = "1.0.0"
edition = "2021"

[lib]
crate-type = ["cdylib"]
path = "src/lib.rs"

[profile.dev]
opt-level = {}
debug = false
rpath = true
lto = false
debug-assertions = true
codegen-units = 16
panic = 'unwind'
incremental = true
overflow-checks = true

[dependencies]
{}
"#,
            CRATE_NAME,
            state.opt_level(),
            crate_imports
        )
    }
}

fn run_cargo(
    mut command: std::process::Command,
    code_block: &CodeBlock,
) -> Result<std::process::Output, Error> {
    let cargo_output = match command.output() {
        Ok(out) => out,
        Err(err) => bail!("Error running 'cargo rustc': {}", err),
    };
    if cargo_output.status.success() {
        Ok(cargo_output)
    } else {
        let (errors, non_json_error) = errors_from_cargo_output(&cargo_output, code_block);
        if errors.is_empty() {
            if let Some(error) = non_json_error {
                bail!(Error::Message(error));
            } else {
                bail!(Error::Message(format!(
                    "Compilation failed, but no parsable errors were found. STDERR:\n\
                     {}\nSTDOUT:{}\n",
                    String::from_utf8_lossy(&cargo_output.stderr),
                    String::from_utf8_lossy(&cargo_output.stdout)
                )));
            }
        } else {
            bail!(Error::CompilationErrors(errors));
        }
    }
}

fn errors_from_cargo_output(
    cargo_output: &std::process::Output,
    code_block: &CodeBlock,
) -> (Vec<CompilationError>, Option<String>) {
    // Our compiler errors should all be in JSON format, but for errors from
    // Cargo errors, we need to add explicit matching for those errors that we
    // expect we might see.
    static KNOWN_NON_JSON_ERRORS: OnceCell<Regex> = OnceCell::new();
    let known_non_json_errors = KNOWN_NON_JSON_ERRORS
        .get_or_init(|| Regex::new("(error: no matching package named)").unwrap());

    let stderr = String::from_utf8_lossy(&cargo_output.stderr);
    let stdout = String::from_utf8_lossy(&cargo_output.stdout);
    let mut non_json_error = None;
    let errors = stderr
        .lines()
        .chain(stdout.lines())
        .filter_map(|line| {
            json::parse(line)
                .ok()
                .and_then(|json| CompilationError::opt_new(json, code_block))
                .or_else(|| {
                    if known_non_json_errors.is_match(line) {
                        non_json_error = Some(line.to_owned());
                    }
                    None
                })
        })
        .collect();
    (errors, non_json_error)
}

pub(crate) struct SoFile {
    pub(crate) path: PathBuf,
}
<commit_msg>Pass --target to cargo rustc<commit_after>// Copyright 2020 The Evcxr Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use crate::{code_block::CodeBlock, eval_context::ContextState};
use crate::{
    errors::{bail, CompilationError, Error},
    eval_context::Config,
};
use once_cell::sync::OnceCell;
use regex::Regex;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;

fn shared_object_name_from_crate_name(crate_name: &str) -> String {
    if cfg!(target_os = "macos") {
        format!("lib{}.dylib", crate_name)
    } else if cfg!(target_os = "windows") {
        format!("{}.dll", crate_name)
    } else {
        format!("lib{}.so", crate_name)
    }
}

fn create_dir(dir: &Path) -> Result<(), Error> {
    if let Err(err) = fs::create_dir_all(dir) {
        bail!("Error creating directory '{:?}': {}", dir, err);
    }
    Ok(())
}

fn write_file(dir: &Path, basename: &str, contents: &str) -> Result<(), Error> {
    create_dir(dir)?;
    let filename = dir.join(basename);
    if let Err(err) = fs::write(&filename, contents) {
        bail!("Error writing '{:?}': {}", filename, err);
    }
    Ok(())
}

/// On Mac, if we copy the dylib, we get intermittent failures where we end up
/// with the previous version of the file when we shouldn't. On windows, if
/// rename the file, we get errors subsequently when something (perhaps the
/// Windows linker) tries to delete the file that it expects to still be there.
/// On Linux either renaming or copying works, but renaming should be more
/// efficient, so we do that.
#[cfg(windows)]
fn rename_or_copy_so_file(src: &Path, dest: &Path) -> Result<(), Error> {
    // Copy file by reading and writing instead of using std::fs::copy. The src
    // is a hard-linked file and we want to make extra sure that we end up with
    // a completely independent copy.
    fn alt_copy(src: &Path, dest: &Path) -> Result<(), std::io::Error> {
        use std::fs::File;
        std::io::copy(&mut File::open(src)?, &mut File::create(dest)?)?;
        Ok(())
    }
    if let Err(err) = alt_copy(src, dest) {
        bail!("Error copying '{:?}' to '{:?}': {}", src, dest, err);
    }
    Ok(())
}

#[cfg(not(windows))]
fn rename_or_copy_so_file(src: &Path, dest: &Path) -> Result<(), Error> {
    if let Err(err) = fs::rename(src, dest) {
        bail!("Error renaming '{:?}' to '{:?}': {}", src, dest, err);
    }
    Ok(())
}

pub(crate) struct Module {
    pub(crate) tmpdir: PathBuf,
    build_num: i32,
    target: String,
}

const CRATE_NAME: &str = "ctx";

impl Module {
    pub(crate) fn new(tmpdir: PathBuf) -> Result<Module, Error> {
        let module = Module {
            tmpdir,
            build_num: 0,
            target: get_host_target()?,
        };
        Ok(module)
    }

    pub(crate) fn deps_dir(&self) -> PathBuf {
        self.target_dir().join("debug").join("deps")
    }

    fn target_dir(&self) -> PathBuf {
        self.tmpdir.join("target").join(&self.target)
    }

    fn so_path(&self) -> PathBuf {
        self.deps_dir()
            .join(shared_object_name_from_crate_name(CRATE_NAME))
    }

    fn src_dir(&self) -> PathBuf {
        self.tmpdir.join("src")
    }

    pub(crate) fn crate_dir(&self) -> &Path {
        &self.tmpdir
    }

    pub fn last_source(&self) -> Result<String, std::io::Error> {
        std::fs::read_to_string(self.src_dir().join("lib.rs"))
    }

    // Writes Cargo.toml. Should be called before compile.
    pub(crate) fn write_cargo_toml(&self, state: &ContextState) -> Result<(), Error> {
        write_file(
            self.crate_dir(),
            "Cargo.toml",
            &self.get_cargo_toml_contents(state),
        )
    }

    pub(crate) fn check(
        &mut self,
        code_block: &CodeBlock,
        config: &Config,
    ) -> Result<Vec<CompilationError>, Error> {
        self.write_code(code_block)?;
        let output = config
            .cargo_command("check")
            .arg("--message-format=json")
            .output();

        let cargo_output = match output {
            Ok(out) => out,
            Err(err) => bail!("Error running 'cargo check': {}", err),
        };
        let (errors, _non_json_error) = errors_from_cargo_output(&cargo_output, code_block);
        Ok(errors)
    }

    pub(crate) fn compile(
        &mut self,
        code_block: &CodeBlock,
        config: &Config,
    ) -> Result<SoFile, Error> {
        let mut command = config.cargo_command("rustc");
        if config.time_passes && config.toolchain != "nightly" {
            bail!("time_passes option requires nightly compiler");
        }

        command
            .arg("--target")
            .arg(&self.target)
            .arg("--message-format=json")
            .arg("--")
            .arg("-C")
            .arg("prefer-dynamic")
            .env("CARGO_TARGET_DIR", "target");
        if config.linker == "lld" {
            command
                .arg("-C")
                .arg(format!("link-arg=-fuse-ld={}", config.linker));
        }
        if let Some(sccache) = &config.sccache {
            command.env("RUSTC_WRAPPER", sccache);
        }
        if config.time_passes {
            command.arg("-Ztime-passes");
        }
        self.write_code(code_block)?;
        let cargo_output = run_cargo(command, code_block)?;
        if config.time_passes {
            let stdout = String::from_utf8_lossy(&cargo_output.stdout);
            eprintln!("{}", stdout);
        }
        self.build_num += 1;
        let copied_so_file = self
            .deps_dir()
            .join(shared_object_name_from_crate_name(&format!(
                "code_{}",
                self.build_num
            )));
        // Every time we compile, the output file is the same. We need to
        // renamed it so that we have a unique filename, otherwise we wouldn't
        // be able to load the result of the next compilation. Also, on Windows,
        // a loaded dll gets locked, so we couldn't even compile a second time
        // if we didn't load a different file.
        rename_or_copy_so_file(&self.so_path(), &copied_so_file)?;
        Ok(SoFile {
            path: copied_so_file,
        })
    }

    fn write_code(&self, code_block: &CodeBlock) -> Result<(), Error> {
        write_file(&self.src_dir(), "lib.rs", &code_block.code_string())?;
        self.maybe_bump_lib_mtime();
        Ok(())
    }

    #[cfg(not(target_os = "macos"))]
    fn maybe_bump_lib_mtime(&self) {}

    #[cfg(target_os = "macos")]
    fn maybe_bump_lib_mtime(&self) {
        // Some Macs use a filesystem that only has 1 second precision on file modification
        // timestamps. Cargo uses these timestamps to see if it needs to recompile things, otherwise
        // it just reuses the previous output. We set the modification timestamp on our source file
        // to be 10 seconds in the future. That way it's guaranteed to be newer than any outputs
        // produced by previous runs. In the event that setting the mtime fails, we just ignore it,
        // as this mostly affects tests and we don't want inability to set mtime to break things for
        // users.
        let _ = filetime::set_file_mtime(
            self.src_dir().join("lib.rs"),
            filetime::FileTime::from_unix_time(filetime::FileTime::now().unix_seconds() + 10, 0),
        );
    }

    fn get_cargo_toml_contents(&self, state: &ContextState) -> String {
        let crate_imports = state.format_cargo_deps();
        format!(
            r#"
[package]
name = "{}"
version = "1.0.0"
edition = "2021"

[lib]
crate-type = ["cdylib"]
path = "src/lib.rs"

[profile.dev]
opt-level = {}
debug = false
rpath = true
lto = false
debug-assertions = true
codegen-units = 16
panic = 'unwind'
incremental = true
overflow-checks = true

[dependencies]
{}
"#,
            CRATE_NAME,
            state.opt_level(),
            crate_imports
        )
    }
}

fn run_cargo(
    mut command: std::process::Command,
    code_block: &CodeBlock,
) -> Result<std::process::Output, Error> {
    let cargo_output = match command.output() {
        Ok(out) => out,
        Err(err) => bail!("Error running 'cargo rustc': {}", err),
    };
    if cargo_output.status.success() {
        Ok(cargo_output)
    } else {
        let (errors, non_json_error) = errors_from_cargo_output(&cargo_output, code_block);
        if errors.is_empty() {
            if let Some(error) = non_json_error {
                bail!(Error::Message(error));
            } else {
                bail!(Error::Message(format!(
                    "Compilation failed, but no parsable errors were found. STDERR:\n\
                     {}\nSTDOUT:{}\n",
                    String::from_utf8_lossy(&cargo_output.stderr),
                    String::from_utf8_lossy(&cargo_output.stdout)
                )));
            }
        } else {
            bail!(Error::CompilationErrors(errors));
        }
    }
}

fn errors_from_cargo_output(
    cargo_output: &std::process::Output,
    code_block: &CodeBlock,
) -> (Vec<CompilationError>, Option<String>) {
    // Our compiler errors should all be in JSON format, but for errors from
    // Cargo errors, we need to add explicit matching for those errors that we
    // expect we might see.
    static KNOWN_NON_JSON_ERRORS: OnceCell<Regex> = OnceCell::new();
    let known_non_json_errors = KNOWN_NON_JSON_ERRORS
        .get_or_init(|| Regex::new("(error: no matching package named)").unwrap());

    let stderr = String::from_utf8_lossy(&cargo_output.stderr);
    let stdout = String::from_utf8_lossy(&cargo_output.stdout);
    let mut non_json_error = None;
    let errors = stderr
        .lines()
        .chain(stdout.lines())
        .filter_map(|line| {
            json::parse(line)
                .ok()
                .and_then(|json| CompilationError::opt_new(json, code_block))
                .or_else(|| {
                    if known_non_json_errors.is_match(line) {
                        non_json_error = Some(line.to_owned());
                    }
                    None
                })
        })
        .collect();
    (errors, non_json_error)
}

pub(crate) struct SoFile {
    pub(crate) path: PathBuf,
}

fn get_host_target() -> Result<String, Error> {
    let output = match Command::new("rustc").arg("-Vv").output() {
        Ok(o) => o,
        Err(error) => bail!("Failed to run rustc: {}", error),
    };
    let stdout = std::str::from_utf8(&output.stdout)?;
    let stderr = std::str::from_utf8(&output.stderr)?;
    for line in stdout.lines() {
        if let Some(host) = line.strip_prefix("host: ") {
            return Ok(host.to_owned());
        }
    }
    bail!(
        "rustc -Vv didn't output a host line.\n{}\n{}",
        stdout,
        stderr
    );
}
<|endoftext|>


<commit_before>extern crate iron;
extern crate staticfile;
extern crate mount;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;
extern crate rustfmt;
extern crate json_sample_shared;
extern crate error_chain;

use std::path::Path;

use iron::prelude::*;
use iron::status;
use staticfile::Static;
use mount::Mount;
use rustfmt::config::{Config, WriteMode};
use error_chain::ChainedError;
use std::env;

use json_sample_shared::{codegen_from_sample, SampleSource};

#[derive(Debug, Deserialize)]
struct ReqBody {
    name: String,
    input: String
}

macro_rules! handle {
    ($result:expr, $err_handler:expr) => (handle!($result, _err => $err_handler));
    ($result:expr, $err_id:ident => $err_handler:expr) => (match $result {
        ::std::result::Result::Ok(val) => val,
        ::std::result::Result::Err($err_id) => return ::std::result::Result::Ok(
            ::iron::Response::with((::iron::status::BadRequest, $err_handler)))
    });
}

fn handle_codegen_request(req: &mut Request) -> IronResult<Response> {
    let req_body: ReqBody = handle!(serde_json::de::from_reader(&mut req.body),
        "Error: Request body was invalid JSON");
    let tokens = handle!(codegen_from_sample(&req_body.name, &SampleSource::Text(&req_body.input)),
        err => format!("{}", err.display()));

    let input = rustfmt::Input::Text(String::from(tokens.as_str()));
    let mut output: Vec<u8> = Vec::new();
    let mut config = Config::default();
    config.write_mode = WriteMode::Plain;
    handle!(rustfmt::format_input(input, &config, Some(&mut output)),
        err => format!("Error formatting output with rustfmt: {:?}", err));

    let s = String::from_utf8_lossy(&output);
    let formatted = fix_rustfmt_issues(&s);

    Ok(Response::with((status::Ok, formatted)))
}

fn fix_rustfmt_issues(input: &str) -> String {
    let mut output = String::new();

    for line in input.lines() {
        if line.starts_with('#') {
            for c in line.chars() {
                match c {
                    ' ' => {},
                    ',' => { output.push_str(", "); }
                    _ => { output.push(c); }
                }
            }
            output.push('\n');
        } else {
            output.push_str(line);
            output.push('\n');
            if line == "}" {
                output.push('\n');
            }
        }
    }

    output
}

fn main() {
    let mut mount = Mount::new();

    let p = env::current_dir().unwrap();
    println!("Working directory is {}", p.display());
    let static_path = Path::new("static/");
    let canonical_static = static_path.canonicalize().unwrap();
    println!("Looking for static files in {}", canonical_static.display());

    mount.mount("/", Static::new(static_path));
    mount.mount("/api", handle_codegen_request);

    let host = env::var("JSONSAMPLE_HOST").unwrap_or_else(|_| String::from("0.0.0.0:5555"));
    let _server = Iron::new(mount).http(&host as &str).unwrap();
    println!("Serving on http://{}", host);
}
<commit_msg>web: apply rustfmt fix on indented lines<commit_after>extern crate iron;
extern crate staticfile;
extern crate mount;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;
extern crate rustfmt;
extern crate json_sample_shared;
extern crate error_chain;

use std::path::Path;

use iron::prelude::*;
use iron::status;
use staticfile::Static;
use mount::Mount;
use rustfmt::config::{Config, WriteMode};
use error_chain::ChainedError;
use std::env;

use json_sample_shared::{codegen_from_sample, SampleSource};

#[derive(Debug, Deserialize)]
struct ReqBody {
    name: String,
    input: String
}

macro_rules! handle {
    ($result:expr, $err_handler:expr) => (handle!($result, _err => $err_handler));
    ($result:expr, $err_id:ident => $err_handler:expr) => (match $result {
        ::std::result::Result::Ok(val) => val,
        ::std::result::Result::Err($err_id) => return ::std::result::Result::Ok(
            ::iron::Response::with((::iron::status::BadRequest, $err_handler)))
    });
}

fn handle_codegen_request(req: &mut Request) -> IronResult<Response> {
    let req_body: ReqBody = handle!(serde_json::de::from_reader(&mut req.body),
        "Error: Request body was invalid JSON");
    let tokens = handle!(codegen_from_sample(&req_body.name, &SampleSource::Text(&req_body.input)),
        err => format!("{}", err.display()));

    let input = rustfmt::Input::Text(String::from(tokens.as_str()));
    let mut output: Vec<u8> = Vec::new();
    let mut config = Config::default();
    config.write_mode = WriteMode::Plain;
    handle!(rustfmt::format_input(input, &config, Some(&mut output)),
        err => format!("Error formatting output with rustfmt: {:?}", err));

    let s = String::from_utf8_lossy(&output);
    let formatted = fix_rustfmt_issues(&s);

    Ok(Response::with((status::Ok, formatted)))
}

fn fix_rustfmt_issues(input: &str) -> String {
    let mut output = String::new();

    for line in input.lines() {
        if line.starts_with('#') || line.starts_with("    #") {
            if line.starts_with("    #") {
                output.push_str("    ");
            }
            for c in line.chars() {
                match c {
                    ' ' => {},
                    ',' => { output.push_str(", "); }
                    _ => { output.push(c); }
                }
            }
            output.push('\n');
        } else {
            output.push_str(line);
            output.push('\n');
            if line == "}" {
                output.push('\n');
            }
        }
    }

    output
}

fn main() {
    let mut mount = Mount::new();

    let p = env::current_dir().unwrap();
    println!("Working directory is {}", p.display());
    let static_path = Path::new("static/");
    let canonical_static = static_path.canonicalize().unwrap();
    println!("Looking for static files in {}", canonical_static.display());

    mount.mount("/", Static::new(static_path));
    mount.mount("/api", handle_codegen_request);

    let host = env::var("JSONSAMPLE_HOST").unwrap_or_else(|_| String::from("0.0.0.0:5555"));
    let _server = Iron::new(mount).http(&host as &str).unwrap();
    println!("Serving on http://{}", host);
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use cssparser::Parser;
use media_queries::CSSErrorReporterTest;
use style::parser::ParserContext;
use style::stylesheets::Origin;

#[test]
fn text_emphasis_style_longhand_should_parse_properly() {
    use style::properties::longhands::text_emphasis_style;
    use style::properties::longhands::text_emphasis_style::{ShapeKeyword, SpecifiedValue, KeywordValue};

    let none = parse_longhand!(text_emphasis_style, "none");
    assert_eq!(none, SpecifiedValue::None);

    let fill = parse_longhand!(text_emphasis_style, "open");
    let fill_struct = SpecifiedValue::Keyword(KeywordValue::Fill(false));
    assert_eq!(fill, fill_struct);

    let shape = parse_longhand!(text_emphasis_style, "triangle");
    let shape_struct = SpecifiedValue::Keyword(KeywordValue::Shape(ShapeKeyword::Triangle));
    assert_eq!(shape, shape_struct);

    let fill_shape = parse_longhand!(text_emphasis_style, "filled dot");
    let fill_shape_struct = SpecifiedValue::Keyword(KeywordValue::FillAndShape(true, ShapeKeyword::Dot));
    assert_eq!(fill_shape, fill_shape_struct);

    let shape_fill = parse_longhand!(text_emphasis_style, "dot filled");
    let shape_fill_struct = SpecifiedValue::Keyword(KeywordValue::FillAndShape(true, ShapeKeyword::Dot));
    assert_eq!(shape_fill, shape_fill_struct);

    let a_string = parse_longhand!(text_emphasis_style, "\"a\"");
    let a_string_struct = SpecifiedValue::String("a".to_string());
    assert_eq!(a_string, a_string_struct);

    let chinese_string = parse_longhand!(text_emphasis_style, "\"点\"");
    let chinese_string_struct = SpecifiedValue::String("点".to_string());
    assert_eq!(chinese_string, chinese_string_struct);

    let unicode_string = parse_longhand!(text_emphasis_style, "\"\\25B2\"");
    let unicode_string_struct = SpecifiedValue::String("▲".to_string());
    assert_eq!(unicode_string, unicode_string_struct);

    let devanagari_string = parse_longhand!(text_emphasis_style, "\"षि\"");
    let devanagari_string_struct = SpecifiedValue::String("षि".to_string());
    assert_eq!(devanagari_string, devanagari_string_struct);
}

#[test]
fn test_text_emphasis_position() {
    use style::properties::longhands::text_emphasis_position;
    use style::properties::longhands::text_emphasis_position::{HorizontalWritingModeValue, VerticalWritingModeValue };
    use style::properties::longhands::text_emphasis_position::SpecifiedValue;

    let over_right = parse_longhand!(text_emphasis_position, "over right");
    assert_eq!(over_right, SpecifiedValue(HorizontalWritingModeValue::Over, VerticalWritingModeValue::Right));

    let over_left = parse_longhand!(text_emphasis_position, "over left");
    assert_eq!(over_left, SpecifiedValue(HorizontalWritingModeValue::Over, VerticalWritingModeValue::Left));

    let under_right = parse_longhand!(text_emphasis_position, "under right");
    assert_eq!(under_right, SpecifiedValue(HorizontalWritingModeValue::Under, VerticalWritingModeValue::Right));

    let under_left = parse_longhand!(text_emphasis_position, "under left");
    assert_eq!(under_left, SpecifiedValue(HorizontalWritingModeValue::Under, VerticalWritingModeValue::Left));

    let right_over = parse_longhand!(text_emphasis_position, "right over");
    assert_eq!(right_over, SpecifiedValue(HorizontalWritingModeValue::Over, VerticalWritingModeValue::Right));

    let left_over = parse_longhand!(text_emphasis_position, "left over");
    assert_eq!(left_over, SpecifiedValue(HorizontalWritingModeValue::Over, VerticalWritingModeValue::Left));

    let right_under = parse_longhand!(text_emphasis_position, "right under");
    assert_eq!(right_under, SpecifiedValue(HorizontalWritingModeValue::Under, VerticalWritingModeValue::Right));

    let left_under = parse_longhand!(text_emphasis_position, "left under");
    assert_eq!(left_under, SpecifiedValue(HorizontalWritingModeValue::Under, VerticalWritingModeValue::Left));
}

#[test]
fn webkit_text_stroke_shorthand_should_parse_properly() {
    use style::properties::longhands::_webkit_text_stroke_color;
    use style::properties::longhands::_webkit_text_stroke_width;
    use style::properties::shorthands::_webkit_text_stroke;
    use media_queries::CSSErrorReporterTest;
    use servo_url::ServoUrl;
    let url = ServoUrl::parse("http://localhost").unwrap();
    let context = ParserContext::new(Origin::Author, &url, Box::new(CSSErrorReporterTest));

    let mut parser = Parser::new("thin red");
    let result = _webkit_text_stroke::parse_value(&context, &mut parser).unwrap();
    assert_eq!(result._webkit_text_stroke_color.unwrap(), parse_longhand!(_webkit_text_stroke_color, "red"));
    assert_eq!(result._webkit_text_stroke_width.unwrap(), parse_longhand!(_webkit_text_stroke_width, "thin"));

    // ensure its no longer sensitive to order
    let mut parser = Parser::new("red thin");
    let result = _webkit_text_stroke::parse_value(&context, &mut parser).unwrap();
    assert_eq!(result._webkit_text_stroke_color.unwrap(), parse_longhand!(_webkit_text_stroke_color, "red"));
    assert_eq!(result._webkit_text_stroke_width.unwrap(), parse_longhand!(_webkit_text_stroke_width, "thin"));
}<commit_msg>forgot to tidy unit test<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use cssparser::Parser;
use media_queries::CSSErrorReporterTest;
use style::parser::ParserContext;
use style::stylesheets::Origin;

#[test]
fn text_emphasis_style_longhand_should_parse_properly() {
    use style::properties::longhands::text_emphasis_style;
    use style::properties::longhands::text_emphasis_style::{ShapeKeyword, SpecifiedValue, KeywordValue};

    let none = parse_longhand!(text_emphasis_style, "none");
    assert_eq!(none, SpecifiedValue::None);

    let fill = parse_longhand!(text_emphasis_style, "open");
    let fill_struct = SpecifiedValue::Keyword(KeywordValue::Fill(false));
    assert_eq!(fill, fill_struct);

    let shape = parse_longhand!(text_emphasis_style, "triangle");
    let shape_struct = SpecifiedValue::Keyword(KeywordValue::Shape(ShapeKeyword::Triangle));
    assert_eq!(shape, shape_struct);

    let fill_shape = parse_longhand!(text_emphasis_style, "filled dot");
    let fill_shape_struct = SpecifiedValue::Keyword(KeywordValue::FillAndShape(true, ShapeKeyword::Dot));
    assert_eq!(fill_shape, fill_shape_struct);

    let shape_fill = parse_longhand!(text_emphasis_style, "dot filled");
    let shape_fill_struct = SpecifiedValue::Keyword(KeywordValue::FillAndShape(true, ShapeKeyword::Dot));
    assert_eq!(shape_fill, shape_fill_struct);

    let a_string = parse_longhand!(text_emphasis_style, "\"a\"");
    let a_string_struct = SpecifiedValue::String("a".to_string());
    assert_eq!(a_string, a_string_struct);

    let chinese_string = parse_longhand!(text_emphasis_style, "\"点\"");
    let chinese_string_struct = SpecifiedValue::String("点".to_string());
    assert_eq!(chinese_string, chinese_string_struct);

    let unicode_string = parse_longhand!(text_emphasis_style, "\"\\25B2\"");
    let unicode_string_struct = SpecifiedValue::String("▲".to_string());
    assert_eq!(unicode_string, unicode_string_struct);

    let devanagari_string = parse_longhand!(text_emphasis_style, "\"षि\"");
    let devanagari_string_struct = SpecifiedValue::String("षि".to_string());
    assert_eq!(devanagari_string, devanagari_string_struct);
}

#[test]
fn test_text_emphasis_position() {
    use style::properties::longhands::text_emphasis_position;
    use style::properties::longhands::text_emphasis_position::{HorizontalWritingModeValue, VerticalWritingModeValue };
    use style::properties::longhands::text_emphasis_position::SpecifiedValue;

    let over_right = parse_longhand!(text_emphasis_position, "over right");
    assert_eq!(over_right, SpecifiedValue(HorizontalWritingModeValue::Over, VerticalWritingModeValue::Right));

    let over_left = parse_longhand!(text_emphasis_position, "over left");
    assert_eq!(over_left, SpecifiedValue(HorizontalWritingModeValue::Over, VerticalWritingModeValue::Left));

    let under_right = parse_longhand!(text_emphasis_position, "under right");
    assert_eq!(under_right, SpecifiedValue(HorizontalWritingModeValue::Under, VerticalWritingModeValue::Right));

    let under_left = parse_longhand!(text_emphasis_position, "under left");
    assert_eq!(under_left, SpecifiedValue(HorizontalWritingModeValue::Under, VerticalWritingModeValue::Left));

    let right_over = parse_longhand!(text_emphasis_position, "right over");
    assert_eq!(right_over, SpecifiedValue(HorizontalWritingModeValue::Over, VerticalWritingModeValue::Right));

    let left_over = parse_longhand!(text_emphasis_position, "left over");
    assert_eq!(left_over, SpecifiedValue(HorizontalWritingModeValue::Over, VerticalWritingModeValue::Left));

    let right_under = parse_longhand!(text_emphasis_position, "right under");
    assert_eq!(right_under, SpecifiedValue(HorizontalWritingModeValue::Under, VerticalWritingModeValue::Right));

    let left_under = parse_longhand!(text_emphasis_position, "left under");
    assert_eq!(left_under, SpecifiedValue(HorizontalWritingModeValue::Under, VerticalWritingModeValue::Left));
}

#[test]
fn webkit_text_stroke_shorthand_should_parse_properly() {
    use media_queries::CSSErrorReporterTest;
    use servo_url::ServoUrl;
    use style::properties::longhands::_webkit_text_stroke_color;
    use style::properties::longhands::_webkit_text_stroke_width;
    use style::properties::shorthands::_webkit_text_stroke;

    let url = ServoUrl::parse("http://localhost").unwrap();
    let context = ParserContext::new(Origin::Author, &url, Box::new(CSSErrorReporterTest));

    let mut parser = Parser::new("thin red");
    let result = _webkit_text_stroke::parse_value(&context, &mut parser).unwrap();
    assert_eq!(result._webkit_text_stroke_color.unwrap(), parse_longhand!(_webkit_text_stroke_color, "red"));
    assert_eq!(result._webkit_text_stroke_width.unwrap(), parse_longhand!(_webkit_text_stroke_width, "thin"));

    // ensure its no longer sensitive to order
    let mut parser = Parser::new("red thin");
    let result = _webkit_text_stroke::parse_value(&context, &mut parser).unwrap();
    assert_eq!(result._webkit_text_stroke_color.unwrap(), parse_longhand!(_webkit_text_stroke_color, "red"));
    assert_eq!(result._webkit_text_stroke_width.unwrap(), parse_longhand!(_webkit_text_stroke_width, "thin"));
}
<|endoftext|>