<commit_before>extern crate rand;
extern crate timely;
extern crate differential_dataflow;

use rand::{Rng, SeedableRng, StdRng};

use timely::dataflow::*;
use timely::dataflow::operators::probe::Handle;
use timely::dataflow::operators::capture::EventWriter;

use differential_dataflow::input::Input;
use differential_dataflow::Collection;
use differential_dataflow::operators::*;
use differential_dataflow::lattice::Lattice;

type Node = u32;
type Edge = (Node, Node);

fn main() {

    let nodes: u32 = std::env::args().nth(1).unwrap().parse().unwrap();
    let edges: u32 = std::env::args().nth(2).unwrap().parse().unwrap();
    let batch: u32 = std::env::args().nth(3).unwrap().parse().unwrap();
    let rounds: u32 = std::env::args().nth(4).unwrap().parse().unwrap();
    let inspect: bool = std::env::args().nth(5).unwrap() == "inspect";

    use timely::logging::{LoggerConfig, EventPusherTee};
    let logger_config = LoggerConfig::new(
        |setup| {
            use std::net::TcpStream;
            let addr = format!("127.0.0.1:{}", 8000 + setup.index);
            let send = TcpStream::connect(addr).unwrap();
            EventWriter::new(send)
        },
        |_setup| {
            // No support for communication threads in this example.
            unimplemented!();
            use std::net::TcpStream;
            let addr = format!("127.0.0.1:{}", 8001);
            let send = TcpStream::connect(addr).unwrap();
            EventWriter::new(send)
        }
    );

    timely::execute_from_args_logging(std::env::args().skip(6), logger_config, move |worker| {

    // // define a new computational scope, in which to run BFS
    // timely::execute_from_args(std::env::args(), move |worker| {

        let timer = ::std::time::Instant::now();

        // define BFS dataflow; return handles to roots and edges inputs
        let mut probe = Handle::new();
        let (mut roots, mut graph) = worker.dataflow(|scope| {

            let (root_input, roots) = scope.new_collection();
            let (edge_input, graph) = scope.new_collection();

            let mut result = bfs(&graph, &roots);

            if !inspect {
                result = result.filter(|_| false);
            }

            result.map(|(_,l)| l)
                  .consolidate()
                  .inspect(|x| println!("\t{:?}", x))
                  .probe_with(&mut probe);

            (root_input, edge_input)
        });

        let seed: &[_] = &[1, 2, 3, 4];
        let mut rng1: StdRng = SeedableRng::from_seed(seed);    // rng for edge additions
        let mut rng2: StdRng = SeedableRng::from_seed(seed);    // rng for edge deletions

        roots.insert(0);
        roots.close();

        println!("performing BFS on {} nodes, {} edges:", nodes, edges);

        if worker.index() == 0 {
            for _ in 0 .. edges {
                graph.insert((rng1.gen_range(0, nodes), rng1.gen_range(0, nodes)));
            }
        }

        println!("{:?}\tloaded", timer.elapsed());

        graph.advance_to(1);
        graph.flush();
        worker.step_while(|| probe.less_than(graph.time()));

        println!("{:?}\tstable", timer.elapsed());

        for round in 0 .. rounds {
            for element in 0 .. batch {
                if worker.index() == 0 {
                    graph.insert((rng1.gen_range(0, nodes), rng1.gen_range(0, nodes)));
                    graph.remove((rng2.gen_range(0, nodes), rng2.gen_range(0, nodes)));
                }
                graph.advance_to(2 + round * batch + element);
            }
            graph.flush();

            let timer2 = ::std::time::Instant::now();
            worker.step_while(|| probe.less_than(&graph.time()));

            if worker.index() == 0 {
                let elapsed = timer2.elapsed();
                println!("{:?}\t{:?}:\t{}", timer.elapsed(), round, elapsed.as_secs() * 1000000000 + (elapsed.subsec_nanos() as u64));
            }
        }
        println!("finished; elapsed: {:?}", timer.elapsed());
    }).unwrap();
}

// returns pairs (n, s) indicating node n can be reached from a root in s steps.
fn bfs<G: Scope>(edges: &Collection<G, Edge>, roots: &Collection<G, Node>) -> Collection<G, (Node, u32)>
where G::Timestamp: Lattice+Ord {

    // initialize roots as reaching themselves at distance 0
    let nodes = roots.map(|x| (x, 0));

    // repeatedly update minimal distances each node can be reached from each root
    nodes.iterate(|inner| {

        let edges = edges.enter(&inner.scope());
        let nodes = nodes.enter(&inner.scope());

        inner.join_map(&edges, |_k,l,d| (*d, l+1))
             .concat(&nodes)
             .group(|_, s, t| t.push((*s[0].0, 1)))
     })
}<commit_msg>undo regression<commit_after>extern crate rand;
extern crate timely;
extern crate differential_dataflow;

use rand::{Rng, SeedableRng, StdRng};

use timely::dataflow::*;
use timely::dataflow::operators::probe::Handle;

use differential_dataflow::input::Input;
use differential_dataflow::Collection;
use differential_dataflow::operators::*;
use differential_dataflow::lattice::Lattice;

type Node = u32;
type Edge = (Node, Node);

fn main() {

    let nodes: u32 = std::env::args().nth(1).unwrap().parse().unwrap();
    let edges: u32 = std::env::args().nth(2).unwrap().parse().unwrap();
    let batch: u32 = std::env::args().nth(3).unwrap().parse().unwrap();
    let rounds: u32 = std::env::args().nth(4).unwrap().parse().unwrap();
    let inspect: bool = std::env::args().nth(5).unwrap() == "inspect";

    // define a new computational scope, in which to run BFS
    timely::execute_from_args(std::env::args(), move |worker| {

        let timer = ::std::time::Instant::now();

        // define BFS dataflow; return handles to roots and edges inputs
        let mut probe = Handle::new();
        let (mut roots, mut graph) = worker.dataflow(|scope| {

            let (root_input, roots) = scope.new_collection();
            let (edge_input, graph) = scope.new_collection();

            let mut result = bfs(&graph, &roots);

            if !inspect {
                result = result.filter(|_| false);
            }

            result.map(|(_,l)| l)
                  .consolidate()
                  .inspect(|x| println!("\t{:?}", x))
                  .probe_with(&mut probe);

            (root_input, edge_input)
        });

        let seed: &[_] = &[1, 2, 3, 4];
        let mut rng1: StdRng = SeedableRng::from_seed(seed);    // rng for edge additions
        let mut rng2: StdRng = SeedableRng::from_seed(seed);    // rng for edge deletions

        roots.insert(0);
        roots.close();

        println!("performing BFS on {} nodes, {} edges:", nodes, edges);

        if worker.index() == 0 {
            for _ in 0 .. edges {
                graph.insert((rng1.gen_range(0, nodes), rng1.gen_range(0, nodes)));
            }
        }

        println!("{:?}\tloaded", timer.elapsed());

        graph.advance_to(1);
        graph.flush();
        worker.step_while(|| probe.less_than(graph.time()));

        println!("{:?}\tstable", timer.elapsed());

        for round in 0 .. rounds {
            for element in 0 .. batch {
                if worker.index() == 0 {
                    graph.insert((rng1.gen_range(0, nodes), rng1.gen_range(0, nodes)));
                    graph.remove((rng2.gen_range(0, nodes), rng2.gen_range(0, nodes)));
                }
                graph.advance_to(2 + round * batch + element);
            }
            graph.flush();

            let timer2 = ::std::time::Instant::now();
            worker.step_while(|| probe.less_than(&graph.time()));

            if worker.index() == 0 {
                let elapsed = timer2.elapsed();
                println!("{:?}\t{:?}:\t{}", timer.elapsed(), round, elapsed.as_secs() * 1000000000 + (elapsed.subsec_nanos() as u64));
            }
        }
        println!("finished; elapsed: {:?}", timer.elapsed());
    }).unwrap();
}

// returns pairs (n, s) indicating node n can be reached from a root in s steps.
fn bfs<G: Scope>(edges: &Collection<G, Edge>, roots: &Collection<G, Node>) -> Collection<G, (Node, u32)>
where G::Timestamp: Lattice+Ord {

    // initialize roots as reaching themselves at distance 0
    let nodes = roots.map(|x| (x, 0));

    // repeatedly update minimal distances each node can be reached from each root
    nodes.iterate(|inner| {

        let edges = edges.enter(&inner.scope());
        let nodes = nodes.enter(&inner.scope());

        inner.join_map(&edges, |_k,l,d| (*d, l+1))
             .concat(&nodes)
             .group(|_, s, t| t.push((*s[0].0, 1)))
     })
}<|endoftext|>


<commit_before>// Translation of the following Go program into Rust.
// Source: https://www.nada.kth.se/~snilsson/concurrency/#Match

/*
func main() {
    people := []string{"Anna", "Bob", "Cody", "Dave", "Eva"}
    match := make(chan string, 1) // Make room for one unmatched send.
    wg := new(sync.WaitGroup)
    for _, name := range people {
        wg.Add(1)
        go Seek(name, match, wg)
    }
    wg.Wait()
    select {
    case name := <-match:
        fmt.Printf("No one received %s’s message.\n", name)
    default:
        // There was no pending send operation.
    }
}

// Seek either sends or receives, whichever possible, a name on the match
// channel and notifies the wait group when done.
func Seek(name string, match chan string, wg *sync.WaitGroup) {
    select {
    case peer := <-match:
        fmt.Printf("%s received a message from %s.\n", name, peer)
    case match <- name:
        // Wait for someone to receive my message.
    }
    wg.Done()
}
*/

extern crate crossbeam;
extern crate channel;

use channel::{select, Receiver, Sender};

fn main() {
    let people = vec!["Anna", "Bob", "Cody", "Dave", "Eva"];
    let (tx, rx) = channel::bounded(1); // Make room for one unmatched send.
    let (tx, rx) = (&tx, &rx);

    crossbeam::scope(|s| for name in people {
        s.spawn(move || seek(name, tx, rx));
    });

    if let Ok(name) = rx.try_recv() {
        println!("No one received {}’s message.", name);
    }
}

// Either sends or receives, whichever possible, a name on the channel.
fn seek<'a>(name: &'a str, tx: &Sender<&'a str>, rx: &Receiver<&'a str>) {
    loop {
        if let Ok(peer) = rx.select() {
            println!("{} received a message from {}.", name, peer);
            break;
        }
        if let Ok(()) = tx.select(name) {
            // Wait for someone to receive my message.
            break;
        }
    }
}
<commit_msg>Remove unused use<commit_after>// Translation of the following Go program into Rust.
// Source: https://www.nada.kth.se/~snilsson/concurrency/#Match

/*
func main() {
    people := []string{"Anna", "Bob", "Cody", "Dave", "Eva"}
    match := make(chan string, 1) // Make room for one unmatched send.
    wg := new(sync.WaitGroup)
    for _, name := range people {
        wg.Add(1)
        go Seek(name, match, wg)
    }
    wg.Wait()
    select {
    case name := <-match:
        fmt.Printf("No one received %s’s message.\n", name)
    default:
        // There was no pending send operation.
    }
}

// Seek either sends or receives, whichever possible, a name on the match
// channel and notifies the wait group when done.
func Seek(name string, match chan string, wg *sync.WaitGroup) {
    select {
    case peer := <-match:
        fmt.Printf("%s received a message from %s.\n", name, peer)
    case match <- name:
        // Wait for someone to receive my message.
    }
    wg.Done()
}
*/

extern crate crossbeam;
extern crate channel;

use channel::{Receiver, Sender};

fn main() {
    let people = vec!["Anna", "Bob", "Cody", "Dave", "Eva"];
    let (tx, rx) = channel::bounded(1); // Make room for one unmatched send.
    let (tx, rx) = (&tx, &rx);

    crossbeam::scope(|s| for name in people {
        s.spawn(move || seek(name, tx, rx));
    });

    if let Ok(name) = rx.try_recv() {
        println!("No one received {}’s message.", name);
    }
}

// Either sends or receives, whichever possible, a name on the channel.
fn seek<'a>(name: &'a str, tx: &Sender<&'a str>, rx: &Receiver<&'a str>) {
    loop {
        if let Ok(peer) = rx.select() {
            println!("{} received a message from {}.", name, peer);
            break;
        }
        if let Ok(()) = tx.select(name) {
            // Wait for someone to receive my message.
            break;
        }
    }
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with
// this file, You can obtain one at https://mozilla.org/MPL/2.0/.

use templates;
use errors::Result;
use errors::ResultExt;
use zip::Zip;
use toc::Toc;
use toc::TocElement;
use epub_content::EpubContent;

use std::io::Read;
use std::io::Write;
use std::path::Path;

use chrono;
use uuid;
use mustache::MapBuilder;

/// Represents a EPUB version
#[derive(Debug, Copy, Clone)]
pub enum EpubVersion {
    /// EPUB 2.0.1 format
    V20,
    /// EPUB 3.0.1 format
    V30,
    /// Hint that destructuring should not be exhaustive
    #[doc(hidden)]
    __NonExhaustive,
}

/// EPUB Metadata
#[derive(Debug)]
struct Metadata {
    pub title: String,
    pub author: String,
    pub lang: String,
    pub generator: String,
    pub toc_name: String,
    pub description: Option<String>,
    pub subject: Option<String>,
    pub license: Option<String>,
}

impl Metadata {
    /// Create new default metadata
    pub fn new() -> Metadata {
        Metadata {
            title: String::new(),
            author: String::new(),
            lang: String::from("en"),
            generator: String::from("Rust EPUB library"),
            toc_name: String::from("Table Of Contents"),
            description: None,
            subject: None,
            license: None,
        }
    }
}

/// A file added in the EPUB
#[derive(Debug)]
struct Content {
    pub file: String,
    pub mime: String,
    pub itemref: bool,
}

impl Content {
    /// Create a new content file
    pub fn new<S1:Into<String>, S2: Into<String>>(file: S1, mime: S2) -> Content {
        Content {
            file: file.into(),
            mime: mime.into(),
            itemref: false,
        }
    }
}

/// Epub Builder
#[derive(Debug)]
pub struct EpubBuilder<Z:Zip> {
    version: EpubVersion,
    zip: Z,
    files: Vec<Content>,
    metadata: Metadata,
    toc: Toc,
    stylesheet: bool,
    inline_toc: bool,
}

impl<Z:Zip> EpubBuilder<Z> {
    /// Create a new default EPUB Builder
    pub fn new(zip: Z) -> Result<EpubBuilder<Z>> {
        let mut epub = EpubBuilder {
            version: EpubVersion::V20,
            zip: zip,
            files: vec!(),
            metadata: Metadata::new(),
            toc: Toc::new(),
            stylesheet: false,
            inline_toc: false,
        };

        // Write mimetype upfront
        epub.zip.write_file("mimetype", "application/epub+zip".as_bytes())?;
        epub.zip.write_file("META-INF/container.xml", templates::CONTAINER)?;
        epub.zip.write_file("META-INF/com.apple.ibooks.display-options.xml", templates::IBOOKS)?;

        Ok(epub)
    }

    /// Set EPUB version (default: V20)
    ///
    /// Supported versions are:
    ///
    /// * `V20`: EPUB 2.0.1
    /// * 'V30`: EPUB 3.0.1
    pub fn epub_version(&mut self, version: EpubVersion) -> &mut Self {
        self.version = version;
        self
    }

    /// Set some EPUB metadata
    ///
    /// # Arguments
    ///
    /// * `metadata`: a (possibly empty) list (or other iterator) of (key, value) tuples
    ///
    /// # Metadata that are used by the EPUB generator
    ///
    /// * `author`: author(s) of the book;
    /// * `title`: title of the book;
    /// * `lang`: the language ot the book, quite important as EPUB renderers rely on it
    ///   for e.g. hyphenating words.
    /// * `subject`;
    /// * `description`;
    /// * `generator`: generator of the book (should be your program name);
    /// * `license`
    /// * `toc_name`
    pub fn metadata<S1: AsRef<str>, S2: Into<String>>(&mut self, key: S1, value: S2) -> Result<&mut Self> {
        match key.as_ref() {
            "author" => self.metadata.author = value.into(),
            "title" => self.metadata.title = value.into(),
            "lang" => self.metadata.lang = value.into(),
            "generator" => self.metadata.generator = value.into(),
            "description" => self.metadata.description = Some(value.into()),
            "subject" => self.metadata.subject = Some(value.into()),
            "license" => self.metadata.license = Some(value.into()),
            "toc_name" => self.metadata.toc_name = value.into(),
            s => bail!("invalid metadata '{}'", s),
        }
        Ok(self)
    }

    /// Sets stylesheet of the EPUB.
    ///
    /// This content will be written in a `stylesheet.css` file; it is used by
    /// some pages (such as nav.xhtml), you don't have use it in your documents though it
    /// makes sense to also do so.
    pub fn stylesheet<R:Read>(&mut self, content: R) -> Result<&mut Self> {
        self.add_resource("stylesheet.css", content, "text/css")?;
        self.stylesheet = true;
        Ok(self)
    }

    /// Adds an inline toc in the document.
    ///
    /// If this method is called it adds a TOC similar to nav.xhtml but listed in the inline document.
    pub fn inline_toc(&mut self) -> &mut Self {
        self.inline_toc = true;
        self.toc.add(TocElement::new("toc.xhtml", self.metadata.toc_name.as_ref()));
        let mut file = Content::new("toc.xhtml", "application/xhtml+xml");
        file.itemref = true;
        self.files.push(file);
        self
    }

    /// Add a resource
    ///
    /// Can be a picture, font, ...
    ///
    /// # Arguments
    ///
    /// * `path`: the path where this file will be writen in the EPUB OEBPS structure,
    ///   e.g. `data/image_0.png`
    /// * `content`: the resource to include
    /// * `mime_type`: the mime type of this file, e.g. "image/png".
    pub fn add_resource<R: Read, P: AsRef<Path>, S: Into<String>>(&mut self,
                                                                     path: P,
                                                                     content: R,
                                                                     mime_type: S) -> Result<&mut Self> {
        self.zip.write_file(Path::new("OEBPS").join(path.as_ref()), content)?;
        self.files.push(Content::new(format!("{}", path.as_ref().display()), mime_type));
        Ok(self)
    }

    /// Add a XHTML content file that will be added to the EPUB.
    ///
    /// # Examples
    ///
    /// ```ignore
    /// epub.add_content(EpubContent::new("intro.xhtml", file));
    /// ```
    ///
    /// ```ignore
    /// epub.add_content(EpubContent::new("chapter_1.xhtml", File::open("chapter_1.xhtml")?)
    ///                      .title("Chapter 1")
    ///                      .child(TocElement::new("chapter_1.xhtml#1", "1.1")));
    /// ```
    ///
    /// * `level`: the level this content will be added in the toc;
    /// * `title`: the title of this content, as it should appear in the TOC;
    /// * `inner_toc`: a table of contents descrbing the inner layout of the content;
    /// * `content`: should be the contents of an XHTML file.
    pub fn add_content<R: Read>(&mut self, content: EpubContent<R>)-> Result<&mut Self> {
        self.zip.write_file(Path::new("OEBPS").join(content.toc.url.as_str()),
                            content.content)?;
        let mut file = Content::new(content.toc.url.as_ref(),
                                "application/xhtml+xml");
        file.itemref = true;
        self.files.push(file);
        if !content.toc.title.is_empty() {
            self.toc.add(content.toc);
        }
        Ok(self)
    }

    /// Generate the EPUB file and write it to the writer
    pub fn generate<W: Write>(&mut self, mut to: W) -> Result<()> {
        // If no styleesheet was provided, generate a dummy one
        if !self.stylesheet {
            self.stylesheet("".as_bytes())?;
        }
        /// Render content.opf
        let bytes = self.render_opf()?;
        self.zip.write_file("OEBPS/content.opf", &bytes as &[u8])?;
        /// Render toc.ncx
        let bytes = self.render_toc()?;
        self.zip.write_file("OEBPS/toc.ncx", &bytes as &[u8])?;
        // Render nav.xhtml
        let bytes = self.render_nav(true)?;
        self.zip.write_file("OEBPS/nav.xhtml", &bytes as &[u8])?;
        // Write inline toc if it needs to
        if self.inline_toc {
            let bytes = self.render_nav(false)?;
            self.zip.write_file("OEBPS/toc.xhtml", &bytes as &[u8])?;
        }

        let res = self.zip.generate()?;
        to.write_all(res.as_ref())
            .chain_err(|| "error writing final result of EPUB content")?;

        Ok(())
    }

    /// Render content.opf file
    fn render_opf(&mut self) -> Result<Vec<u8>> {
        let mut optional = String::new();
        if let Some(ref desc) = self.metadata.description {
            optional.push_str(&format!("<dc:description>{}</dc:description>\n", desc));
        }
        if let Some(ref subject) = self.metadata.subject {
            optional.push_str(&format!("<dc:subject>{}</dc:subject>\n", subject));
        }
        if let Some(ref rights) = self.metadata.license {
            optional.push_str(&format!("<dc:rights>{}</dc:rights>\n", rights));
        }
        let date = chrono::UTC::now().format("%Y-%m-%dT%H:%M:%SZ");
        let uuid = uuid::Uuid::new_v4().urn().to_string();

        let mut items = String::new();
        let mut itemrefs = String::new();

        for content in self.files.iter() {
            items.push_str(&format!("<item media-type = \"{mime}\" \
                                     id = \"{id}\" \
                                     href = \"{href}\" />\n",
                                    mime = content.mime,
                                    id = to_id(&content.file),
                                    href = content.file));
            if content.itemref {
                itemrefs.push_str(&format!("<itemref idref = \"{id}\" />\n",
                                           id = to_id(&content.file)));

            }
        }

        let data = MapBuilder::new()
            .insert_str("lang", &self.metadata.lang)
            .insert_str("author", &self.metadata.author)
            .insert_str("title", &self.metadata.title)
            .insert_str("generator", &self.metadata.generator)
            .insert_str("optional", optional)
            .insert_str("items", items)
            .insert_str("itemrefs", itemrefs)
            .insert_str("date", date)
            .insert_str("uuid", uuid)
            .build();

        let mut content = vec!();
        let res = match self.version {
            EpubVersion::V20 => templates::v2::CONTENT_OPF.render_data(&mut content, &data),
            EpubVersion::V30 => templates::v3::CONTENT_OPF.render_data(&mut content, &data),
            EpubVersion::__NonExhaustive => unreachable!(),
        };

        res
            .chain_err(|| "could not render template for content.opf")?;

        Ok(content)
    }

        /// Render toc.ncx
    fn render_toc(&mut self) -> Result<Vec<u8>> {
        let mut nav_points = String::new();

        nav_points.push_str(&self.toc.render_epub());

        let data = MapBuilder::new()
            .insert_str("toc_name", &self.metadata.toc_name)
            .insert_str("nav_points", nav_points)
            .build();
        let mut res: Vec<u8> = vec![];
        templates::TOC_NCX.render_data(&mut res, &data)
            .chain_err(|| "error rendering toc.ncx template")?;
        Ok(res)
    }

    /// Render nav.xhtml
    fn render_nav(&mut self, numbered: bool) -> Result<Vec<u8>> {
        let content = self.toc.render(numbered);
        let data = MapBuilder::new()
            .insert_str("content", content)
            .insert_str("toc_name", &self.metadata.toc_name)
            .insert_str("generator", &self.metadata.generator)
            .build();

        let mut res = vec!();
        let eh = match self.version {
            EpubVersion::V20 => templates::v2::NAV_XHTML.render_data(&mut res, &data),
            EpubVersion::V30 => templates::v3::NAV_XHTML.render_data(&mut res, &data),
            EpubVersion::__NonExhaustive => unreachable!(),
        };

        eh.chain_err(|| "error rendering nav.xhtml template")?;
        Ok(res)
    }
}

// generate an id compatible string, replacing / and . by _
fn to_id(s: &str) -> String {
    s.replace(".", "_").replace("/", "_")
}
<commit_msg>Document epub.rs file<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with
// this file, You can obtain one at https://mozilla.org/MPL/2.0/.

use templates;
use errors::Result;
use errors::ResultExt;
use zip::Zip;
use toc::Toc;
use toc::TocElement;
use epub_content::EpubContent;

use std::io::Read;
use std::io::Write;
use std::path::Path;

use chrono;
use uuid;
use mustache::MapBuilder;

/// Represents the EPUB version.
///
/// Currently, this library supports EPUB 2.0.1 and 3.0.1.
#[derive(Debug, Copy, Clone)]
pub enum EpubVersion {
    /// EPUB 2.0.1 format
    V20,
    /// EPUB 3.0.1 format
    V30,
    /// Hint that destructuring should not be exhaustive
    #[doc(hidden)]
    __NonExhaustive,
}

/// EPUB Metadata
#[derive(Debug)]
struct Metadata {
    pub title: String,
    pub author: String,
    pub lang: String,
    pub generator: String,
    pub toc_name: String,
    pub description: Option<String>,
    pub subject: Option<String>,
    pub license: Option<String>,
}

impl Metadata {
    /// Create new default metadata
    pub fn new() -> Metadata {
        Metadata {
            title: String::new(),
            author: String::new(),
            lang: String::from("en"),
            generator: String::from("Rust EPUB library"),
            toc_name: String::from("Table Of Contents"),
            description: None,
            subject: None,
            license: None,
        }
    }
}

/// A file added in the EPUB
#[derive(Debug)]
struct Content {
    pub file: String,
    pub mime: String,
    pub itemref: bool,
}

impl Content {
    /// Create a new content file
    pub fn new<S1:Into<String>, S2: Into<String>>(file: S1, mime: S2) -> Content {
        Content {
            file: file.into(),
            mime: mime.into(),
            itemref: false,
        }
    }
}

/// Epub Builder
///
/// The main struct you'll need to use in this library. It is first created using
/// a wrapper to zip files; then you add content to it, and finally you generate
/// the EPUB file by calling the `generate` method.
///
/// ```
/// use epub_builder::EpubBuilder;
/// use epub_builder::ZipCommand;
/// use std::io;
///
/// // "Empty" EPUB file
/// let mut builder = EpubBuilder::new(ZipCommand::new().unwrap()).unwrap();
/// builder.metadata("title", "Empty EPUB").unwrap();
/// builder.generate(&mut io::stdout()).unwrap();
/// ```
#[derive(Debug)]
pub struct EpubBuilder<Z:Zip> {
    version: EpubVersion,
    zip: Z,
    files: Vec<Content>,
    metadata: Metadata,
    toc: Toc,
    stylesheet: bool,
    inline_toc: bool,
}

impl<Z:Zip> EpubBuilder<Z> {
    /// Create a new default EPUB Builder
    pub fn new(zip: Z) -> Result<EpubBuilder<Z>> {
        let mut epub = EpubBuilder {
            version: EpubVersion::V20,
            zip: zip,
            files: vec!(),
            metadata: Metadata::new(),
            toc: Toc::new(),
            stylesheet: false,
            inline_toc: false,
        };

        // Write mimetype upfront
        epub.zip.write_file("mimetype", "application/epub+zip".as_bytes())?;
        epub.zip.write_file("META-INF/container.xml", templates::CONTAINER)?;
        epub.zip.write_file("META-INF/com.apple.ibooks.display-options.xml", templates::IBOOKS)?;

        Ok(epub)
    }

    /// Set EPUB version (default: V20)
    ///
    /// Supported versions are:
    ///
    /// * `V20`: EPUB 2.0.1
    /// * 'V30`: EPUB 3.0.1
    pub fn epub_version(&mut self, version: EpubVersion) -> &mut Self {
        self.version = version;
        self
    }

    /// Set some EPUB metadata
    ///
    /// # Valid keys used by the EPUB builder
    ///
    /// * `author`: author(s) of the book;
    /// * `title`: title of the book;
    /// * `lang`: the language ot the book, quite important as EPUB renderers rely on it
    ///   for e.g. hyphenating words.
    /// * `generator`: generator of the book (should be your program name);
    /// * `toc_name`: the name to use for table of contents (by default, "Table of Contents");
    /// * `subject`;
    /// * `description`;
    /// * `license`.

    pub fn metadata<S1: AsRef<str>, S2: Into<String>>(&mut self, key: S1, value: S2) -> Result<&mut Self> {
        match key.as_ref() {
            "author" => self.metadata.author = value.into(),
            "title" => self.metadata.title = value.into(),
            "lang" => self.metadata.lang = value.into(),
            "generator" => self.metadata.generator = value.into(),
            "description" => self.metadata.description = Some(value.into()),
            "subject" => self.metadata.subject = Some(value.into()),
            "license" => self.metadata.license = Some(value.into()),
            "toc_name" => self.metadata.toc_name = value.into(),
            s => bail!("invalid metadata '{}'", s),
        }
        Ok(self)
    }

    /// Sets stylesheet of the EPUB.
    ///
    /// This content will be written in a `stylesheet.css` file; it is used by
    /// some pages (such as nav.xhtml), you don't have use it in your documents though it
    /// makes sense to also do so.
    pub fn stylesheet<R:Read>(&mut self, content: R) -> Result<&mut Self> {
        self.add_resource("stylesheet.css", content, "text/css")?;
        self.stylesheet = true;
        Ok(self)
    }

    /// Adds an inline toc in the document.
    ///
    /// If this method is called it adds a page that contains the table of contents
    /// that appears in the document.
    ///
    /// The position where this table of contents will be inserted depends on when
    /// you call this method: if you call it before adding any content, it will be
    /// at the beginning, if you call it after, it will be at the end.
    pub fn inline_toc(&mut self) -> &mut Self {
        self.inline_toc = true;
        self.toc.add(TocElement::new("toc.xhtml", self.metadata.toc_name.as_ref()));
        let mut file = Content::new("toc.xhtml", "application/xhtml+xml");
        file.itemref = true;
        self.files.push(file);
        self
    }

    /// Add a resource to the EPUB file
    ///
    /// This resource be a picture, a font, some CSS file, .... Unlike
    /// `add_content`, files added this way won't appear in the linear
    /// document.
    ///
    /// # Arguments
    ///
    /// * `path`: the path where this file will be writen in the EPUB OEBPS structure,
    ///   e.g. `data/image_0.png`
    /// * `content`: the resource to include
    /// * `mime_type`: the mime type of this file, e.g. "image/png".
    pub fn add_resource<R: Read, P: AsRef<Path>, S: Into<String>>(&mut self,
                                                                     path: P,
                                                                     content: R,
                                                                     mime_type: S) -> Result<&mut Self> {
        self.zip.write_file(Path::new("OEBPS").join(path.as_ref()), content)?;
        self.files.push(Content::new(format!("{}", path.as_ref().display()), mime_type));
        Ok(self)
    }

    /// Add a XHTML content file that will be added to the EPUB.
    ///
    /// # Examples
    ///
    /// ```
    /// # use epub_builder::{EpubBuilder, ZipLibrary, EpubContent};
    /// let content = "Some content";
    /// let mut builder = EpubBuilder::new(ZipLibrary::new()).unwrap();
    /// // Add a chapter that won't be added to the Table of Contents
    /// builder.add_content(EpubContent::new("intro.xhtml", content.as_bytes())).unwrap();
    /// ```
    ///
    /// ```
    /// # use epub_builder::{EpubBuilder, ZipLibrary, EpubContent, TocElement};
    /// # let mut builder = EpubBuilder::new(ZipLibrary::new()).unwrap();
    /// # let content = "Some content";
    /// // Sets the title of a chapter so it is added to the Table of contents
    /// // Also add information about its structure
    /// builder.add_content(EpubContent::new("chapter_1.xhtml", content.as_bytes())
    ///                      .title("Chapter 1")
    ///                      .child(TocElement::new("chapter_1.xhtml#1", "1.1"))).unwrap();
    /// ```
    ///
    /// ```
    /// # use epub_builder::{EpubBuilder, ZipLibrary, EpubContent};
    /// # let mut builder = EpubBuilder::new(ZipLibrary::new()).unwrap();
    /// # let content = "Some content";
    /// // Add a section, by setting the level to 2 (instead of the default value 1)
    /// builder.add_content(EpubContent::new("section.xhtml", content.as_bytes())
    ///                      .title("Section 1")
    ///                      .level(2)).unwrap();
    /// ```
    ///
    /// # See also
    ///
    /// [`EpubContent`](struct.EpubContent.html)
    pub fn add_content<R: Read>(&mut self, content: EpubContent<R>)-> Result<&mut Self> {
        self.zip.write_file(Path::new("OEBPS").join(content.toc.url.as_str()),
                            content.content)?;
        let mut file = Content::new(content.toc.url.as_ref(),
                                "application/xhtml+xml");
        file.itemref = true;
        self.files.push(file);
        if !content.toc.title.is_empty() {
            self.toc.add(content.toc);
        }
        Ok(self)
    }

    /// Generate the EPUB file and write it to the writer
    ///
    /// # Example
    ///
    /// ```
    /// # use epub_builder::{EpubBuilder, ZipLibrary};
    /// let mut builder = EpubBuilder::new(ZipLibrary::new()).unwrap();
    /// // Write the EPUB file into a Vec<u8>
    /// let mut epub: Vec<u8> = vec!();
    /// builder.generate(&mut epub).unwrap();
    /// ```
    pub fn generate<W: Write>(&mut self, mut to: W) -> Result<()> {
        // If no styleesheet was provided, generate a dummy one
        if !self.stylesheet {
            self.stylesheet("".as_bytes())?;
        }
        /// Render content.opf
        let bytes = self.render_opf()?;
        self.zip.write_file("OEBPS/content.opf", &bytes as &[u8])?;
        /// Render toc.ncx
        let bytes = self.render_toc()?;
        self.zip.write_file("OEBPS/toc.ncx", &bytes as &[u8])?;
        // Render nav.xhtml
        let bytes = self.render_nav(true)?;
        self.zip.write_file("OEBPS/nav.xhtml", &bytes as &[u8])?;
        // Write inline toc if it needs to
        if self.inline_toc {
            let bytes = self.render_nav(false)?;
            self.zip.write_file("OEBPS/toc.xhtml", &bytes as &[u8])?;
        }

        let res = self.zip.generate()?;
        to.write_all(res.as_ref())
            .chain_err(|| "error writing final result of EPUB content")?;

        Ok(())
    }

    /// Render content.opf file
    fn render_opf(&mut self) -> Result<Vec<u8>> {
        let mut optional = String::new();
        if let Some(ref desc) = self.metadata.description {
            optional.push_str(&format!("<dc:description>{}</dc:description>\n", desc));
        }
        if let Some(ref subject) = self.metadata.subject {
            optional.push_str(&format!("<dc:subject>{}</dc:subject>\n", subject));
        }
        if let Some(ref rights) = self.metadata.license {
            optional.push_str(&format!("<dc:rights>{}</dc:rights>\n", rights));
        }
        let date = chrono::UTC::now().format("%Y-%m-%dT%H:%M:%SZ");
        let uuid = uuid::Uuid::new_v4().urn().to_string();

        let mut items = String::new();
        let mut itemrefs = String::new();

        for content in self.files.iter() {
            items.push_str(&format!("<item media-type = \"{mime}\" \
                                     id = \"{id}\" \
                                     href = \"{href}\" />\n",
                                    mime = content.mime,
                                    id = to_id(&content.file),
                                    href = content.file));
            if content.itemref {
                itemrefs.push_str(&format!("<itemref idref = \"{id}\" />\n",
                                           id = to_id(&content.file)));

            }
        }

        let data = MapBuilder::new()
            .insert_str("lang", &self.metadata.lang)
            .insert_str("author", &self.metadata.author)
            .insert_str("title", &self.metadata.title)
            .insert_str("generator", &self.metadata.generator)
            .insert_str("optional", optional)
            .insert_str("items", items)
            .insert_str("itemrefs", itemrefs)
            .insert_str("date", date)
            .insert_str("uuid", uuid)
            .build();

        let mut content = vec!();
        let res = match self.version {
            EpubVersion::V20 => templates::v2::CONTENT_OPF.render_data(&mut content, &data),
            EpubVersion::V30 => templates::v3::CONTENT_OPF.render_data(&mut content, &data),
            EpubVersion::__NonExhaustive => unreachable!(),
        };

        res
            .chain_err(|| "could not render template for content.opf")?;

        Ok(content)
    }

        /// Render toc.ncx
    fn render_toc(&mut self) -> Result<Vec<u8>> {
        let mut nav_points = String::new();

        nav_points.push_str(&self.toc.render_epub());

        let data = MapBuilder::new()
            .insert_str("toc_name", &self.metadata.toc_name)
            .insert_str("nav_points", nav_points)
            .build();
        let mut res: Vec<u8> = vec![];
        templates::TOC_NCX.render_data(&mut res, &data)
            .chain_err(|| "error rendering toc.ncx template")?;
        Ok(res)
    }

    /// Render nav.xhtml
    fn render_nav(&mut self, numbered: bool) -> Result<Vec<u8>> {
        let content = self.toc.render(numbered);
        let data = MapBuilder::new()
            .insert_str("content", content)
            .insert_str("toc_name", &self.metadata.toc_name)
            .insert_str("generator", &self.metadata.generator)
            .build();

        let mut res = vec!();
        let eh = match self.version {
            EpubVersion::V20 => templates::v2::NAV_XHTML.render_data(&mut res, &data),
            EpubVersion::V30 => templates::v3::NAV_XHTML.render_data(&mut res, &data),
            EpubVersion::__NonExhaustive => unreachable!(),
        };

        eh.chain_err(|| "error rendering nav.xhtml template")?;
        Ok(res)
    }
}

// generate an id compatible string, replacing / and . by _
fn to_id(s: &str) -> String {
    s.replace(".", "_").replace("/", "_")
}
<|endoftext|>


<commit_before>//! This pass checks HIR bodies that may be evaluated at compile-time (e.g., `const`, `static`,
//! `const fn`) for structured control flow (e.g. `if`, `while`), which is forbidden in a const
//! context.
//!
//! By the time the MIR const-checker runs, these high-level constructs have been lowered to
//! control-flow primitives (e.g., `Goto`, `SwitchInt`), making it tough to properly attribute
//! errors. We still look for those primitives in the MIR const-checker to ensure nothing slips
//! through, but errors for structured control flow in a `const` should be emitted here.

use rustc_ast::ast::Mutability;
use rustc_errors::struct_span_err;
use rustc_hir as hir;
use rustc_hir::def_id::DefId;
use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
use rustc_middle::hir::map::Map;
use rustc_middle::ty::query::Providers;
use rustc_middle::ty::TyCtxt;
use rustc_session::config::nightly_options;
use rustc_session::parse::feature_err;
use rustc_span::{sym, Span, Symbol};

use std::fmt;

/// An expression that is not *always* legal in a const context.
#[derive(Clone, Copy)]
enum NonConstExpr {
    Loop(hir::LoopSource),
    Match(hir::MatchSource),
    OrPattern,
}

impl NonConstExpr {
    fn name(self) -> String {
        match self {
            Self::Loop(src) => format!("`{}`", src.name()),
            Self::Match(src) => format!("`{}`", src.name()),
            Self::OrPattern => "or-pattern".to_string(),
        }
    }

    fn required_feature_gates(self) -> Option<&'static [Symbol]> {
        use hir::LoopSource::*;
        use hir::MatchSource::*;

        let gates: &[_] = match self {
            Self::Match(Normal)
            | Self::Match(IfDesugar { .. })
            | Self::Match(IfLetDesugar { .. })
            | Self::OrPattern => &[sym::const_if_match],

            Self::Loop(Loop) => &[sym::const_loop],

            Self::Loop(While)
            | Self::Loop(WhileLet)
            | Self::Match(WhileDesugar)
            | Self::Match(WhileLetDesugar) => &[sym::const_loop, sym::const_if_match],

            // A `for` loop's desugaring contains a call to `IntoIterator::into_iter`,
            // so they are not yet allowed with `#![feature(const_loop)]`.
            _ => return None,
        };

        Some(gates)
    }
}

#[derive(Copy, Clone)]
enum ConstKind {
    Static,
    StaticMut,
    ConstFn,
    Const,
    AnonConst,
}

impl ConstKind {
    fn for_body(body: &hir::Body<'_>, hir_map: Map<'_>) -> Option<Self> {
        let is_const_fn = |id| hir_map.fn_sig_by_hir_id(id).unwrap().header.is_const();

        let owner = hir_map.body_owner(body.id());
        let const_kind = match hir_map.body_owner_kind(owner) {
            hir::BodyOwnerKind::Const => Self::Const,
            hir::BodyOwnerKind::Static(Mutability::Mut) => Self::StaticMut,
            hir::BodyOwnerKind::Static(Mutability::Not) => Self::Static,

            hir::BodyOwnerKind::Fn if is_const_fn(owner) => Self::ConstFn,
            hir::BodyOwnerKind::Fn | hir::BodyOwnerKind::Closure => return None,
        };

        Some(const_kind)
    }
}

impl fmt::Display for ConstKind {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let s = match self {
            Self::Static => "static",
            Self::StaticMut => "static mut",
            Self::Const | Self::AnonConst => "const",
            Self::ConstFn => "const fn",
        };

        write!(f, "{}", s)
    }
}

fn check_mod_const_bodies(tcx: TyCtxt<'_>, module_def_id: DefId) {
    let mut vis = CheckConstVisitor::new(tcx);
    tcx.hir().visit_item_likes_in_module(module_def_id, &mut vis.as_deep_visitor());
}

pub(crate) fn provide(providers: &mut Providers<'_>) {
    *providers = Providers { check_mod_const_bodies, ..*providers };
}

#[derive(Copy, Clone)]
struct CheckConstVisitor<'tcx> {
    tcx: TyCtxt<'tcx>,
    const_kind: Option<ConstKind>,
}

impl<'tcx> CheckConstVisitor<'tcx> {
    fn new(tcx: TyCtxt<'tcx>) -> Self {
        CheckConstVisitor { tcx, const_kind: None }
    }

    /// Emits an error when an unsupported expression is found in a const context.
    fn const_check_violated(&self, expr: NonConstExpr, span: Span) {
        let features = self.tcx.features();
        let required_gates = expr.required_feature_gates();
        match required_gates {
            // Don't emit an error if the user has enabled the requisite feature gates.
            Some(gates) if gates.iter().all(|&g| features.enabled(g)) => return,

            // `-Zunleash-the-miri-inside-of-you` only works for expressions that don't have a
            // corresponding feature gate. This encourages nightly users to use feature gates when
            // possible.
            None if self.tcx.sess.opts.debugging_opts.unleash_the_miri_inside_of_you => {
                self.tcx.sess.span_warn(span, "skipping const checks");
                return;
            }

            _ => {}
        }

        let const_kind = self
            .const_kind
            .expect("`const_check_violated` may only be called inside a const context");
        let msg = format!("{} is not allowed in a `{}`", expr.name(), const_kind);

        let required_gates = required_gates.unwrap_or(&[]);
        let missing_gates: Vec<_> =
            required_gates.iter().copied().filter(|&g| !features.enabled(g)).collect();

        match missing_gates.as_slice() {
            &[] => struct_span_err!(self.tcx.sess, span, E0744, "{}", msg).emit(),

            // If the user enabled `#![feature(const_loop)]` but not `#![feature(const_if_match)]`,
            // explain why their `while` loop is being rejected.
            &[gate @ sym::const_if_match] if required_gates.contains(&sym::const_loop) => {
                feature_err(&self.tcx.sess.parse_sess, gate, span, &msg)
                    .note(
                        "`#![feature(const_loop)]` alone is not sufficient, \
                           since this loop expression contains an implicit conditional",
                    )
                    .emit();
            }

            &[missing_primary, ref missing_secondary @ ..] => {
                let mut err = feature_err(&self.tcx.sess.parse_sess, missing_primary, span, &msg);

                // If multiple feature gates would be required to enable this expression, include
                // them as help messages. Don't emit a separate error for each missing feature gate.
                //
                // FIXME(ecstaticmorse): Maybe this could be incorporated into `feature_err`? This
                // is a pretty narrow case, however.
                if nightly_options::is_nightly_build() {
                    for gate in missing_secondary {
                        let note = format!(
                            "add `#![feature({})]` to the crate attributes to enable",
                            gate,
                        );
                        err.help(&note);
                    }
                }

                err.emit();
            }
        }
    }

    /// Saves the parent `const_kind` before calling `f` and restores it afterwards.
    fn recurse_into(&mut self, kind: Option<ConstKind>, f: impl FnOnce(&mut Self)) {
        let parent_kind = self.const_kind;
        self.const_kind = kind;
        f(self);
        self.const_kind = parent_kind;
    }
}

impl<'tcx> Visitor<'tcx> for CheckConstVisitor<'tcx> {
    type Map = Map<'tcx>;

    fn nested_visit_map(&mut self) -> intravisit::NestedVisitorMap<Self::Map> {
        NestedVisitorMap::OnlyBodies(self.tcx.hir())
    }

    fn visit_anon_const(&mut self, anon: &'tcx hir::AnonConst) {
        let kind = Some(ConstKind::AnonConst);
        self.recurse_into(kind, |this| intravisit::walk_anon_const(this, anon));
    }

    fn visit_body(&mut self, body: &'tcx hir::Body<'tcx>) {
        let kind = ConstKind::for_body(body, self.tcx.hir());
        self.recurse_into(kind, |this| intravisit::walk_body(this, body));
    }

    fn visit_pat(&mut self, p: &'tcx hir::Pat<'tcx>) {
        if self.const_kind.is_some() {
            if let hir::PatKind::Or { .. } = p.kind {
                self.const_check_violated(NonConstExpr::OrPattern, p.span);
            }
        }
        intravisit::walk_pat(self, p)
    }

    fn visit_expr(&mut self, e: &'tcx hir::Expr<'tcx>) {
        match &e.kind {
            // Skip the following checks if we are not currently in a const context.
            _ if self.const_kind.is_none() => {}

            hir::ExprKind::Loop(_, _, source) => {
                self.const_check_violated(NonConstExpr::Loop(*source), e.span);
            }

            hir::ExprKind::Match(_, _, source) => {
                let non_const_expr = match source {
                    // These are handled by `ExprKind::Loop` above.
                    hir::MatchSource::WhileDesugar
                    | hir::MatchSource::WhileLetDesugar
                    | hir::MatchSource::ForLoopDesugar => None,

                    _ => Some(NonConstExpr::Match(*source)),
                };

                if let Some(expr) = non_const_expr {
                    self.const_check_violated(expr, e.span);
                }
            }

            _ => {}
        }

        intravisit::walk_expr(self, e);
    }
}
<commit_msg>Use `is_const_fn_raw` query for HIR const-checking<commit_after>//! This pass checks HIR bodies that may be evaluated at compile-time (e.g., `const`, `static`,
//! `const fn`) for structured control flow (e.g. `if`, `while`), which is forbidden in a const
//! context.
//!
//! By the time the MIR const-checker runs, these high-level constructs have been lowered to
//! control-flow primitives (e.g., `Goto`, `SwitchInt`), making it tough to properly attribute
//! errors. We still look for those primitives in the MIR const-checker to ensure nothing slips
//! through, but errors for structured control flow in a `const` should be emitted here.

use rustc_ast::ast::Mutability;
use rustc_errors::struct_span_err;
use rustc_hir as hir;
use rustc_hir::def_id::DefId;
use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
use rustc_middle::hir::map::Map;
use rustc_middle::ty::query::Providers;
use rustc_middle::ty::TyCtxt;
use rustc_session::config::nightly_options;
use rustc_session::parse::feature_err;
use rustc_span::{sym, Span, Symbol};

use std::fmt;

/// An expression that is not *always* legal in a const context.
#[derive(Clone, Copy)]
enum NonConstExpr {
    Loop(hir::LoopSource),
    Match(hir::MatchSource),
    OrPattern,
}

impl NonConstExpr {
    fn name(self) -> String {
        match self {
            Self::Loop(src) => format!("`{}`", src.name()),
            Self::Match(src) => format!("`{}`", src.name()),
            Self::OrPattern => "or-pattern".to_string(),
        }
    }

    fn required_feature_gates(self) -> Option<&'static [Symbol]> {
        use hir::LoopSource::*;
        use hir::MatchSource::*;

        let gates: &[_] = match self {
            Self::Match(Normal)
            | Self::Match(IfDesugar { .. })
            | Self::Match(IfLetDesugar { .. })
            | Self::OrPattern => &[sym::const_if_match],

            Self::Loop(Loop) => &[sym::const_loop],

            Self::Loop(While)
            | Self::Loop(WhileLet)
            | Self::Match(WhileDesugar)
            | Self::Match(WhileLetDesugar) => &[sym::const_loop, sym::const_if_match],

            // A `for` loop's desugaring contains a call to `IntoIterator::into_iter`,
            // so they are not yet allowed with `#![feature(const_loop)]`.
            _ => return None,
        };

        Some(gates)
    }
}

#[derive(Copy, Clone)]
enum ConstKind {
    Static,
    StaticMut,
    ConstFn,
    Const,
    AnonConst,
}

impl ConstKind {
    fn for_body(body: &hir::Body<'_>, tcx: TyCtxt<'_>) -> Option<Self> {
        let owner = tcx.hir().body_owner(body.id());
        let const_kind = match tcx.hir().body_owner_kind(owner) {
            hir::BodyOwnerKind::Const => Self::Const,
            hir::BodyOwnerKind::Static(Mutability::Mut) => Self::StaticMut,
            hir::BodyOwnerKind::Static(Mutability::Not) => Self::Static,

            hir::BodyOwnerKind::Fn if tcx.is_const_fn_raw(tcx.hir().local_def_id(owner)) => {
                Self::ConstFn
            }
            hir::BodyOwnerKind::Fn | hir::BodyOwnerKind::Closure => return None,
        };

        Some(const_kind)
    }
}

impl fmt::Display for ConstKind {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let s = match self {
            Self::Static => "static",
            Self::StaticMut => "static mut",
            Self::Const | Self::AnonConst => "const",
            Self::ConstFn => "const fn",
        };

        write!(f, "{}", s)
    }
}

fn check_mod_const_bodies(tcx: TyCtxt<'_>, module_def_id: DefId) {
    let mut vis = CheckConstVisitor::new(tcx);
    tcx.hir().visit_item_likes_in_module(module_def_id, &mut vis.as_deep_visitor());
}

pub(crate) fn provide(providers: &mut Providers<'_>) {
    *providers = Providers { check_mod_const_bodies, ..*providers };
}

#[derive(Copy, Clone)]
struct CheckConstVisitor<'tcx> {
    tcx: TyCtxt<'tcx>,
    const_kind: Option<ConstKind>,
}

impl<'tcx> CheckConstVisitor<'tcx> {
    fn new(tcx: TyCtxt<'tcx>) -> Self {
        CheckConstVisitor { tcx, const_kind: None }
    }

    /// Emits an error when an unsupported expression is found in a const context.
    fn const_check_violated(&self, expr: NonConstExpr, span: Span) {
        let features = self.tcx.features();
        let required_gates = expr.required_feature_gates();
        match required_gates {
            // Don't emit an error if the user has enabled the requisite feature gates.
            Some(gates) if gates.iter().all(|&g| features.enabled(g)) => return,

            // `-Zunleash-the-miri-inside-of-you` only works for expressions that don't have a
            // corresponding feature gate. This encourages nightly users to use feature gates when
            // possible.
            None if self.tcx.sess.opts.debugging_opts.unleash_the_miri_inside_of_you => {
                self.tcx.sess.span_warn(span, "skipping const checks");
                return;
            }

            _ => {}
        }

        let const_kind = self
            .const_kind
            .expect("`const_check_violated` may only be called inside a const context");
        let msg = format!("{} is not allowed in a `{}`", expr.name(), const_kind);

        let required_gates = required_gates.unwrap_or(&[]);
        let missing_gates: Vec<_> =
            required_gates.iter().copied().filter(|&g| !features.enabled(g)).collect();

        match missing_gates.as_slice() {
            &[] => struct_span_err!(self.tcx.sess, span, E0744, "{}", msg).emit(),

            // If the user enabled `#![feature(const_loop)]` but not `#![feature(const_if_match)]`,
            // explain why their `while` loop is being rejected.
            &[gate @ sym::const_if_match] if required_gates.contains(&sym::const_loop) => {
                feature_err(&self.tcx.sess.parse_sess, gate, span, &msg)
                    .note(
                        "`#![feature(const_loop)]` alone is not sufficient, \
                           since this loop expression contains an implicit conditional",
                    )
                    .emit();
            }

            &[missing_primary, ref missing_secondary @ ..] => {
                let mut err = feature_err(&self.tcx.sess.parse_sess, missing_primary, span, &msg);

                // If multiple feature gates would be required to enable this expression, include
                // them as help messages. Don't emit a separate error for each missing feature gate.
                //
                // FIXME(ecstaticmorse): Maybe this could be incorporated into `feature_err`? This
                // is a pretty narrow case, however.
                if nightly_options::is_nightly_build() {
                    for gate in missing_secondary {
                        let note = format!(
                            "add `#![feature({})]` to the crate attributes to enable",
                            gate,
                        );
                        err.help(&note);
                    }
                }

                err.emit();
            }
        }
    }

    /// Saves the parent `const_kind` before calling `f` and restores it afterwards.
    fn recurse_into(&mut self, kind: Option<ConstKind>, f: impl FnOnce(&mut Self)) {
        let parent_kind = self.const_kind;
        self.const_kind = kind;
        f(self);
        self.const_kind = parent_kind;
    }
}

impl<'tcx> Visitor<'tcx> for CheckConstVisitor<'tcx> {
    type Map = Map<'tcx>;

    fn nested_visit_map(&mut self) -> intravisit::NestedVisitorMap<Self::Map> {
        NestedVisitorMap::OnlyBodies(self.tcx.hir())
    }

    fn visit_anon_const(&mut self, anon: &'tcx hir::AnonConst) {
        let kind = Some(ConstKind::AnonConst);
        self.recurse_into(kind, |this| intravisit::walk_anon_const(this, anon));
    }

    fn visit_body(&mut self, body: &'tcx hir::Body<'tcx>) {
        let kind = ConstKind::for_body(body, self.tcx);
        self.recurse_into(kind, |this| intravisit::walk_body(this, body));
    }

    fn visit_pat(&mut self, p: &'tcx hir::Pat<'tcx>) {
        if self.const_kind.is_some() {
            if let hir::PatKind::Or { .. } = p.kind {
                self.const_check_violated(NonConstExpr::OrPattern, p.span);
            }
        }
        intravisit::walk_pat(self, p)
    }

    fn visit_expr(&mut self, e: &'tcx hir::Expr<'tcx>) {
        match &e.kind {
            // Skip the following checks if we are not currently in a const context.
            _ if self.const_kind.is_none() => {}

            hir::ExprKind::Loop(_, _, source) => {
                self.const_check_violated(NonConstExpr::Loop(*source), e.span);
            }

            hir::ExprKind::Match(_, _, source) => {
                let non_const_expr = match source {
                    // These are handled by `ExprKind::Loop` above.
                    hir::MatchSource::WhileDesugar
                    | hir::MatchSource::WhileLetDesugar
                    | hir::MatchSource::ForLoopDesugar => None,

                    _ => Some(NonConstExpr::Match(*source)),
                };

                if let Some(expr) = non_const_expr {
                    self.const_check_violated(expr, e.span);
                }
            }

            _ => {}
        }

        intravisit::walk_expr(self, e);
    }
}
<|endoftext|>


<commit_before>use std::io::{Error, ErrorKind, Read};
use std::io::{BufReader, BufRead};
use std::io::prelude::*;
use std::iter::Iterator;
use std::fs::File;
use std::path::Path;
use std::string::String;
use std::convert::*;

extern crate csv; // TODO:why do I need to this here as well as top level lib.rs ??

use super::csv::StringRecord;

use super::rel::*;

#[derive(Debug)]
pub enum ExecutionError {
    IoError(Error),
    Custom(String)
}

impl From<Error> for ExecutionError {
    fn from(e: Error) -> Self {
        ExecutionError::IoError(e)
    }
}

#[derive(Debug)]
struct InMemoryRelation<'a> {
    tuples: &'a Vec<Tuple>,
    schema: &'a TupleType
}

/// Represents a csv file with a known schema
#[derive(Debug)]
pub struct CsvRelation {
    file: File,
    schema: TupleType
}

//pub struct FilterRelation<'a> {
//    schema: &'a TupleType,
//    input: Box<SimpleRelation<'a>>,
//    expr: &'a Rex
//}

impl<'a> CsvRelation {

    pub fn open(file: File, schema: TupleType) -> Result<Self,ExecutionError> {
        Ok(CsvRelation { file, schema })
    }

    /// Convert StringRecord into our internal tuple type based on the known schema
    fn create_tuple(&self, r: &StringRecord) -> Result<Tuple,ExecutionError> {
        assert_eq!(self.schema.columns.len(), r.len());
        let values = self.schema.columns.iter().zip(r.into_iter()).map(|(c,s)| match c.data_type {
            //TODO: remove unwrap use here
            DataType::UnsignedLong => Value::UnsignedLong(s.parse::<u64>().unwrap()),
            DataType::String => Value::String(s.to_string()),
            DataType::Double => Value::Double(s.parse::<f64>().unwrap()),
        }).collect();
        Ok(Tuple::new(values))
    }
}

pub trait SimpleRelation<'a> {

    /// scan all records in this relation
    fn scan(&'a self) -> Box<Iterator<Item=Result<Tuple,ExecutionError>> + 'a>;

    fn schema(&'a self) -> &'a TupleType;
}

impl<'a> SimpleRelation<'a> for CsvRelation {

    fn scan(&'a self) -> Box<Iterator<Item=Result<Tuple,ExecutionError>> + 'a> {

        //let CsvRelation { file, schema } = *self;

        let buf_reader = BufReader::new(&self.file);
        let csv_reader = csv::Reader::from_reader(buf_reader);
        let record_iter = csv_reader.into_records();

        let tuple_iter = record_iter.map(move|r| match r {
            Ok(record) => self.create_tuple(&record),
            Err(_) => Err(ExecutionError::Custom("TODO".to_string()))
        });

        Box::new(tuple_iter)
    }

    fn schema(&'a self) -> &'a TupleType {
        &self.schema
    }

}

impl<'a> SimpleRelation<'a> for InMemoryRelation<'a> {

    fn scan(&'a self) -> Box<Iterator<Item=Result<Tuple,ExecutionError>> + 'a> {
        let tuple_iter = self.tuples.iter().map(move |t| Ok(t.clone()));
        Box::new(tuple_iter)
    }

    fn schema(&'a self) -> &'a TupleType {
        &self.schema
    }

}

//impl<'a> SimpleRelation<'a> for FilterRelation<'a> {
//
//    fn scan(&'a self) -> Box<Iterator<Item=Result<Tuple, ExecutionError>> + 'a> {
//        Box::new(self.input.scan().filter(|t| {
//            //let x = evaluate(t, &self.schema, &self.expr);
//            //TODO:
//            true
//        }))
//    }
//
//    fn schema(&'a self) -> &'a TupleType {
//        unimplemented!()
//    }
//}
//

pub fn create_execution_plan<'a>(plan: &'a Rel) -> Result<Box<SimpleRelation + 'a>,ExecutionError> {
    match plan {

        &Rel::CsvFile { ref filename, ref schema } => {
            let file = File::open(filename)?;
            let rel = CsvRelation::open(file, schema.clone())?;
            Ok(Box::new(rel))
        },

//        &Rel::Selection { ref expr, ref input, ref schema } => {
//            let input_rel = execute(&input)?;
//            Ok(Box::new(FilterRelation {
//                input: input_rel,
//                expr: expr,
//                schema: schema
//            }))
//        },

        _ => Err(ExecutionError::Custom("not implemented".to_string()))
    }
}

/// Evaluate a relational expression against a tuple
pub fn evaluate(tuple: &Tuple, tt: &TupleType, rex: &Rex) -> Result<Value, Box<Error>> {

    match rex {
        &Rex::BinaryExpr { box ref left, ref op, box ref right } => {
            let left_value = evaluate(tuple, tt, left)?;
            let right_value = evaluate(tuple, tt, right)?;
            match op {
                &Operator::Eq => Ok(Value::Boolean(left_value == right_value)),
                &Operator::NotEq => Ok(Value::Boolean(left_value != right_value)),
                &Operator::Lt => Ok(Value::Boolean(left_value < right_value)),
                &Operator::LtEq => Ok(Value::Boolean(left_value <= right_value)),
                &Operator::Gt => Ok(Value::Boolean(left_value > right_value)),
                &Operator::GtEq => Ok(Value::Boolean(left_value >= right_value)),
            }
        },
        &Rex::TupleValue(index) => Ok(tuple.values[index].clone()),
        &Rex::Literal(ref value) => Ok(value.clone()),
    }

}

<commit_msg>simple_select works<commit_after>use std::io::{Error, ErrorKind, Read};
use std::io::{BufReader, BufRead};
use std::io::prelude::*;
use std::iter::Iterator;
use std::fs::File;
use std::path::Path;
use std::string::String;
use std::convert::*;

extern crate csv; // TODO:why do I need to this here as well as top level lib.rs ??

use super::csv::StringRecord;

use super::rel::*;

#[derive(Debug)]
pub enum ExecutionError {
    IoError(Error),
    Custom(String)
}

impl From<Error> for ExecutionError {
    fn from(e: Error) -> Self {
        ExecutionError::IoError(e)
    }
}

//#[derive(Debug)]
//struct InMemoryRelation<'a> {
//    tuples: &'a Vec<Tuple>,
//    schema: &'a TupleType
//}

/// Represents a csv file with a known schema
#[derive(Debug)]
pub struct CsvRelation {
    file: File,
    schema: TupleType
}

pub struct FilterRelation {
    schema: TupleType,
    input: Box<SimpleRelation>,
    expr: Rex
}

impl<'a> CsvRelation {

    pub fn open(file: File, schema: TupleType) -> Result<Self,ExecutionError> {
        Ok(CsvRelation { file, schema })
    }

    /// Convert StringRecord into our internal tuple type based on the known schema
    fn create_tuple(&self, r: &StringRecord) -> Result<Tuple,ExecutionError> {
        assert_eq!(self.schema.columns.len(), r.len());
        let values = self.schema.columns.iter().zip(r.into_iter()).map(|(c,s)| match c.data_type {
            //TODO: remove unwrap use here
            DataType::UnsignedLong => Value::UnsignedLong(s.parse::<u64>().unwrap()),
            DataType::String => Value::String(s.to_string()),
            DataType::Double => Value::Double(s.parse::<f64>().unwrap()),
        }).collect();
        Ok(Tuple::new(values))
    }
}

pub trait SimpleRelation {

    /// scan all records in this relation
    fn scan<'a>(&'a self) -> Box<Iterator<Item=Result<Tuple,ExecutionError>> + 'a>;

    //fn schema(&'a self) -> &'a TupleType;
}

impl SimpleRelation for CsvRelation {

    fn scan<'a>(&'a self) -> Box<Iterator<Item=Result<Tuple,ExecutionError>> + 'a> {

        //let CsvRelation { file, schema } = *self;

        let buf_reader = BufReader::new(&self.file);
        let csv_reader = csv::Reader::from_reader(buf_reader);
        let record_iter = csv_reader.into_records();

        let tuple_iter = record_iter.map(move|r| match r {
            Ok(record) => self.create_tuple(&record),
            Err(_) => Err(ExecutionError::Custom("TODO".to_string()))
        });

        Box::new(tuple_iter)
    }

//    fn schema(&'a self) -> &'a TupleType {
//        &self.schema
//    }

}

//impl<'a> SimpleRelation<'a> for InMemoryRelation<'a> {
//
//    fn scan<'a>(&'a self) -> Box<Iterator<Item=Result<Tuple,ExecutionError>> + 'a> {
//        let tuple_iter = self.tuples.iter().map(move |t| Ok(t.clone()));
//        Box::new(tuple_iter)
//    }
//
////    fn schema(&'a self) -> &'a TupleType {
////        &self.schema
////    }
//
//}

impl SimpleRelation for FilterRelation {

    fn scan<'a>(&'a self) -> Box<Iterator<Item=Result<Tuple, ExecutionError>> + 'a> {
        Box::new(self.input.scan().filter(|t| {
            //let x = evaluate(t, &self.schema, &self.expr);
            //TODO:
            true
        }))
    }

}

pub fn create_execution_plan<'a>(plan: &'a Rel) -> Result<Box<SimpleRelation + 'a>,ExecutionError> {
    match *plan {

        Rel::CsvFile { ref filename, ref schema } => {
            let file = File::open(filename)?;
            let rel = CsvRelation::open(file, schema.clone())?;
            Ok(Box::new(rel))
        },

//        Rel::Selection { expr, ref input, schema } => {
//            let input_rel = create_execution_plan(input)?;
//            Ok(Box::new(FilterRelation {
//                input: input_rel,
//                expr: expr,
//                schema: schema
//            }))
//        },

        _ => Err(ExecutionError::Custom("not implemented".to_string()))
    }
}

/// Evaluate a relational expression against a tuple
pub fn evaluate(tuple: &Tuple, tt: &TupleType, rex: &Rex) -> Result<Value, Box<Error>> {

    match rex {
        &Rex::BinaryExpr { box ref left, ref op, box ref right } => {
            let left_value = evaluate(tuple, tt, left)?;
            let right_value = evaluate(tuple, tt, right)?;
            match op {
                &Operator::Eq => Ok(Value::Boolean(left_value == right_value)),
                &Operator::NotEq => Ok(Value::Boolean(left_value != right_value)),
                &Operator::Lt => Ok(Value::Boolean(left_value < right_value)),
                &Operator::LtEq => Ok(Value::Boolean(left_value <= right_value)),
                &Operator::Gt => Ok(Value::Boolean(left_value > right_value)),
                &Operator::GtEq => Ok(Value::Boolean(left_value >= right_value)),
            }
        },
        &Rex::TupleValue(index) => Ok(tuple.values[index].clone()),
        &Rex::Literal(ref value) => Ok(value.clone()),
    }

}

<|endoftext|>


<commit_before>//! Defines the trait that must be implemented by ISPC task execution systems
//! and provides a default threaded one for use.

use libc;
use num_cpus;

use std::mem;
use std::cell::RefCell;
use std::sync::{Arc, RwLock, Mutex};
use std::sync::atomic::{self, AtomicUsize};
use std::thread::{self, JoinHandle};

use task::{ISPCTaskFn, Context};

/// Trait to be implemented to provide ISPC task execution functionality.
///
/// The runtime [required functions](http://ispc.github.io/ispc.html#task-parallelism-runtime-requirements)
/// will be forwarded directly to your struct, making this interface unsafe.
pub trait TaskSystem {
    /// Alloc is called when memory must be allocated to store parameters to pass to a task
    /// and must return a pointer to an allocation of `size` bytes aligned to `align`.
    ///
    /// The `handle_ptr` will be `NULL` if this is the first time launch has been called in
    /// the function or is the first launch call after an explicit `sync` statement. Both
    /// situations should be treated equivalently as creating a new exeuction context for tasks.
    /// The `handle_ptr` should be set to some context tracking facility so that you can later
    /// track task groups launched in the context and perform finer grained synchronization in
    /// `sync`.
    unsafe fn alloc(&self, handle_ptr: *mut *mut libc::c_void, size: i64, align: i32) -> *mut libc::c_void;
    /// Launch is called when a new group of tasks is being launched and should schedule them to
    /// be executed in some way.
    ///
    /// The `handle_ptr` will point to the same handle you set up in `alloc` and can be used to
    /// associate groups of tasks with a context of execution as mentioned before. The function `f`
    /// should be executed `count0 * count1 * count2` times and indices passed to the function
    /// should be as if running in a nested for loop:
    ///
    /// ```ignore
    /// let total_tasks = count0 * count1 * count2;
    /// for z in 0..count2 {
    ///     for y in 0..count1 {
    ///         for x in 0..count0 {
    ///             let task_id = x + y * count0 + z * count0 * count1;
    ///             f(data, thread_id, total_threads, task_id, total_tasks,
    ///               x, y, z, count0, count1, count2);
    ///         }
    ///     }
    /// }
    /// ```
    ///
    /// The `data` pointer points to the ISPC task specific parameter pointer and should be passed
    /// through to the function.
    unsafe fn launch(&self, handle_ptr: *mut *mut libc::c_void, f: ISPCTaskFn, data: *mut libc::c_void,
                     count0: i32, count1: i32, count2: i32);
    /// Synchronize an execution context with the tasks it's launched. Use `handle` to determine
    /// the task context being synchronized.
    ///
    /// This function should not return until all tasks launched within the context being
    /// synchronized with have been completed. You can use the `handle` to determine which context
    /// is being synchronized with and thus which tasks must be completed before returning.
    unsafe fn sync(&self, handle: *mut libc::c_void);
}

/// Thread local storage to store the thread's id, otherwise we don't know
/// who we are in sync. The thread id starts at an invalid value but will be set
/// upon thread launch
thread_local!(static THREAD_ID: RefCell<usize> = RefCell::new(0));

/// A multithreaded execution environment for the tasks.
pub struct Parallel {
    context_list: RwLock<Vec<Arc<Context>>>,
    next_context_id: AtomicUsize,
    threads: Mutex<Vec<JoinHandle<()>>>,
}

impl Parallel {
    pub fn new() -> Parallel {
        let par = Parallel { context_list: RwLock::new(Vec::new()),
                             next_context_id: AtomicUsize::new(0),
                             threads: Mutex::new(Vec::new()) };
        {
            let mut threads = par.threads.lock().unwrap();
            let num_threads = num_cpus::get();
            println!("Launching {} threads", num_threads);
            for i in 0..num_threads {
                // Note that the spawned thread ids start at 1 since the main thread is 0
                threads.push(thread::spawn(move || worker_thread(i + 1, num_threads + 1)));
            }
        }
        par
    }
}

impl TaskSystem for Parallel {
    unsafe fn alloc(&self, handle_ptr: *mut *mut libc::c_void, size: i64, align: i32) -> *mut libc::c_void {
        println!("ISPCAlloc, size: {}, align: {}", size, align);
        // If the handle is null this is the first time this function has spawned tasks
        // and we should create a new Context structure in the TASK_LIST for it, otherwise
        // it's the pointer to where we should append the new Group
        let mut context_list = self.context_list.write().unwrap();
        let context = if (*handle_ptr).is_null() {
            println!("handle ptr is null");
            // This is a bit hairy. We allocate the new task context in a box, then
            // unbox it into a raw ptr to get a ptr we can pass back to ISPC through
            // the handle_ptr and then re-box it into our TASK_LIST so it will
            // be free'd properly when we erase it from the vector in ISPCSync
            let c = Arc::new(Context::new(self.next_context_id.fetch_add(1, atomic::Ordering::SeqCst)));
            {
                let h = &*c;
                *handle_ptr = mem::transmute(h);
            }
            context_list.push(c);
            context_list.last_mut().unwrap()
        } else {
            // TODO: I'm not sure if it'd be worth optimizing this path to just require a read lock
            println!("handle ptr is not null");
            let handle_ctx: *mut Context = mem::transmute(*handle_ptr);
            context_list.iter_mut().find(|c| (*handle_ctx).id == c.id).unwrap()
        };
        println!("context.id = {}", context.id);
        context.alloc(size as usize, align as usize)
    }
    unsafe fn launch(&self, handle_ptr: *mut *mut libc::c_void, f: ISPCTaskFn, data: *mut libc::c_void,
                     count0: i32, count1: i32, count2: i32) {
        // Push the tasks being launched on to the list of task groups for this function
        let context: &mut Context = mem::transmute(*handle_ptr);
        println!("ISPCLaunch, context.id = {}, counts: [{}, {}, {}]", context.id, count0, count1, count2);
        context.launch((count0, count1, count2), data, f);
        // Unpark any sleeping threads since we have jobs for them
        let threads = self.threads.lock().unwrap();
        for t in threads.iter() {
            t.thread().unpark();
        }
    }
    unsafe fn sync(&self, handle: *mut libc::c_void) {
        let context: &mut Context = mem::transmute(handle);
        let thread = THREAD_ID.with(|f| *f.borrow());
        let total_threads = num_cpus::get();
        println!("Thread {} is syncing", thread);
        // Make sure all tasks are done, and execute them if not for this simple
        // serial version. TODO: In the future we'd wait on each Group's semaphore or atomic bool
        // Maybe the waiting thread could help execute tasks as well, otherwise it might be
        // possible to deadlock, where all threads are waiting for some enqueue'd tasks but no
        // threads are available to run them. Just running tasks in our context is not sufficient
        // to prevent deadlock actually, because those tasks could in turn launch & sync and get stuck
        // so if our tasks aren't done and there's none left to run in our context we should start
        // running tasks from other contexts to help out
        println!("ISPCSync, context.id = {}", context.id);
        for tg in context.iter() {
            for chunk in tg.chunks(1) {
                println!("syncing thread {} running local chunk {:?}", thread, chunk);
                // TODO: We need to figure out which thread we are
                chunk.execute(thread as i32, total_threads as i32);
            }
        }
        // If all the tasks for this context have been finished we're done sync'ing and can
        // clean up memory and remove the context from the TASK_LIST. Otherwise there are some
        // unfinished groups further down the the tree that were spawned by our direct tasks that
        // those are now sync'ing on and we need to help out. However since we don't know the tree
        // our best option is to just start grabbing chunks from unfinished groups in the TASK_LIST
        // and running them to at least ensure global forward progress, which will eventually get
        // the stuff we're waiting on to finish. After each chunk execution we should check if
        // our sync'ing context is done and break
        while !context.current_tasks_done() {
            println!("Not all tasks for context {} are done, thread {} helping out!", context.id, thread);
            // Get a task group to run
            let ctx = {
                // Do it in a block to limit the scope of the lock
                ::get_task_system().context_list.read().unwrap().iter()
                    .find(|c| !c.current_tasks_done()).map(|c| c.clone())
            };
            if let Some(c) = ctx {
                println!("syncing thread {} got a context {:?}", thread, c);
                for tg in c.iter() {
                    println!("syncing thread {} looking at task group {:?}", thread, tg);
                    for chunk in tg.chunks(1) {
                        println!("syncing thread {} running nonlocal chunk {:?}", thread, chunk);
                        chunk.execute(thread as i32, total_threads as i32);
                    }
                }
            } else {
                println!("Syncing thread didn't get a context");
            }
        }
        // Now erase this context from our vector
        let mut context_list = self.context_list.write().unwrap();
        let pos = context_list.iter().position(|c| context.id == c.id).unwrap();
        context_list.remove(pos);
    }
}

fn worker_thread(thread: usize, total_threads: usize) {
    THREAD_ID.with(|f| *f.borrow_mut() = thread);
    println!("thread {} is spawned", thread);
    loop {
        thread::park();
        println!("thread {} was woken up to work!", thread);
        // Get a task group to run
        let ctx = {
            // Do it in a block to limit the scope of the lock
            ::get_task_system().context_list.read().unwrap().iter()
                .find(|c| !c.current_tasks_done()).map(|c| c.clone())
        };
        if let Some(c) = ctx {
            for tg in c.iter() {
                for chunk in tg.chunks(1) {
                    println!("thread {} running chunk {:?}", thread, chunk);
                    chunk.execute(thread as i32, total_threads as i32);
                }
            }
        }
    }
}

<commit_msg>Get and cache the task system in the worker thread<commit_after>//! Defines the trait that must be implemented by ISPC task execution systems
//! and provides a default threaded one for use.

use libc;
use num_cpus;

use std::mem;
use std::cell::RefCell;
use std::sync::{Arc, RwLock, Mutex};
use std::sync::atomic::{self, AtomicUsize};
use std::thread::{self, JoinHandle};

use task::{ISPCTaskFn, Context};

/// Trait to be implemented to provide ISPC task execution functionality.
///
/// The runtime [required functions](http://ispc.github.io/ispc.html#task-parallelism-runtime-requirements)
/// will be forwarded directly to your struct, making this interface unsafe.
pub trait TaskSystem {
    /// Alloc is called when memory must be allocated to store parameters to pass to a task
    /// and must return a pointer to an allocation of `size` bytes aligned to `align`.
    ///
    /// The `handle_ptr` will be `NULL` if this is the first time launch has been called in
    /// the function or is the first launch call after an explicit `sync` statement. Both
    /// situations should be treated equivalently as creating a new exeuction context for tasks.
    /// The `handle_ptr` should be set to some context tracking facility so that you can later
    /// track task groups launched in the context and perform finer grained synchronization in
    /// `sync`.
    unsafe fn alloc(&self, handle_ptr: *mut *mut libc::c_void, size: i64, align: i32) -> *mut libc::c_void;
    /// Launch is called when a new group of tasks is being launched and should schedule them to
    /// be executed in some way.
    ///
    /// The `handle_ptr` will point to the same handle you set up in `alloc` and can be used to
    /// associate groups of tasks with a context of execution as mentioned before. The function `f`
    /// should be executed `count0 * count1 * count2` times and indices passed to the function
    /// should be as if running in a nested for loop:
    ///
    /// ```ignore
    /// let total_tasks = count0 * count1 * count2;
    /// for z in 0..count2 {
    ///     for y in 0..count1 {
    ///         for x in 0..count0 {
    ///             let task_id = x + y * count0 + z * count0 * count1;
    ///             f(data, thread_id, total_threads, task_id, total_tasks,
    ///               x, y, z, count0, count1, count2);
    ///         }
    ///     }
    /// }
    /// ```
    ///
    /// The `data` pointer points to the ISPC task specific parameter pointer and should be passed
    /// through to the function.
    unsafe fn launch(&self, handle_ptr: *mut *mut libc::c_void, f: ISPCTaskFn, data: *mut libc::c_void,
                     count0: i32, count1: i32, count2: i32);
    /// Synchronize an execution context with the tasks it's launched. Use `handle` to determine
    /// the task context being synchronized.
    ///
    /// This function should not return until all tasks launched within the context being
    /// synchronized with have been completed. You can use the `handle` to determine which context
    /// is being synchronized with and thus which tasks must be completed before returning.
    unsafe fn sync(&self, handle: *mut libc::c_void);
}

/// Thread local storage to store the thread's id, otherwise we don't know
/// who we are in sync. The thread id starts at an invalid value but will be set
/// upon thread launch
thread_local!(static THREAD_ID: RefCell<usize> = RefCell::new(0));

/// A multithreaded execution environment for the tasks.
pub struct Parallel {
    context_list: RwLock<Vec<Arc<Context>>>,
    next_context_id: AtomicUsize,
    threads: Mutex<Vec<JoinHandle<()>>>,
}

impl Parallel {
    pub fn new() -> Parallel {
        let par = Parallel { context_list: RwLock::new(Vec::new()),
                             next_context_id: AtomicUsize::new(0),
                             threads: Mutex::new(Vec::new()) };
        {
            let mut threads = par.threads.lock().unwrap();
            let num_threads = num_cpus::get();
            println!("Launching {} threads", num_threads);
            for i in 0..num_threads {
                // Note that the spawned thread ids start at 1 since the main thread is 0
                threads.push(thread::spawn(move || worker_thread(i + 1, num_threads + 1)));
            }
        }
        par
    }
}

impl TaskSystem for Parallel {
    unsafe fn alloc(&self, handle_ptr: *mut *mut libc::c_void, size: i64, align: i32) -> *mut libc::c_void {
        println!("ISPCAlloc, size: {}, align: {}", size, align);
        // If the handle is null this is the first time this function has spawned tasks
        // and we should create a new Context structure in the TASK_LIST for it, otherwise
        // it's the pointer to where we should append the new Group
        let mut context_list = self.context_list.write().unwrap();
        let context = if (*handle_ptr).is_null() {
            println!("handle ptr is null");
            // This is a bit hairy. We allocate the new task context in a box, then
            // unbox it into a raw ptr to get a ptr we can pass back to ISPC through
            // the handle_ptr and then re-box it into our TASK_LIST so it will
            // be free'd properly when we erase it from the vector in ISPCSync
            let c = Arc::new(Context::new(self.next_context_id.fetch_add(1, atomic::Ordering::SeqCst)));
            {
                let h = &*c;
                *handle_ptr = mem::transmute(h);
            }
            context_list.push(c);
            context_list.last_mut().unwrap()
        } else {
            // TODO: I'm not sure if it'd be worth optimizing this path to just require a read lock
            println!("handle ptr is not null");
            let handle_ctx: *mut Context = mem::transmute(*handle_ptr);
            context_list.iter_mut().find(|c| (*handle_ctx).id == c.id).unwrap()
        };
        println!("context.id = {}", context.id);
        context.alloc(size as usize, align as usize)
    }
    unsafe fn launch(&self, handle_ptr: *mut *mut libc::c_void, f: ISPCTaskFn, data: *mut libc::c_void,
                     count0: i32, count1: i32, count2: i32) {
        // Push the tasks being launched on to the list of task groups for this function
        let context: &mut Context = mem::transmute(*handle_ptr);
        println!("ISPCLaunch, context.id = {}, counts: [{}, {}, {}]", context.id, count0, count1, count2);
        context.launch((count0, count1, count2), data, f);
        // Unpark any sleeping threads since we have jobs for them
        let threads = self.threads.lock().unwrap();
        for t in threads.iter() {
            t.thread().unpark();
        }
    }
    unsafe fn sync(&self, handle: *mut libc::c_void) {
        let context: &mut Context = mem::transmute(handle);
        let thread = THREAD_ID.with(|f| *f.borrow());
        let total_threads = num_cpus::get();
        println!("Thread {} is syncing", thread);
        // Make sure all tasks are done, and execute them if not for this simple
        // serial version. TODO: In the future we'd wait on each Group's semaphore or atomic bool
        // Maybe the waiting thread could help execute tasks as well, otherwise it might be
        // possible to deadlock, where all threads are waiting for some enqueue'd tasks but no
        // threads are available to run them. Just running tasks in our context is not sufficient
        // to prevent deadlock actually, because those tasks could in turn launch & sync and get stuck
        // so if our tasks aren't done and there's none left to run in our context we should start
        // running tasks from other contexts to help out
        println!("ISPCSync, context.id = {}", context.id);
        for tg in context.iter() {
            for chunk in tg.chunks(1) {
                println!("syncing thread {} running local chunk {:?}", thread, chunk);
                // TODO: We need to figure out which thread we are
                chunk.execute(thread as i32, total_threads as i32);
            }
        }
        // If all the tasks for this context have been finished we're done sync'ing and can
        // clean up memory and remove the context from the TASK_LIST. Otherwise there are some
        // unfinished groups further down the the tree that were spawned by our direct tasks that
        // those are now sync'ing on and we need to help out. However since we don't know the tree
        // our best option is to just start grabbing chunks from unfinished groups in the TASK_LIST
        // and running them to at least ensure global forward progress, which will eventually get
        // the stuff we're waiting on to finish. After each chunk execution we should check if
        // our sync'ing context is done and break
        while !context.current_tasks_done() {
            println!("Not all tasks for context {} are done, thread {} helping out!", context.id, thread);
            // Get a task group to run
            let ctx = {
                // Do it in a block to limit the scope of the lock
                ::get_task_system().context_list.read().unwrap().iter()
                    .find(|c| !c.current_tasks_done()).map(|c| c.clone())
            };
            if let Some(c) = ctx {
                println!("syncing thread {} got a context {:?}", thread, c);
                for tg in c.iter() {
                    println!("syncing thread {} looking at task group {:?}", thread, tg);
                    for chunk in tg.chunks(1) {
                        println!("syncing thread {} running nonlocal chunk {:?}", thread, chunk);
                        chunk.execute(thread as i32, total_threads as i32);
                    }
                }
            } else {
                println!("Syncing thread didn't get a context");
            }
        }
        // Now erase this context from our vector
        let mut context_list = self.context_list.write().unwrap();
        let pos = context_list.iter().position(|c| context.id == c.id).unwrap();
        context_list.remove(pos);
    }
}

fn worker_thread(thread: usize, total_threads: usize) {
    THREAD_ID.with(|f| *f.borrow_mut() = thread);
    let task_system = ::get_task_system();
    println!("thread {} is spawned", thread);
    loop {
        thread::park();
        println!("thread {} was woken up to work!", thread);
        // Get a task group to run
        let ctx = {
            // Do it in a block to limit the scope of the lock
            task_system.context_list.read().unwrap().iter()
                .find(|c| !c.current_tasks_done()).map(|c| c.clone())
        };
        if let Some(c) = ctx {
            for tg in c.iter() {
                for chunk in tg.chunks(1) {
                    println!("thread {} running chunk {:?}", thread, chunk);
                    chunk.execute(thread as i32, total_threads as i32);
                }
            }
        }
    }
}

<|endoftext|>