<commit_before>//! The master module provides the Master struct which instructs Workers which
//! portions of the image they should render and collects their results to combine
//! into the final image.

use std::path::PathBuf;
use std::io::prelude::*;
use std::collections::HashMap;
use std::net::ToSocketAddrs;
use std::iter;

use bincode::rustc_serialize::decode;
use image;
use mio::tcp::{TcpStream, Shutdown};
use mio::*;

use film::Image;
use exec::Config;
use exec::distrib::{worker, Instructions};
use sampler::BlockQueue;

pub struct DistributedFrame {
    pub frame: usize,
    // The number of workers who have reported results for this
    // frame so far
    pub num_reporting: usize,
    pub render: Image,
    pub completed: bool,
}

#[derive(Clone, Debug)]
struct WorkerBuffer {
    pub buf: Vec<u8>,
    pub expected_size: usize,
    pub currently_read: usize,
}

impl WorkerBuffer {
    pub fn new() -> WorkerBuffer {
        WorkerBuffer { buf: Vec::new(), expected_size: 8, currently_read: 0 }
    }
}

pub struct Master {
    // Hostnames of the workers to send work too
    workers: Vec<String>,
    connections: Vec<TcpStream>,
    // Temporary buffers to store worker results in as they're
    // read in over TCP
    worker_buffers: Vec<WorkerBuffer>,
    config: Config,
    frames: HashMap<usize, DistributedFrame>,
    selected_frames: Option<(usize, usize)>,
    img_dim: (usize, usize),
    blocks_per_worker: usize,
    blocks_remainder: usize,
}

impl Master {
    /// Create a new master that will contact the worker nodes passed and
    /// send instructions on what parts of the scene to start rendering
    pub fn start_workers(workers: Vec<String>, config: Config, img_dim: (usize, usize),
                         frames: Option<(usize, usize)>) -> (Master, EventLoop<Master>) {
        // Figure out how many blocks we have for this image and assign them to our workers
        let queue = BlockQueue::new((img_dim.0 as u32, img_dim.1 as u32), (8, 8), (0, 0));
        let blocks_per_worker = queue.len() / workers.len();
        let blocks_remainder = queue.len() % workers.len();
        println!("Have {} workers, each worker does {} blocks w/ {} remainder",
                 workers.len(), blocks_per_worker, blocks_remainder);

        let mut event_loop = EventLoop::<Master>::new().unwrap();
        let mut connections = Vec::new();

        // Connect to each worker and send instructions on what to render
        // I guess this should also be queue'd up and the writes actually performed asynchronously?
        // Otherwise we need to loop on write_all while waiting for the connection to open
        for (i, host) in workers.iter().enumerate() {
            let addr = (&host[..], worker::PORT).to_socket_addrs().unwrap().next().unwrap();
            println!("Connecting too {:?}", addr);
            match TcpStream::connect(&addr) {
                Ok(stream) => {
                    println!("Connected and writing");
                    println!("Adding stream to event loop");
                    // Each worker is identified in the event loop by their index in the vec
                    match event_loop.register(&stream, Token(i)) {
                        Err(e) => println!("Error registering stream from {}: {}", host, e),
                        Ok(_) => {},
                    }
                    connections.push(stream);
                },
                Err(e) => println!("Failed to contact worker {}: {:?}", host, e),
            }
        }
        let worker_buffers: Vec<_> = iter::repeat(WorkerBuffer::new()).take(workers.len()).collect();
        let master = Master { workers: workers, connections: connections,
                              worker_buffers: worker_buffers, config: config,
                              frames: HashMap::new(), selected_frames: frames,
                              img_dim: img_dim,
                              blocks_per_worker: blocks_per_worker,
                              blocks_remainder: blocks_remainder };
        (master, event_loop)
    }
    /// Read a result frame from a worker and save it into the list of frames we're collecting from
    /// all workers. Will save out the final render if all workers have reported results for this
    /// frame.
    fn save_results(&mut self, frame: worker::Frame) {
        // Find the frame this worker is referring to and add its results
        let frame_num = frame.frame as usize;
        println!("Worker reporting frame {}", frame_num);
        if let Some(df) = self.frames.get_mut(&frame_num) {
            df.render.add_blocks(frame.block_size, &frame.blocks, &frame.pixels);
            df.num_reporting += 1;
            println!("We have parts of this frame already");
        }
        // TODO: Better way here? We get stuck because self.frames is counted as borrowed
        // from the if above, making this really awkward.
        if self.frames.get_mut(&frame_num).is_none() {
            // TODO: I need to seperate the render target from the actual image as the
            // filter here isn't used in this path and it doesn't make sense to have one
            let render = Image::new(self.img_dim);
            let mut df = DistributedFrame { frame: frame_num, num_reporting: 1, render: render, completed: false };
            df.render.add_blocks(frame.block_size, &frame.blocks, &frame.pixels);
            self.frames.insert(df.frame, df);
            println!("This is a new frame");
        }
        if let Some(df) = self.frames.get_mut(&frame_num) {
            if df.num_reporting == self.workers.len() {
                df.completed = true;
                println!("We have completed frame {}", frame_num);
                let out_file = match self.config.out_path.extension() {
                    Some(_) => self.config.out_path.clone(),
                    None => self.config.out_path.join(PathBuf::from(format!("frame{:05}.png", df.frame))),
                };
                println!("Frame {}: rendered to '{}'\n--------------------", frame_num, out_file.display());
                let img = df.render.get_srgb8();
                let dim = df.render.dimensions();
                match image::save_buffer(&out_file.as_path(), &img[..], dim.0 as u32, dim.1 as u32, image::RGB(8)) {
                    Ok(_) => {},
                    Err(e) => println!("Error saving image, {}", e),
                };
            }
        }
    }
    fn read_worker_buffer(&mut self, worker: usize) -> bool {
        let mut buf = &mut self.worker_buffers[worker];
        // If we haven't read the size of data being sent, read that now
        if buf.currently_read < 8 {
            // First 8 bytes are a u64 specifying the number of bytes being sent
            buf.buf.extend(iter::repeat(0u8).take(8));
            println!("Reading size header...");
            match self.connections[worker].read(&mut buf.buf[buf.currently_read..]) {
                Ok(n) => buf.currently_read += n,
                Err(e) => println!("Error reading results from worker {}: {}", self.workers[worker], e),
            }
            println!("Read {} bytes so far, must read {} in total", buf.currently_read, buf.expected_size);
            if buf.currently_read == buf.expected_size {
                println!("Got size header");
                // How many bytes we expect to get from the worker for a frame
                buf.expected_size = decode(&buf.buf[..]).unwrap();
                println!("Expecting {} bytes", buf.expected_size);
                // Extend the Vec so we've got enough room for the remaning bytes, minus the 8 for the
                // encoded size header
                buf.buf.extend(iter::repeat(0u8).take(buf.expected_size - 8));
            }
        }
        // If we've finished reading the size header we can now start reading the frame data
        if buf.currently_read >= 8 {
            println!("Reading from worker");
            match self.connections[worker].read(&mut buf.buf[buf.currently_read..]) {
                Ok(n) => buf.currently_read += n,
                Err(e) => println!("Error reading results from worker {}: {}", self.workers[worker], e),
            }
            println!("Read {} bytes so far, must read {} in total", buf.currently_read, buf.expected_size);
        }
        buf.currently_read == buf.expected_size
    }
}

impl Handler for Master {
    type Timeout = ();
    type Message = ();

    fn ready(&mut self, event_loop: &mut EventLoop<Master>, token: Token, event: EventSet) {
        let worker = token.as_usize();
        println!("Event from {}: {:?}", worker, event);
        if event.is_error() {
            println!("Error connecting too {}", self.workers[worker]);
            match self.connections[worker].shutdown(Shutdown::Both) {
                Err(e) => println!("Error shutting down worker {}: {}", worker, e),
                Ok(_) => {},
            }
            // Remove the connection from the event loop
            match event_loop.deregister(&self.connections[worker]) {
                Err(e) => println!("Error deregistering worker {}: {}", worker, e),
                Ok(_) => {},
            }
        }
        // A worker is ready to receive instructions from us
        if event.is_writable() {
            let b_start = worker * self.blocks_per_worker;
            let b_count =
                if worker == self.workers.len() - 1 {
                    self.blocks_per_worker + self.blocks_remainder
                } else {
                    self.blocks_per_worker
                };
            let instr = Instructions::new(&self.config.scene_file, self.selected_frames, b_start, b_count);
            println!("Sending instructions {:?} to {}", instr, self.workers[worker]);
            let bytes = instr.to_json().into_bytes();
            // Loop until we successfully write the byes
            match self.connections[worker].write_all(&bytes[..]) {
                Err(e) => {
                    println!("Failed to send instructions to {}: {:?}", self.workers[worker], e);
                    return;
                },
                Ok(_) => println!("Instructions sent"),
            }
            // Register that we no longer care about writable events on this connection
            event_loop.reregister(&self.connections[worker], token,
                                  EventSet::readable() | EventSet::error() | EventSet::hup(),
                                  PollOpt::level()).expect("Re-registering failed");
            // We no longer need to write anything, so close the write end
            match self.connections[worker].shutdown(Shutdown::Write) {
                Err(e) => panic!(format!("Failed to shutdown write end to worker {}: {}",
                                         self.workers[worker], e)),
                Ok(_) => {},
            }
        }
        // Some results are available from a worker
        if event.is_readable() {
            println!("Readable event from worker {}", worker);
            if self.read_worker_buffer(worker) {
                println!("Got all frame data, now parsing");
                let frame: worker::Frame = decode(&self.worker_buffers[worker].buf[..]).unwrap();
                println!("Saving frame results");
                self.save_results(frame);
                // Clean up our read buffer
                self.worker_buffers[worker].buf.clear();
                self.worker_buffers[worker].expected_size = 8;
                self.worker_buffers[worker].currently_read = 0;
            }
        }
        // If the worker has terminated, shutdown the read end of the connection
        if event.is_hup() {
            println!("Worker {} has hung up", worker);
            match self.connections[worker].shutdown(Shutdown::Read) {
                Err(e) => println!("Error shutting down worker {}: {}", worker, e),
                Ok(_) => {},
            }
            // Remove the connection from the event loop
            match event_loop.deregister(&self.connections[worker]) {
                Err(e) => println!("Error deregistering worker {}: {}", worker, e),
                Ok(_) => {},
            }
        }
        // After getting results from the worker we check if we've completed all our frames
        // and exit if so
        if self.frames.len() == self.config.frame_info.frames
            && self.frames.values().fold(true, |all, v| all && v.completed) {
                println!("All frames have been rendered, master exiting");
                event_loop.shutdown();
        }
    }
}

<commit_msg>Have master mark frames as completed now and release the Image when done<commit_after>//! The master module provides the Master struct which instructs Workers which
//! portions of the image they should render and collects their results to combine
//! into the final image.

use std::path::PathBuf;
use std::io::prelude::*;
use std::collections::HashMap;
use std::net::ToSocketAddrs;
use std::iter;

use bincode::rustc_serialize::decode;
use image;
use mio::tcp::{TcpStream, Shutdown};
use mio::*;

use film::Image;
use exec::Config;
use exec::distrib::{worker, Instructions};
use sampler::BlockQueue;

enum DistributedFrame {
    InProgress {
        frame: usize,
        // The number of workers who have reported results for this
        // frame so far
        num_reporting: usize,
        render: Image,
    },
    Completed,
}

#[derive(Clone, Debug)]
struct WorkerBuffer {
    pub buf: Vec<u8>,
    pub expected_size: usize,
    pub currently_read: usize,
}

impl WorkerBuffer {
    pub fn new() -> WorkerBuffer {
        WorkerBuffer { buf: Vec::new(), expected_size: 8, currently_read: 0 }
    }
}

pub struct Master {
    // Hostnames of the workers to send work too
    workers: Vec<String>,
    connections: Vec<TcpStream>,
    // Temporary buffers to store worker results in as they're
    // read in over TCP
    worker_buffers: Vec<WorkerBuffer>,
    config: Config,
    frames: HashMap<usize, DistributedFrame>,
    selected_frames: Option<(usize, usize)>,
    img_dim: (usize, usize),
    blocks_per_worker: usize,
    blocks_remainder: usize,
}

impl Master {
    /// Create a new master that will contact the worker nodes passed and
    /// send instructions on what parts of the scene to start rendering
    pub fn start_workers(workers: Vec<String>, config: Config, img_dim: (usize, usize),
                         frames: Option<(usize, usize)>) -> (Master, EventLoop<Master>) {
        // Figure out how many blocks we have for this image and assign them to our workers
        let queue = BlockQueue::new((img_dim.0 as u32, img_dim.1 as u32), (8, 8), (0, 0));
        let blocks_per_worker = queue.len() / workers.len();
        let blocks_remainder = queue.len() % workers.len();
        println!("Have {} workers, each worker does {} blocks w/ {} remainder",
                 workers.len(), blocks_per_worker, blocks_remainder);

        let mut event_loop = EventLoop::<Master>::new().unwrap();
        let mut connections = Vec::new();

        // Connect to each worker and send instructions on what to render
        // I guess this should also be queue'd up and the writes actually performed asynchronously?
        // Otherwise we need to loop on write_all while waiting for the connection to open
        for (i, host) in workers.iter().enumerate() {
            let addr = (&host[..], worker::PORT).to_socket_addrs().unwrap().next().unwrap();
            println!("Connecting too {:?}", addr);
            match TcpStream::connect(&addr) {
                Ok(stream) => {
                    println!("Connected and writing");
                    println!("Adding stream to event loop");
                    // Each worker is identified in the event loop by their index in the vec
                    match event_loop.register(&stream, Token(i)) {
                        Err(e) => println!("Error registering stream from {}: {}", host, e),
                        Ok(_) => {},
                    }
                    connections.push(stream);
                },
                Err(e) => println!("Failed to contact worker {}: {:?}", host, e),
            }
        }
        let worker_buffers: Vec<_> = iter::repeat(WorkerBuffer::new()).take(workers.len()).collect();
        let master = Master { workers: workers, connections: connections,
                              worker_buffers: worker_buffers, config: config,
                              frames: HashMap::new(), selected_frames: frames,
                              img_dim: img_dim,
                              blocks_per_worker: blocks_per_worker,
                              blocks_remainder: blocks_remainder };
        (master, event_loop)
    }
    /// Read a result frame from a worker and save it into the list of frames we're collecting from
    /// all workers. Will save out the final render if all workers have reported results for this
    /// frame.
    fn save_results(&mut self, frame: worker::Frame) {
        // Find the frame this worker is referring to and add its results
        let frame_num = frame.frame as usize;
        println!("Worker reporting frame {}", frame_num);
        if let Some(df) = self.frames.get_mut(&frame_num) {
            match df {
                DistributedFrame::InProgress { frame: _, num_reporting: num_reporting, render: render } => {
                    render.add_blocks(frame.block_size, &frame.blocks, &frame.pixels);
                    num_reporting += 1;
                    println!("We have parts of this frame already");
                },
                DistributedFrame::Completed => println!("Worker reporting on completed frame!?"),
            }
        }
        // TODO: Better way here? We get stuck because self.frames is counted as borrowed
        // from the if above, making this really awkward.
        if self.frames.get_mut(&frame_num).is_none() {
            // TODO: I need to seperate the render target from the actual image as the
            // filter here isn't used in this path and it doesn't make sense to have one
            let render = Image::new(self.img_dim);
            let mut df = DistributedFrame::InProgress { frame: frame_num, num_reporting: 1, render: render};
            df.render.add_blocks(frame.block_size, &frame.blocks, &frame.pixels);
            self.frames.insert(df.frame, df);
            println!("This is a new frame");
        }
        if let Some(df) = self.frames.get_mut(&frame_num) {
            match df {
                DistributedFrame::InProgress { frame: frame, num_reporting: num_reporting, render: render } => {
                    if df.num_reporting == self.workers.len() {
                        df.completed = true;
                        println!("We have completed frame {}", frame_num);
                        let out_file = match self.config.out_path.extension() {
                            Some(_) => self.config.out_path.clone(),
                            None => self.config.out_path.join(
                                PathBuf::from(format!("frame{:05}.png", df.frame))),
                        };
                        println!("Frame {}: rendered to '{}'\n--------------------",
                                 frame_num, out_file.display());
                        let img = df.render.get_srgb8();
                        let dim = df.render.dimensions();
                        match image::save_buffer(&out_file.as_path(), &img[..], dim.0 as u32,
                                                 dim.1 as u32, image::RGB(8)) {
                            Ok(_) => {},
                            Err(e) => println!("Error saving image, {}", e),
                        };
                    }
                },
                DistributedFrame::Completed => println!("Can not save out already completed frame"),
            }
            df = DistributedFrame::Completed;
        }
    }
    fn read_worker_buffer(&mut self, worker: usize) -> bool {
        let mut buf = &mut self.worker_buffers[worker];
        // If we haven't read the size of data being sent, read that now
        if buf.currently_read < 8 {
            // First 8 bytes are a u64 specifying the number of bytes being sent
            buf.buf.extend(iter::repeat(0u8).take(8));
            println!("Reading size header...");
            match self.connections[worker].read(&mut buf.buf[buf.currently_read..]) {
                Ok(n) => buf.currently_read += n,
                Err(e) => println!("Error reading results from worker {}: {}", self.workers[worker], e),
            }
            println!("Read {} bytes so far, must read {} in total", buf.currently_read, buf.expected_size);
            if buf.currently_read == buf.expected_size {
                println!("Got size header");
                // How many bytes we expect to get from the worker for a frame
                buf.expected_size = decode(&buf.buf[..]).unwrap();
                println!("Expecting {} bytes", buf.expected_size);
                // Extend the Vec so we've got enough room for the remaning bytes, minus the 8 for the
                // encoded size header
                buf.buf.extend(iter::repeat(0u8).take(buf.expected_size - 8));
            }
        }
        // If we've finished reading the size header we can now start reading the frame data
        if buf.currently_read >= 8 {
            println!("Reading from worker");
            match self.connections[worker].read(&mut buf.buf[buf.currently_read..]) {
                Ok(n) => buf.currently_read += n,
                Err(e) => println!("Error reading results from worker {}: {}", self.workers[worker], e),
            }
            println!("Read {} bytes so far, must read {} in total", buf.currently_read, buf.expected_size);
        }
        buf.currently_read == buf.expected_size
    }
}

impl Handler for Master {
    type Timeout = ();
    type Message = ();

    fn ready(&mut self, event_loop: &mut EventLoop<Master>, token: Token, event: EventSet) {
        let worker = token.as_usize();
        println!("Event from {}: {:?}", worker, event);
        if event.is_error() {
            println!("Error connecting too {}", self.workers[worker]);
            match self.connections[worker].shutdown(Shutdown::Both) {
                Err(e) => println!("Error shutting down worker {}: {}", worker, e),
                Ok(_) => {},
            }
            // Remove the connection from the event loop
            match event_loop.deregister(&self.connections[worker]) {
                Err(e) => println!("Error deregistering worker {}: {}", worker, e),
                Ok(_) => {},
            }
        }
        // A worker is ready to receive instructions from us
        if event.is_writable() {
            let b_start = worker * self.blocks_per_worker;
            let b_count =
                if worker == self.workers.len() - 1 {
                    self.blocks_per_worker + self.blocks_remainder
                } else {
                    self.blocks_per_worker
                };
            let instr = Instructions::new(&self.config.scene_file, self.selected_frames, b_start, b_count);
            println!("Sending instructions {:?} to {}", instr, self.workers[worker]);
            let bytes = instr.to_json().into_bytes();
            // Loop until we successfully write the byes
            match self.connections[worker].write_all(&bytes[..]) {
                Err(e) => {
                    println!("Failed to send instructions to {}: {:?}", self.workers[worker], e);
                    return;
                },
                Ok(_) => println!("Instructions sent"),
            }
            // Register that we no longer care about writable events on this connection
            event_loop.reregister(&self.connections[worker], token,
                                  EventSet::readable() | EventSet::error() | EventSet::hup(),
                                  PollOpt::level()).expect("Re-registering failed");
            // We no longer need to write anything, so close the write end
            match self.connections[worker].shutdown(Shutdown::Write) {
                Err(e) => panic!(format!("Failed to shutdown write end to worker {}: {}",
                                         self.workers[worker], e)),
                Ok(_) => {},
            }
        }
        // Some results are available from a worker
        if event.is_readable() {
            println!("Readable event from worker {}", worker);
            if self.read_worker_buffer(worker) {
                println!("Got all frame data, now parsing");
                let frame: worker::Frame = decode(&self.worker_buffers[worker].buf[..]).unwrap();
                println!("Saving frame results");
                self.save_results(frame);
                // Clean up our read buffer
                self.worker_buffers[worker].buf.clear();
                self.worker_buffers[worker].expected_size = 8;
                self.worker_buffers[worker].currently_read = 0;
            }
        }
        // If the worker has terminated, shutdown the read end of the connection
        if event.is_hup() {
            println!("Worker {} has hung up", worker);
            match self.connections[worker].shutdown(Shutdown::Read) {
                Err(e) => println!("Error shutting down worker {}: {}", worker, e),
                Ok(_) => {},
            }
            // Remove the connection from the event loop
            match event_loop.deregister(&self.connections[worker]) {
                Err(e) => println!("Error deregistering worker {}: {}", worker, e),
                Ok(_) => {},
            }
        }
        // After getting results from the worker we check if we've completed all our frames
        // and exit if so
        let all_complete = self.frames.values().fold(true,
                                |all, v| {
                                    match v {
                                        DistributedFrame::Completed => true && v,
                                        _ => false,
                                    }
                                });
        if self.frames.len() == self.config.frame_info.frames && all_complete {
            println!("All frames have been rendered, master exiting");
            event_loop.shutdown();
        }
    }
}

<|endoftext|>


<commit_before>use super::super::IntoBytes;
use frame::*;

/// The structure which represents a body of a frame of type `options`.
#[derive(Debug)]
pub struct BodyReqOptions;

impl BodyReqOptions {
    /// Creates new body of a frame of type `options`
    pub fn new() -> BodyReqOptions {
        BodyReqOptions {}
    }
}

impl IntoBytes for BodyReqOptions {
    fn into_cbytes(&self) -> Vec<u8> {
        vec![]
    }
}

// Frame implementation related to BodyReqStartup

impl Frame {
    /// Creates new frame of type `options`.
    pub fn new_req_options() -> Frame {
        let version = Version::Request;
        let flag = Flag::Ignore;
        // sync client
        let stream: u64 = 0;
        let opcode = Opcode::Options;
        let body = BodyReqOptions::new();

        Frame {
            version: version,
            flags: vec![flag],
            stream: stream,
            opcode: opcode,
            body: body.into_cbytes(),
            // for request frames it's always None
            tracing_id: None,
            warnings: vec![],
        }
    }
}
<commit_msg>updates to clippy<commit_after>use super::super::IntoBytes;
use frame::*;

/// The structure which represents a body of a frame of type `options`.
#[derive(Debug,Default)]
pub struct BodyReqOptions;

impl IntoBytes for BodyReqOptions {
    fn into_cbytes(&self) -> Vec<u8> {
        vec![]
    }
}

// Frame implementation related to BodyReqStartup

impl Frame {
    /// Creates new frame of type `options`.
    pub fn new_req_options() -> Frame {
        let version = Version::Request;
        let flag = Flag::Ignore;
        // sync client
        let stream: u64 = 0;
        let opcode = Opcode::Options;
        let body: BodyReqOptions = Default::default();

        Frame {
            version: version,
            flags: vec![flag],
            stream: stream,
            opcode: opcode,
            body: body.into_cbytes(),
            // for request frames it's always None
            tracing_id: None,
            warnings: vec![],
        }
    }
}
<|endoftext|>


<commit_before>use crate::core::{GitReference, PackageId, SourceId};
use crate::sources::git;
use crate::sources::registry::download;
use crate::sources::registry::MaybeLock;
use crate::sources::registry::{LoadResponse, RegistryConfig, RegistryData};
use crate::util::errors::CargoResult;
use crate::util::interning::InternedString;
use crate::util::{Config, Filesystem};
use anyhow::Context as _;
use cargo_util::paths;
use lazycell::LazyCell;
use log::{debug, trace};
use std::cell::{Cell, Ref, RefCell};
use std::fs::File;
use std::mem;
use std::path::Path;
use std::str;
use std::task::Poll;

/// A remote registry is a registry that lives at a remote URL (such as
/// crates.io). The git index is cloned locally, and `.crate` files are
/// downloaded as needed and cached locally.
pub struct RemoteRegistry<'cfg> {
    index_path: Filesystem,
    /// Path to the cache of `.crate` files (`$CARGO_HOME/registry/path/$REG-HASH`).
    cache_path: Filesystem,
    source_id: SourceId,
    index_git_ref: GitReference,
    config: &'cfg Config,
    tree: RefCell<Option<git2::Tree<'static>>>,
    repo: LazyCell<git2::Repository>,
    head: Cell<Option<git2::Oid>>,
    current_sha: Cell<Option<InternedString>>,
    needs_update: bool, // Does this registry need to be updated?
    updated: bool,      // Has this registry been updated this session?
}

impl<'cfg> RemoteRegistry<'cfg> {
    pub fn new(source_id: SourceId, config: &'cfg Config, name: &str) -> RemoteRegistry<'cfg> {
        RemoteRegistry {
            index_path: config.registry_index_path().join(name),
            cache_path: config.registry_cache_path().join(name),
            source_id,
            config,
            // TODO: we should probably make this configurable
            index_git_ref: GitReference::DefaultBranch,
            tree: RefCell::new(None),
            repo: LazyCell::new(),
            head: Cell::new(None),
            current_sha: Cell::new(None),
            needs_update: false,
            updated: false,
        }
    }

    fn repo(&self) -> CargoResult<&git2::Repository> {
        self.repo.try_borrow_with(|| {
            let path = self.config.assert_package_cache_locked(&self.index_path);

            // Fast path without a lock
            if let Ok(repo) = git2::Repository::open(&path) {
                trace!("opened a repo without a lock");
                return Ok(repo);
            }

            // Ok, now we need to lock and try the whole thing over again.
            trace!("acquiring registry index lock");
            match git2::Repository::open(&path) {
                Ok(repo) => Ok(repo),
                Err(_) => {
                    drop(paths::remove_dir_all(&path));
                    paths::create_dir_all(&path)?;

                    // Note that we'd actually prefer to use a bare repository
                    // here as we're not actually going to check anything out.
                    // All versions of Cargo, though, share the same CARGO_HOME,
                    // so for compatibility with older Cargo which *does* do
                    // checkouts we make sure to initialize a new full
                    // repository (not a bare one).
                    //
                    // We should change this to `init_bare` whenever we feel
                    // like enough time has passed or if we change the directory
                    // that the folder is located in, such as by changing the
                    // hash at the end of the directory.
                    //
                    // Note that in the meantime we also skip `init.templatedir`
                    // as it can be misconfigured sometimes or otherwise add
                    // things that we don't want.
                    let mut opts = git2::RepositoryInitOptions::new();
                    opts.external_template(false);
                    Ok(git2::Repository::init_opts(&path, &opts).with_context(|| {
                        format!("failed to initialize index git repository (in {:?})", path)
                    })?)
                }
            }
        })
    }

    fn head(&self) -> CargoResult<git2::Oid> {
        if self.head.get().is_none() {
            let repo = self.repo()?;
            let oid = self.index_git_ref.resolve(repo)?;
            self.head.set(Some(oid));
        }
        Ok(self.head.get().unwrap())
    }

    fn tree(&self) -> CargoResult<Ref<'_, git2::Tree<'_>>> {
        {
            let tree = self.tree.borrow();
            if tree.is_some() {
                return Ok(Ref::map(tree, |s| s.as_ref().unwrap()));
            }
        }
        let repo = self.repo()?;
        let commit = repo.find_commit(self.head()?)?;
        let tree = commit.tree()?;

        // Unfortunately in libgit2 the tree objects look like they've got a
        // reference to the repository object which means that a tree cannot
        // outlive the repository that it came from. Here we want to cache this
        // tree, though, so to accomplish this we transmute it to a static
        // lifetime.
        //
        // Note that we don't actually hand out the static lifetime, instead we
        // only return a scoped one from this function. Additionally the repo
        // we loaded from (above) lives as long as this object
        // (`RemoteRegistry`) so we then just need to ensure that the tree is
        // destroyed first in the destructor, hence the destructor on
        // `RemoteRegistry` below.
        let tree = unsafe { mem::transmute::<git2::Tree<'_>, git2::Tree<'static>>(tree) };
        *self.tree.borrow_mut() = Some(tree);
        Ok(Ref::map(self.tree.borrow(), |s| s.as_ref().unwrap()))
    }

    fn current_version(&self) -> Option<InternedString> {
        if let Some(sha) = self.current_sha.get() {
            return Some(sha);
        }
        let sha = InternedString::new(&self.head().ok()?.to_string());
        self.current_sha.set(Some(sha));
        Some(sha)
    }
}

const LAST_UPDATED_FILE: &str = ".last-updated";

impl<'cfg> RegistryData for RemoteRegistry<'cfg> {
    fn prepare(&self) -> CargoResult<()> {
        self.repo()?; // create intermediate dirs and initialize the repo
        Ok(())
    }

    fn index_path(&self) -> &Filesystem {
        &self.index_path
    }

    fn assert_index_locked<'a>(&self, path: &'a Filesystem) -> &'a Path {
        self.config.assert_package_cache_locked(path)
    }

    fn load(
        &mut self,
        _root: &Path,
        path: &Path,
        index_version: Option<&str>,
    ) -> Poll<CargoResult<LoadResponse>> {
        if self.needs_update {
            return Poll::Pending;
        }
        // Check if the cache is valid.
        let current_version = self.current_version();
        if current_version.is_some() && current_version.as_deref() == index_version {
            return Poll::Ready(Ok(LoadResponse::CacheValid));
        }
        // Note that the index calls this method and the filesystem is locked
        // in the index, so we don't need to worry about an `update_index`
        // happening in a different process.
        fn load_helper(
            registry: &RemoteRegistry<'_>,
            path: &Path,
            current_version: Option<&str>,
        ) -> CargoResult<LoadResponse> {
            let repo = registry.repo()?;
            let tree = registry.tree()?;
            let entry = tree.get_path(path);
            let entry = entry?;
            let object = entry.to_object(repo)?;
            let blob = match object.as_blob() {
                Some(blob) => blob,
                None => anyhow::bail!("path `{}` is not a blob in the git repo", path.display()),
            };

            Ok(LoadResponse::Data {
                raw_data: blob.content().to_vec(),
                index_version: current_version.map(String::from),
            })
        }

        match load_helper(&self, path, current_version.as_deref()) {
            Ok(result) => Poll::Ready(Ok(result)),
            Err(_) if !self.updated => {
                // If git returns an error and we haven't updated the repo, return
                // pending to allow an update to try again.
                self.needs_update = true;
                Poll::Pending
            }
            Err(e)
                if e.downcast_ref::<git2::Error>()
                    .map(|e| e.code() == git2::ErrorCode::NotFound)
                    .unwrap_or_default() =>
            {
                // The repo has been updated and the file does not exist.
                Poll::Ready(Ok(LoadResponse::NotFound))
            }
            Err(e) => Poll::Ready(Err(e)),
        }
    }

    fn config(&mut self) -> Poll<CargoResult<Option<RegistryConfig>>> {
        debug!("loading config");
        self.prepare()?;
        self.config.assert_package_cache_locked(&self.index_path);
        match self.load(Path::new(""), Path::new("config.json"), None)? {
            Poll::Ready(LoadResponse::Data { raw_data, .. }) => {
                trace!("config loaded");
                Poll::Ready(Ok(Some(serde_json::from_slice(&raw_data)?)))
            }
            Poll::Ready(_) => Poll::Ready(Ok(None)),
            Poll::Pending => Poll::Pending,
        }
    }

    fn block_until_ready(&mut self) -> CargoResult<()> {
        if !self.needs_update {
            return Ok(());
        }

        self.updated = true;
        self.needs_update = false;

        if self.config.offline() {
            return Ok(());
        }
        if self.config.cli_unstable().no_index_update {
            return Ok(());
        }
        // Make sure the index is only updated once per session since it is an
        // expensive operation. This generally only happens when the resolver
        // is run multiple times, such as during `cargo publish`.
        if self.config.updated_sources().contains(&self.source_id) {
            return Ok(());
        }

        debug!("updating the index");

        // Ensure that we'll actually be able to acquire an HTTP handle later on
        // once we start trying to download crates. This will weed out any
        // problems with `.cargo/config` configuration related to HTTP.
        //
        // This way if there's a problem the error gets printed before we even
        // hit the index, which may not actually read this configuration.
        self.config.http()?;

        self.prepare()?;
        self.head.set(None);
        *self.tree.borrow_mut() = None;
        self.current_sha.set(None);
        let path = self.config.assert_package_cache_locked(&self.index_path);
        self.config
            .shell()
            .status("Updating", self.source_id.display_index())?;

        // Fetch the latest version of our `index_git_ref` into the index
        // checkout.
        let url = self.source_id.url();
        let repo = self.repo.borrow_mut().unwrap();
        git::fetch(repo, url.as_str(), &self.index_git_ref, self.config)
            .with_context(|| format!("failed to fetch `{}`", url))?;
        self.config.updated_sources().insert(self.source_id);

        // Create a dummy file to record the mtime for when we updated the
        // index.
        paths::create(&path.join(LAST_UPDATED_FILE))?;

        Ok(())
    }

    fn invalidate_cache(&mut self) {
        if !self.updated {
            self.needs_update = true;
        }
    }

    fn is_updated(&self) -> bool {
        self.updated
    }

    fn download(&mut self, pkg: PackageId, checksum: &str) -> CargoResult<MaybeLock> {
        let registry_config = loop {
            match self.config()? {
                Poll::Pending => self.block_until_ready()?,
                Poll::Ready(cfg) => break cfg.unwrap(),
            }
        };

        download::download(
            &self.cache_path,
            &self.config,
            pkg,
            checksum,
            registry_config,
        )
    }

    fn finish_download(
        &mut self,
        pkg: PackageId,
        checksum: &str,
        data: &[u8],
    ) -> CargoResult<File> {
        download::finish_download(&self.cache_path, &self.config, pkg, checksum, data)
    }

    fn is_crate_downloaded(&self, pkg: PackageId) -> bool {
        download::is_crate_downloaded(&self.cache_path, &self.config, pkg)
    }
}

impl<'cfg> Drop for RemoteRegistry<'cfg> {
    fn drop(&mut self) {
        // Just be sure to drop this before our other fields
        self.tree.borrow_mut().take();
    }
}
<commit_msg>Auto merge of #10507 - Eh2406:finer_grained_cashe, r=weihanglo<commit_after>use crate::core::{GitReference, PackageId, SourceId};
use crate::sources::git;
use crate::sources::registry::download;
use crate::sources::registry::MaybeLock;
use crate::sources::registry::{LoadResponse, RegistryConfig, RegistryData};
use crate::util::errors::CargoResult;
use crate::util::interning::InternedString;
use crate::util::{Config, Filesystem};
use anyhow::Context as _;
use cargo_util::paths;
use lazycell::LazyCell;
use log::{debug, trace};
use std::cell::{Cell, Ref, RefCell};
use std::fs::File;
use std::mem;
use std::path::Path;
use std::str;
use std::task::Poll;

/// A remote registry is a registry that lives at a remote URL (such as
/// crates.io). The git index is cloned locally, and `.crate` files are
/// downloaded as needed and cached locally.
pub struct RemoteRegistry<'cfg> {
    index_path: Filesystem,
    /// Path to the cache of `.crate` files (`$CARGO_HOME/registry/path/$REG-HASH`).
    cache_path: Filesystem,
    source_id: SourceId,
    index_git_ref: GitReference,
    config: &'cfg Config,
    tree: RefCell<Option<git2::Tree<'static>>>,
    repo: LazyCell<git2::Repository>,
    head: Cell<Option<git2::Oid>>,
    current_sha: Cell<Option<InternedString>>,
    needs_update: bool, // Does this registry need to be updated?
    updated: bool,      // Has this registry been updated this session?
}

impl<'cfg> RemoteRegistry<'cfg> {
    pub fn new(source_id: SourceId, config: &'cfg Config, name: &str) -> RemoteRegistry<'cfg> {
        RemoteRegistry {
            index_path: config.registry_index_path().join(name),
            cache_path: config.registry_cache_path().join(name),
            source_id,
            config,
            // TODO: we should probably make this configurable
            index_git_ref: GitReference::DefaultBranch,
            tree: RefCell::new(None),
            repo: LazyCell::new(),
            head: Cell::new(None),
            current_sha: Cell::new(None),
            needs_update: false,
            updated: false,
        }
    }

    fn repo(&self) -> CargoResult<&git2::Repository> {
        self.repo.try_borrow_with(|| {
            let path = self.config.assert_package_cache_locked(&self.index_path);

            // Fast path without a lock
            if let Ok(repo) = git2::Repository::open(&path) {
                trace!("opened a repo without a lock");
                return Ok(repo);
            }

            // Ok, now we need to lock and try the whole thing over again.
            trace!("acquiring registry index lock");
            match git2::Repository::open(&path) {
                Ok(repo) => Ok(repo),
                Err(_) => {
                    drop(paths::remove_dir_all(&path));
                    paths::create_dir_all(&path)?;

                    // Note that we'd actually prefer to use a bare repository
                    // here as we're not actually going to check anything out.
                    // All versions of Cargo, though, share the same CARGO_HOME,
                    // so for compatibility with older Cargo which *does* do
                    // checkouts we make sure to initialize a new full
                    // repository (not a bare one).
                    //
                    // We should change this to `init_bare` whenever we feel
                    // like enough time has passed or if we change the directory
                    // that the folder is located in, such as by changing the
                    // hash at the end of the directory.
                    //
                    // Note that in the meantime we also skip `init.templatedir`
                    // as it can be misconfigured sometimes or otherwise add
                    // things that we don't want.
                    let mut opts = git2::RepositoryInitOptions::new();
                    opts.external_template(false);
                    Ok(git2::Repository::init_opts(&path, &opts).with_context(|| {
                        format!("failed to initialize index git repository (in {:?})", path)
                    })?)
                }
            }
        })
    }

    fn head(&self) -> CargoResult<git2::Oid> {
        if self.head.get().is_none() {
            let repo = self.repo()?;
            let oid = self.index_git_ref.resolve(repo)?;
            self.head.set(Some(oid));
        }
        Ok(self.head.get().unwrap())
    }

    fn tree(&self) -> CargoResult<Ref<'_, git2::Tree<'_>>> {
        {
            let tree = self.tree.borrow();
            if tree.is_some() {
                return Ok(Ref::map(tree, |s| s.as_ref().unwrap()));
            }
        }
        let repo = self.repo()?;
        let commit = repo.find_commit(self.head()?)?;
        let tree = commit.tree()?;

        // Unfortunately in libgit2 the tree objects look like they've got a
        // reference to the repository object which means that a tree cannot
        // outlive the repository that it came from. Here we want to cache this
        // tree, though, so to accomplish this we transmute it to a static
        // lifetime.
        //
        // Note that we don't actually hand out the static lifetime, instead we
        // only return a scoped one from this function. Additionally the repo
        // we loaded from (above) lives as long as this object
        // (`RemoteRegistry`) so we then just need to ensure that the tree is
        // destroyed first in the destructor, hence the destructor on
        // `RemoteRegistry` below.
        let tree = unsafe { mem::transmute::<git2::Tree<'_>, git2::Tree<'static>>(tree) };
        *self.tree.borrow_mut() = Some(tree);
        Ok(Ref::map(self.tree.borrow(), |s| s.as_ref().unwrap()))
    }

    fn current_version(&self) -> Option<InternedString> {
        if let Some(sha) = self.current_sha.get() {
            return Some(sha);
        }
        let sha = InternedString::new(&self.head().ok()?.to_string());
        self.current_sha.set(Some(sha));
        Some(sha)
    }
}

const LAST_UPDATED_FILE: &str = ".last-updated";

impl<'cfg> RegistryData for RemoteRegistry<'cfg> {
    fn prepare(&self) -> CargoResult<()> {
        self.repo()?; // create intermediate dirs and initialize the repo
        Ok(())
    }

    fn index_path(&self) -> &Filesystem {
        &self.index_path
    }

    fn assert_index_locked<'a>(&self, path: &'a Filesystem) -> &'a Path {
        self.config.assert_package_cache_locked(path)
    }

    // `index_version` Is a string representing the version of the file used to construct the cached copy.
    // Older versions of Cargo used the single value of the hash of the HEAD commit as a `index_version`.
    // This is technically correct but a little too conservative. If a new commit is fetched all cached
    // files need to be regenerated even if a particular file was not changed.
    // Cargo now reads the `index_version` in two parts the cache file is considered valid if `index_version`
    // ends with the hash of the HEAD commit OR if it starts with the hash of the file's contents.
    // In the future cargo can write cached files with `index_version` = `git_file_hash + ":" + `git_commit_hash`,
    // but for now it still uses `git_commit_hash` to be compatible with older Cargoes.
    fn load(
        &mut self,
        _root: &Path,
        path: &Path,
        index_version: Option<&str>,
    ) -> Poll<CargoResult<LoadResponse>> {
        if self.needs_update {
            return Poll::Pending;
        }
        // Check if the cache is valid.
        let git_commit_hash = self.current_version();
        if let (Some(c), Some(i)) = (git_commit_hash, index_version) {
            if i.ends_with(c.as_str()) {
                return Poll::Ready(Ok(LoadResponse::CacheValid));
            }
        }
        // Note that the index calls this method and the filesystem is locked
        // in the index, so we don't need to worry about an `update_index`
        // happening in a different process.
        fn load_helper(
            registry: &RemoteRegistry<'_>,
            path: &Path,
            index_version: Option<&str>,
            git_commit_hash: Option<&str>,
        ) -> CargoResult<LoadResponse> {
            let repo = registry.repo()?;
            let tree = registry.tree()?;
            let entry = tree.get_path(path);
            let entry = entry?;
            let git_file_hash = entry.id().to_string();

            if let Some(i) = index_version {
                if i.starts_with(git_file_hash.as_str()) {
                    return Ok(LoadResponse::CacheValid);
                }
            }

            let object = entry.to_object(repo)?;
            let blob = match object.as_blob() {
                Some(blob) => blob,
                None => anyhow::bail!("path `{}` is not a blob in the git repo", path.display()),
            };

            Ok(LoadResponse::Data {
                raw_data: blob.content().to_vec(),
                index_version: git_commit_hash.map(String::from),
                // TODO: When the reading code has been stable for long enough (Say 8/2022)
                // change to `git_file_hash + ":" + git_commit_hash`
            })
        }

        match load_helper(&self, path, index_version, git_commit_hash.as_deref()) {
            Ok(result) => Poll::Ready(Ok(result)),
            Err(_) if !self.updated => {
                // If git returns an error and we haven't updated the repo, return
                // pending to allow an update to try again.
                self.needs_update = true;
                Poll::Pending
            }
            Err(e)
                if e.downcast_ref::<git2::Error>()
                    .map(|e| e.code() == git2::ErrorCode::NotFound)
                    .unwrap_or_default() =>
            {
                // The repo has been updated and the file does not exist.
                Poll::Ready(Ok(LoadResponse::NotFound))
            }
            Err(e) => Poll::Ready(Err(e)),
        }
    }

    fn config(&mut self) -> Poll<CargoResult<Option<RegistryConfig>>> {
        debug!("loading config");
        self.prepare()?;
        self.config.assert_package_cache_locked(&self.index_path);
        match self.load(Path::new(""), Path::new("config.json"), None)? {
            Poll::Ready(LoadResponse::Data { raw_data, .. }) => {
                trace!("config loaded");
                Poll::Ready(Ok(Some(serde_json::from_slice(&raw_data)?)))
            }
            Poll::Ready(_) => Poll::Ready(Ok(None)),
            Poll::Pending => Poll::Pending,
        }
    }

    fn block_until_ready(&mut self) -> CargoResult<()> {
        if !self.needs_update {
            return Ok(());
        }

        self.updated = true;
        self.needs_update = false;

        if self.config.offline() {
            return Ok(());
        }
        if self.config.cli_unstable().no_index_update {
            return Ok(());
        }
        // Make sure the index is only updated once per session since it is an
        // expensive operation. This generally only happens when the resolver
        // is run multiple times, such as during `cargo publish`.
        if self.config.updated_sources().contains(&self.source_id) {
            return Ok(());
        }

        debug!("updating the index");

        // Ensure that we'll actually be able to acquire an HTTP handle later on
        // once we start trying to download crates. This will weed out any
        // problems with `.cargo/config` configuration related to HTTP.
        //
        // This way if there's a problem the error gets printed before we even
        // hit the index, which may not actually read this configuration.
        self.config.http()?;

        self.prepare()?;
        self.head.set(None);
        *self.tree.borrow_mut() = None;
        self.current_sha.set(None);
        let path = self.config.assert_package_cache_locked(&self.index_path);
        self.config
            .shell()
            .status("Updating", self.source_id.display_index())?;

        // Fetch the latest version of our `index_git_ref` into the index
        // checkout.
        let url = self.source_id.url();
        let repo = self.repo.borrow_mut().unwrap();
        git::fetch(repo, url.as_str(), &self.index_git_ref, self.config)
            .with_context(|| format!("failed to fetch `{}`", url))?;
        self.config.updated_sources().insert(self.source_id);

        // Create a dummy file to record the mtime for when we updated the
        // index.
        paths::create(&path.join(LAST_UPDATED_FILE))?;

        Ok(())
    }

    fn invalidate_cache(&mut self) {
        if !self.updated {
            self.needs_update = true;
        }
    }

    fn is_updated(&self) -> bool {
        self.updated
    }

    fn download(&mut self, pkg: PackageId, checksum: &str) -> CargoResult<MaybeLock> {
        let registry_config = loop {
            match self.config()? {
                Poll::Pending => self.block_until_ready()?,
                Poll::Ready(cfg) => break cfg.unwrap(),
            }
        };

        download::download(
            &self.cache_path,
            &self.config,
            pkg,
            checksum,
            registry_config,
        )
    }

    fn finish_download(
        &mut self,
        pkg: PackageId,
        checksum: &str,
        data: &[u8],
    ) -> CargoResult<File> {
        download::finish_download(&self.cache_path, &self.config, pkg, checksum, data)
    }

    fn is_crate_downloaded(&self, pkg: PackageId) -> bool {
        download::is_crate_downloaded(&self.cache_path, &self.config, pkg)
    }
}

impl<'cfg> Drop for RemoteRegistry<'cfg> {
    fn drop(&mut self) {
        // Just be sure to drop this before our other fields
        self.tree.borrow_mut().take();
    }
}
<|endoftext|>


<commit_before>use alloc::vec::Vec;
use std::fmt;

use super::lazy_buffer::LazyBuffer;

/// An iterator to iterate through all the `n`-length combinations in an iterator, with replacement.
///
/// See [`.combinations_with_replacement()`](../trait.Itertools.html#method.combinations_with_replacement) for more information.
#[derive(Clone)]
pub struct CombinationsWithReplacement<I>
where
    I: Iterator,
    I::Item: Clone,
{
    indices: Vec<usize>,
    // The current known max index value. This increases as pool grows.
    max_index: usize,
    pool: LazyBuffer<I>,
    first: bool,
}

impl<I> fmt::Debug for CombinationsWithReplacement<I>
where
    I: Iterator + fmt::Debug,
    I::Item: fmt::Debug + Clone,
{
    debug_fmt_fields!(Combinations, indices, max_index, pool, first);
}

impl<I> CombinationsWithReplacement<I>
where
    I: Iterator,
    I::Item: Clone,
{
    /// Map the current mask over the pool to get an output combination
    fn current(&self) -> Vec<I::Item> {
        self.indices.iter().map(|i| self.pool[*i].clone()).collect()
    }
}

/// Create a new `CombinationsWithReplacement` from a clonable iterator.
pub fn combinations_with_replacement<I>(iter: I, k: usize) -> CombinationsWithReplacement<I>
where
    I: Iterator,
    I::Item: Clone,
{
    let indices: Vec<usize> = alloc::vec![0; k];
    let pool: LazyBuffer<I> = LazyBuffer::new(iter);

    CombinationsWithReplacement {
        indices,
        max_index: 0,
        pool,
        first: true,
    }
}

impl<I> Iterator for CombinationsWithReplacement<I>
where
    I: Iterator,
    I::Item: Clone,
{
    type Item = Vec<I::Item>;
    fn next(&mut self) -> Option<Self::Item> {
        // If this is the first iteration, return early
        if self.first {
            // In empty edge cases, stop iterating immediately
            return if self.indices.len() != 0 && !self.pool.get_next() {
                None
            // Otherwise, yield the initial state
            } else {
                self.first = false;
                Some(self.current())
            };
        }

        // Check if we need to consume more from the iterator
        // This will run while we increment our first index digit
        if self.pool.get_next() {
            self.max_index = self.pool.len() - 1;
        }

        // Work out where we need to update our indices
        let mut increment: Option<(usize, usize)> = None;
        for (i, indices_int) in self.indices.iter().enumerate().rev() {
            if indices_int < &self.max_index {
                increment = Some((i, indices_int + 1));
                break;
            }
        }

        match increment {
            // If we can update the indices further
            Some((increment_from, increment_value)) => {
                // We need to update the rightmost non-max value
                // and all those to the right
                for indices_index in increment_from..self.indices.len() {
                    self.indices[indices_index] = increment_value
                }
                Some(self.current())
            }
            // Otherwise, we're done
            None => None,
        }
    }
}
<commit_msg>Eliminate redundant CombinationsWithReplacement::max_index<commit_after>use alloc::vec::Vec;
use std::fmt;

use super::lazy_buffer::LazyBuffer;

/// An iterator to iterate through all the `n`-length combinations in an iterator, with replacement.
///
/// See [`.combinations_with_replacement()`](../trait.Itertools.html#method.combinations_with_replacement) for more information.
#[derive(Clone)]
pub struct CombinationsWithReplacement<I>
where
    I: Iterator,
    I::Item: Clone,
{
    indices: Vec<usize>,
    pool: LazyBuffer<I>,
    first: bool,
}

impl<I> fmt::Debug for CombinationsWithReplacement<I>
where
    I: Iterator + fmt::Debug,
    I::Item: fmt::Debug + Clone,
{
    debug_fmt_fields!(Combinations, indices, pool, first);
}

impl<I> CombinationsWithReplacement<I>
where
    I: Iterator,
    I::Item: Clone,
{
    /// Map the current mask over the pool to get an output combination
    fn current(&self) -> Vec<I::Item> {
        self.indices.iter().map(|i| self.pool[*i].clone()).collect()
    }
}

/// Create a new `CombinationsWithReplacement` from a clonable iterator.
pub fn combinations_with_replacement<I>(iter: I, k: usize) -> CombinationsWithReplacement<I>
where
    I: Iterator,
    I::Item: Clone,
{
    let indices: Vec<usize> = alloc::vec![0; k];
    let pool: LazyBuffer<I> = LazyBuffer::new(iter);

    CombinationsWithReplacement {
        indices,
        pool,
        first: true,
    }
}

impl<I> Iterator for CombinationsWithReplacement<I>
where
    I: Iterator,
    I::Item: Clone,
{
    type Item = Vec<I::Item>;
    fn next(&mut self) -> Option<Self::Item> {
        // If this is the first iteration, return early
        if self.first {
            // In empty edge cases, stop iterating immediately
            return if self.indices.len() != 0 && !self.pool.get_next() {
                None
            // Otherwise, yield the initial state
            } else {
                self.first = false;
                Some(self.current())
            };
        }

        // Check if we need to consume more from the iterator
        // This will run while we increment our first index digit
        self.pool.get_next();

        // Work out where we need to update our indices
        let mut increment: Option<(usize, usize)> = None;
        for (i, indices_int) in self.indices.iter().enumerate().rev() {
            if *indices_int < self.pool.len()-1 {
                increment = Some((i, indices_int + 1));
                break;
            }
        }

        match increment {
            // If we can update the indices further
            Some((increment_from, increment_value)) => {
                // We need to update the rightmost non-max value
                // and all those to the right
                for indices_index in increment_from..self.indices.len() {
                    self.indices[indices_index] = increment_value
                }
                Some(self.current())
            }
            // Otherwise, we're done
            None => None,
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Infrastructure for compiler plugins.
//!
//! Plugins are Rust libraries which extend the behavior of `rustc`
//! in various ways.
//!
//! Plugin authors will use the `Registry` type re-exported by
//! this module, along with its methods.  The rest of the module
//! is for use by `rustc` itself.
//!
//! To define a plugin, build a dylib crate with a
//! `#[plugin_registrar]` function:
//!
//! ```rust,ignore
//! #![crate_name = "myplugin"]
//! #![crate_type = "dylib"]
//! #![feature(plugin_registrar)]
//!
//! extern crate rustc;
//!
//! use rustc::plugin::Registry;
//!
//! #[plugin_registrar]
//! pub fn plugin_registrar(reg: &mut Registry) {
//!     reg.register_macro("mymacro", expand_mymacro);
//! }
//!
//! fn expand_mymacro(...) {  // details elided
//! ```
//!
//! WARNING: We currently don't check that the registrar function
//! has the appropriate type!
//!
//! To use a plugin while compiling another crate:
//!
//! ```rust
//! #![feature(plugin)]
//! #![plugin(myplugin)]
//! ```
//!
//! See the [Plugins Chapter](../../book/compiler-plugins.html) of the book
//! for more examples.

#![crate_name = "rustc_plugin"]
#![unstable(feature = "rustc_private", issue = "27812")]
#![crate_type = "dylib"]
#![crate_type = "rlib"]
#![doc(html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
       html_favicon_url = "https://doc.rust-lang.org/favicon.ico",
       html_root_url = "https://doc.rust-lang.org/nightly/")]
#![cfg_attr(not(stage0), deny(warnings))]

#![feature(staged_api)]
#![feature(rustc_diagnostic_macros)]
#![feature(rustc_private)]

#[macro_use] extern crate log;
#[macro_use] extern crate syntax;
#[macro_use] #[no_link] extern crate rustc_bitflags;

extern crate rustc;
extern crate rustc_back;
extern crate rustc_metadata;
extern crate syntax_pos;
extern crate rustc_errors as errors;

pub use self::registry::Registry;

pub mod diagnostics;
pub mod registry;
pub mod load;
pub mod build;
<commit_msg>Change 'rustc::plugin' to 'rustc_plugin' in doc comment<commit_after>// Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Infrastructure for compiler plugins.
//!
//! Plugins are Rust libraries which extend the behavior of `rustc`
//! in various ways.
//!
//! Plugin authors will use the `Registry` type re-exported by
//! this module, along with its methods.  The rest of the module
//! is for use by `rustc` itself.
//!
//! To define a plugin, build a dylib crate with a
//! `#[plugin_registrar]` function:
//!
//! ```rust,ignore
//! #![crate_name = "myplugin"]
//! #![crate_type = "dylib"]
//! #![feature(plugin_registrar)]
//!
//! extern crate rustc;
//!
//! use rustc_plugin::Registry;
//!
//! #[plugin_registrar]
//! pub fn plugin_registrar(reg: &mut Registry) {
//!     reg.register_macro("mymacro", expand_mymacro);
//! }
//!
//! fn expand_mymacro(...) {  // details elided
//! ```
//!
//! WARNING: We currently don't check that the registrar function
//! has the appropriate type!
//!
//! To use a plugin while compiling another crate:
//!
//! ```rust
//! #![feature(plugin)]
//! #![plugin(myplugin)]
//! ```
//!
//! See the [Plugins Chapter](../../book/compiler-plugins.html) of the book
//! for more examples.

#![crate_name = "rustc_plugin"]
#![unstable(feature = "rustc_private", issue = "27812")]
#![crate_type = "dylib"]
#![crate_type = "rlib"]
#![doc(html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
       html_favicon_url = "https://doc.rust-lang.org/favicon.ico",
       html_root_url = "https://doc.rust-lang.org/nightly/")]
#![cfg_attr(not(stage0), deny(warnings))]

#![feature(staged_api)]
#![feature(rustc_diagnostic_macros)]
#![feature(rustc_private)]

#[macro_use] extern crate log;
#[macro_use] extern crate syntax;
#[macro_use] #[no_link] extern crate rustc_bitflags;

extern crate rustc;
extern crate rustc_back;
extern crate rustc_metadata;
extern crate syntax_pos;
extern crate rustc_errors as errors;

pub use self::registry::Registry;

pub mod diagnostics;
pub mod registry;
pub mod load;
pub mod build;
<|endoftext|>


<commit_before>// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! The main parser interface

use ast::node_id;
use ast;
use codemap::{span, CodeMap, FileMap, CharPos, BytePos};
use codemap;
use diagnostic::{span_handler, mk_span_handler, mk_handler, Emitter};
use parse::attr::parser_attr;
use parse::lexer::{reader, StringReader};
use parse::parser::Parser;
use parse::token::{ident_interner, mk_ident_interner};

use core::io;
use core::option::{None, Option, Some};
use core::path::Path;
use core::result::{Err, Ok, Result};

pub mod lexer;
pub mod parser;
pub mod token;
pub mod comments;
pub mod attr;

/// Common routines shared by parser mods
pub mod common;

/// Functions dealing with operator precedence
pub mod prec;

/// Routines the parser uses to classify AST nodes
pub mod classify;

/// Reporting obsolete syntax
pub mod obsolete;

pub struct ParseSess {
    cm: @codemap::CodeMap,
    next_id: node_id,
    span_diagnostic: @span_handler,
    interner: @ident_interner,
}

pub fn new_parse_sess(demitter: Option<Emitter>) -> @mut ParseSess {
    let cm = @CodeMap::new();
    @mut ParseSess {
        cm: cm,
        next_id: 1,
        span_diagnostic: mk_span_handler(mk_handler(demitter), cm),
        interner: mk_ident_interner(),
    }
}

pub fn new_parse_sess_special_handler(sh: @span_handler,
                                      cm: @codemap::CodeMap)
                                   -> @mut ParseSess {
    @mut ParseSess {
        cm: cm,
        next_id: 1,
        span_diagnostic: sh,
        interner: mk_ident_interner(),
    }
}

// a bunch of utility functions of the form parse_<thing>_from_<source>
// where <thing> includes crate, expr, item, stmt, tts, and one that
// uses a HOF to parse anything, and <source> includes file and
// source_str.

// this appears to be the main entry point for rust parsing by
// rustc and crate:
pub fn parse_crate_from_file(
    input: &Path,
    cfg: ast::crate_cfg,
    sess: @mut ParseSess
) -> @ast::crate {
    let p = new_parser_from_file(sess, /*bad*/ copy cfg, input);
    p.parse_crate_mod(/*bad*/ copy cfg)
    // why is there no p.abort_if_errors here?
}

pub fn parse_crate_from_source_str(
    name: ~str,
    source: @~str,
    cfg: ast::crate_cfg,
    sess: @mut ParseSess
) -> @ast::crate {
    let p = new_parser_from_source_str(
        sess,
        /*bad*/ copy cfg,
        /*bad*/ copy name,
        codemap::FssNone,
        source
    );
    maybe_aborted(p.parse_crate_mod(/*bad*/ copy cfg),p)
}

pub fn parse_expr_from_source_str(
    name: ~str,
    source: @~str,
    +cfg: ast::crate_cfg,
    sess: @mut ParseSess
) -> @ast::expr {
    let p = new_parser_from_source_str(
        sess,
        cfg,
        /*bad*/ copy name,
        codemap::FssNone,
        source
    );
    maybe_aborted(p.parse_expr(), p)
}

pub fn parse_item_from_source_str(
    name: ~str,
    source: @~str,
    +cfg: ast::crate_cfg,
    +attrs: ~[ast::attribute],
    sess: @mut ParseSess
) -> Option<@ast::item> {
    let p = new_parser_from_source_str(
        sess,
        cfg,
        /*bad*/ copy name,
        codemap::FssNone,
        source
    );
    maybe_aborted(p.parse_item(attrs),p)
}

pub fn parse_stmt_from_source_str(
    name: ~str,
    source: @~str,
    +cfg: ast::crate_cfg,
    +attrs: ~[ast::attribute],
    sess: @mut ParseSess
) -> @ast::stmt {
    let p = new_parser_from_source_str(
        sess,
        cfg,
        /*bad*/ copy name,
        codemap::FssNone,
        source
    );
    maybe_aborted(p.parse_stmt(attrs),p)
}

pub fn parse_tts_from_source_str(
    name: ~str,
    source: @~str,
    +cfg: ast::crate_cfg,
    sess: @mut ParseSess
) -> ~[ast::token_tree] {
    let p = new_parser_from_source_str(
        sess,
        cfg,
        /*bad*/ copy name,
        codemap::FssNone,
        source
    );
    *p.quote_depth += 1u;
    maybe_aborted(p.parse_all_token_trees(),p)
}

pub fn parse_from_source_str<T>(
    f: &fn (Parser) -> T,
    name: ~str, ss: codemap::FileSubstr,
    source: @~str,
    +cfg: ast::crate_cfg,
    sess: @mut ParseSess
) -> T {
    let p = new_parser_from_source_str(
        sess,
        cfg,
        /*bad*/ copy name,
        /*bad*/ copy ss,
        source
    );
    let r = f(p);
    if !p.reader.is_eof() {
        p.reader.fatal(~"expected end-of-string");
    }
    maybe_aborted(r,p)
}

pub fn next_node_id(sess: @mut ParseSess) -> node_id {
    let rv = sess.next_id;
    sess.next_id += 1;
    // ID 0 is reserved for the crate and doesn't actually exist in the AST
    fail_unless!(rv != 0);
    return rv;
}

pub fn new_parser_from_source_str(sess: @mut ParseSess,
                                  +cfg: ast::crate_cfg,
                                  +name: ~str,
                                  +ss: codemap::FileSubstr,
                                  source: @~str)
                               -> Parser {
    let filemap = sess.cm.new_filemap_w_substr(name, ss, source);
    let srdr = lexer::new_string_reader(
        copy sess.span_diagnostic,
        filemap,
        sess.interner
    );
    Parser(sess, cfg, srdr as @reader)
}

/// Read the entire source file, return a parser
/// that draws from that string
pub fn new_parser_result_from_file(
    sess: @mut ParseSess,
    +cfg: ast::crate_cfg,
    path: &Path
) -> Result<Parser, ~str> {
    match io::read_whole_file_str(path) {
        Ok(src) => {
            let filemap = sess.cm.new_filemap(path.to_str(), @src);
            let srdr = lexer::new_string_reader(copy sess.span_diagnostic,
                                                filemap,
                                                sess.interner);
            Ok(Parser(sess, cfg, srdr as @reader))

        }
        Err(e) => Err(e)
    }
}

/// Create a new parser, handling errors as appropriate
/// if the file doesn't exist
pub fn new_parser_from_file(
    sess: @mut ParseSess,
    +cfg: ast::crate_cfg,
    path: &Path
) -> Parser {
    match new_parser_result_from_file(sess, cfg, path) {
        Ok(parser) => parser,
        Err(e) => {
            sess.span_diagnostic.handler().fatal(e)
        }
    }
}

/// Create a new parser based on a span from an existing parser. Handles
/// error messages correctly when the file does not exist.
pub fn new_sub_parser_from_file(
    sess: @mut ParseSess,
    +cfg: ast::crate_cfg,
    path: &Path,
    sp: span
) -> Parser {
    match new_parser_result_from_file(sess, cfg, path) {
        Ok(parser) => parser,
        Err(e) => {
            sess.span_diagnostic.span_fatal(sp, e)
        }
    }
}

pub fn new_parser_from_tts(
    sess: @mut ParseSess,
    +cfg: ast::crate_cfg,
    +tts: ~[ast::token_tree]
) -> Parser {
    let trdr = lexer::new_tt_reader(
        copy sess.span_diagnostic,
        sess.interner,
        None,
        tts
    );
    Parser(sess, cfg, trdr as @reader)
}

// abort if necessary
pub fn maybe_aborted<T>(+result : T, p: Parser) -> T {
    p.abort_if_errors();
    result
}

#[cfg(test)]
mod test {
    use super::*;
    use std::serialize::Encodable;
    use std;
    use core::io;
    use core::option::None;
    use core::str;
    use util::testing::*;

    #[test] fn to_json_str (val: @Encodable<std::json::Encoder>) -> ~str {
        do io::with_str_writer |writer| {
            val.encode(~std::json::Encoder(writer));
        }
    }

    #[test] fn alltts () {
        let tts = parse_tts_from_source_str(
            ~"bogofile",
            @~"fn foo (x : int) { x; }",
            ~[],
            new_parse_sess(None));
        check_equal(to_json_str(@tts as @Encodable<std::json::Encoder>),
                    ~"[[\"tt_tok\",[,[\"IDENT\",[\"fn\",false]]]],\
                      [\"tt_tok\",[,[\"IDENT\",[\"foo\",false]]]],\
                      [\"tt_delim\",[[[\"tt_tok\",[,[\"LPAREN\",[]]]],\
                      [\"tt_tok\",[,[\"IDENT\",[\"x\",false]]]],\
                      [\"tt_tok\",[,[\"COLON\",[]]]],\
                      [\"tt_tok\",[,[\"IDENT\",[\"int\",false]]]],\
                      [\"tt_tok\",[,[\"RPAREN\",[]]]]]]],\
                      [\"tt_delim\",[[[\"tt_tok\",[,[\"LBRACE\",[]]]],\
                      [\"tt_tok\",[,[\"IDENT\",[\"x\",false]]]],\
                      [\"tt_tok\",[,[\"SEMI\",[]]]],\
                      [\"tt_tok\",[,[\"RBRACE\",[]]]]]]]]"
                   );
        let ast1 = new_parser_from_tts(new_parse_sess(None),~[],tts)
            .parse_item(~[]);
        let ast2 = parse_item_from_source_str(
            ~"bogofile",
            @~"fn foo (x : int) { x; }",
            ~[],~[],
            new_parse_sess(None));
        check_equal(ast1,ast2);
    }
}

//
// Local Variables:
// mode: rust
// fill-column: 78;
// indent-tabs-mode: nil
// c-basic-offset: 4
// buffer-file-coding-system: utf-8-unix
// End:
//
<commit_msg>Test case fixup (old one tested the old bad behavior).<commit_after>// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! The main parser interface

use ast::node_id;
use ast;
use codemap::{span, CodeMap, FileMap, CharPos, BytePos};
use codemap;
use diagnostic::{span_handler, mk_span_handler, mk_handler, Emitter};
use parse::attr::parser_attr;
use parse::lexer::{reader, StringReader};
use parse::parser::Parser;
use parse::token::{ident_interner, mk_ident_interner};

use core::io;
use core::option::{None, Option, Some};
use core::path::Path;
use core::result::{Err, Ok, Result};

pub mod lexer;
pub mod parser;
pub mod token;
pub mod comments;
pub mod attr;

/// Common routines shared by parser mods
pub mod common;

/// Functions dealing with operator precedence
pub mod prec;

/// Routines the parser uses to classify AST nodes
pub mod classify;

/// Reporting obsolete syntax
pub mod obsolete;

pub struct ParseSess {
    cm: @codemap::CodeMap,
    next_id: node_id,
    span_diagnostic: @span_handler,
    interner: @ident_interner,
}

pub fn new_parse_sess(demitter: Option<Emitter>) -> @mut ParseSess {
    let cm = @CodeMap::new();
    @mut ParseSess {
        cm: cm,
        next_id: 1,
        span_diagnostic: mk_span_handler(mk_handler(demitter), cm),
        interner: mk_ident_interner(),
    }
}

pub fn new_parse_sess_special_handler(sh: @span_handler,
                                      cm: @codemap::CodeMap)
                                   -> @mut ParseSess {
    @mut ParseSess {
        cm: cm,
        next_id: 1,
        span_diagnostic: sh,
        interner: mk_ident_interner(),
    }
}

// a bunch of utility functions of the form parse_<thing>_from_<source>
// where <thing> includes crate, expr, item, stmt, tts, and one that
// uses a HOF to parse anything, and <source> includes file and
// source_str.

// this appears to be the main entry point for rust parsing by
// rustc and crate:
pub fn parse_crate_from_file(
    input: &Path,
    cfg: ast::crate_cfg,
    sess: @mut ParseSess
) -> @ast::crate {
    let p = new_parser_from_file(sess, /*bad*/ copy cfg, input);
    p.parse_crate_mod(/*bad*/ copy cfg)
    // why is there no p.abort_if_errors here?
}

pub fn parse_crate_from_source_str(
    name: ~str,
    source: @~str,
    cfg: ast::crate_cfg,
    sess: @mut ParseSess
) -> @ast::crate {
    let p = new_parser_from_source_str(
        sess,
        /*bad*/ copy cfg,
        /*bad*/ copy name,
        codemap::FssNone,
        source
    );
    maybe_aborted(p.parse_crate_mod(/*bad*/ copy cfg),p)
}

pub fn parse_expr_from_source_str(
    name: ~str,
    source: @~str,
    +cfg: ast::crate_cfg,
    sess: @mut ParseSess
) -> @ast::expr {
    let p = new_parser_from_source_str(
        sess,
        cfg,
        /*bad*/ copy name,
        codemap::FssNone,
        source
    );
    maybe_aborted(p.parse_expr(), p)
}

pub fn parse_item_from_source_str(
    name: ~str,
    source: @~str,
    +cfg: ast::crate_cfg,
    +attrs: ~[ast::attribute],
    sess: @mut ParseSess
) -> Option<@ast::item> {
    let p = new_parser_from_source_str(
        sess,
        cfg,
        /*bad*/ copy name,
        codemap::FssNone,
        source
    );
    maybe_aborted(p.parse_item(attrs),p)
}

pub fn parse_stmt_from_source_str(
    name: ~str,
    source: @~str,
    +cfg: ast::crate_cfg,
    +attrs: ~[ast::attribute],
    sess: @mut ParseSess
) -> @ast::stmt {
    let p = new_parser_from_source_str(
        sess,
        cfg,
        /*bad*/ copy name,
        codemap::FssNone,
        source
    );
    maybe_aborted(p.parse_stmt(attrs),p)
}

pub fn parse_tts_from_source_str(
    name: ~str,
    source: @~str,
    +cfg: ast::crate_cfg,
    sess: @mut ParseSess
) -> ~[ast::token_tree] {
    let p = new_parser_from_source_str(
        sess,
        cfg,
        /*bad*/ copy name,
        codemap::FssNone,
        source
    );
    *p.quote_depth += 1u;
    maybe_aborted(p.parse_all_token_trees(),p)
}

pub fn parse_from_source_str<T>(
    f: &fn (Parser) -> T,
    name: ~str, ss: codemap::FileSubstr,
    source: @~str,
    +cfg: ast::crate_cfg,
    sess: @mut ParseSess
) -> T {
    let p = new_parser_from_source_str(
        sess,
        cfg,
        /*bad*/ copy name,
        /*bad*/ copy ss,
        source
    );
    let r = f(p);
    if !p.reader.is_eof() {
        p.reader.fatal(~"expected end-of-string");
    }
    maybe_aborted(r,p)
}

pub fn next_node_id(sess: @mut ParseSess) -> node_id {
    let rv = sess.next_id;
    sess.next_id += 1;
    // ID 0 is reserved for the crate and doesn't actually exist in the AST
    fail_unless!(rv != 0);
    return rv;
}

pub fn new_parser_from_source_str(sess: @mut ParseSess,
                                  +cfg: ast::crate_cfg,
                                  +name: ~str,
                                  +ss: codemap::FileSubstr,
                                  source: @~str)
                               -> Parser {
    let filemap = sess.cm.new_filemap_w_substr(name, ss, source);
    let srdr = lexer::new_string_reader(
        copy sess.span_diagnostic,
        filemap,
        sess.interner
    );
    Parser(sess, cfg, srdr as @reader)
}

/// Read the entire source file, return a parser
/// that draws from that string
pub fn new_parser_result_from_file(
    sess: @mut ParseSess,
    +cfg: ast::crate_cfg,
    path: &Path
) -> Result<Parser, ~str> {
    match io::read_whole_file_str(path) {
        Ok(src) => {
            let filemap = sess.cm.new_filemap(path.to_str(), @src);
            let srdr = lexer::new_string_reader(copy sess.span_diagnostic,
                                                filemap,
                                                sess.interner);
            Ok(Parser(sess, cfg, srdr as @reader))

        }
        Err(e) => Err(e)
    }
}

/// Create a new parser, handling errors as appropriate
/// if the file doesn't exist
pub fn new_parser_from_file(
    sess: @mut ParseSess,
    +cfg: ast::crate_cfg,
    path: &Path
) -> Parser {
    match new_parser_result_from_file(sess, cfg, path) {
        Ok(parser) => parser,
        Err(e) => {
            sess.span_diagnostic.handler().fatal(e)
        }
    }
}

/// Create a new parser based on a span from an existing parser. Handles
/// error messages correctly when the file does not exist.
pub fn new_sub_parser_from_file(
    sess: @mut ParseSess,
    +cfg: ast::crate_cfg,
    path: &Path,
    sp: span
) -> Parser {
    match new_parser_result_from_file(sess, cfg, path) {
        Ok(parser) => parser,
        Err(e) => {
            sess.span_diagnostic.span_fatal(sp, e)
        }
    }
}

pub fn new_parser_from_tts(
    sess: @mut ParseSess,
    +cfg: ast::crate_cfg,
    +tts: ~[ast::token_tree]
) -> Parser {
    let trdr = lexer::new_tt_reader(
        copy sess.span_diagnostic,
        sess.interner,
        None,
        tts
    );
    Parser(sess, cfg, trdr as @reader)
}

// abort if necessary
pub fn maybe_aborted<T>(+result : T, p: Parser) -> T {
    p.abort_if_errors();
    result
}

#[cfg(test)]
mod test {
    use super::*;
    use std::serialize::Encodable;
    use std;
    use core::io;
    use core::option::None;
    use core::str;
    use util::testing::*;

    #[test] fn to_json_str (val: @Encodable<std::json::Encoder>) -> ~str {
        do io::with_str_writer |writer| {
            val.encode(~std::json::Encoder(writer));
        }
    }

    #[test] fn alltts () {
        let tts = parse_tts_from_source_str(
            ~"bogofile",
            @~"fn foo (x : int) { x; }",
            ~[],
            new_parse_sess(None));
        check_equal(to_json_str(@tts as Encodable::<std::json::Encoder>),
                    ~"[[\"tt_tok\",[null,[\"IDENT\",[\"fn\",false]]]],\
                      [\"tt_tok\",[null,[\"IDENT\",[\"foo\",false]]]],\
                      [\"tt_delim\",[[[\"tt_tok\",[null,[\"LPAREN\",[]]]],\
                      [\"tt_tok\",[null,[\"IDENT\",[\"x\",false]]]],\
                      [\"tt_tok\",[null,[\"COLON\",[]]]],\
                      [\"tt_tok\",[null,[\"IDENT\",[\"int\",false]]]],\
                      [\"tt_tok\",[null,[\"RPAREN\",[]]]]]]],\
                      [\"tt_delim\",[[[\"tt_tok\",[null,[\"LBRACE\",[]]]],\
                      [\"tt_tok\",[null,[\"IDENT\",[\"x\",false]]]],\
                      [\"tt_tok\",[null,[\"SEMI\",[]]]],\
                      [\"tt_tok\",[null,[\"RBRACE\",[]]]]]]]]"
                   );
        let ast1 = new_parser_from_tts(new_parse_sess(None),~[],tts)
            .parse_item(~[]);
        let ast2 = parse_item_from_source_str(
            ~"bogofile",
            @~"fn foo (x : int) { x; }",
            ~[],~[],
            new_parse_sess(None));
        check_equal(ast1,ast2);
    }
}

//
// Local Variables:
// mode: rust
// fill-column: 78;
// indent-tabs-mode: nil
// c-basic-offset: 4
// buffer-file-coding-system: utf-8-unix
// End:
//
<|endoftext|>


<commit_before>pub mod spin;
<commit_msg>libuni: Now reexports alloc::arc::Arc<commit_after>pub use alloc::arc::Arc;

pub mod spin;
<|endoftext|>