<commit_before>//! `pagecache` is a lock-free pagecache and log for building high-performance
//! databases.

pub mod constants;
pub mod logger;

mod blob_io;
mod diskptr;
mod iobuf;
mod iterator;
mod materializer;
mod pagecache;
mod parallel_io;
mod reader;
mod reservation;
mod segment;
mod snapshot;

#[cfg(feature = "measure_allocs")]
mod measure_allocs;

#[cfg(feature = "measure_allocs")]
#[global_allocator]
static ALLOCATOR: measure_allocs::TrackingAllocator =
    measure_allocs::TrackingAllocator;

use crate::{debug, AtomicLsn, DeserializeOwned, SeqCst, Serialize};

use self::{
    blob_io::{gc_blobs, read_blob, remove_blob, write_blob},
    constants::{
        BATCH_MANIFEST_PID, CONFIG_PID, COUNTER_PID, META_PID,
        PAGE_CONSOLIDATION_THRESHOLD,
    },
    iobuf::{IoBuf, IoBufs},
    iterator::{raw_segment_iter_from, LogIter},
    pagecache::Update,
    parallel_io::Pio,
    segment::SegmentAccountant,
    snapshot::advance_snapshot,
};

pub(crate) use self::{
    logger::{MessageHeader, SegmentHeader},
    reader::{read_message, read_segment_header},
    reservation::Reservation,
    snapshot::{read_snapshot_or_default, PageState, Snapshot},
};

pub use self::{
    constants::{
        BATCH_MANIFEST_INLINE_LEN, BLOB_INLINE_LEN, MAX_SPACE_AMPLIFICATION,
        MINIMUM_ITEMS_PER_SEGMENT, MSG_HEADER_LEN, SEG_HEADER_LEN,
    },
    diskptr::DiskPtr,
    logger::{Log, LogRead},
    materializer::Materializer,
    pagecache::{PageCache, PagePtr, RecoveryGuard},
    segment::SegmentMode,
};

/// A segment id == Segment.LogOffset / Config.SegmentSize.
pub type SegmentId = usize;

/// An offset in the file storage sometimes called LogId (lid).
pub type LogOffset = u64;

/// A pointer to an blob blob.
pub type BlobPointer = Lsn;

/// A logical sequence number of a log entry ==
/// aligned log entry LogOffset ==
/// Segment.LogOffset & ~SegmentSize
/// Represent an Id of a log segment.
pub type Lsn = i64;

/// A page identifier.
pub type PageId = u64;

/// A byte used to disambiguate log message types
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
#[repr(u8)]
pub enum MessageKind {
    /// The EVIL_BYTE is written as a canary to help
    /// detect torn writes.
    Corrupted = 0,
    /// Indicates that the following buffer corresponds
    /// to a reservation for an in-memory operation that
    /// failed to complete. It should be skipped during
    /// recovery.
    Cancelled = 1,
    /// Indicates that the following buffer is used
    /// as padding to fill out the rest of the segment
    /// before sealing it.
    Pad = 2,
    /// Indicates that the following buffer contains
    /// an Lsn for the last write in an atomic writebatch.
    BatchManifest = 3,
    /// Indicates that this page was freed from the pagetable.
    Free = 4,
    /// Indicates that the last persisted ID was at least
    /// this high.
    Counter = 5,
    /// The meta page, stored inline
    InlineMeta = 6,
    /// The meta page, stored blobly
    BlobMeta = 7,
    /// The config page, stored inline
    InlineConfig = 8,
    /// The config page, stored blobly
    BlobConfig = 9,
    /// A consolidated page replacement, stored inline
    InlineReplace = 10,
    /// A consolidated page replacement, stored blobly
    BlobReplace = 11,
    /// A partial page update, stored inline
    InlineAppend = 12,
    /// A partial page update, stored blobly
    BlobAppend = 13,
}

impl MessageKind {
    pub(crate) const fn into(self) -> u8 {
        self as u8
    }
}

impl From<u8> for MessageKind {
    fn from(byte: u8) -> Self {
        use MessageKind::*;
        match byte {
            0 => Corrupted,
            1 => Cancelled,
            2 => Pad,
            3 => BatchManifest,
            4 => Free,
            5 => Counter,
            6 => InlineMeta,
            7 => BlobMeta,
            8 => InlineConfig,
            9 => BlobConfig,
            10 => InlineReplace,
            11 => BlobReplace,
            12 => InlineAppend,
            13 => BlobAppend,
            other => {
                debug!("encountered unexpected message kind byte {}", other);
                Corrupted
            }
        }
    }
}

/// The high-level types of stored information
/// about pages and their mutations
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum LogKind {
    /// Persisted data containing a page replacement
    Replace,
    /// Persisted immutable update
    Append,
    /// Freeing of a page
    Free,
    /// Some state indicating this should be skipped
    Skip,
    /// Unexpected corruption
    Corrupted,
}

fn log_kind_from_update<PageFrag>(update: &Update<PageFrag>) -> LogKind
where
    PageFrag: DeserializeOwned + Serialize,
{
    use Update::*;

    match update {
        Free => LogKind::Free,
        Append(..) => LogKind::Append,
        Compact(..) | Counter(..) | Meta(..) | Config(..) => LogKind::Replace,
    }
}

impl From<MessageKind> for LogKind {
    fn from(kind: MessageKind) -> Self {
        use MessageKind::*;
        match kind {
            Free => LogKind::Free,

            InlineReplace | Counter | BlobReplace | InlineMeta | BlobMeta
            | InlineConfig | BlobConfig => LogKind::Replace,

            InlineAppend | BlobAppend => LogKind::Append,

            Cancelled | Pad | BatchManifest => LogKind::Skip,
            other => {
                debug!("encountered unexpected message kind byte {:?}", other);
                LogKind::Corrupted
            }
        }
    }
}

fn assert_usize<T>(from: T) -> usize
where
    usize: TryFrom<T, Error = std::num::TryFromIntError>,
{
    usize::try_from(from).expect("lost data cast while converting to usize")
}

// TODO remove this when atomic fetch_max stabilizes in #48655
fn bump_atomic_lsn(atomic_lsn: &AtomicLsn, to: Lsn) {
    let mut current = atomic_lsn.load(SeqCst);
    loop {
        if current >= to {
            return;
        }
        let last = atomic_lsn.compare_and_swap(current, to, SeqCst);
        if last == current {
            // we succeeded.
            return;
        }
        current = last;
    }
}

use std::convert::{TryFrom, TryInto};

#[cfg(feature = "compression")]
use zstd::block::decompress;

#[inline]
pub(crate) fn u64_to_arr(number: u64) -> [u8; 8] {
    number.to_le_bytes()
}

#[inline]
pub(crate) fn arr_to_u64(arr: &[u8]) -> u64 {
    arr.try_into().map(u64::from_le_bytes).unwrap()
}

#[inline]
pub(crate) fn arr_to_u32(arr: &[u8]) -> u32 {
    arr.try_into().map(u32::from_le_bytes).unwrap()
}

#[inline]
pub(crate) fn u32_to_arr(number: u32) -> [u8; 4] {
    number.to_le_bytes()
}

pub(crate) fn maybe_decompress(buf: Vec<u8>) -> std::io::Result<Vec<u8>> {
    #[cfg(feature = "compression")]
    {
        use std::sync::atomic::AtomicUsize;

        use super::*;

        static MAX_COMPRESSION_RATIO: AtomicUsize = AtomicUsize::new(1);
        use std::sync::atomic::Ordering::{Acquire, Release};

        let _measure = Measure::new(&M.decompress);
        loop {
            let ratio = MAX_COMPRESSION_RATIO.load(Acquire);
            match decompress(&*buf, buf.len() * ratio) {
                Err(ref e) if e.kind() == std::io::ErrorKind::Other => {
                    debug!(
                        "bumping expected compression \
                         ratio up from {} to {}: {:?}",
                        ratio,
                        ratio + 1,
                        e
                    );
                    let _who_cares = MAX_COMPRESSION_RATIO.compare_and_swap(
                        ratio,
                        ratio + 1,
                        Release,
                    );
                }
                other => return other,
            }
        }
    }

    #[cfg(not(feature = "compression"))]
    Ok(buf)
}
<commit_msg>fix basic types docs<commit_after>//! `pagecache` is a lock-free pagecache and log for building high-performance
//! databases.

pub mod constants;
pub mod logger;

mod blob_io;
mod diskptr;
mod iobuf;
mod iterator;
mod materializer;
mod pagecache;
mod parallel_io;
mod reader;
mod reservation;
mod segment;
mod snapshot;

#[cfg(feature = "measure_allocs")]
mod measure_allocs;

#[cfg(feature = "measure_allocs")]
#[global_allocator]
static ALLOCATOR: measure_allocs::TrackingAllocator =
    measure_allocs::TrackingAllocator;

use crate::{debug, AtomicLsn, DeserializeOwned, SeqCst, Serialize};

use self::{
    blob_io::{gc_blobs, read_blob, remove_blob, write_blob},
    constants::{
        BATCH_MANIFEST_PID, CONFIG_PID, COUNTER_PID, META_PID,
        PAGE_CONSOLIDATION_THRESHOLD,
    },
    iobuf::{IoBuf, IoBufs},
    iterator::{raw_segment_iter_from, LogIter},
    pagecache::Update,
    parallel_io::Pio,
    segment::SegmentAccountant,
    snapshot::advance_snapshot,
};

pub(crate) use self::{
    logger::{MessageHeader, SegmentHeader},
    reader::{read_message, read_segment_header},
    reservation::Reservation,
    snapshot::{read_snapshot_or_default, PageState, Snapshot},
};

pub use self::{
    constants::{
        BATCH_MANIFEST_INLINE_LEN, BLOB_INLINE_LEN, MAX_SPACE_AMPLIFICATION,
        MINIMUM_ITEMS_PER_SEGMENT, MSG_HEADER_LEN, SEG_HEADER_LEN,
    },
    diskptr::DiskPtr,
    logger::{Log, LogRead},
    materializer::Materializer,
    pagecache::{PageCache, PagePtr, RecoveryGuard},
    segment::SegmentMode,
};

/// The offset of a segment. This equals its LogOffset (or the offset of any
/// item contained inside it) divided by the configured segment_size.
pub type SegmentId = usize;

/// A file offset in the database log.
pub type LogOffset = u64;

/// A pointer to an blob blob.
pub type BlobPointer = Lsn;

/// A logical sequence number of a log entry.
/// A segment lsn equals to an aligned LogOffset that is
/// Segment.LogOffset & ~SegmentSize.
pub type Lsn = i64;

/// A page identifier.
pub type PageId = u64;

/// A byte used to disambiguate log message types
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
#[repr(u8)]
pub enum MessageKind {
    /// The EVIL_BYTE is written as a canary to help
    /// detect torn writes.
    Corrupted = 0,
    /// Indicates that the following buffer corresponds
    /// to a reservation for an in-memory operation that
    /// failed to complete. It should be skipped during
    /// recovery.
    Cancelled = 1,
    /// Indicates that the following buffer is used
    /// as padding to fill out the rest of the segment
    /// before sealing it.
    Pad = 2,
    /// Indicates that the following buffer contains
    /// an Lsn for the last write in an atomic writebatch.
    BatchManifest = 3,
    /// Indicates that this page was freed from the pagetable.
    Free = 4,
    /// Indicates that the last persisted ID was at least
    /// this high.
    Counter = 5,
    /// The meta page, stored inline
    InlineMeta = 6,
    /// The meta page, stored blobly
    BlobMeta = 7,
    /// The config page, stored inline
    InlineConfig = 8,
    /// The config page, stored blobly
    BlobConfig = 9,
    /// A consolidated page replacement, stored inline
    InlineReplace = 10,
    /// A consolidated page replacement, stored blobly
    BlobReplace = 11,
    /// A partial page update, stored inline
    InlineAppend = 12,
    /// A partial page update, stored blobly
    BlobAppend = 13,
}

impl MessageKind {
    pub(crate) const fn into(self) -> u8 {
        self as u8
    }
}

impl From<u8> for MessageKind {
    fn from(byte: u8) -> Self {
        use MessageKind::*;
        match byte {
            0 => Corrupted,
            1 => Cancelled,
            2 => Pad,
            3 => BatchManifest,
            4 => Free,
            5 => Counter,
            6 => InlineMeta,
            7 => BlobMeta,
            8 => InlineConfig,
            9 => BlobConfig,
            10 => InlineReplace,
            11 => BlobReplace,
            12 => InlineAppend,
            13 => BlobAppend,
            other => {
                debug!("encountered unexpected message kind byte {}", other);
                Corrupted
            }
        }
    }
}

/// The high-level types of stored information
/// about pages and their mutations
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum LogKind {
    /// Persisted data containing a page replacement
    Replace,
    /// Persisted immutable update
    Append,
    /// Freeing of a page
    Free,
    /// Some state indicating this should be skipped
    Skip,
    /// Unexpected corruption
    Corrupted,
}

fn log_kind_from_update<PageFrag>(update: &Update<PageFrag>) -> LogKind
where
    PageFrag: DeserializeOwned + Serialize,
{
    use Update::*;

    match update {
        Free => LogKind::Free,
        Append(..) => LogKind::Append,
        Compact(..) | Counter(..) | Meta(..) | Config(..) => LogKind::Replace,
    }
}

impl From<MessageKind> for LogKind {
    fn from(kind: MessageKind) -> Self {
        use MessageKind::*;
        match kind {
            Free => LogKind::Free,

            InlineReplace | Counter | BlobReplace | InlineMeta | BlobMeta
            | InlineConfig | BlobConfig => LogKind::Replace,

            InlineAppend | BlobAppend => LogKind::Append,

            Cancelled | Pad | BatchManifest => LogKind::Skip,
            other => {
                debug!("encountered unexpected message kind byte {:?}", other);
                LogKind::Corrupted
            }
        }
    }
}

fn assert_usize<T>(from: T) -> usize
where
    usize: TryFrom<T, Error = std::num::TryFromIntError>,
{
    usize::try_from(from).expect("lost data cast while converting to usize")
}

// TODO remove this when atomic fetch_max stabilizes in #48655
fn bump_atomic_lsn(atomic_lsn: &AtomicLsn, to: Lsn) {
    let mut current = atomic_lsn.load(SeqCst);
    loop {
        if current >= to {
            return;
        }
        let last = atomic_lsn.compare_and_swap(current, to, SeqCst);
        if last == current {
            // we succeeded.
            return;
        }
        current = last;
    }
}

use std::convert::{TryFrom, TryInto};

#[cfg(feature = "compression")]
use zstd::block::decompress;

#[inline]
pub(crate) fn u64_to_arr(number: u64) -> [u8; 8] {
    number.to_le_bytes()
}

#[inline]
pub(crate) fn arr_to_u64(arr: &[u8]) -> u64 {
    arr.try_into().map(u64::from_le_bytes).unwrap()
}

#[inline]
pub(crate) fn arr_to_u32(arr: &[u8]) -> u32 {
    arr.try_into().map(u32::from_le_bytes).unwrap()
}

#[inline]
pub(crate) fn u32_to_arr(number: u32) -> [u8; 4] {
    number.to_le_bytes()
}

pub(crate) fn maybe_decompress(buf: Vec<u8>) -> std::io::Result<Vec<u8>> {
    #[cfg(feature = "compression")]
    {
        use std::sync::atomic::AtomicUsize;

        use super::*;

        static MAX_COMPRESSION_RATIO: AtomicUsize = AtomicUsize::new(1);
        use std::sync::atomic::Ordering::{Acquire, Release};

        let _measure = Measure::new(&M.decompress);
        loop {
            let ratio = MAX_COMPRESSION_RATIO.load(Acquire);
            match decompress(&*buf, buf.len() * ratio) {
                Err(ref e) if e.kind() == std::io::ErrorKind::Other => {
                    debug!(
                        "bumping expected compression \
                         ratio up from {} to {}: {:?}",
                        ratio,
                        ratio + 1,
                        e
                    );
                    let _who_cares = MAX_COMPRESSION_RATIO.compare_and_swap(
                        ratio,
                        ratio + 1,
                        Release,
                    );
                }
                other => return other,
            }
        }
    }

    #[cfg(not(feature = "compression"))]
    Ok(buf)
}
<|endoftext|>


<commit_before>use std::collections::HashSet;
use std::collections::hash_map::HashMap;
use std::fmt;
use std::io;
use std::mem;
use std::sync::mpsc::{channel, Receiver, Sender};

use crossbeam::{self, Scope};
use jobserver::{Acquired, HelperThread};

use core::{PackageId, Profile, Target};
use util::{Config, DependencyQueue, Dirty, Fresh, Freshness};
use util::{internal, profile, CargoResult, CargoResultExt, ProcessBuilder};
use handle_error;

use super::{Context, Kind, Unit};
use super::job::Job;

/// A management structure of the entire dependency graph to compile.
///
/// This structure is backed by the `DependencyQueue` type and manages the
/// actual compilation step of each package. Packages enqueue units of work and
/// then later on the entire graph is processed and compiled.
pub struct JobQueue<'a> {
    queue: DependencyQueue<Key<'a>, Vec<(Job, Freshness)>>,
    tx: Sender<Message<'a>>,
    rx: Receiver<Message<'a>>,
    active: usize,
    pending: HashMap<Key<'a>, PendingBuild>,
    compiled: HashSet<&'a PackageId>,
    documented: HashSet<&'a PackageId>,
    counts: HashMap<&'a PackageId, usize>,
    is_release: bool,
}

/// A helper structure for metadata about the state of a building package.
struct PendingBuild {
    /// Number of jobs currently active
    amt: usize,
    /// Current freshness state of this package. Any dirty target within a
    /// package will cause the entire package to become dirty.
    fresh: Freshness,
}

#[derive(Clone, Copy, Eq, PartialEq, Hash)]
struct Key<'a> {
    pkg: &'a PackageId,
    target: &'a Target,
    profile: &'a Profile,
    kind: Kind,
}

pub struct JobState<'a> {
    tx: Sender<Message<'a>>,
}

enum Message<'a> {
    Run(String),
    Stdout(String),
    Stderr(String),
    Token(io::Result<Acquired>),
    Finish(Key<'a>, CargoResult<()>),
}

impl<'a> JobState<'a> {
    pub fn running(&self, cmd: &ProcessBuilder) {
        let _ = self.tx.send(Message::Run(cmd.to_string()));
    }

    pub fn stdout(&self, out: &str) {
        let _ = self.tx.send(Message::Stdout(out.to_string()));
    }

    pub fn stderr(&self, err: &str) {
        let _ = self.tx.send(Message::Stderr(err.to_string()));
    }
}

impl<'a> JobQueue<'a> {
    pub fn new<'cfg>(cx: &Context<'a, 'cfg>) -> JobQueue<'a> {
        let (tx, rx) = channel();
        JobQueue {
            queue: DependencyQueue::new(),
            tx,
            rx,
            active: 0,
            pending: HashMap::new(),
            compiled: HashSet::new(),
            documented: HashSet::new(),
            counts: HashMap::new(),
            is_release: cx.build_config.release,
        }
    }

    pub fn enqueue<'cfg>(
        &mut self,
        cx: &Context<'a, 'cfg>,
        unit: &Unit<'a>,
        job: Job,
        fresh: Freshness,
    ) -> CargoResult<()> {
        let key = Key::new(unit);
        let deps = key.dependencies(cx)?;
        self.queue
            .queue(Fresh, key, Vec::new(), &deps)
            .push((job, fresh));
        *self.counts.entry(key.pkg).or_insert(0) += 1;
        Ok(())
    }

    /// Execute all jobs necessary to build the dependency graph.
    ///
    /// This function will spawn off `config.jobs()` workers to build all of the
    /// necessary dependencies, in order. Freshness is propagated as far as
    /// possible along each dependency chain.
    pub fn execute(&mut self, cx: &mut Context) -> CargoResult<()> {
        let _p = profile::start("executing the job graph");
        self.queue.queue_finished();

        // We need to give a handle to the send half of our message queue to the
        // jobserver helper thread. Unfortunately though we need the handle to be
        // `'static` as that's typically what's required when spawning a
        // thread!
        //
        // To work around this we transmute the `Sender` to a static lifetime.
        // we're only sending "longer living" messages and we should also
        // destroy all references to the channel before this function exits as
        // the destructor for the `helper` object will ensure the associated
        // thread is no longer running.
        //
        // As a result, this `transmute` to a longer lifetime should be safe in
        // practice.
        let tx = self.tx.clone();
        let tx = unsafe { mem::transmute::<Sender<Message<'a>>, Sender<Message<'static>>>(tx) };
        let helper = cx.jobserver
            .clone()
            .into_helper_thread(move |token| {
                drop(tx.send(Message::Token(token)));
            })
            .chain_err(|| "failed to create helper thread for jobserver management")?;

        crossbeam::scope(|scope| self.drain_the_queue(cx, scope, &helper))
    }

    fn drain_the_queue(
        &mut self,
        cx: &mut Context,
        scope: &Scope<'a>,
        jobserver_helper: &HelperThread,
    ) -> CargoResult<()> {
        let mut tokens = Vec::new();
        let mut queue = Vec::new();
        trace!("queue: {:#?}", self.queue);

        // Iteratively execute the entire dependency graph. Each turn of the
        // loop starts out by scheduling as much work as possible (up to the
        // maximum number of parallel jobs we have tokens for). A local queue
        // is maintained separately from the main dependency queue as one
        // dequeue may actually dequeue quite a bit of work (e.g. 10 binaries
        // in one project).
        //
        // After a job has finished we update our internal state if it was
        // successful and otherwise wait for pending work to finish if it failed
        // and then immediately return.
        let mut error = None;
        loop {
            // Dequeue as much work as we can, learning about everything
            // possible that can run. Note that this is also the point where we
            // start requesting job tokens. Each job after the first needs to
            // request a token.
            while let Some((fresh, key, jobs)) = self.queue.dequeue() {
                let total_fresh = jobs.iter().fold(fresh, |fresh, &(_, f)| f.combine(fresh));
                self.pending.insert(
                    key,
                    PendingBuild {
                        amt: jobs.len(),
                        fresh: total_fresh,
                    },
                );
                for (job, f) in jobs {
                    queue.push((key, job, f.combine(fresh)));
                    if self.active + queue.len() > 0 {
                        jobserver_helper.request_token();
                    }
                }
            }

            // Now that we've learned of all possible work that we can execute
            // try to spawn it so long as we've got a jobserver token which says
            // we're able to perform some parallel work.
            while error.is_none() && self.active < tokens.len() + 1 && !queue.is_empty() {
                let (key, job, fresh) = queue.remove(0);
                self.run(key, fresh, job, cx.config, scope)?;
            }

            // If after all that we're not actually running anything then we're
            // done!
            if self.active == 0 {
                break;
            }

            // And finally, before we block waiting for the next event, drop any
            // excess tokens we may have accidentally acquired. Due to how our
            // jobserver interface is architected we may acquire a token that we
            // don't actually use, and if this happens just relinquish it back
            // to the jobserver itself.
            tokens.truncate(self.active - 1);

            match self.rx.recv().unwrap() {
                Message::Run(cmd) => {
                    cx.config.shell().verbose(|c| c.status("Running", &cmd))?;
                }
                Message::Stdout(out) => {
                    if cx.config.extra_verbose() {
                        println!("{}", out);
                    }
                }
                Message::Stderr(err) => {
                    if cx.config.extra_verbose() {
                        writeln!(cx.config.shell().err(), "{}", err)?;
                    }
                }
                Message::Finish(key, result) => {
                    info!("end: {:?}", key);
                    self.active -= 1;
                    if self.active > 0 {
                        assert!(!tokens.is_empty());
                        drop(tokens.pop());
                    }
                    match result {
                        Ok(()) => self.finish(key, cx)?,
                        Err(e) => {
                            let msg = "The following warnings were emitted during compilation:";
                            self.emit_warnings(Some(msg), key, cx)?;

                            if self.active > 0 {
                                error = Some(format_err!("build failed"));
                                handle_error(e, &mut *cx.config.shell());
                                cx.config.shell().warn(
                                    "build failed, waiting for other \
                                     jobs to finish...",
                                )?;
                            } else {
                                error = Some(e);
                            }
                        }
                    }
                }
                Message::Token(acquired_token) => {
                    tokens.push(acquired_token.chain_err(|| "failed to acquire jobserver token")?);
                }
            }
        }

        let build_type = if self.is_release { "release" } else { "dev" };
        let profile = cx.lib_profile();
        let mut opt_type = String::from(if profile.opt_level == "0" {
            "unoptimized"
        } else {
            "optimized"
        });
        if profile.debuginfo.is_some() {
            opt_type += " + debuginfo";
        }
        let duration = cx.config.creation_time().elapsed();
        let time_elapsed = format!(
            "{}.{:02} secs",
            duration.as_secs(),
            duration.subsec_nanos() / 10_000_000
        );
        if self.queue.is_empty() {
            let message = format!(
                "{} [{}] target(s) in {}",
                build_type, opt_type, time_elapsed
            );
            cx.config.shell().status("Finished", message)?;
            Ok(())
        } else if let Some(e) = error {
            Err(e)
        } else {
            debug!("queue: {:#?}", self.queue);
            Err(internal("finished with jobs still left in the queue"))
        }
    }

    /// Executes a job in the `scope` given, pushing the spawned thread's
    /// handled onto `threads`.
    fn run(
        &mut self,
        key: Key<'a>,
        fresh: Freshness,
        job: Job,
        config: &Config,
        scope: &Scope<'a>,
    ) -> CargoResult<()> {
        info!("start: {:?}", key);

        self.active += 1;
        *self.counts.get_mut(key.pkg).unwrap() -= 1;

        let my_tx = self.tx.clone();
        let doit = move || {
            let res = job.run(fresh, &JobState { tx: my_tx.clone() });
            my_tx.send(Message::Finish(key, res)).unwrap();
        };
        match fresh {
            Freshness::Fresh => doit(),
            Freshness::Dirty => {
                scope.spawn(doit);
            }
        }

        // Print out some nice progress information
        self.note_working_on(config, &key, fresh)?;

        Ok(())
    }

    fn emit_warnings(&self, msg: Option<&str>, key: Key<'a>, cx: &mut Context) -> CargoResult<()> {
        let output = cx.build_state.outputs.lock().unwrap();
        if let Some(output) = output.get(&(key.pkg.clone(), key.kind)) {
            if let Some(msg) = msg {
                if !output.warnings.is_empty() {
                    writeln!(cx.config.shell().err(), "{}\n", msg)?;
                }
            }

            for warning in output.warnings.iter() {
                cx.config.shell().warn(warning)?;
            }

            if !output.warnings.is_empty() && msg.is_some() {
                // Output an empty line.
                writeln!(cx.config.shell().err(), "")?;
            }
        }

        Ok(())
    }

    fn finish(&mut self, key: Key<'a>, cx: &mut Context) -> CargoResult<()> {
        if key.profile.run_custom_build && cx.show_warnings(key.pkg) {
            self.emit_warnings(None, key, cx)?;
        }

        let state = self.pending.get_mut(&key).unwrap();
        state.amt -= 1;
        if state.amt == 0 {
            self.queue.finish(&key, state.fresh);
        }
        Ok(())
    }

    // This isn't super trivial because we don't want to print loads and
    // loads of information to the console, but we also want to produce a
    // faithful representation of what's happening. This is somewhat nuanced
    // as a package can start compiling *very* early on because of custom
    // build commands and such.
    //
    // In general, we try to print "Compiling" for the first nontrivial task
    // run for a package, regardless of when that is. We then don't print
    // out any more information for a package after we've printed it once.
    fn note_working_on(
        &mut self,
        config: &Config,
        key: &Key<'a>,
        fresh: Freshness,
    ) -> CargoResult<()> {
        if (self.compiled.contains(key.pkg) && !key.profile.doc)
            || (self.documented.contains(key.pkg) && key.profile.doc)
        {
            return Ok(());
        }

        match fresh {
            // Any dirty stage which runs at least one command gets printed as
            // being a compiled package
            Dirty => {
                if key.profile.doc {
                    if !key.profile.test {
                        self.documented.insert(key.pkg);
                        config.shell().status("Documenting", key.pkg)?;
                    }
                } else {
                    self.compiled.insert(key.pkg);
                    config.shell().status("Compiling", key.pkg)?;
                }
            }
            Fresh if self.counts[key.pkg] == 0 => {
                self.compiled.insert(key.pkg);
                config.shell().verbose(|c| c.status("Fresh", key.pkg))?;
            }
            Fresh => {}
        }
        Ok(())
    }
}

impl<'a> Key<'a> {
    fn new(unit: &Unit<'a>) -> Key<'a> {
        Key {
            pkg: unit.pkg.package_id(),
            target: unit.target,
            profile: unit.profile,
            kind: unit.kind,
        }
    }

    fn dependencies<'cfg>(&self, cx: &Context<'a, 'cfg>) -> CargoResult<Vec<Key<'a>>> {
        let unit = Unit {
            pkg: cx.get_package(self.pkg)?,
            target: self.target,
            profile: self.profile,
            kind: self.kind,
        };
        let targets = cx.dep_targets(&unit);
        Ok(targets
            .iter()
            .filter_map(|unit| {
                // Binaries aren't actually needed to *compile* tests, just to run
                // them, so we don't include this dependency edge in the job graph.
                if self.target.is_test() && unit.target.is_bin() {
                    None
                } else {
                    Some(Key::new(unit))
                }
            })
            .collect())
    }
}

impl<'a> fmt::Debug for Key<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(
            f,
            "{} => {}/{} => {:?}",
            self.pkg, self.target, self.profile, self.kind
        )
    }
}
<commit_msg>Print "Checking" for cargo check<commit_after>use std::collections::HashSet;
use std::collections::hash_map::HashMap;
use std::fmt;
use std::io;
use std::mem;
use std::sync::mpsc::{channel, Receiver, Sender};

use crossbeam::{self, Scope};
use jobserver::{Acquired, HelperThread};

use core::{PackageId, Profile, Target};
use util::{Config, DependencyQueue, Dirty, Fresh, Freshness};
use util::{internal, profile, CargoResult, CargoResultExt, ProcessBuilder};
use handle_error;

use super::{Context, Kind, Unit};
use super::job::Job;

/// A management structure of the entire dependency graph to compile.
///
/// This structure is backed by the `DependencyQueue` type and manages the
/// actual compilation step of each package. Packages enqueue units of work and
/// then later on the entire graph is processed and compiled.
pub struct JobQueue<'a> {
    queue: DependencyQueue<Key<'a>, Vec<(Job, Freshness)>>,
    tx: Sender<Message<'a>>,
    rx: Receiver<Message<'a>>,
    active: usize,
    pending: HashMap<Key<'a>, PendingBuild>,
    compiled: HashSet<&'a PackageId>,
    documented: HashSet<&'a PackageId>,
    counts: HashMap<&'a PackageId, usize>,
    is_release: bool,
}

/// A helper structure for metadata about the state of a building package.
struct PendingBuild {
    /// Number of jobs currently active
    amt: usize,
    /// Current freshness state of this package. Any dirty target within a
    /// package will cause the entire package to become dirty.
    fresh: Freshness,
}

#[derive(Clone, Copy, Eq, PartialEq, Hash)]
struct Key<'a> {
    pkg: &'a PackageId,
    target: &'a Target,
    profile: &'a Profile,
    kind: Kind,
}

pub struct JobState<'a> {
    tx: Sender<Message<'a>>,
}

enum Message<'a> {
    Run(String),
    Stdout(String),
    Stderr(String),
    Token(io::Result<Acquired>),
    Finish(Key<'a>, CargoResult<()>),
}

impl<'a> JobState<'a> {
    pub fn running(&self, cmd: &ProcessBuilder) {
        let _ = self.tx.send(Message::Run(cmd.to_string()));
    }

    pub fn stdout(&self, out: &str) {
        let _ = self.tx.send(Message::Stdout(out.to_string()));
    }

    pub fn stderr(&self, err: &str) {
        let _ = self.tx.send(Message::Stderr(err.to_string()));
    }
}

impl<'a> JobQueue<'a> {
    pub fn new<'cfg>(cx: &Context<'a, 'cfg>) -> JobQueue<'a> {
        let (tx, rx) = channel();
        JobQueue {
            queue: DependencyQueue::new(),
            tx,
            rx,
            active: 0,
            pending: HashMap::new(),
            compiled: HashSet::new(),
            documented: HashSet::new(),
            counts: HashMap::new(),
            is_release: cx.build_config.release,
        }
    }

    pub fn enqueue<'cfg>(
        &mut self,
        cx: &Context<'a, 'cfg>,
        unit: &Unit<'a>,
        job: Job,
        fresh: Freshness,
    ) -> CargoResult<()> {
        let key = Key::new(unit);
        let deps = key.dependencies(cx)?;
        self.queue
            .queue(Fresh, key, Vec::new(), &deps)
            .push((job, fresh));
        *self.counts.entry(key.pkg).or_insert(0) += 1;
        Ok(())
    }

    /// Execute all jobs necessary to build the dependency graph.
    ///
    /// This function will spawn off `config.jobs()` workers to build all of the
    /// necessary dependencies, in order. Freshness is propagated as far as
    /// possible along each dependency chain.
    pub fn execute(&mut self, cx: &mut Context) -> CargoResult<()> {
        let _p = profile::start("executing the job graph");
        self.queue.queue_finished();

        // We need to give a handle to the send half of our message queue to the
        // jobserver helper thread. Unfortunately though we need the handle to be
        // `'static` as that's typically what's required when spawning a
        // thread!
        //
        // To work around this we transmute the `Sender` to a static lifetime.
        // we're only sending "longer living" messages and we should also
        // destroy all references to the channel before this function exits as
        // the destructor for the `helper` object will ensure the associated
        // thread is no longer running.
        //
        // As a result, this `transmute` to a longer lifetime should be safe in
        // practice.
        let tx = self.tx.clone();
        let tx = unsafe { mem::transmute::<Sender<Message<'a>>, Sender<Message<'static>>>(tx) };
        let helper = cx.jobserver
            .clone()
            .into_helper_thread(move |token| {
                drop(tx.send(Message::Token(token)));
            })
            .chain_err(|| "failed to create helper thread for jobserver management")?;

        crossbeam::scope(|scope| self.drain_the_queue(cx, scope, &helper))
    }

    fn drain_the_queue(
        &mut self,
        cx: &mut Context,
        scope: &Scope<'a>,
        jobserver_helper: &HelperThread,
    ) -> CargoResult<()> {
        let mut tokens = Vec::new();
        let mut queue = Vec::new();
        trace!("queue: {:#?}", self.queue);

        // Iteratively execute the entire dependency graph. Each turn of the
        // loop starts out by scheduling as much work as possible (up to the
        // maximum number of parallel jobs we have tokens for). A local queue
        // is maintained separately from the main dependency queue as one
        // dequeue may actually dequeue quite a bit of work (e.g. 10 binaries
        // in one project).
        //
        // After a job has finished we update our internal state if it was
        // successful and otherwise wait for pending work to finish if it failed
        // and then immediately return.
        let mut error = None;
        loop {
            // Dequeue as much work as we can, learning about everything
            // possible that can run. Note that this is also the point where we
            // start requesting job tokens. Each job after the first needs to
            // request a token.
            while let Some((fresh, key, jobs)) = self.queue.dequeue() {
                let total_fresh = jobs.iter().fold(fresh, |fresh, &(_, f)| f.combine(fresh));
                self.pending.insert(
                    key,
                    PendingBuild {
                        amt: jobs.len(),
                        fresh: total_fresh,
                    },
                );
                for (job, f) in jobs {
                    queue.push((key, job, f.combine(fresh)));
                    if self.active + queue.len() > 0 {
                        jobserver_helper.request_token();
                    }
                }
            }

            // Now that we've learned of all possible work that we can execute
            // try to spawn it so long as we've got a jobserver token which says
            // we're able to perform some parallel work.
            while error.is_none() && self.active < tokens.len() + 1 && !queue.is_empty() {
                let (key, job, fresh) = queue.remove(0);
                self.run(key, fresh, job, cx.config, scope)?;
            }

            // If after all that we're not actually running anything then we're
            // done!
            if self.active == 0 {
                break;
            }

            // And finally, before we block waiting for the next event, drop any
            // excess tokens we may have accidentally acquired. Due to how our
            // jobserver interface is architected we may acquire a token that we
            // don't actually use, and if this happens just relinquish it back
            // to the jobserver itself.
            tokens.truncate(self.active - 1);

            match self.rx.recv().unwrap() {
                Message::Run(cmd) => {
                    cx.config.shell().verbose(|c| c.status("Running", &cmd))?;
                }
                Message::Stdout(out) => {
                    if cx.config.extra_verbose() {
                        println!("{}", out);
                    }
                }
                Message::Stderr(err) => {
                    if cx.config.extra_verbose() {
                        writeln!(cx.config.shell().err(), "{}", err)?;
                    }
                }
                Message::Finish(key, result) => {
                    info!("end: {:?}", key);
                    self.active -= 1;
                    if self.active > 0 {
                        assert!(!tokens.is_empty());
                        drop(tokens.pop());
                    }
                    match result {
                        Ok(()) => self.finish(key, cx)?,
                        Err(e) => {
                            let msg = "The following warnings were emitted during compilation:";
                            self.emit_warnings(Some(msg), key, cx)?;

                            if self.active > 0 {
                                error = Some(format_err!("build failed"));
                                handle_error(e, &mut *cx.config.shell());
                                cx.config.shell().warn(
                                    "build failed, waiting for other \
                                     jobs to finish...",
                                )?;
                            } else {
                                error = Some(e);
                            }
                        }
                    }
                }
                Message::Token(acquired_token) => {
                    tokens.push(acquired_token.chain_err(|| "failed to acquire jobserver token")?);
                }
            }
        }

        let build_type = if self.is_release { "release" } else { "dev" };
        let profile = cx.lib_profile();
        let mut opt_type = String::from(if profile.opt_level == "0" {
            "unoptimized"
        } else {
            "optimized"
        });
        if profile.debuginfo.is_some() {
            opt_type += " + debuginfo";
        }
        let duration = cx.config.creation_time().elapsed();
        let time_elapsed = format!(
            "{}.{:02} secs",
            duration.as_secs(),
            duration.subsec_nanos() / 10_000_000
        );
        if self.queue.is_empty() {
            let message = format!(
                "{} [{}] target(s) in {}",
                build_type, opt_type, time_elapsed
            );
            cx.config.shell().status("Finished", message)?;
            Ok(())
        } else if let Some(e) = error {
            Err(e)
        } else {
            debug!("queue: {:#?}", self.queue);
            Err(internal("finished with jobs still left in the queue"))
        }
    }

    /// Executes a job in the `scope` given, pushing the spawned thread's
    /// handled onto `threads`.
    fn run(
        &mut self,
        key: Key<'a>,
        fresh: Freshness,
        job: Job,
        config: &Config,
        scope: &Scope<'a>,
    ) -> CargoResult<()> {
        info!("start: {:?}", key);

        self.active += 1;
        *self.counts.get_mut(key.pkg).unwrap() -= 1;

        let my_tx = self.tx.clone();
        let doit = move || {
            let res = job.run(fresh, &JobState { tx: my_tx.clone() });
            my_tx.send(Message::Finish(key, res)).unwrap();
        };
        match fresh {
            Freshness::Fresh => doit(),
            Freshness::Dirty => {
                scope.spawn(doit);
            }
        }

        // Print out some nice progress information
        self.note_working_on(config, &key, fresh)?;

        Ok(())
    }

    fn emit_warnings(&self, msg: Option<&str>, key: Key<'a>, cx: &mut Context) -> CargoResult<()> {
        let output = cx.build_state.outputs.lock().unwrap();
        if let Some(output) = output.get(&(key.pkg.clone(), key.kind)) {
            if let Some(msg) = msg {
                if !output.warnings.is_empty() {
                    writeln!(cx.config.shell().err(), "{}\n", msg)?;
                }
            }

            for warning in output.warnings.iter() {
                cx.config.shell().warn(warning)?;
            }

            if !output.warnings.is_empty() && msg.is_some() {
                // Output an empty line.
                writeln!(cx.config.shell().err(), "")?;
            }
        }

        Ok(())
    }

    fn finish(&mut self, key: Key<'a>, cx: &mut Context) -> CargoResult<()> {
        if key.profile.run_custom_build && cx.show_warnings(key.pkg) {
            self.emit_warnings(None, key, cx)?;
        }

        let state = self.pending.get_mut(&key).unwrap();
        state.amt -= 1;
        if state.amt == 0 {
            self.queue.finish(&key, state.fresh);
        }
        Ok(())
    }

    // This isn't super trivial because we don't want to print loads and
    // loads of information to the console, but we also want to produce a
    // faithful representation of what's happening. This is somewhat nuanced
    // as a package can start compiling *very* early on because of custom
    // build commands and such.
    //
    // In general, we try to print "Compiling" for the first nontrivial task
    // run for a package, regardless of when that is. We then don't print
    // out any more information for a package after we've printed it once.
    fn note_working_on(
        &mut self,
        config: &Config,
        key: &Key<'a>,
        fresh: Freshness,
    ) -> CargoResult<()> {
        if (self.compiled.contains(key.pkg) && !key.profile.doc)
            || (self.documented.contains(key.pkg) && key.profile.doc)
        {
            return Ok(());
        }

        match fresh {
            // Any dirty stage which runs at least one command gets printed as
            // being a compiled package
            Dirty => {
                if key.profile.doc {
                    if !key.profile.test {
                        self.documented.insert(key.pkg);
                        config.shell().status("Documenting", key.pkg)?;
                    }
                } else {
                    self.compiled.insert(key.pkg);
                    if key.profile.check {
                        config.shell().status("Checking", key.pkg)?;
                    } else {
                        config.shell().status("Compiling", key.pkg)?;
                    }
                }
            }
            Fresh if self.counts[key.pkg] == 0 => {
                self.compiled.insert(key.pkg);
                config.shell().verbose(|c| c.status("Fresh", key.pkg))?;
            }
            Fresh => {}
        }
        Ok(())
    }
}

impl<'a> Key<'a> {
    fn new(unit: &Unit<'a>) -> Key<'a> {
        Key {
            pkg: unit.pkg.package_id(),
            target: unit.target,
            profile: unit.profile,
            kind: unit.kind,
        }
    }

    fn dependencies<'cfg>(&self, cx: &Context<'a, 'cfg>) -> CargoResult<Vec<Key<'a>>> {
        let unit = Unit {
            pkg: cx.get_package(self.pkg)?,
            target: self.target,
            profile: self.profile,
            kind: self.kind,
        };
        let targets = cx.dep_targets(&unit);
        Ok(targets
            .iter()
            .filter_map(|unit| {
                // Binaries aren't actually needed to *compile* tests, just to run
                // them, so we don't include this dependency edge in the job graph.
                if self.target.is_test() && unit.target.is_bin() {
                    None
                } else {
                    Some(Key::new(unit))
                }
            })
            .collect())
    }
}

impl<'a> fmt::Debug for Key<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(
            f,
            "{} => {}/{} => {:?}",
            self.pkg, self.target, self.profile, self.kind
        )
    }
}
<|endoftext|>


<commit_before>use log;
use collections::String;

const PORT: u16 = 0x3f8;

struct SerialLogger(());

fn outw(port: u16, data: u16) {
    unsafe {
        asm!("outw %ax, %dx" : : "{dx}" (port as u16), "{ax}" (data));
    }
}

fn outb(port: u16, data: u8) {
    unsafe {
        asm!("outb %al, %dx" : : "{dx}" (port as u16), "{al}" (data));
    }
}

fn inb(port: u16) -> u8 {
    let data: u8;
    unsafe {
        asm!("inb %dx, %al" : "={al}"(data) : "{dx}"(port as u16));
    }
    data
}

impl SerialLogger {
    fn init(&self) {
        outw(PORT + 1, 0x00);
        outw(PORT + 3, 0x80);
        outw(PORT + 0, 0x01);
        outw(PORT + 1, 0x00);
        outw(PORT + 3, 0x03);
        outw(PORT + 2, 0xc7);
        outw(PORT + 4, 0x0b);
        outw(PORT + 1, 0x01);
    }

    fn write_string(&self, s: String) {
        for c in s.chars() {
            while (inb(PORT + 5) & 0x20) == 0 {}
            outb(PORT, c as u8);
        }
    }
}

impl log::Log for SerialLogger {
    fn enabled(&self, metadata: &log::LogMetadata) -> bool {
        metadata.level() <= log::LogLevel::Info
    }

    fn log(&self, record: &log::LogRecord) {
        if self.enabled(record.metadata()) {
            let log_message = format!("{}: {}\n\0", record.level(), record.args());
            self.write_string(log_message);
        }
    }
}

pub fn init() -> Result<(), log::SetLoggerError> {
    unsafe {
        log::set_logger_raw(|max_log_level| {
            static LOGGER: SerialLogger = SerialLogger(());
            LOGGER.init();
            max_log_level.set(log::LogLevelFilter::Info);
            &LOGGER
        })
    }
}

pub fn shutdown() -> Result<(), log::ShutdownLoggerError> {
    log::shutdown_logger_raw().map(|_logger| {})
}
<commit_msg>Make clippy happy<commit_after>use log;
use collections::String;

const PORT: u16 = 0x3f8;

struct SerialLogger(());

fn outw(port: u16, data: u16) {
    unsafe {
        asm!("outw %ax, %dx" : : "{dx}" (port as u16), "{ax}" (data));
    }
}

fn outb(port: u16, data: u8) {
    unsafe {
        asm!("outb %al, %dx" : : "{dx}" (port as u16), "{al}" (data));
    }
}

fn inb(port: u16) -> u8 {
    let data: u8;
    unsafe {
        asm!("inb %dx, %al" : "={al}"(data) : "{dx}"(port as u16));
    }
    data
}

impl SerialLogger {
    fn init(&self) {
        outw(PORT + 1, 0x00);
        outw(PORT + 3, 0x80);
        outw(PORT, 0x01);
        outw(PORT + 1, 0x00);
        outw(PORT + 3, 0x03);
        outw(PORT + 2, 0xc7);
        outw(PORT + 4, 0x0b);
        outw(PORT + 1, 0x01);
    }

    fn write_string(&self, s: String) {
        for c in s.chars() {
            while (inb(PORT + 5) & 0x20) == 0 {}
            outb(PORT, c as u8);
        }
    }
}

impl log::Log for SerialLogger {
    fn enabled(&self, metadata: &log::LogMetadata) -> bool {
        metadata.level() <= log::LogLevel::Info
    }

    fn log(&self, record: &log::LogRecord) {
        if self.enabled(record.metadata()) {
            let log_message = format!("{}: {}\n\0", record.level(), record.args());
            self.write_string(log_message);
        }
    }
}

pub fn init() -> Result<(), log::SetLoggerError> {
    unsafe {
        log::set_logger_raw(|max_log_level| {
            static LOGGER: SerialLogger = SerialLogger(());
            LOGGER.init();
            max_log_level.set(log::LogLevelFilter::Info);
            &LOGGER
        })
    }
}

pub fn shutdown() -> Result<(), log::ShutdownLoggerError> {
    log::shutdown_logger_raw().map(|_logger| {})
}
<|endoftext|>


<commit_before>use gl::types::*;
use std::f32::consts::PI;
use std::ops::{Index, IndexMut, Mul};

pub const TAU: GLfloat = 2.0 * PI;

macro_rules! define_vec {
    ($name:ident, $size:expr) => (
        #[derive(Copy, Clone, Debug, PartialEq)]
        pub struct $name(pub [GLfloat; $size]);

        impl $name {
            pub fn zero() -> Self {
                $name([0.0; $size])
            }

            /// Calculate the square of the length (or norm) of the vector. Slightly faster than
            /// `length`.
            pub fn length_squared(&self) -> GLfloat {
                let mut sum = 0.0;

                for i in 0..$size {
                    sum += self[i] * self[i];
                }

                sum
            }

            /// Calculate the the length (or norm) of the vector.
            pub fn length(&self) -> GLfloat {
                self.length_squared().sqrt()
            }

            /// Normalize the vector so that it has the same orientation but a length of 1.
            pub fn normalize(&mut self) {
                let length = self.length();

                for i in 0..$size {
                    self[i] /= length;
                }
            }
        }

        impl Index<usize> for $name {
            type Output = GLfloat;

            fn index(&self, i: usize) -> &GLfloat {
                &self.0[i]
            }
        }

        impl IndexMut<usize> for $name {
            fn index_mut(&mut self, i: usize) -> &mut GLfloat {
                &mut self.0[i]
            }
        }
    );
}

define_vec!(Vec3, 3);
define_vec!(Vec4, 4);

#[derive(Copy, Clone, Debug, PartialEq)]
pub struct Mat4(pub [[GLfloat; 4]; 4]);

impl Mat4 {
    /// The zero matrix.
    pub fn zero() -> Self {
        Mat4([
            [0.0, 0.0, 0.0, 0.0],
            [0.0, 0.0, 0.0, 0.0],
            [0.0, 0.0, 0.0, 0.0],
            [0.0, 0.0, 0.0, 0.0],
        ])
    }

    /// The identity matrix.
    pub fn identity() -> Self {
        Mat4([
            [1.0, 0.0, 0.0, 0.0],
            [0.0, 1.0, 0.0, 0.0],
            [0.0, 0.0, 1.0, 0.0],
            [0.0, 0.0, 0.0, 1.0],
        ])
    }

    /// Build a matrix representing a scaling by the given factors.
    pub fn scale(x: GLfloat, y: GLfloat, z: GLfloat) -> Self {
        Mat4([
            [x,   0.0, 0.0, 0.0],
            [0.0, y,   0.0, 0.0],
            [0.0, 0.0, z,   0.0],
            [0.0, 0.0, 0.0, 1.0],
        ])
    }

    /// Build a matrix representing a translation.
    pub fn translate(x: GLfloat, y: GLfloat, z: GLfloat) -> Self {
        Mat4([
            [1.0, 0.0, 0.0, x  ],
            [0.0, 1.0, 0.0, y  ],
            [0.0, 0.0, 1.0, z  ],
            [0.0, 0.0, 0.0, 1.0],
        ])
    }

    /// Build a matrix representing a rotation around the X-axis by the given angle (in radians).
    pub fn rotate_x(angle: GLfloat) -> Self {
        let cos = angle.cos();
        let sin = angle.sin();

        Mat4([
            [1.0, 0.0,  0.0, 0.0],
            [0.0, cos, -sin, 0.0],
            [0.0, sin,  cos, 0.0],
            [0.0, 0.0,  0.0, 1.0],
        ])
    }

    /// Build a matrix representing a rotation around the Y-axis by the given angle (in radians).
    pub fn rotate_y(angle: GLfloat) -> Self {
        let cos = angle.cos();
        let sin = angle.sin();

        Mat4([
            [ cos, 0.0, sin, 0.0],
            [ 0.0, 1.0, 0.0, 0.0],
            [-sin, 0.0, cos, 0.0],
            [ 0.0, 0.0, 0.0, 1.0],
        ])
    }

    /// Build a matrix representing a rotation around the Z-axis by the given angle (in radians).
    pub fn rotate_z(angle: GLfloat) -> Self {
        let cos = angle.cos();
        let sin = angle.sin();

        Mat4([
            [cos, -sin, 0.0, 0.0],
            [sin,  cos, 0.0, 0.0],
            [0.0,  0.0, 1.0, 0.0],
            [0.0,  0.0, 0.0, 1.0],
        ])
    }
}

impl Index<usize> for Mat4 {
    type Output = [GLfloat; 4];

    fn index(&self, i: usize) -> &[GLfloat; 4] {
        &self.0[i]
    }
}

impl IndexMut<usize> for Mat4 {
    fn index_mut(&mut self, i: usize) -> &mut [GLfloat; 4] {
        &mut self.0[i]
    }
}

impl Mul<Mat4> for Mat4 {
    type Output = Mat4;

    fn mul(self, other: Mat4) -> Mat4 {
        let mut result = Mat4::zero();

        for i in 0..4 {
            for j in 0..4 {
                for k in 0..4 {
                    result[i][j] += self[i][k] * other[k][j];
                }
            }
        }

        result
    }
}

impl Mul<Vec4> for Mat4 {
    type Output = Vec4;

    fn mul(self, vec: Vec4) -> Vec4 {
        let mut result = Vec4::zero();

        for i in 0..4 {
            for j in 0..4 {
                result[i] += self[i][j] * vec[j];
            }
        }

        result
    }
}

#[test]
fn test_math() {
    let scale = Mat4::scale(2.0, 2.0, 2.0);
    let trans = Mat4::translate(1.0, 2.0, 3.0);
    let combined = trans * scale;

    let original = Vec4([3.0, 3.0, 3.0, 1.0]);
    let expected = Vec4([7.0, 8.0, 9.0, 1.0]);

    assert_eq!(expected, combined * original);
}
<commit_msg>Implement Add and Sub for Vec3/Vec4.<commit_after>use gl::types::*;
use std::f32::consts::PI;
use std::ops::{Add, Index, IndexMut, Mul, Sub};

pub const TAU: GLfloat = 2.0 * PI;

macro_rules! define_vec {
    ($name:ident, $size:expr) => (
        #[derive(Copy, Clone, Debug, PartialEq)]
        pub struct $name(pub [GLfloat; $size]);

        impl $name {
            pub fn zero() -> Self {
                $name([0.0; $size])
            }

            /// Calculate the square of the length (or norm) of the vector. Slightly faster than
            /// `length`.
            pub fn length_squared(&self) -> GLfloat {
                let mut sum = 0.0;

                for i in 0..$size {
                    sum += self[i] * self[i];
                }

                sum
            }

            /// Calculate the the length (or norm) of the vector.
            pub fn length(&self) -> GLfloat {
                self.length_squared().sqrt()
            }

            /// Normalize the vector so that it has the same orientation but a length of 1.
            pub fn normalize(&mut self) {
                let length = self.length();

                for i in 0..$size {
                    self[i] /= length;
                }
            }
        }

        impl Index<usize> for $name {
            type Output = GLfloat;

            fn index(&self, i: usize) -> &GLfloat {
                &self.0[i]
            }
        }

        impl IndexMut<usize> for $name {
            fn index_mut(&mut self, i: usize) -> &mut GLfloat {
                &mut self.0[i]
            }
        }

        impl Add for $name {
            type Output = Self;

            fn add(self, other: Self) -> Self {
                let mut result = $name::zero();

                for i in 0..$size {
                    result[i] = self[i] + other[i];
                }

                result
            }
        }

        impl Sub for $name {
            type Output = Self;

            fn sub(self, other: Self) -> Self {
                let mut result = $name::zero();

                for i in 0..$size {
                    result[i] = self[i] - other[i];
                }

                result
            }
        }
    );
}

define_vec!(Vec3, 3);
define_vec!(Vec4, 4);

#[derive(Copy, Clone, Debug, PartialEq)]
pub struct Mat4(pub [[GLfloat; 4]; 4]);

impl Mat4 {
    /// The zero matrix.
    pub fn zero() -> Self {
        Mat4([
            [0.0, 0.0, 0.0, 0.0],
            [0.0, 0.0, 0.0, 0.0],
            [0.0, 0.0, 0.0, 0.0],
            [0.0, 0.0, 0.0, 0.0],
        ])
    }

    /// The identity matrix.
    pub fn identity() -> Self {
        Mat4([
            [1.0, 0.0, 0.0, 0.0],
            [0.0, 1.0, 0.0, 0.0],
            [0.0, 0.0, 1.0, 0.0],
            [0.0, 0.0, 0.0, 1.0],
        ])
    }

    /// Build a matrix representing a scaling by the given factors.
    pub fn scale(x: GLfloat, y: GLfloat, z: GLfloat) -> Self {
        Mat4([
            [x,   0.0, 0.0, 0.0],
            [0.0, y,   0.0, 0.0],
            [0.0, 0.0, z,   0.0],
            [0.0, 0.0, 0.0, 1.0],
        ])
    }

    /// Build a matrix representing a translation.
    pub fn translate(x: GLfloat, y: GLfloat, z: GLfloat) -> Self {
        Mat4([
            [1.0, 0.0, 0.0, x  ],
            [0.0, 1.0, 0.0, y  ],
            [0.0, 0.0, 1.0, z  ],
            [0.0, 0.0, 0.0, 1.0],
        ])
    }

    /// Build a matrix representing a rotation around the X-axis by the given angle (in radians).
    pub fn rotate_x(angle: GLfloat) -> Self {
        let cos = angle.cos();
        let sin = angle.sin();

        Mat4([
            [1.0, 0.0,  0.0, 0.0],
            [0.0, cos, -sin, 0.0],
            [0.0, sin,  cos, 0.0],
            [0.0, 0.0,  0.0, 1.0],
        ])
    }

    /// Build a matrix representing a rotation around the Y-axis by the given angle (in radians).
    pub fn rotate_y(angle: GLfloat) -> Self {
        let cos = angle.cos();
        let sin = angle.sin();

        Mat4([
            [ cos, 0.0, sin, 0.0],
            [ 0.0, 1.0, 0.0, 0.0],
            [-sin, 0.0, cos, 0.0],
            [ 0.0, 0.0, 0.0, 1.0],
        ])
    }

    /// Build a matrix representing a rotation around the Z-axis by the given angle (in radians).
    pub fn rotate_z(angle: GLfloat) -> Self {
        let cos = angle.cos();
        let sin = angle.sin();

        Mat4([
            [cos, -sin, 0.0, 0.0],
            [sin,  cos, 0.0, 0.0],
            [0.0,  0.0, 1.0, 0.0],
            [0.0,  0.0, 0.0, 1.0],
        ])
    }
}

impl Index<usize> for Mat4 {
    type Output = [GLfloat; 4];

    fn index(&self, i: usize) -> &[GLfloat; 4] {
        &self.0[i]
    }
}

impl IndexMut<usize> for Mat4 {
    fn index_mut(&mut self, i: usize) -> &mut [GLfloat; 4] {
        &mut self.0[i]
    }
}

impl Mul<Mat4> for Mat4 {
    type Output = Mat4;

    fn mul(self, other: Mat4) -> Mat4 {
        let mut result = Mat4::zero();

        for i in 0..4 {
            for j in 0..4 {
                for k in 0..4 {
                    result[i][j] += self[i][k] * other[k][j];
                }
            }
        }

        result
    }
}

impl Mul<Vec4> for Mat4 {
    type Output = Vec4;

    fn mul(self, vec: Vec4) -> Vec4 {
        let mut result = Vec4::zero();

        for i in 0..4 {
            for j in 0..4 {
                result[i] += self[i][j] * vec[j];
            }
        }

        result
    }
}

#[test]
fn test_math() {
    let scale = Mat4::scale(2.0, 2.0, 2.0);
    let trans = Mat4::translate(1.0, 2.0, 3.0);
    let combined = trans * scale;

    let original = Vec4([3.0, 3.0, 3.0, 1.0]);
    let expected = Vec4([7.0, 8.0, 9.0, 1.0]);

    assert_eq!(expected, combined * original);
}
<|endoftext|>


<commit_before>#![feature(async_closure)]
#![feature(box_patterns)]
#![feature(generic_associated_types)]
#![feature(let_chains)]
#![feature(let_else)]
#![feature(never_type)]
#![feature(try_trait_v2)]
#![allow(clippy::module_inception)]

mod builtin_functions;
mod compiler;
mod database;
mod fuzzer;
mod language_server;
mod module;
mod vm;

use crate::{
    compiler::{
        ast_to_hir::AstToHir,
        cst_to_ast::CstToAst,
        error::CompilerError,
        hir::{self, CollectErrors, Id},
        hir_to_lir::HirToLir,
        rcst_to_cst::RcstToCst,
        string_to_rcst::StringToRcst,
    },
    database::Database,
    language_server::utils::LspPositionConversion,
    module::{Module, ModuleKind},
    vm::{
        context::{DbUseProvider, RunForever},
        tracer::{FullTracer, Tracer},
        Closure, ExecutionResult, FiberId, Status, Struct, Vm,
    },
};
use compiler::lir::Lir;
use itertools::Itertools;
use language_server::CandyLanguageServer;
use notify::{watcher, RecursiveMode, Watcher};
use std::{
    collections::HashMap,
    convert::TryInto,
    env::current_dir,
    path::PathBuf,
    sync::{mpsc::channel, Arc},
    time::Duration,
};
use structopt::StructOpt;
use tower_lsp::{LspService, Server};
use tracing::{debug, error, info, warn, Level, Metadata};
use tracing_subscriber::{filter, fmt::format::FmtSpan, prelude::*};
use vm::{ChannelId, CompletedOperation, OperationId};

#[derive(StructOpt, Debug)]
#[structopt(name = "candy", about = "The 🍭 Candy CLI.")]
enum CandyOptions {
    Build(CandyBuildOptions),
    Run(CandyRunOptions),
    Fuzz(CandyFuzzOptions),
    Lsp,
}

#[derive(StructOpt, Debug)]
struct CandyBuildOptions {
    #[structopt(long)]
    debug: bool,

    #[structopt(long)]
    watch: bool,

    #[structopt(parse(from_os_str))]
    file: PathBuf,
}

#[derive(StructOpt, Debug)]
struct CandyRunOptions {
    #[structopt(long)]
    debug: bool,

    #[structopt(parse(from_os_str))]
    file: PathBuf,
}

#[derive(StructOpt, Debug)]
struct CandyFuzzOptions {
    #[structopt(parse(from_os_str))]
    file: PathBuf,
}

#[tokio::main]
async fn main() {
    init_logger();
    match CandyOptions::from_args() {
        CandyOptions::Build(options) => build(options),
        CandyOptions::Run(options) => run(options),
        CandyOptions::Fuzz(options) => fuzz(options).await,
        CandyOptions::Lsp => lsp().await,
    }
}

fn build(options: CandyBuildOptions) {
    let module = Module::from_package_root_and_file(
        current_dir().unwrap(),
        options.file.clone(),
        ModuleKind::Code,
    );
    raw_build(module.clone(), options.debug);

    if options.watch {
        let (tx, rx) = channel();
        let mut watcher = watcher(tx, Duration::from_secs(1)).unwrap();
        watcher
            .watch(&options.file, RecursiveMode::Recursive)
            .unwrap();
        loop {
            match rx.recv() {
                Ok(_) => {
                    raw_build(module.clone(), options.debug);
                }
                Err(e) => error!("watch error: {e:#?}"),
            }
        }
    }
}
fn raw_build(module: Module, debug: bool) -> Option<Arc<Lir>> {
    let db = Database::default();

    tracing::span!(Level::DEBUG, "Parsing string to RCST").in_scope(|| {
        let rcst = db
            .rcst(module.clone())
            .unwrap_or_else(|err| panic!("Error parsing file `{}`: {:?}", module, err));
        if debug {
            module.dump_associated_debug_file("rcst", &format!("{:#?}\n", rcst));
        }
    });

    tracing::span!(Level::DEBUG, "Turning RCST to CST").in_scope(|| {
        let cst = db.cst(module.clone()).unwrap();
        if debug {
            module.dump_associated_debug_file("cst", &format!("{:#?}\n", cst));
        }
    });

    tracing::span!(Level::DEBUG, "Abstracting CST to AST").in_scope(|| {
        let (asts, ast_cst_id_map) = db.ast(module.clone()).unwrap();
        if debug {
            module.dump_associated_debug_file(
                "ast",
                &format!("{}\n", asts.iter().map(|ast| format!("{}", ast)).join("\n")),
            );
            module.dump_associated_debug_file(
                "ast_to_cst_ids",
                &ast_cst_id_map
                    .keys()
                    .into_iter()
                    .sorted_by_key(|it| it.local)
                    .map(|key| format!("{key} -> {}\n", ast_cst_id_map[key].0))
                    .join(""),
            );
        }
    });

    tracing::span!(Level::DEBUG, "Turning AST to HIR").in_scope(|| {
        let (hir, hir_ast_id_map) = db.hir(module.clone()).unwrap();
        if debug {
            module.dump_associated_debug_file("hir", &format!("{}", hir));
            module.dump_associated_debug_file(
                "hir_to_ast_ids",
                &hir_ast_id_map
                    .keys()
                    .into_iter()
                    .map(|key| format!("{key} -> {}\n", hir_ast_id_map[key]))
                    .join(""),
            );
        }
        let mut errors = vec![];
        hir.collect_errors(&mut errors);
        for CompilerError { span, payload, .. } in errors {
            let (start_line, start_col) = db.offset_to_lsp(module.clone(), span.start);
            let (end_line, end_col) = db.offset_to_lsp(module.clone(), span.end);
            warn!("{start_line}:{start_col} – {end_line}:{end_col}: {payload:?}");
        }
    });

    let lir = tracing::span!(Level::DEBUG, "Lowering HIR to LIR").in_scope(|| {
        let lir = db.lir(module.clone()).unwrap();
        if debug {
            module.dump_associated_debug_file("lir", &format!("{lir}"));
        }
        lir
    });

    Some(lir)
}

fn run(options: CandyRunOptions) {
    let module = Module::from_package_root_and_file(
        current_dir().unwrap(),
        options.file.clone(),
        ModuleKind::Code,
    );
    let db = Database::default();

    if raw_build(module.clone(), false).is_none() {
        warn!("Build failed.");
        return;
    };
    // TODO: Optimize the code before running.

    let path_string = options.file.to_string_lossy();
    debug!("Running `{path_string}`.");

    let module_closure = Closure::of_module(&db, module.clone()).unwrap();
    let mut tracer = FullTracer::new();

    let mut vm = Vm::new();
    vm.set_up_for_running_module_closure(module_closure);
    vm.run(&mut DbUseProvider { db: &db }, &mut RunForever, &mut tracer);
    if let Status::WaitingForOperations = vm.status() {
        error!("The module waits on channel operations. Perhaps, the code tried to read from a channel without sending a packet into it.");
        // TODO: Show stack traces of all fibers?
    }
    let result = vm.tear_down();

    if options.debug {
        module.dump_associated_debug_file("trace", &format!("{tracer:?}"));
    }

    let (mut heap, exported_definitions): (_, Struct) = match result {
        ExecutionResult::Finished(return_value) => {
            debug!("The module exports these definitions: {return_value:?}",);
            let exported = return_value
                .heap
                .get(return_value.address)
                .data
                .clone()
                .try_into()
                .unwrap();
            (return_value.heap, exported)
        }
        ExecutionResult::Panicked {
            reason,
            responsible,
        } => {
            error!("The module panicked because {reason}.");
            if let Some(responsible) = responsible {
                error!("{responsible} is responsible.");
            } else {
                error!("Some top-level code panics.");
            }
            error!(
                "This is the stack trace:\n{}",
                tracer.format_panic_stack_trace_to_root_fiber(&db)
            );
            return;
        }
    };

    let main = heap.create_symbol("Main".to_string());
    let main = match exported_definitions.get(&heap, main) {
        Some(main) => main,
        None => {
            error!("The module doesn't contain a main function.");
            return;
        }
    };

    debug!("Running main function.");
    // TODO: Add more environment stuff.
    let mut vm = Vm::new();
    let mut stdout = StdoutService::new(&mut vm);
    let environment = {
        let stdout_symbol = heap.create_symbol("Stdout".to_string());
        let stdout_port = heap.create_send_port(stdout.channel);
        heap.create_struct(HashMap::from([(stdout_symbol, stdout_port)]))
    };
    tracer.in_fiber_tracer(FiberId::root()).call_started(
        &heap,
        Id::new(module, vec!["main".to_string()]),
        main,
        vec![environment],
    );
    vm.set_up_for_running_closure(heap, main, &[environment]);
    loop {
        match vm.status() {
            Status::CanRun => {
                debug!("VM still running.");
                vm.run(&mut DbUseProvider { db: &db }, &mut RunForever, &mut tracer);
            }
            Status::WaitingForOperations => {
                todo!("VM can't proceed until some operations complete.");
            }
            _ => break,
        }
        stdout.run(&mut vm);
        for channel in vm.unreferenced_channels.iter().copied().collect_vec() {
            if channel != stdout.channel {
                vm.free_channel(channel);
            }
        }
    }
    match vm.tear_down() {
        ExecutionResult::Finished(return_value) => {
            tracer
                .in_fiber_tracer(FiberId::root())
                .call_ended(&return_value.heap, return_value.address);
            debug!("The main function returned: {return_value:?}");
        }
        ExecutionResult::Panicked {
            reason,
            responsible,
        } => {
            error!("The main function panicked because {reason}.");
            if let Some(responsible) = responsible {
                error!("{responsible} is responsible.");
            } else {
                error!("A needs directly in the main function panicks. Perhaps the main functions expects more in the environment.");
            }
            error!(
                "This is the stack trace:\n{}",
                tracer.format_panic_stack_trace_to_root_fiber(&db)
            );
        }
    }
}

/// A state machine that corresponds to a loop that always calls `receive` on
/// the stdout channel and then logs that packet.
struct StdoutService {
    channel: ChannelId,
    current_receive: OperationId,
}
impl StdoutService {
    fn new(vm: &mut Vm) -> Self {
        let channel = vm.create_channel(1);
        let current_receive = vm.receive(channel);
        Self {
            channel,
            current_receive,
        }
    }
    fn run(&mut self, vm: &mut Vm) {
        if let Some(CompletedOperation::Received { packet }) =
            vm.completed_operations.remove(&self.current_receive)
        {
            info!("Sent to stdout: {packet:?}");
            self.current_receive = vm.receive(self.channel);
        }
    }
}

async fn fuzz(options: CandyFuzzOptions) {
    let module = Module::from_package_root_and_file(
        current_dir().unwrap(),
        options.file.clone(),
        ModuleKind::Code,
    );

    if raw_build(module.clone(), false).is_none() {
        warn!("Build failed.");
        return;
    }

    debug!("Fuzzing `{module}`.");
    let db = Database::default();
    fuzzer::fuzz(&db, module).await;
}

async fn lsp() {
    info!("Starting language server…");
    let (service, socket) = LspService::new(CandyLanguageServer::from_client);
    Server::new(tokio::io::stdin(), tokio::io::stdout(), socket)
        .serve(service)
        .await;
}

fn init_logger() {
    let console_log = tracing_subscriber::fmt::layer()
        .compact()
        .with_span_events(FmtSpan::ENTER)
        .with_filter(filter::filter_fn(|metadata| {
            // For external packages, show only the error logs.
            metadata.level() <= &Level::ERROR
                || metadata
                    .module_path()
                    .unwrap_or_default()
                    .starts_with("candy")
        }))
        .with_filter(filter::filter_fn(level_for("candy::compiler", Level::WARN)))
        .with_filter(filter::filter_fn(level_for(
            "candy::language_server",
            Level::DEBUG,
        )))
        .with_filter(filter::filter_fn(level_for("candy::vm", Level::DEBUG)))
        .with_filter(filter::filter_fn(level_for(
            "candy::vm::heap",
            Level::DEBUG,
        )));
    tracing_subscriber::registry().with(console_log).init();
}
fn level_for(module: &'static str, level: Level) -> impl Fn(&Metadata) -> bool {
    move |metadata| {
        if metadata
            .module_path()
            .unwrap_or_default()
            .starts_with(module)
        {
            metadata.level() <= &level
        } else {
            true
        }
    }
}
<commit_msg>Remove let_else feature flag<commit_after>#![feature(async_closure)]
#![feature(box_patterns)]
#![feature(generic_associated_types)]
#![feature(let_chains)]
#![feature(never_type)]
#![feature(try_trait_v2)]
#![allow(clippy::module_inception)]

mod builtin_functions;
mod compiler;
mod database;
mod fuzzer;
mod language_server;
mod module;
mod vm;

use crate::{
    compiler::{
        ast_to_hir::AstToHir,
        cst_to_ast::CstToAst,
        error::CompilerError,
        hir::{self, CollectErrors, Id},
        hir_to_lir::HirToLir,
        rcst_to_cst::RcstToCst,
        string_to_rcst::StringToRcst,
    },
    database::Database,
    language_server::utils::LspPositionConversion,
    module::{Module, ModuleKind},
    vm::{
        context::{DbUseProvider, RunForever},
        tracer::{FullTracer, Tracer},
        Closure, ExecutionResult, FiberId, Status, Struct, Vm,
    },
};
use compiler::lir::Lir;
use itertools::Itertools;
use language_server::CandyLanguageServer;
use notify::{watcher, RecursiveMode, Watcher};
use std::{
    collections::HashMap,
    convert::TryInto,
    env::current_dir,
    path::PathBuf,
    sync::{mpsc::channel, Arc},
    time::Duration,
};
use structopt::StructOpt;
use tower_lsp::{LspService, Server};
use tracing::{debug, error, info, warn, Level, Metadata};
use tracing_subscriber::{filter, fmt::format::FmtSpan, prelude::*};
use vm::{ChannelId, CompletedOperation, OperationId};

#[derive(StructOpt, Debug)]
#[structopt(name = "candy", about = "The 🍭 Candy CLI.")]
enum CandyOptions {
    Build(CandyBuildOptions),
    Run(CandyRunOptions),
    Fuzz(CandyFuzzOptions),
    Lsp,
}

#[derive(StructOpt, Debug)]
struct CandyBuildOptions {
    #[structopt(long)]
    debug: bool,

    #[structopt(long)]
    watch: bool,

    #[structopt(parse(from_os_str))]
    file: PathBuf,
}

#[derive(StructOpt, Debug)]
struct CandyRunOptions {
    #[structopt(long)]
    debug: bool,

    #[structopt(parse(from_os_str))]
    file: PathBuf,
}

#[derive(StructOpt, Debug)]
struct CandyFuzzOptions {
    #[structopt(parse(from_os_str))]
    file: PathBuf,
}

#[tokio::main]
async fn main() {
    init_logger();
    match CandyOptions::from_args() {
        CandyOptions::Build(options) => build(options),
        CandyOptions::Run(options) => run(options),
        CandyOptions::Fuzz(options) => fuzz(options).await,
        CandyOptions::Lsp => lsp().await,
    }
}

fn build(options: CandyBuildOptions) {
    let module = Module::from_package_root_and_file(
        current_dir().unwrap(),
        options.file.clone(),
        ModuleKind::Code,
    );
    raw_build(module.clone(), options.debug);

    if options.watch {
        let (tx, rx) = channel();
        let mut watcher = watcher(tx, Duration::from_secs(1)).unwrap();
        watcher
            .watch(&options.file, RecursiveMode::Recursive)
            .unwrap();
        loop {
            match rx.recv() {
                Ok(_) => {
                    raw_build(module.clone(), options.debug);
                }
                Err(e) => error!("watch error: {e:#?}"),
            }
        }
    }
}
fn raw_build(module: Module, debug: bool) -> Option<Arc<Lir>> {
    let db = Database::default();

    tracing::span!(Level::DEBUG, "Parsing string to RCST").in_scope(|| {
        let rcst = db
            .rcst(module.clone())
            .unwrap_or_else(|err| panic!("Error parsing file `{}`: {:?}", module, err));
        if debug {
            module.dump_associated_debug_file("rcst", &format!("{:#?}\n", rcst));
        }
    });

    tracing::span!(Level::DEBUG, "Turning RCST to CST").in_scope(|| {
        let cst = db.cst(module.clone()).unwrap();
        if debug {
            module.dump_associated_debug_file("cst", &format!("{:#?}\n", cst));
        }
    });

    tracing::span!(Level::DEBUG, "Abstracting CST to AST").in_scope(|| {
        let (asts, ast_cst_id_map) = db.ast(module.clone()).unwrap();
        if debug {
            module.dump_associated_debug_file(
                "ast",
                &format!("{}\n", asts.iter().map(|ast| format!("{}", ast)).join("\n")),
            );
            module.dump_associated_debug_file(
                "ast_to_cst_ids",
                &ast_cst_id_map
                    .keys()
                    .into_iter()
                    .sorted_by_key(|it| it.local)
                    .map(|key| format!("{key} -> {}\n", ast_cst_id_map[key].0))
                    .join(""),
            );
        }
    });

    tracing::span!(Level::DEBUG, "Turning AST to HIR").in_scope(|| {
        let (hir, hir_ast_id_map) = db.hir(module.clone()).unwrap();
        if debug {
            module.dump_associated_debug_file("hir", &format!("{}", hir));
            module.dump_associated_debug_file(
                "hir_to_ast_ids",
                &hir_ast_id_map
                    .keys()
                    .into_iter()
                    .map(|key| format!("{key} -> {}\n", hir_ast_id_map[key]))
                    .join(""),
            );
        }
        let mut errors = vec![];
        hir.collect_errors(&mut errors);
        for CompilerError { span, payload, .. } in errors {
            let (start_line, start_col) = db.offset_to_lsp(module.clone(), span.start);
            let (end_line, end_col) = db.offset_to_lsp(module.clone(), span.end);
            warn!("{start_line}:{start_col} – {end_line}:{end_col}: {payload:?}");
        }
    });

    let lir = tracing::span!(Level::DEBUG, "Lowering HIR to LIR").in_scope(|| {
        let lir = db.lir(module.clone()).unwrap();
        if debug {
            module.dump_associated_debug_file("lir", &format!("{lir}"));
        }
        lir
    });

    Some(lir)
}

fn run(options: CandyRunOptions) {
    let module = Module::from_package_root_and_file(
        current_dir().unwrap(),
        options.file.clone(),
        ModuleKind::Code,
    );
    let db = Database::default();

    if raw_build(module.clone(), false).is_none() {
        warn!("Build failed.");
        return;
    };
    // TODO: Optimize the code before running.

    let path_string = options.file.to_string_lossy();
    debug!("Running `{path_string}`.");

    let module_closure = Closure::of_module(&db, module.clone()).unwrap();
    let mut tracer = FullTracer::new();

    let mut vm = Vm::new();
    vm.set_up_for_running_module_closure(module_closure);
    vm.run(&mut DbUseProvider { db: &db }, &mut RunForever, &mut tracer);
    if let Status::WaitingForOperations = vm.status() {
        error!("The module waits on channel operations. Perhaps, the code tried to read from a channel without sending a packet into it.");
        // TODO: Show stack traces of all fibers?
    }
    let result = vm.tear_down();

    if options.debug {
        module.dump_associated_debug_file("trace", &format!("{tracer:?}"));
    }

    let (mut heap, exported_definitions): (_, Struct) = match result {
        ExecutionResult::Finished(return_value) => {
            debug!("The module exports these definitions: {return_value:?}",);
            let exported = return_value
                .heap
                .get(return_value.address)
                .data
                .clone()
                .try_into()
                .unwrap();
            (return_value.heap, exported)
        }
        ExecutionResult::Panicked {
            reason,
            responsible,
        } => {
            error!("The module panicked because {reason}.");
            if let Some(responsible) = responsible {
                error!("{responsible} is responsible.");
            } else {
                error!("Some top-level code panics.");
            }
            error!(
                "This is the stack trace:\n{}",
                tracer.format_panic_stack_trace_to_root_fiber(&db)
            );
            return;
        }
    };

    let main = heap.create_symbol("Main".to_string());
    let main = match exported_definitions.get(&heap, main) {
        Some(main) => main,
        None => {
            error!("The module doesn't contain a main function.");
            return;
        }
    };

    debug!("Running main function.");
    // TODO: Add more environment stuff.
    let mut vm = Vm::new();
    let mut stdout = StdoutService::new(&mut vm);
    let environment = {
        let stdout_symbol = heap.create_symbol("Stdout".to_string());
        let stdout_port = heap.create_send_port(stdout.channel);
        heap.create_struct(HashMap::from([(stdout_symbol, stdout_port)]))
    };
    tracer.in_fiber_tracer(FiberId::root()).call_started(
        &heap,
        Id::new(module, vec!["main".to_string()]),
        main,
        vec![environment],
    );
    vm.set_up_for_running_closure(heap, main, &[environment]);
    loop {
        match vm.status() {
            Status::CanRun => {
                debug!("VM still running.");
                vm.run(&mut DbUseProvider { db: &db }, &mut RunForever, &mut tracer);
            }
            Status::WaitingForOperations => {
                todo!("VM can't proceed until some operations complete.");
            }
            _ => break,
        }
        stdout.run(&mut vm);
        for channel in vm.unreferenced_channels.iter().copied().collect_vec() {
            if channel != stdout.channel {
                vm.free_channel(channel);
            }
        }
    }
    match vm.tear_down() {
        ExecutionResult::Finished(return_value) => {
            tracer
                .in_fiber_tracer(FiberId::root())
                .call_ended(&return_value.heap, return_value.address);
            debug!("The main function returned: {return_value:?}");
        }
        ExecutionResult::Panicked {
            reason,
            responsible,
        } => {
            error!("The main function panicked because {reason}.");
            if let Some(responsible) = responsible {
                error!("{responsible} is responsible.");
            } else {
                error!("A needs directly in the main function panicks. Perhaps the main functions expects more in the environment.");
            }
            error!(
                "This is the stack trace:\n{}",
                tracer.format_panic_stack_trace_to_root_fiber(&db)
            );
        }
    }
}

/// A state machine that corresponds to a loop that always calls `receive` on
/// the stdout channel and then logs that packet.
struct StdoutService {
    channel: ChannelId,
    current_receive: OperationId,
}
impl StdoutService {
    fn new(vm: &mut Vm) -> Self {
        let channel = vm.create_channel(1);
        let current_receive = vm.receive(channel);
        Self {
            channel,
            current_receive,
        }
    }
    fn run(&mut self, vm: &mut Vm) {
        if let Some(CompletedOperation::Received { packet }) =
            vm.completed_operations.remove(&self.current_receive)
        {
            info!("Sent to stdout: {packet:?}");
            self.current_receive = vm.receive(self.channel);
        }
    }
}

async fn fuzz(options: CandyFuzzOptions) {
    let module = Module::from_package_root_and_file(
        current_dir().unwrap(),
        options.file.clone(),
        ModuleKind::Code,
    );

    if raw_build(module.clone(), false).is_none() {
        warn!("Build failed.");
        return;
    }

    debug!("Fuzzing `{module}`.");
    let db = Database::default();
    fuzzer::fuzz(&db, module).await;
}

async fn lsp() {
    info!("Starting language server…");
    let (service, socket) = LspService::new(CandyLanguageServer::from_client);
    Server::new(tokio::io::stdin(), tokio::io::stdout(), socket)
        .serve(service)
        .await;
}

fn init_logger() {
    let console_log = tracing_subscriber::fmt::layer()
        .compact()
        .with_span_events(FmtSpan::ENTER)
        .with_filter(filter::filter_fn(|metadata| {
            // For external packages, show only the error logs.
            metadata.level() <= &Level::ERROR
                || metadata
                    .module_path()
                    .unwrap_or_default()
                    .starts_with("candy")
        }))
        .with_filter(filter::filter_fn(level_for("candy::compiler", Level::WARN)))
        .with_filter(filter::filter_fn(level_for(
            "candy::language_server",
            Level::DEBUG,
        )))
        .with_filter(filter::filter_fn(level_for("candy::vm", Level::DEBUG)))
        .with_filter(filter::filter_fn(level_for(
            "candy::vm::heap",
            Level::DEBUG,
        )));
    tracing_subscriber::registry().with(console_log).init();
}
fn level_for(module: &'static str, level: Level) -> impl Fn(&Metadata) -> bool {
    move |metadata| {
        if metadata
            .module_path()
            .unwrap_or_default()
            .starts_with(module)
        {
            metadata.level() <= &level
        } else {
            true
        }
    }
}
<|endoftext|>


<commit_before>#![feature(plugin)]
#![allow(unstable)]

extern crate regex;
#[plugin] #[no_link] extern crate regex_macros;
extern crate gcc;
extern crate toml;

use std::collections::HashSet;
use std::default::Default;
use std::io::fs::{File,readdir};
use std::os;
use regex::Regex;

// Fetch the 'package.exclude' list from our Cargo.toml file.  We'll
// use this to decide what sources to admit.
fn get_excluded_sources(manifest: &Path) -> HashSet<String> {
    let text = File::open(manifest).read_to_string().unwrap();
    let toml = toml::Parser::new(text.as_slice()).parse().unwrap();
    let package = toml.get("package").unwrap().as_table().unwrap();
    let exclude = package.get("exclude").unwrap().as_slice().unwrap();
    exclude.iter().map(|e| {
        let str = e.as_str().unwrap();
        Path::new(str).filename_str().unwrap().to_string()
    }).collect()
}

static CC_FILE: Regex = regex!(r"\.cc\z");

// Get all the *.cc files in path that aren't excluded.
fn get_cc_files(dir: &Path, excluded: &HashSet<String>) -> Vec<Path> {
    readdir(dir).unwrap().iter().filter(|p| {
        let filename = p.filename_str().unwrap();
        CC_FILE.is_match(filename) && !excluded.contains(filename)
    }).map(|e| e.clone()).collect()
}

fn main() {
    let src = Path::new(os::getenv("CARGO_MANIFEST_DIR").unwrap());
    let includes = vec![Path::new("cld2/public"), Path::new("cld2/internal")];

    // Decide what sources to build.
    let excluded = get_excluded_sources(&src.join("Cargo.toml"));
    let mut sources =
        get_cc_files(&src.join("cld2").join("internal"), &excluded);
    sources.push(src.join("src").join("wrapper.cpp"));

    // Convert the sources back to relative path &str values.
    let rel_sources: Vec<Path> = sources.iter().map(|p| {
        p.path_relative_from(&src).unwrap()
    }).collect();
    let rel_sources_str: Vec<&str> = rel_sources.iter().map(|p| {
        p.as_str().unwrap()
    }).collect();

    // Run the build.
    gcc::compile_library("libcld2.a", &gcc::Config {
        include_directories: includes,
        .. Default::default()
    }, rel_sources_str.as_slice());

    // Decide how to link our C++ runtime.  Feel free to submit patches
    // to make this work on your platform.  Other likely options are "c++"
    // and "c++abi" depending on OS and compiler.
    let cxx_abi = "stdc++";
    println!("cargo:rustc-flags=-l {}", cxx_abi);
}
<commit_msg>Rename io -> old_io<commit_after>#![feature(plugin)]
#![allow(unstable)]

extern crate regex;
#[plugin] #[no_link] extern crate regex_macros;
extern crate gcc;
extern crate toml;

use std::collections::HashSet;
use std::default::Default;
use std::old_io::fs::{File,readdir};
use std::os;
use regex::Regex;

// Fetch the 'package.exclude' list from our Cargo.toml file.  We'll
// use this to decide what sources to admit.
fn get_excluded_sources(manifest: &Path) -> HashSet<String> {
    let text = File::open(manifest).read_to_string().unwrap();
    let toml = toml::Parser::new(text.as_slice()).parse().unwrap();
    let package = toml.get("package").unwrap().as_table().unwrap();
    let exclude = package.get("exclude").unwrap().as_slice().unwrap();
    exclude.iter().map(|e| {
        let str = e.as_str().unwrap();
        Path::new(str).filename_str().unwrap().to_string()
    }).collect()
}

static CC_FILE: Regex = regex!(r"\.cc\z");

// Get all the *.cc files in path that aren't excluded.
fn get_cc_files(dir: &Path, excluded: &HashSet<String>) -> Vec<Path> {
    readdir(dir).unwrap().iter().filter(|p| {
        let filename = p.filename_str().unwrap();
        CC_FILE.is_match(filename) && !excluded.contains(filename)
    }).map(|e| e.clone()).collect()
}

fn main() {
    let src = Path::new(os::getenv("CARGO_MANIFEST_DIR").unwrap());
    let includes = vec![Path::new("cld2/public"), Path::new("cld2/internal")];

    // Decide what sources to build.
    let excluded = get_excluded_sources(&src.join("Cargo.toml"));
    let mut sources =
        get_cc_files(&src.join("cld2").join("internal"), &excluded);
    sources.push(src.join("src").join("wrapper.cpp"));

    // Convert the sources back to relative path &str values.
    let rel_sources: Vec<Path> = sources.iter().map(|p| {
        p.path_relative_from(&src).unwrap()
    }).collect();
    let rel_sources_str: Vec<&str> = rel_sources.iter().map(|p| {
        p.as_str().unwrap()
    }).collect();

    // Run the build.
    gcc::compile_library("libcld2.a", &gcc::Config {
        include_directories: includes,
        .. Default::default()
    }, rel_sources_str.as_slice());

    // Decide how to link our C++ runtime.  Feel free to submit patches
    // to make this work on your platform.  Other likely options are "c++"
    // and "c++abi" depending on OS and compiler.
    let cxx_abi = "stdc++";
    println!("cargo:rustc-flags=-l {}", cxx_abi);
}
<|endoftext|>


<commit_before>// https://leetcode.com/problems/maximum-number-of-groups-entering-a-competition/
pub fn maximum_groups(grades: Vec<i32>) -> i32 {
    todo!()
}

fn main() {
    println!("{}", maximum_groups(vec![10, 6, 12, 7, 3, 5])); // 3
    println!("{}", maximum_groups(vec![8, 8])); // 1
}
<commit_msg>Solve problem 2358<commit_after>// https://leetcode.com/problems/maximum-number-of-groups-entering-a-competition/
pub fn maximum_groups(grades: Vec<i32>) -> i32 {
    let mut answer = 0;
    let mut i = 1;
    let mut j = i;
    while j <= grades.len() {
        i += 1;
        j += i;
        answer += 1;
    }
    answer
}

fn main() {
    println!("{}", maximum_groups(vec![10, 6, 12, 7, 3, 5])); // 3
    println!("{}", maximum_groups(vec![8, 8])); // 1
    println!("{}", maximum_groups(vec![1, 1, 1, 1, 1])); // 2
    println!("{}", maximum_groups(vec![1, 3, 5, 6, 1])); // 2
    println!("{}", maximum_groups(vec![1, 3, 5, 6, 1, 1])); // 3
}
<|endoftext|>