<commit_before>use std::io::{Read, Write};

#[cfg(feature = "zstd")]
use zstd::block::{compress, decompress};

use super::*;

/// A snapshot of the state required to quickly restart
/// the `PageCache` and `SegmentAccountant`.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct Snapshot {
    /// The last read message lsn
    pub last_lsn: Lsn,
    /// The last read message lid
    pub last_lid: LogId,
    /// the mapping from pages to (lsn, lid)
    pub pt: FastMap8<PageId, PageState>,
    /// The highest stable offset persisted
    /// into a segment header
    pub max_header_stable_lsn: Lsn,
}

#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum PageState {
    Present(Vec<(Lsn, DiskPtr, usize)>),
    Free(Lsn, DiskPtr),
}

impl PageState {
    fn push(&mut self, item: (Lsn, DiskPtr, usize)) {
        match *self {
            PageState::Present(ref mut items) => items.push(item),
            PageState::Free(_, _) => {
                panic!("pushed items to a PageState::Free")
            }
        }
    }

    /// Iterate over the (lsn, lid) pairs that hold this page's state.
    pub fn iter(&self) -> impl Iterator<Item = (Lsn, DiskPtr, usize)> {
        match *self {
            PageState::Present(ref items) => items.clone().into_iter(),
            PageState::Free(lsn, ptr) => {
                vec![(lsn, ptr, MSG_HEADER_LEN)].into_iter()
            }
        }
    }

    fn is_free(&self) -> bool {
        match *self {
            PageState::Free(_, _) => true,
            _ => false,
        }
    }
}

impl Snapshot {
    fn apply(
        &mut self,
        log_kind: LogKind,
        pid: PageId,
        lsn: Lsn,
        disk_ptr: DiskPtr,
        sz: usize,
    ) {
        // unwrapping this because it's already passed the crc check
        // in the log iterator
        trace!("trying to deserialize buf for ptr {} lsn {}", disk_ptr, lsn);

        match log_kind {
            LogKind::Replace => {
                trace!(
                    "compact of pid {} at ptr {} lsn {}",
                    pid,
                    disk_ptr,
                    lsn,
                );

                self.pt
                    .insert(pid, PageState::Present(vec![(lsn, disk_ptr, sz)]));
            }
            LogKind::Append => {
                // Because we rewrite pages over time, we may have relocated
                // a page's initial Compact to a later segment. We should skip
                // over pages here unless we've encountered a Compact for them.
                if let Some(lids) = self.pt.get_mut(&pid) {
                    trace!(
                        "append of pid {} at lid {} lsn {}",
                        pid,
                        disk_ptr,
                        lsn,
                    );

                    if lids.is_free() {
                        // this can happen if the allocate or replace
                        // has been moved to a later segment.

                        trace!(
                            "we have not yet encountered an \
                             allocation of this page, skipping push"
                        );

                        return;
                    }

                    lids.push((lsn, disk_ptr, sz));
                }
            }
            LogKind::Free => {
                trace!("free of pid {} at ptr {} lsn {}", pid, disk_ptr, lsn);
                self.pt.insert(pid, PageState::Free(lsn, disk_ptr));
            }
            LogKind::Corrupted | LogKind::Skip => panic!(
                "unexppected messagekind in snapshot application: {:?}",
                log_kind
            ),
        }
    }
}

pub(super) fn advance_snapshot(
    iter: LogIter,
    mut snapshot: Snapshot,
    config: &Config,
) -> Result<Snapshot> {
    let _measure = Measure::new(&M.advance_snapshot);

    trace!("building on top of old snapshot: {:?}", snapshot);

    for (log_kind, pid, lsn, ptr, sz) in iter {
        trace!(
            "in advance_snapshot looking at item with lsn {} ptr {}",
            lsn,
            ptr
        );

        if lsn <= snapshot.last_lsn {
            // don't process already-processed Lsn's. last_lsn is for the last
            // item ALREADY INCLUDED lsn in the snapshot.
            trace!(
                "continuing in advance_snapshot, lsn {} ptr {} last_lsn {}",
                lsn,
                ptr,
                snapshot.last_lsn
            );
            continue;
        }

        assert!(lsn > snapshot.last_lsn);
        snapshot.last_lsn = lsn;
        snapshot.last_lid = ptr.lid();

        snapshot.apply(log_kind, pid, lsn, ptr, sz);
    }

    write_snapshot(config, &snapshot)?;

    trace!("generated new snapshot: {:?}", snapshot);

    Ok(snapshot)
}

/// Read a `Snapshot` or generate a default, then advance it to
/// the tip of the data file, if present.
pub fn read_snapshot_or_default(config: &Config) -> Result<Snapshot> {
    let mut last_snap =
        read_snapshot(config)?.unwrap_or_else(Snapshot::default);

    let (log_iter, max_header_stable_lsn) =
        raw_segment_iter_from(last_snap.last_lsn, config)?;

    last_snap.max_header_stable_lsn = max_header_stable_lsn;

    advance_snapshot(log_iter, last_snap, config)
}

/// Read a `Snapshot` from disk.
fn read_snapshot(config: &Config) -> std::io::Result<Option<Snapshot>> {
    let mut candidates = config.get_snapshot_files()?;
    if candidates.is_empty() {
        debug!("no previous snapshot found");
        return Ok(None);
    }

    candidates.sort();

    let path = candidates.pop().unwrap();

    let mut f = std::fs::OpenOptions::new().read(true).open(&path)?;
    if f.metadata()?.len() <= 12 {
        warn!("empty/corrupt snapshot file found");
        return Ok(None);
    }

    let mut buf = vec![];
    f.read_to_end(&mut buf)?;
    let len = buf.len();
    let mut len_expected_bytes = [0u8; 8];
    len_expected_bytes.copy_from_slice(&buf[len - 12..len - 4]);

    let mut crc_expected_bytes = [0u8; 4];
    crc_expected_bytes.copy_from_slice(&buf[len - 4..]);

    buf.split_off(len - 12);
    let crc_expected: u32 = arr_to_u32(&crc_expected_bytes);

    let crc_actual = crc32(&buf);

    if crc_expected != crc_actual {
        return Ok(None);
    }

    #[cfg(feature = "zstd")]
    let bytes = if config.use_compression {
        let len_expected: u64 = arr_to_u64(&len_expected_bytes);
        decompress(&*buf, len_expected as usize).unwrap()
    } else {
        buf
    };

    #[cfg(not(feature = "zstd"))]
    let bytes = buf;

    Ok(deserialize::<Snapshot>(&*bytes).ok())
}

fn write_snapshot(config: &Config, snapshot: &Snapshot) -> Result<()> {
    let raw_bytes = serialize(&snapshot).unwrap();
    let decompressed_len = raw_bytes.len();

    #[cfg(feature = "zstd")]
    let bytes = if config.use_compression {
        compress(&*raw_bytes, config.compression_factor).unwrap()
    } else {
        raw_bytes
    };

    #[cfg(not(feature = "zstd"))]
    let bytes = raw_bytes;

    let crc32: [u8; 4] = u32_to_arr(crc32(&bytes));
    let len_bytes: [u8; 8] = u64_to_arr(decompressed_len as u64);

    let path_1_suffix = format!("snap.{:016X}.generating", snapshot.last_lsn);

    let mut path_1 = config.snapshot_prefix();
    path_1.push(path_1_suffix);

    let path_2_suffix = format!("snap.{:016X}", snapshot.last_lsn);

    let mut path_2 = config.snapshot_prefix();
    path_2.push(path_2_suffix);

    let parent = path_1.parent().unwrap();
    std::fs::create_dir_all(parent)?;
    let mut f = std::fs::OpenOptions::new()
        .write(true)
        .create(true)
        .open(&path_1)?;

    // write the snapshot bytes, followed by a crc64 checksum at the end
    maybe_fail!("snap write");
    f.write_all(&*bytes)?;
    maybe_fail!("snap write len");
    f.write_all(&len_bytes)?;
    maybe_fail!("snap write crc");
    f.write_all(&crc32)?;
    maybe_fail!("snap write post");

    trace!("wrote snapshot to {}", path_1.to_string_lossy());

    maybe_fail!("snap write mv");
    std::fs::rename(&path_1, &path_2)?;
    maybe_fail!("snap write mv post");

    trace!("renamed snapshot to {}", path_2.to_string_lossy());

    // clean up any old snapshots
    let candidates = config.get_snapshot_files()?;
    for path in candidates {
        let path_str = path.file_name().unwrap().to_str().unwrap();
        if !path_2.to_string_lossy().ends_with(&*path_str) {
            debug!("removing old snapshot file {:?}", path);

            maybe_fail!("snap write rm old");

            if let Err(_e) = std::fs::remove_file(&path) {
                // TODO should this just be a try return?
                warn!(
                    "failed to remove old snapshot file, maybe snapshot race? {}",
                    _e
                );
            }
        }
    }
    Ok(())
}
<commit_msg>only write snapshots when they change<commit_after>use std::io::{Read, Write};

#[cfg(feature = "zstd")]
use zstd::block::{compress, decompress};

use super::*;

/// A snapshot of the state required to quickly restart
/// the `PageCache` and `SegmentAccountant`.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct Snapshot {
    /// The last read message lsn
    pub last_lsn: Lsn,
    /// The last read message lid
    pub last_lid: LogId,
    /// the mapping from pages to (lsn, lid)
    pub pt: FastMap8<PageId, PageState>,
    /// The highest stable offset persisted
    /// into a segment header
    pub max_header_stable_lsn: Lsn,
}

#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum PageState {
    Present(Vec<(Lsn, DiskPtr, usize)>),
    Free(Lsn, DiskPtr),
}

impl PageState {
    fn push(&mut self, item: (Lsn, DiskPtr, usize)) {
        match *self {
            PageState::Present(ref mut items) => items.push(item),
            PageState::Free(_, _) => {
                panic!("pushed items to a PageState::Free")
            }
        }
    }

    /// Iterate over the (lsn, lid) pairs that hold this page's state.
    pub fn iter(&self) -> impl Iterator<Item = (Lsn, DiskPtr, usize)> {
        match *self {
            PageState::Present(ref items) => items.clone().into_iter(),
            PageState::Free(lsn, ptr) => {
                vec![(lsn, ptr, MSG_HEADER_LEN)].into_iter()
            }
        }
    }

    fn is_free(&self) -> bool {
        match *self {
            PageState::Free(_, _) => true,
            _ => false,
        }
    }
}

impl Snapshot {
    fn apply(
        &mut self,
        log_kind: LogKind,
        pid: PageId,
        lsn: Lsn,
        disk_ptr: DiskPtr,
        sz: usize,
    ) {
        // unwrapping this because it's already passed the crc check
        // in the log iterator
        trace!("trying to deserialize buf for ptr {} lsn {}", disk_ptr, lsn);

        match log_kind {
            LogKind::Replace => {
                trace!(
                    "compact of pid {} at ptr {} lsn {}",
                    pid,
                    disk_ptr,
                    lsn,
                );

                self.pt
                    .insert(pid, PageState::Present(vec![(lsn, disk_ptr, sz)]));
            }
            LogKind::Append => {
                // Because we rewrite pages over time, we may have relocated
                // a page's initial Compact to a later segment. We should skip
                // over pages here unless we've encountered a Compact for them.
                if let Some(lids) = self.pt.get_mut(&pid) {
                    trace!(
                        "append of pid {} at lid {} lsn {}",
                        pid,
                        disk_ptr,
                        lsn,
                    );

                    if lids.is_free() {
                        // this can happen if the allocate or replace
                        // has been moved to a later segment.

                        trace!(
                            "we have not yet encountered an \
                             allocation of this page, skipping push"
                        );

                        return;
                    }

                    lids.push((lsn, disk_ptr, sz));
                }
            }
            LogKind::Free => {
                trace!("free of pid {} at ptr {} lsn {}", pid, disk_ptr, lsn);
                self.pt.insert(pid, PageState::Free(lsn, disk_ptr));
            }
            LogKind::Corrupted | LogKind::Skip => panic!(
                "unexppected messagekind in snapshot application: {:?}",
                log_kind
            ),
        }
    }
}

pub(super) fn advance_snapshot(
    iter: LogIter,
    mut snapshot: Snapshot,
    config: &Config,
) -> Result<Snapshot> {
    let _measure = Measure::new(&M.advance_snapshot);

    trace!("building on top of old snapshot: {:?}", snapshot);

    let old_lsn = snapshot.last_lsn;

    for (log_kind, pid, lsn, ptr, sz) in iter {
        trace!(
            "in advance_snapshot looking at item with lsn {} ptr {}",
            lsn,
            ptr
        );

        if lsn <= snapshot.last_lsn {
            // don't process already-processed Lsn's. last_lsn is for the last
            // item ALREADY INCLUDED lsn in the snapshot.
            trace!(
                "continuing in advance_snapshot, lsn {} ptr {} last_lsn {}",
                lsn,
                ptr,
                snapshot.last_lsn
            );
            continue;
        }

        assert!(lsn > snapshot.last_lsn);
        snapshot.last_lsn = lsn;
        snapshot.last_lid = ptr.lid();

        snapshot.apply(log_kind, pid, lsn, ptr, sz);
    }

    if snapshot.last_lsn != old_lsn {
        write_snapshot(config, &snapshot)?;
    }

    trace!("generated new snapshot: {:?}", snapshot);

    Ok(snapshot)
}

/// Read a `Snapshot` or generate a default, then advance it to
/// the tip of the data file, if present.
pub fn read_snapshot_or_default(config: &Config) -> Result<Snapshot> {
    let mut last_snap =
        read_snapshot(config)?.unwrap_or_else(Snapshot::default);

    let (log_iter, max_header_stable_lsn) =
        raw_segment_iter_from(last_snap.last_lsn, config)?;

    last_snap.max_header_stable_lsn = max_header_stable_lsn;

    advance_snapshot(log_iter, last_snap, config)
}

/// Read a `Snapshot` from disk.
fn read_snapshot(config: &Config) -> std::io::Result<Option<Snapshot>> {
    let mut candidates = config.get_snapshot_files()?;
    if candidates.is_empty() {
        debug!("no previous snapshot found");
        return Ok(None);
    }

    candidates.sort();

    let path = candidates.pop().unwrap();

    let mut f = std::fs::OpenOptions::new().read(true).open(&path)?;
    if f.metadata()?.len() <= 12 {
        warn!("empty/corrupt snapshot file found");
        return Ok(None);
    }

    let mut buf = vec![];
    f.read_to_end(&mut buf)?;
    let len = buf.len();
    let mut len_expected_bytes = [0u8; 8];
    len_expected_bytes.copy_from_slice(&buf[len - 12..len - 4]);

    let mut crc_expected_bytes = [0u8; 4];
    crc_expected_bytes.copy_from_slice(&buf[len - 4..]);

    buf.split_off(len - 12);
    let crc_expected: u32 = arr_to_u32(&crc_expected_bytes);

    let crc_actual = crc32(&buf);

    if crc_expected != crc_actual {
        return Ok(None);
    }

    #[cfg(feature = "zstd")]
    let bytes = if config.use_compression {
        let len_expected: u64 = arr_to_u64(&len_expected_bytes);
        decompress(&*buf, len_expected as usize).unwrap()
    } else {
        buf
    };

    #[cfg(not(feature = "zstd"))]
    let bytes = buf;

    Ok(deserialize::<Snapshot>(&*bytes).ok())
}

fn write_snapshot(config: &Config, snapshot: &Snapshot) -> Result<()> {
    let raw_bytes = serialize(&snapshot).unwrap();
    let decompressed_len = raw_bytes.len();

    #[cfg(feature = "zstd")]
    let bytes = if config.use_compression {
        compress(&*raw_bytes, config.compression_factor).unwrap()
    } else {
        raw_bytes
    };

    #[cfg(not(feature = "zstd"))]
    let bytes = raw_bytes;

    let crc32: [u8; 4] = u32_to_arr(crc32(&bytes));
    let len_bytes: [u8; 8] = u64_to_arr(decompressed_len as u64);

    let path_1_suffix = format!("snap.{:016X}.generating", snapshot.last_lsn);

    let mut path_1 = config.snapshot_prefix();
    path_1.push(path_1_suffix);

    let path_2_suffix = format!("snap.{:016X}", snapshot.last_lsn);

    let mut path_2 = config.snapshot_prefix();
    path_2.push(path_2_suffix);

    let parent = path_1.parent().unwrap();
    std::fs::create_dir_all(parent)?;
    let mut f = std::fs::OpenOptions::new()
        .write(true)
        .create(true)
        .open(&path_1)?;

    // write the snapshot bytes, followed by a crc64 checksum at the end
    maybe_fail!("snap write");
    f.write_all(&*bytes)?;
    maybe_fail!("snap write len");
    f.write_all(&len_bytes)?;
    maybe_fail!("snap write crc");
    f.write_all(&crc32)?;
    maybe_fail!("snap write post");

    trace!("wrote snapshot to {}", path_1.to_string_lossy());

    maybe_fail!("snap write mv");
    std::fs::rename(&path_1, &path_2)?;
    maybe_fail!("snap write mv post");

    trace!("renamed snapshot to {}", path_2.to_string_lossy());

    // clean up any old snapshots
    let candidates = config.get_snapshot_files()?;
    for path in candidates {
        let path_str = path.file_name().unwrap().to_str().unwrap();
        if !path_2.to_string_lossy().ends_with(&*path_str) {
            debug!("removing old snapshot file {:?}", path);

            maybe_fail!("snap write rm old");

            if let Err(_e) = std::fs::remove_file(&path) {
                // TODO should this just be a try return?
                warn!(
                    "failed to remove old snapshot file, maybe snapshot race? {}",
                    _e
                );
            }
        }
    }
    Ok(())
}
<|endoftext|>


<commit_before>/// Wrapper around the afl-fuzz binary

use libc;
use std::env;
use std::ffi::CString;
use std::path::PathBuf;

extern "C" {
    // `main` function with afl-fuzz.c
    fn afl_fuzz_main(argc: libc::c_int,
                     argv: *const *const libc::c_char) -> libc::c_int;
}

fn call_afl_fuzz_main(args: Vec<CString>) -> Result<(), libc::c_int> {
    assert!(!args.is_empty());

    // `main` functions in C expect an array of pointers to the arguments
    let args_ptrs = args.iter()
                        .map(|arg| arg.as_ptr())
                        .collect::<Vec<_>>();

    let ret = unsafe {
        afl_fuzz_main(args_ptrs.len() as libc::c_int,
                      args_ptrs.as_ptr())
    };

    match ret {
        0 => Ok(()),
        n => Err(n),
    }
}

// TODO: use builder pattern
pub struct AflFuzzConfig {
    pub in_dir: PathBuf,
    pub out_dir: PathBuf,
    pub target: PathBuf,
}

impl AflFuzzConfig {
    fn into_args(self) -> Vec<CString> {
        let mut args = vec![];

        // -i
        args.push("-i");
        args.push(self.in_dir.to_str().unwrap());

        // -o
        args.push("-o");
        args.push(self.out_dir.to_str().unwrap());

        // target
        args.push(self.target.to_str().unwrap());

        // Convert args from `str`s to to `CString`s
        args.into_iter()
            .map(|s| CString::new(s).unwrap())
            .collect()
    }
}

pub fn afl_fuzz_env() -> Result<(), libc::c_int> {
    let args = env::args();

    // don't include anything before "afl-fuzz"
    let args = args.skip_while(|a| a != "afl-fuzz")
                   .map(|arg| CString::new(arg).unwrap())
                   .collect::<Vec<_>>();

    call_afl_fuzz_main(args)
}

pub fn afl_fuzz(config: AflFuzzConfig) -> Result<(), libc::c_int> {
    let mut args = vec![];

    // Fake the first argument
    let arg0 = CString::new("afl-fuzz").unwrap();
    args.push(arg0);

    args.extend(config.into_args());

    call_afl_fuzz_main(args)
}

/// FIXME: figure out a way to make it not run indefinitely
#[test]
fn testsomething() {
    let config = AflFuzzConfig {
        in_dir: ".".into(),
        out_dir: "testout".into(),
        target: "../target/debug/examples/hello".into(),
    };
    afl_fuzz(config);
}
<commit_msg>Attempt to improve executable argument detection.<commit_after>/// Wrapper around the afl-fuzz binary

use libc;
use std::env;
use std::ffi::CString;
use std::path::PathBuf;

extern "C" {
    // `main` function with afl-fuzz.c
    fn afl_fuzz_main(argc: libc::c_int,
                     argv: *const *const libc::c_char) -> libc::c_int;
}

fn call_afl_fuzz_main(args: Vec<CString>) -> Result<(), libc::c_int> {
    assert!(!args.is_empty());

    // `main` functions in C expect an array of pointers to the arguments
    let args_ptrs = args.iter()
                        .map(|arg| arg.as_ptr())
                        .collect::<Vec<_>>();

    let ret = unsafe {
        afl_fuzz_main(args_ptrs.len() as libc::c_int,
                      args_ptrs.as_ptr())
    };

    match ret {
        0 => Ok(()),
        n => Err(n),
    }
}

// TODO: use builder pattern
pub struct AflFuzzConfig {
    pub in_dir: PathBuf,
    pub out_dir: PathBuf,
    pub target: PathBuf,
}

impl AflFuzzConfig {
    fn into_args(self) -> Vec<CString> {
        let mut args = vec![];

        // -i
        args.push("-i");
        args.push(self.in_dir.to_str().unwrap());

        // -o
        args.push("-o");
        args.push(self.out_dir.to_str().unwrap());

        // target
        args.push(self.target.to_str().unwrap());

        // Convert args from `str`s to to `CString`s
        args.into_iter()
            .map(|s| CString::new(s).unwrap())
            .collect()
    }
}

pub fn afl_fuzz_env() -> Result<(), libc::c_int> {
    let args = env::args();

    // don't include anything before 'afl-fuzz' or 'cargo-afl-fuzz'
    let args = args.skip_while(|a| a != "afl-fuzz" && a != "cargo-afl-fuzz")
                   .map(|arg| CString::new(arg).unwrap())
                   .collect::<Vec<_>>();
    assert!(!args.is_empty(), "Error generating afl-fuzz arguments");

    call_afl_fuzz_main(args)
}

pub fn afl_fuzz(config: AflFuzzConfig) -> Result<(), libc::c_int> {
    let mut args = vec![];

    // Fake the first argument
    let arg0 = CString::new("afl-fuzz").unwrap();
    args.push(arg0);

    args.extend(config.into_args());

    call_afl_fuzz_main(args)
}

/// FIXME: figure out a way to make it not run indefinitely
#[test]
fn testsomething() {
    let config = AflFuzzConfig {
        in_dir: ".".into(),
        out_dir: "testout".into(),
        target: "../target/debug/examples/hello".into(),
    };
    afl_fuzz(config);
}
<|endoftext|>


<commit_before>//! Handle network connections for a varlink service

#![allow(dead_code)]

#[cfg(unix)]
use libc::{close, dup2, getpid};
use std::io::{Read, Write};
use std::net::{Shutdown, TcpStream};
use std::process::Child;
use tempfile::TempDir;

#[cfg(unix)]
use std::os::unix::io::IntoRawFd;
#[cfg(unix)]
use std::os::unix::net::UnixStream;
#[cfg(windows)]
use uds_windows::UnixStream;

use crate::{ErrorKind, Result};

pub enum VarlinkStream {
    TCP(TcpStream),
    UNIX(UnixStream),
}

#[cfg(windows)]
pub fn varlink_exec<S: ?Sized + AsRef<str>>(
    _address: &S,
) -> Result<(Child, String, Option<TempDir>)> {
    return Err(ErrorKind::MethodNotImplemented("varlink_exec".into()).into());
}

#[cfg(unix)]
pub fn varlink_exec<S: ?Sized + AsRef<str>>(
    address: &S,
) -> Result<(Child, String, Option<TempDir>)> {
    use std::env;
    use std::os::unix::process::CommandExt;
    use std::process::Command;
    use tempfile::tempdir;

    let executable = String::from("exec ") + address.as_ref();

    use unix_socket::UnixListener;

    let dir = tempdir()?;
    let file_path = dir.path().join("varlink-socket");

    let listener = UnixListener::bind(file_path.clone())?;
    let fd = listener.into_raw_fd();

    let child = Command::new("sh")
        .arg("-c")
        .arg(executable)
        .before_exec({
            let file_path = file_path.clone();
            move || {
                unsafe {
                    if fd != 3 {
                        close(3);
                        dup2(fd, 3);
                    }
                    env::set_var("VARLINK_ADDRESS", format!("unix:{}", file_path.display()));
                    env::set_var("LISTEN_FDS", "1");
                    env::set_var("LISTEN_FDNAMES", "varlink");
                    env::set_var("LISTEN_PID", format!("{}", getpid()));
                }
                Ok(())
            }
        })
        .spawn()?;
    Ok((child, format!("unix:{}", file_path.display()), Some(dir)))
}

#[cfg(windows)]
pub fn varlink_bridge<S: ?Sized + AsRef<str>>(address: &S) -> Result<(Child, VarlinkStream)> {
    use std::os::windows::io::IntoRawSocket;
    use std::os::windows::io::{FromRawHandle, RawHandle};
    use std::process::Command;

    let (stream0, stream1) = UnixStream::pair()?;
    let executable = address.as_ref();
    let fd = stream1.into_raw_socket();
    let childin: ::std::fs::File = unsafe { ::std::fs::File::from_raw_handle(fd as RawHandle) };
    let childout: ::std::fs::File = unsafe { ::std::fs::File::from_raw_handle(fd as RawHandle) };
    let child = Command::new("cmd")
        .arg("/C")
        .arg(executable)
        .stdin(childin)
        .stdout(childout)
        .spawn()?;
    Ok((child, VarlinkStream::UNIX(stream0)))
}

#[cfg(unix)]
pub fn varlink_bridge<S: ?Sized + AsRef<str>>(address: &S) -> Result<(Child, VarlinkStream)> {
    use std::os::unix::io::FromRawFd;
    use std::process::Command;

    let executable = address.as_ref();
    // use unix_socket::UnixStream;
    let (stream0, stream1) = UnixStream::pair()?;
    let fd = stream1.into_raw_fd();
    let childin = unsafe { ::std::fs::File::from_raw_fd(fd) };
    let childout = unsafe { ::std::fs::File::from_raw_fd(fd) };

    let child = Command::new("sh")
        .arg("-c")
        .arg(executable)
        .stdin(childin)
        .stdout(childout)
        .spawn()?;
    Ok((child, VarlinkStream::UNIX(stream0)))
}

#[cfg(any(target_os = "linux", target_os = "android"))]
fn get_abstract_unixstream(addr: &str) -> Result<UnixStream> {
    // FIXME: abstract unix domains sockets still not in std
    // FIXME: https://github.com/rust-lang/rust/issues/14194
    use std::os::unix::io::FromRawFd;
    use unix_socket::UnixStream as AbstractStream;

    unsafe {
        Ok(UnixStream::from_raw_fd(
            AbstractStream::connect(addr)?.into_raw_fd(),
        ))
    }
}

#[cfg(not(any(target_os = "linux", target_os = "android")))]
fn get_abstract_unixstream(_addr: &str) -> Result<UnixStream> {
    Err(ErrorKind::InvalidAddress.into())
}

impl<'a> VarlinkStream {
    pub fn connect<S: ?Sized + AsRef<str>>(address: &S) -> Result<(Self, String)> {
        let address = address.as_ref();
        let new_address: String = address.into();

        if new_address.starts_with("tcp:") {
            Ok((
                VarlinkStream::TCP(TcpStream::connect(&new_address[4..])?),
                new_address,
            ))
        } else if new_address.starts_with("unix:") {
            let mut addr = String::from(new_address[5..].split(';').next().unwrap());
            if addr.starts_with('@') {
                addr = addr.replacen('@', "\0", 1);
                return get_abstract_unixstream(&addr)
                    .and_then(|v| Ok((VarlinkStream::UNIX(v), new_address)));
            }
            Ok((VarlinkStream::UNIX(UnixStream::connect(addr)?), new_address))
        } else {
            Err(ErrorKind::InvalidAddress)?
        }
    }

    pub fn split(&mut self) -> Result<(Box<Read + Send + Sync>, Box<Write + Send + Sync>)> {
        match *self {
            VarlinkStream::TCP(ref mut s) => {
                Ok((Box::new(s.try_clone()?), Box::new(s.try_clone()?)))
            }
            VarlinkStream::UNIX(ref mut s) => {
                Ok((Box::new(s.try_clone()?), Box::new(s.try_clone()?)))
            }
        }
    }

    pub fn shutdown(&mut self) -> Result<()> {
        match *self {
            VarlinkStream::TCP(ref mut s) => s.shutdown(Shutdown::Both)?,
            VarlinkStream::UNIX(ref mut s) => s.shutdown(Shutdown::Both)?,
        }
        Ok(())
    }

    pub fn set_nonblocking(&self, b: bool) -> Result<()> {
        match *self {
            VarlinkStream::TCP(ref l) => l.set_nonblocking(b)?,
            VarlinkStream::UNIX(ref l) => l.set_nonblocking(b)?,
        }
        Ok(())
    }
}

impl Drop for VarlinkStream {
    fn drop(&mut self) {
        let _r = self.shutdown();
    }
}
<commit_msg>varlink/src/client.rs: fix windows varlink_bridge()<commit_after>//! Handle network connections for a varlink service

#![allow(dead_code)]

#[cfg(unix)]
use libc::{close, dup2, getpid};
use std::io::{Read, Write};
use std::net::{Shutdown, TcpStream};
use std::process::Child;
use tempfile::TempDir;

#[cfg(unix)]
use std::os::unix::io::IntoRawFd;
#[cfg(unix)]
use std::os::unix::net::UnixStream;
#[cfg(windows)]
use uds_windows::UnixStream;

use crate::{ErrorKind, Result};

pub enum VarlinkStream {
    TCP(TcpStream),
    UNIX(UnixStream),
}

#[cfg(windows)]
pub fn varlink_exec<S: ?Sized + AsRef<str>>(
    _address: &S,
) -> Result<(Child, String, Option<TempDir>)> {
    return Err(ErrorKind::MethodNotImplemented("varlink_exec".into()).into());
}

#[cfg(unix)]
pub fn varlink_exec<S: ?Sized + AsRef<str>>(
    address: &S,
) -> Result<(Child, String, Option<TempDir>)> {
    use std::env;
    use std::os::unix::process::CommandExt;
    use std::process::Command;
    use tempfile::tempdir;

    let executable = String::from("exec ") + address.as_ref();

    use unix_socket::UnixListener;

    let dir = tempdir()?;
    let file_path = dir.path().join("varlink-socket");

    let listener = UnixListener::bind(file_path.clone())?;
    let fd = listener.into_raw_fd();

    let child = Command::new("sh")
        .arg("-c")
        .arg(executable)
        .before_exec({
            let file_path = file_path.clone();
            move || {
                unsafe {
                    if fd != 3 {
                        close(3);
                        dup2(fd, 3);
                    }
                    env::set_var("VARLINK_ADDRESS", format!("unix:{}", file_path.display()));
                    env::set_var("LISTEN_FDS", "1");
                    env::set_var("LISTEN_FDNAMES", "varlink");
                    env::set_var("LISTEN_PID", format!("{}", getpid()));
                }
                Ok(())
            }
        })
        .spawn()?;
    Ok((child, format!("unix:{}", file_path.display()), Some(dir)))
}

#[cfg(windows)]
pub fn varlink_bridge<S: ?Sized + AsRef<str>>(address: &S) -> Result<(Child, VarlinkStream)> {
    use std::process::{Command, Stdio};
    use std::thread;
    use std::io::copy;

    let (stream0, stream1) = UnixStream::pair()?;
    let executable = address.as_ref();

    let mut child = Command::new("cmd")
        .arg("/C")
        .arg(executable)
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .spawn()?;

    let mut client_writer = child.stdin.take().unwrap();
    let mut client_reader = child.stdout.take().unwrap();
    let mut service_writer =  stream1.try_clone()?;
    let mut service_reader = stream1;

    thread::spawn(move || copy(&mut client_reader, &mut service_writer));
    thread::spawn(move || copy(&mut service_reader, &mut client_writer));

    Ok((child, VarlinkStream::UNIX(stream0)))
}

#[cfg(unix)]
pub fn varlink_bridge<S: ?Sized + AsRef<str>>(address: &S) -> Result<(Child, VarlinkStream)> {
    use std::os::unix::io::FromRawFd;
    use std::process::Command;

    let executable = address.as_ref();
    // use unix_socket::UnixStream;
    let (stream0, stream1) = UnixStream::pair()?;
    let fd = stream1.into_raw_fd();
    let childin = unsafe { ::std::fs::File::from_raw_fd(fd) };
    let childout = unsafe { ::std::fs::File::from_raw_fd(fd) };

    let child = Command::new("sh")
        .arg("-c")
        .arg(executable)
        .stdin(childin)
        .stdout(childout)
        .spawn()?;
    Ok((child, VarlinkStream::UNIX(stream0)))
}

#[cfg(any(target_os = "linux", target_os = "android"))]
fn get_abstract_unixstream(addr: &str) -> Result<UnixStream> {
    // FIXME: abstract unix domains sockets still not in std
    // FIXME: https://github.com/rust-lang/rust/issues/14194
    use std::os::unix::io::FromRawFd;
    use unix_socket::UnixStream as AbstractStream;

    unsafe {
        Ok(UnixStream::from_raw_fd(
            AbstractStream::connect(addr)?.into_raw_fd(),
        ))
    }
}

#[cfg(not(any(target_os = "linux", target_os = "android")))]
fn get_abstract_unixstream(_addr: &str) -> Result<UnixStream> {
    Err(ErrorKind::InvalidAddress.into())
}

impl<'a> VarlinkStream {
    pub fn connect<S: ?Sized + AsRef<str>>(address: &S) -> Result<(Self, String)> {
        let address = address.as_ref();
        let new_address: String = address.into();

        if new_address.starts_with("tcp:") {
            Ok((
                VarlinkStream::TCP(TcpStream::connect(&new_address[4..])?),
                new_address,
            ))
        } else if new_address.starts_with("unix:") {
            let mut addr = String::from(new_address[5..].split(';').next().unwrap());
            if addr.starts_with('@') {
                addr = addr.replacen('@', "\0", 1);
                return get_abstract_unixstream(&addr)
                    .and_then(|v| Ok((VarlinkStream::UNIX(v), new_address)));
            }
            Ok((VarlinkStream::UNIX(UnixStream::connect(addr)?), new_address))
        } else {
            Err(ErrorKind::InvalidAddress)?
        }
    }

    pub fn split(&mut self) -> Result<(Box<Read + Send + Sync>, Box<Write + Send + Sync>)> {
        match *self {
            VarlinkStream::TCP(ref mut s) => {
                Ok((Box::new(s.try_clone()?), Box::new(s.try_clone()?)))
            }
            VarlinkStream::UNIX(ref mut s) => {
                Ok((Box::new(s.try_clone()?), Box::new(s.try_clone()?)))
            }
        }
    }

    pub fn shutdown(&mut self) -> Result<()> {
        match *self {
            VarlinkStream::TCP(ref mut s) => s.shutdown(Shutdown::Both)?,
            VarlinkStream::UNIX(ref mut s) => s.shutdown(Shutdown::Both)?,
        }
        Ok(())
    }

    pub fn set_nonblocking(&self, b: bool) -> Result<()> {
        match *self {
            VarlinkStream::TCP(ref l) => l.set_nonblocking(b)?,
            VarlinkStream::UNIX(ref l) => l.set_nonblocking(b)?,
        }
        Ok(())
    }
}

impl Drop for VarlinkStream {
    fn drop(&mut self) {
        let _r = self.shutdown();
    }
}
<|endoftext|>


<commit_before>// Copyright 2016 Mozilla
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

use std::rc::Rc;

use std::sync::atomic::{
    AtomicUsize,
    Ordering,
};

#[derive(Clone)]
pub struct RcCounter {
    c: Rc<AtomicUsize>,
}

impl RcCounter {
    pub fn new() -> Self {
        RcCounter { c: Rc::new(AtomicUsize::new(0)) }
    }

    pub fn next(&self) -> usize {
        self.c.fetch_add(1, Ordering::SeqCst)
    }
}

#[cfg(test)]
mod tests {
    use super::RcCounter;

    #[test]
    fn test_rc_counter() {
        let c = RcCounter::new();
        assert_eq!(c.next(), 0);
        assert_eq!(c.next(), 1);

        let d = c.clone();
        assert_eq!(d.next(), 2);
        assert_eq!(c.next(), 3);
    }
}<commit_msg>Pre: comment RcCounter.<commit_after>// Copyright 2016 Mozilla
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

use std::rc::Rc;

use std::sync::atomic::{
    AtomicUsize,
    Ordering,
};

#[derive(Clone)]
pub struct RcCounter {
    c: Rc<AtomicUsize>,
}

/// A simple shared counter.
impl RcCounter {
    pub fn new() -> Self {
        RcCounter { c: Rc::new(AtomicUsize::new(0)) }
    }

    /// Return the next value in the sequence.
    ///
    /// ```
    /// use mentat_core::counter::RcCounter;
    ///
    /// let c = RcCounter::with_initial(3);
    /// assert_eq!(c.next(), 3);
    /// assert_eq!(c.next(), 4);
    /// let d = c.clone();
    /// assert_eq!(d.next(), 5);
    /// assert_eq!(c.next(), 6);
    /// ```
    pub fn next(&self) -> usize {
        self.c.fetch_add(1, Ordering::SeqCst)
    }
}

#[cfg(test)]
mod tests {
    use super::RcCounter;

    #[test]
    fn test_rc_counter() {
        let c = RcCounter::new();
        assert_eq!(c.next(), 0);
        assert_eq!(c.next(), 1);

        let d = c.clone();
        assert_eq!(d.next(), 2);
        assert_eq!(c.next(), 3);
    }
}<|endoftext|>


<commit_before>//! This module holds the machinery behind Generic.
//!
//! It contains the Generic trait and some helper methods for using the Generic
//! trait without having to use universal function call syntax.
//!
//! # Examples
//!
//! ```rust
//! #[macro_use] extern crate frunk;
//! #[macro_use] extern crate frunk_core;
//!
//! # fn main() {
//! #[derive(Generic)]
//! struct ApiPerson<'a> {
//!     FirstName: &'a str,
//!     LastName: &'a str,
//!     Age: usize,
//! }
//!
//! #[derive(Generic)]
//! struct DomainPerson<'a> {
//!     first_name: &'a str,
//!     last_name: &'a str,
//!     age: usize,
//! }
//!
//! let a_person = ApiPerson {
//!     FirstName: "Joe",
//!     LastName: "Blow",
//!     Age: 30,
//! };
//! let d_person: DomainPerson = frunk::convert_from(a_person); // done
//! # }

/// A trait that converts from a type to a generic representation
///
/// For the most part, you should be using the derivation that is available through
/// frunk_derive to generate instances of this trait for your types.
///
/// # Laws
///
/// Any implementation of `Generic` must satisfy the following two laws:
///
/// 1. `forall x : Self. x == Generic::from(Generic::into(x))`
/// 2. `forall y : Repr. y == Generic::into(Generic::from(y))`
///
/// That is, `from` and `into` should make up an isomorphism between
/// `Self` and the representation type `Repr`.
///
/// # Examples
///
/// ```rust
/// #[macro_use] extern crate frunk;
/// #[macro_use] extern crate frunk_core;
///
/// # fn main() {
/// #[derive(Generic)]
/// struct ApiPerson<'a> {
///     FirstName: &'a str,
///     LastName: &'a str,
///     Age: usize,
/// }
///
/// #[derive(Generic)]
/// struct DomainPerson<'a> {
///     first_name: &'a str,
///     last_name: &'a str,
///     age: usize,
/// }
///
/// let a_person = ApiPerson {
///     FirstName: "Joe",
///     LastName: "Blow",
///     Age: 30,
/// };
/// let d_person: DomainPerson = frunk::convert_from(a_person); // done
/// # }
/// ```
pub trait Generic {
    /// The generic representation type.
    type Repr;

    /// Convert an object to its representation type `Repr`.
    fn into(self) -> Self::Repr;

    /// Convert an object's representation type `Repr` to the object's type.
    fn from(repr: Self::Repr) -> Self;

    /// Convert an object to a another type provided that they have
    /// a compatible representation type.
    fn convert_from<Src>(src: Src) -> Self
    where
        Src: Generic<Repr = Self::Repr>,
        Self: Sized,
    {
        let repr = <Src as Generic>::into(src);
        <Self as Generic>::from(repr)
    }

    /// Maps the given object of type `Self` by first transforming it to
    /// the representation type, then applying a `mapper` function
    /// on the representation type and finally transforming it back to
    /// an object of type `Self`.
    fn map_repr<Repr, Mapper>(self, mapper: Mapper) -> Self
    where
        Self: Generic<Repr = Repr> + Sized,
        Mapper: FnOnce(Repr) -> Repr
    {
        Self::from(mapper(self.into()))
    }
}

/// Given a generic representation `Repr` of a `Dst`, returns `Dst`.
pub fn from_generic<Dst, Repr>(repr: Repr) -> Dst
where
    Dst: Generic<Repr = Repr>,
{
    <Dst as Generic>::from(repr)
}

/// Given a object of type `Src`, returns its generic representation `Repr`.
pub fn into_generic<Src, Repr>(src: Src) -> Repr
where
    Src: Generic<Repr = Repr>,
{
    <Src as Generic>::into(src)
}

/// Converts one type `Src` into another type `Dst` assuming they have the same
/// generic representation.
pub fn convert_from<Src, Dst, Repr>(src: Src) -> Dst
where
    Src: Generic<Repr = Repr>,
    Dst: Generic<Repr = Repr>,
{
    <Dst as Generic>::convert_from(src)
}

/// Maps an object of the element type `Elt` using a function on the
/// representation type `Repr` of `Elt`.
pub fn map_repr<Elt, Repr, Mapper>(src: Elt, mapper: Mapper) -> Elt
where
    Elt: Generic<Repr = Repr>,
    Mapper: FnOnce(Repr) -> Repr
{
    <Elt as Generic>::map_repr(src, mapper)
}<commit_msg>fix nits in generic.rs<commit_after>//! This module holds the machinery behind `Generic`.
//!
//! It contains the `Generic` trait and some helper methods for using the
//! `Generic` trait without having to use universal function call syntax.
//!
//! # Examples
//!
//! ```rust
//! #[macro_use] extern crate frunk;
//! #[macro_use] extern crate frunk_core;
//!
//! # fn main() {
//! #[derive(Generic)]
//! struct ApiPerson<'a> {
//!     FirstName: &'a str,
//!     LastName: &'a str,
//!     Age: usize,
//! }
//!
//! #[derive(Generic)]
//! struct DomainPerson<'a> {
//!     first_name: &'a str,
//!     last_name: &'a str,
//!     age: usize,
//! }
//!
//! let a_person = ApiPerson {
//!     FirstName: "Joe",
//!     LastName: "Blow",
//!     Age: 30,
//! };
//! let d_person: DomainPerson = frunk::convert_from(a_person); // done
//! # }

/// A trait that converts from a type to a generic representation.
///
/// For the most part, you should be using the derivation that is available
/// through `frunk_derive` to generate instances of this trait for your types.
///
/// # Laws
///
/// Any implementation of `Generic` must satisfy the following two laws:
///
/// 1. `forall x : Self. x == Generic::from(Generic::into(x))`
/// 2. `forall y : Repr. y == Generic::into(Generic::from(y))`
///
/// That is, `from` and `into` should make up an isomorphism between
/// `Self` and the representation type `Repr`.
///
/// # Examples
///
/// ```rust
/// #[macro_use] extern crate frunk;
/// #[macro_use] extern crate frunk_core;
///
/// # fn main() {
/// #[derive(Generic)]
/// struct ApiPerson<'a> {
///     FirstName: &'a str,
///     LastName: &'a str,
///     Age: usize,
/// }
///
/// #[derive(Generic)]
/// struct DomainPerson<'a> {
///     first_name: &'a str,
///     last_name: &'a str,
///     age: usize,
/// }
///
/// let a_person = ApiPerson {
///     FirstName: "Joe",
///     LastName: "Blow",
///     Age: 30,
/// };
/// let d_person: DomainPerson = frunk::convert_from(a_person); // done
/// # }
/// ```
pub trait Generic {
    /// The generic representation type.
    type Repr;

    /// Convert a value to its representation type `Repr`.
    fn into(self) -> Self::Repr;

    /// Convert a value's representation type `Repr` to the values's type.
    fn from(repr: Self::Repr) -> Self;

    /// Convert a value to an another type provided that they have
    /// the same representation type.
    fn convert_from<Src>(src: Src) -> Self
    where
        Src: Generic<Repr = Self::Repr>,
        Self: Sized,
    {
        let repr = <Src as Generic>::into(src);
        <Self as Generic>::from(repr)
    }

    /// Maps the given value of type `Self` by first transforming it to
    /// the representation type `Repr`, then applying a `mapper` function
    /// on `Repr` and finally transforming it back to a value of type `Self`.
    fn map_repr<Repr, Mapper>(self, mapper: Mapper) -> Self
    where
        Self: Generic<Repr = Repr> + Sized,
        Mapper: FnOnce(Repr) -> Repr
    {
        Self::from(mapper(self.into()))
    }
}

/// Given a generic representation `Repr` of a `Dst`, returns `Dst`.
pub fn from_generic<Dst, Repr>(repr: Repr) -> Dst
where
    Dst: Generic<Repr = Repr>,
{
    <Dst as Generic>::from(repr)
}

/// Given a value of type `Src`, returns its generic representation `Repr`.
pub fn into_generic<Src, Repr>(src: Src) -> Repr
where
    Src: Generic<Repr = Repr>,
{
    <Src as Generic>::into(src)
}

/// Converts one type `Src` into another type `Dst` assuming they have the same
/// representation type `Repr`.
pub fn convert_from<Src, Dst, Repr>(src: Src) -> Dst
where
    Src: Generic<Repr = Repr>,
    Dst: Generic<Repr = Repr>,
{
    <Dst as Generic>::convert_from(src)
}

/// Maps a value of a given type `A` using a function on the
/// representation type `Repr` of `A`.
pub fn map_repr<A, Repr, Mapper>(val: A, mapper: Mapper) -> A
where
    A: Generic<Repr = Repr>,
    Mapper: FnOnce(Repr) -> Repr
{
    <A as Generic>::map_repr(val, mapper)
}
<|endoftext|>


<commit_before>#![feature(plugin, custom_derive)]
#![plugin(serde_macros)]

#[macro_use]
extern crate lazy_static;

extern crate hyper;
extern crate regex;
extern crate rust_rosetta;
extern crate serde;
extern crate url;
extern crate walkdir;

use std::collections::{HashMap, HashSet, VecDeque};
use std::fs::File;
use std::io::BufReader;
use std::io::prelude::*;
use std::iter::FromIterator;
use std::path::PathBuf;

use hyper::Client;
use hyper::header::Connection;
use regex::Regex;
use url::Url;
use walkdir::WalkDir;

use rust_rosetta::rosetta_code::find_unimplemented_tasks;

lazy_static!{
    static ref TASK_COMMENT_RE: Regex = Regex::new("// http://rosettacode.org/wiki/(.+)").unwrap();
    static ref LIB_OR_MOD: Regex = Regex::new("^(lib|mod)$").unwrap();

    /// Extracts code from the first Rust section from Rosetta Code wiki markup.
    static ref RUST_WIKI_SECTION_RE: Regex =
        Regex::new(r"==\{\{header\|Rust\}\}==(?s:.*?)<lang rust>((?s:.*?))</lang>").unwrap();

    /// The location of implemented tasks.
    ///
    /// Uses the `RUST_ROSETTA_SRC` environment variable if specified, otherwise uses `src` in the
    /// parent directory.
    static ref RUST_ROSETTA_SRC: PathBuf =
        PathBuf::from(option_env!("RUST_ROSETTA_SRC").unwrap_or("../src"));
}

#[derive(Debug, Serialize, Deserialize)]
pub struct Task {
    /// The title of the task.
    pub title: String,

    /// The code of the task implemented locally. If the tasks is not implemented in the
    /// repository, then it will be `None`.
    pub local_code: Option<String>,

    /// The code of the task implemented on the Rosetta Code wiki. If the task is not yet
    /// implemented online, it will be `None`.
    pub remote_code: Option<String>,

    /// The URL of the task on the RosettaCode wiki.
    pub url: String, // FIXME: Should be a url::Url
}

impl Task {
    /// True if and only if the task is only implemented on the Rosetta Code wiki.
    pub fn is_remote_only(&self) -> bool {
        self.local_code.is_none() && self.remote_code.is_some()
    }

    /// True if and only if the task is is only implemented in the repository.
    pub fn is_local_only(&self) -> bool {
        self.local_code.is_some() && self.remote_code.is_none()
    }

    /// True if and only if the task is neither implemented on the Rosetta code wiki or on the
    /// repository.
    pub fn is_unimplemented(&self) -> bool {
        self.local_code.is_none() && self.local_code.is_none()
    }
}

/// Transforms a task title to a URL task title.
fn normalize(title: &str) -> String {
    title.replace(" ", "_")
}

/// Returns the titles of every task on Rosetta Code.
fn all_task_titles() -> Vec<String> {
    find_unimplemented_tasks::all_tasks().iter().map(|task| task.title.to_owned()).collect()
}

/// Iterator over task information. One `Task` will be returned each iteration.
pub struct TaskIterator {
    task_titles: VecDeque<String>,
    local_tasks: HashMap<String, PathBuf>,
    client: Client,
}

impl TaskIterator {
    pub fn new(titles: &[String]) -> Self {
        // Determine which tasks are implemented locally by walking the src folder and reading the
        // comment at the top of the file.
        let mut local_tasks = HashMap::new();
        for entry in WalkDir::new(RUST_ROSETTA_SRC.as_path()) {
            let entry = entry.unwrap();
            let file_stem = entry.path().file_stem().unwrap().to_str().unwrap();

            // If we find a non-Rust file (or a lib or mod) skip it.
            match entry.path().extension().and_then(|s| s.to_str()) {
                Some("rs") if !LIB_OR_MOD.is_match(file_stem) => (),
                _ => continue,
            }

            let file = File::open(entry.path()).unwrap();
            let first_line = BufReader::new(file).lines().next().unwrap().unwrap();
            let task_name = TASK_COMMENT_RE.captures(&first_line)
                                           .and_then(|c| c.at(1))
                                           .expect(&format!("could not parse task name for {:?}",
                                                            entry.path()));

            local_tasks.insert(task_name.to_owned(), entry.path().to_owned());
        }

        TaskIterator {
            client: Client::new(),
            task_titles: titles.iter().cloned().collect(),
            local_tasks: local_tasks,
        }
    }
}

impl Iterator for TaskIterator {
    type Item = Task;

    fn next(&mut self) -> Option<Self::Item> {
        self.task_titles.pop_front().map(|title| {
            let normalized_title = normalize(&title);

            let mut task_url = Url::parse(&format!("http://rosettacode.org/wiki/{}",
                                                   normalized_title))
                                   .unwrap();
            task_url.set_query_from_pairs(&[("action", "raw")]);

            let local_code = self.local_tasks
                                 .get(&normalized_title)
                                 .and_then(|path| File::open(path).ok())
                                 .map(|mut file| {
                                     let mut local_code = String::new();
                                     file.read_to_string(&mut local_code).unwrap();
                                     local_code
                                 });

            let mut res = self.client
                              .get(&task_url.serialize())
                              .header(Connection::close())
                              .send()
                              .unwrap();

            let mut body = String::new();
            res.read_to_string(&mut body).unwrap();
            let remote_code = RUST_WIKI_SECTION_RE.captures(&body)
                                                  .map(|captures| captures.at(1).unwrap())
                                                  .map(|code| code.to_owned());

            let mut wiki_url = task_url.clone();
            wiki_url.query = None;

            Task {
                title: title.to_owned(),
                local_code: local_code,
                remote_code: remote_code,
                url: wiki_url.serialize(),
            }
        })
    }
}

/// Retrieves data for every task on Rosetta Code.
pub fn fetch_all_tasks() -> TaskIterator {
    fetch_tasks(&all_task_titles())
}

/// Parses both local (implemented in this repository) and remote (implemented on the wiki) tasks,
/// and returns the code of each.
pub fn fetch_tasks(tasks: &[String]) -> TaskIterator {
    let all_task_titles: HashSet<String> = HashSet::from_iter(all_task_titles());
    let requested_task_titles = HashSet::from_iter(tasks.iter().cloned());
    let mut task_titles: Vec<String> = all_task_titles.intersection(&requested_task_titles)
                                                      .cloned()
                                                      .collect();
    task_titles.sort();

    TaskIterator::new(&task_titles)
}
<commit_msg>add local code path to task struct<commit_after>#![feature(plugin, custom_derive)]
#![plugin(serde_macros)]

#[macro_use]
extern crate lazy_static;

extern crate hyper;
extern crate regex;
extern crate rust_rosetta;
extern crate serde;
extern crate url;
extern crate walkdir;

use std::collections::{HashMap, HashSet, VecDeque};
use std::fs::File;
use std::io::BufReader;
use std::io::prelude::*;
use std::iter::FromIterator;
use std::path::PathBuf;

use hyper::Client;
use hyper::header::Connection;
use regex::Regex;
use url::Url;
use walkdir::WalkDir;

use rust_rosetta::rosetta_code::find_unimplemented_tasks;

lazy_static!{
    static ref TASK_COMMENT_RE: Regex = Regex::new("// http://rosettacode.org/wiki/(.+)").unwrap();
    static ref LIB_OR_MOD: Regex = Regex::new("^(lib|mod)$").unwrap();

    /// Extracts code from the first Rust section from Rosetta Code wiki markup.
    static ref RUST_WIKI_SECTION_RE: Regex =
        Regex::new(r"==\{\{header\|Rust\}\}==(?s:.*?)<lang rust>((?s:.*?))</lang>").unwrap();

    /// The location of implemented tasks.
    ///
    /// Uses the `RUST_ROSETTA_SRC` environment variable if specified, otherwise uses `src` in the
    /// parent directory.
    static ref RUST_ROSETTA_SRC: PathBuf =
        PathBuf::from(option_env!("RUST_ROSETTA_SRC").unwrap_or("../src"));
}

#[derive(Debug, Serialize, Deserialize)]
pub struct Task {
    /// The title of the task.
    pub title: String,

    /// The code of the task implemented locally. If the tasks is not implemented in the
    /// repository, then it will be `None`.
    pub local_code: Option<String>,

    /// The code of the task implemented on the Rosetta Code wiki. If the task is not yet
    /// implemented online, it will be `None`.
    pub remote_code: Option<String>,

    /// The URL of the task on the RosettaCode wiki.
    pub url: String, // FIXME: Should be a url::Url

    /// The relative path to where the task is in the repository, if implemented.
    pub path: Option<PathBuf>,
}

impl Task {
    /// True if and only if the task is only implemented on the Rosetta Code wiki.
    pub fn is_remote_only(&self) -> bool {
        self.local_code.is_none() && self.remote_code.is_some()
    }

    /// True if and only if the task is is only implemented in the repository.
    pub fn is_local_only(&self) -> bool {
        self.local_code.is_some() && self.remote_code.is_none()
    }

    /// True if and only if the task is neither implemented on the Rosetta code wiki or on the
    /// repository.
    pub fn is_unimplemented(&self) -> bool {
        self.local_code.is_none() && self.local_code.is_none()
    }
}

/// Transforms a task title to a URL task title.
fn normalize(title: &str) -> String {
    title.replace(" ", "_")
}

/// Returns the titles of every task on Rosetta Code.
fn all_task_titles() -> Vec<String> {
    find_unimplemented_tasks::all_tasks().iter().map(|task| task.title.to_owned()).collect()
}

/// Iterator over task information. One `Task` will be returned each iteration.
pub struct TaskIterator {
    task_titles: VecDeque<String>,
    local_tasks: HashMap<String, PathBuf>,
    client: Client,
}

impl TaskIterator {
    pub fn new(titles: &[String]) -> Self {
        // Determine which tasks are implemented locally by walking the src folder and reading the
        // comment at the top of the file.
        let mut local_tasks = HashMap::new();
        for entry in WalkDir::new(RUST_ROSETTA_SRC.as_path()) {
            let entry = entry.unwrap();
            let path = entry.path();
            let file_stem = path.file_stem().unwrap().to_str().unwrap();

            // If we find a non-Rust file (or a lib or mod) skip it.
            match path.extension().and_then(|s| s.to_str()) {
                Some("rs") if !LIB_OR_MOD.is_match(file_stem) => (),
                _ => continue,
            }

            let file = File::open(path).unwrap();
            let first_line = BufReader::new(file).lines().next().unwrap().unwrap();
            let task_name = TASK_COMMENT_RE.captures(&first_line)
                                           .and_then(|c| c.at(1))
                                           .expect(&format!("could not parse task name for {:?}",
                                                            path));

            local_tasks.insert(task_name.to_owned(), path.to_owned());
        }

        TaskIterator {
            client: Client::new(),
            task_titles: titles.iter().cloned().collect(),
            local_tasks: local_tasks,
        }
    }
}

impl Iterator for TaskIterator {
    type Item = Task;

    fn next(&mut self) -> Option<Self::Item> {
        self.task_titles.pop_front().map(|title| {
            let normalized_title = normalize(&title);

            let mut task_url = Url::parse(&format!("http://rosettacode.org/wiki/{}",
                                                   normalized_title))
                                   .unwrap();
            task_url.set_query_from_pairs(&[("action", "raw")]);

            let path = self.local_tasks.remove(&normalized_title);
            let local_code = path.clone().and_then(|path| {
                File::open(path).ok().map(|mut file| {
                    let mut local_code = String::new();
                    file.read_to_string(&mut local_code).unwrap();
                    local_code
                })
            });

            let mut res = self.client
                              .get(&task_url.serialize())
                              .header(Connection::close())
                              .send()
                              .unwrap();

            let mut body = String::new();
            res.read_to_string(&mut body).unwrap();
            let remote_code = RUST_WIKI_SECTION_RE.captures(&body)
                                                  .map(|captures| captures.at(1).unwrap())
                                                  .map(|code| code.to_owned());

            let mut wiki_url = task_url.clone();
            wiki_url.query = None;

            let relative_path = path.map(|path| {
                path.strip_prefix(RUST_ROSETTA_SRC.parent().unwrap()).unwrap().to_owned()
            });

            Task {
                title: title.to_owned(),
                local_code: local_code,
                remote_code: remote_code,
                url: wiki_url.serialize(),
                path: relative_path,
            }
        })
    }
}

/// Retrieves data for every task on Rosetta Code.
pub fn fetch_all_tasks() -> TaskIterator {
    fetch_tasks(&all_task_titles())
}

/// Parses both local (implemented in this repository) and remote (implemented on the wiki) tasks,
/// and returns the code of each.
pub fn fetch_tasks(tasks: &[String]) -> TaskIterator {
    let all_task_titles: HashSet<String> = HashSet::from_iter(all_task_titles());
    let requested_task_titles = HashSet::from_iter(tasks.iter().cloned());
    let mut task_titles: Vec<String> = all_task_titles.intersection(&requested_task_titles)
                                                      .cloned()
                                                      .collect();
    task_titles.sort();

    TaskIterator::new(&task_titles)
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013, David Renshaw (dwrenshaw@gmail.com)
 *
 * See the LICENSE file in the capnproto-rust root directory.
 */

use std;
use serialize_packed::{PackedOutputStream, PackedInputStream};
use io;

pub fn expect_packs_to(unpacked : &[u8],
                       packed : &[u8]) {

    use std::io::{Reader, Writer};

    // --------
    // write

    let mut bytes : std::vec::Vec<u8> = ::std::iter::repeat(0u8).take(packed.len()).collect();
    {
        let mut writer = io::ArrayOutputStream::new(bytes.as_mut_slice());
        let mut packed_output_stream = PackedOutputStream {inner : &mut writer};
        packed_output_stream.write(unpacked).unwrap();
        packed_output_stream.flush().unwrap();
    }

    assert!(bytes.as_slice().eq(packed));
//            "expected: {}, got: {}", packed, bytes.as_slice());

    // --------
    // read

    let mut reader = io::ArrayInputStream::new(packed);
    let mut packed_input_stream = PackedInputStream {inner : &mut reader};

    let bytes = packed_input_stream.read_exact(unpacked.len()).unwrap();

//    assert!(packed_input_stream.eof());
    assert!(bytes.slice(0, bytes.len()).eq(unpacked));
//            "expected: {}, got: {}", unpacked, bytes.as_slice());

}

static ZEROES : &'static[u8] = &[0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0];

#[test]
pub fn simple_packing() {
    expect_packs_to(&[], &[]);
    expect_packs_to(ZEROES.slice(0, 8), &[0,0]);
    expect_packs_to(&[0,0,12,0,0,34,0,0], &[0x24,12,34]);
    expect_packs_to(&[1,3,2,4,5,7,6,8], &[0xff,1,3,2,4,5,7,6,8,0]);
    expect_packs_to(&[0,0,0,0,0,0,0,0,1,3,2,4,5,7,6,8], &[0,0,0xff,1,3,2,4,5,7,6,8,0]);
    expect_packs_to(&[0,0,12,0,0,34,0,0,1,3,2,4,5,7,6,8], &[0x24,12,34,0xff,1,3,2,4,5,7,6,8,0]);
    expect_packs_to(&[1,3,2,4,5,7,6,8,8,6,7,4,5,2,3,1], &[0xff,1,3,2,4,5,7,6,8,1,8,6,7,4,5,2,3,1]);

    expect_packs_to(
        &[1,2,3,4,5,6,7,8, 1,2,3,4,5,6,7,8, 1,2,3,4,5,6,7,8, 1,2,3,4,5,6,7,8, 0,2,4,0,9,0,5,1],
        &[0xff,1,2,3,4,5,6,7,8, 3, 1,2,3,4,5,6,7,8, 1,2,3,4,5,6,7,8, 1,2,3,4,5,6,7,8,
          0xd6,2,4,9,5,1]);
    expect_packs_to(
        &[1,2,3,4,5,6,7,8, 1,2,3,4,5,6,7,8, 6,2,4,3,9,0,5,1, 1,2,3,4,5,6,7,8, 0,2,4,0,9,0,5,1],
        &[0xff,1,2,3,4,5,6,7,8, 3, 1,2,3,4,5,6,7,8, 6,2,4,3,9,0,5,1, 1,2,3,4,5,6,7,8,
          0xd6,2,4,9,5,1]);

    expect_packs_to(
        &[8,0,100,6,0,1,1,2, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,1,0,2,0,3,1],
        &[0xed,8,100,6,1,1,2, 0,2, 0xd4,1,2,3,1]);

    expect_packs_to(ZEROES.slice(0,16), &[0,1]);
    expect_packs_to(&[0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0], &[0,2]);

}
<commit_msg>{} -> {:?}<commit_after>/*
 * Copyright (c) 2013, David Renshaw (dwrenshaw@gmail.com)
 *
 * See the LICENSE file in the capnproto-rust root directory.
 */

use std;
use serialize_packed::{PackedOutputStream, PackedInputStream};
use io;

pub fn expect_packs_to(unpacked : &[u8],
                       packed : &[u8]) {

    use std::io::{Reader, Writer};

    // --------
    // write

    let mut bytes : std::vec::Vec<u8> = ::std::iter::repeat(0u8).take(packed.len()).collect();
    {
        let mut writer = io::ArrayOutputStream::new(bytes.as_mut_slice());
        let mut packed_output_stream = PackedOutputStream {inner : &mut writer};
        packed_output_stream.write(unpacked).unwrap();
        packed_output_stream.flush().unwrap();
    }

    assert!(bytes.as_slice().eq(packed),
            "expected: {:?}, got: {:?}", packed, bytes);

    // --------
    // read

    let mut reader = io::ArrayInputStream::new(packed);
    let mut packed_input_stream = PackedInputStream {inner : &mut reader};

    let bytes = packed_input_stream.read_exact(unpacked.len()).unwrap();

//    assert!(packed_input_stream.eof());
    assert!(bytes.slice(0, bytes.len()).eq(unpacked),
            "expected: {:?}, got: {:?}", unpacked, bytes);

}

static ZEROES : &'static[u8] = &[0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0];

#[test]
pub fn simple_packing() {
    expect_packs_to(&[], &[]);
    expect_packs_to(ZEROES.slice(0, 8), &[0,0]);
    expect_packs_to(&[0,0,12,0,0,34,0,0], &[0x24,12,34]);
    expect_packs_to(&[1,3,2,4,5,7,6,8], &[0xff,1,3,2,4,5,7,6,8,0]);
    expect_packs_to(&[0,0,0,0,0,0,0,0,1,3,2,4,5,7,6,8], &[0,0,0xff,1,3,2,4,5,7,6,8,0]);
    expect_packs_to(&[0,0,12,0,0,34,0,0,1,3,2,4,5,7,6,8], &[0x24,12,34,0xff,1,3,2,4,5,7,6,8,0]);
    expect_packs_to(&[1,3,2,4,5,7,6,8,8,6,7,4,5,2,3,1], &[0xff,1,3,2,4,5,7,6,8,1,8,6,7,4,5,2,3,1]);

    expect_packs_to(
        &[1,2,3,4,5,6,7,8, 1,2,3,4,5,6,7,8, 1,2,3,4,5,6,7,8, 1,2,3,4,5,6,7,8, 0,2,4,0,9,0,5,1],
        &[0xff,1,2,3,4,5,6,7,8, 3, 1,2,3,4,5,6,7,8, 1,2,3,4,5,6,7,8, 1,2,3,4,5,6,7,8,
          0xd6,2,4,9,5,1]);
    expect_packs_to(
        &[1,2,3,4,5,6,7,8, 1,2,3,4,5,6,7,8, 6,2,4,3,9,0,5,1, 1,2,3,4,5,6,7,8, 0,2,4,0,9,0,5,1],
        &[0xff,1,2,3,4,5,6,7,8, 3, 1,2,3,4,5,6,7,8, 6,2,4,3,9,0,5,1, 1,2,3,4,5,6,7,8,
          0xd6,2,4,9,5,1]);

    expect_packs_to(
        &[8,0,100,6,0,1,1,2, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,1,0,2,0,3,1],
        &[0xed,8,100,6,1,1,2, 0,2, 0xd4,1,2,3,1]);

    expect_packs_to(ZEROES.slice(0,16), &[0,1]);
    expect_packs_to(&[0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0], &[0,2]);

}
<|endoftext|>


<commit_before>//! Code for building the standard library.

use crate::core::compiler::UnitInterner;
use crate::core::compiler::{CompileKind, CompileMode, RustcTargetData, Unit};
use crate::core::profiles::{Profiles, UnitFor};
use crate::core::resolver::features::{CliFeatures, FeaturesFor, ResolvedFeatures};
use crate::core::resolver::HasDevUnits;
use crate::core::{Dependency, PackageId, PackageSet, Resolve, SourceId, Workspace};
use crate::ops::{self, Packages};
use crate::util::errors::CargoResult;
use std::collections::{HashMap, HashSet};
use std::env;
use std::path::PathBuf;

/// Parse the `-Zbuild-std` flag.
pub fn parse_unstable_flag(value: Option<&str>) -> Vec<String> {
    // This is a temporary hack until there is a more principled way to
    // declare dependencies in Cargo.toml.
    let value = value.unwrap_or("std");
    let mut crates: HashSet<&str> = value.split(',').collect();
    if crates.contains("std") {
        crates.insert("core");
        crates.insert("alloc");
        crates.insert("proc_macro");
        crates.insert("panic_unwind");
        crates.insert("compiler_builtins");
    } else if crates.contains("core") {
        crates.insert("compiler_builtins");
    }
    crates.into_iter().map(|s| s.to_string()).collect()
}

/// Resolve the standard library dependencies.
pub fn resolve_std<'cfg>(
    ws: &Workspace<'cfg>,
    target_data: &RustcTargetData<'cfg>,
    requested_targets: &[CompileKind],
    crates: &[String],
) -> CargoResult<(PackageSet<'cfg>, Resolve, ResolvedFeatures)> {
    let src_path = detect_sysroot_src_path(target_data)?;
    let to_patch = [
        "rustc-std-workspace-core",
        "rustc-std-workspace-alloc",
        "rustc-std-workspace-std",
    ];
    let patches = to_patch
        .iter()
        .map(|&name| {
            let source_path = SourceId::for_path(&src_path.join("library").join(name))?;
            let dep = Dependency::parse(name, None, source_path)?;
            Ok(dep)
        })
        .collect::<CargoResult<Vec<_>>>()?;
    let crates_io_url = crate::sources::CRATES_IO_INDEX.parse().unwrap();
    let mut patch = HashMap::new();
    patch.insert(crates_io_url, patches);
    let members = vec![
        String::from("library/std"),
        String::from("library/core"),
        String::from("library/alloc"),
        String::from("library/test"),
    ];
    let ws_config = crate::core::WorkspaceConfig::Root(crate::core::WorkspaceRootConfig::new(
        &src_path,
        &Some(members),
        /*default_members*/ &None,
        /*exclude*/ &None,
        /*custom_metadata*/ &None,
    ));
    let virtual_manifest = crate::core::VirtualManifest::new(
        /*replace*/ Vec::new(),
        patch,
        ws_config,
        /*profiles*/ None,
        crate::core::Features::default(),
        None,
    );

    let config = ws.config();
    // This is a delicate hack. In order for features to resolve correctly,
    // the resolver needs to run a specific "current" member of the workspace.
    // Thus, in order to set the features for `std`, we need to set `libtest`
    // to be the "current" member. `libtest` is the root, and all other
    // standard library crates are dependencies from there. Since none of the
    // other crates need to alter their features, this should be fine, for
    // now. Perhaps in the future features will be decoupled from the resolver
    // and it will be easier to control feature selection.
    let current_manifest = src_path.join("library/test/Cargo.toml");
    // TODO: Consider doing something to enforce --locked? Or to prevent the
    // lock file from being written, such as setting ephemeral.
    let mut std_ws = Workspace::new_virtual(src_path, current_manifest, virtual_manifest, config)?;
    // Don't require optional dependencies in this workspace, aka std's own
    // `[dev-dependencies]`. No need for us to generate a `Resolve` which has
    // those included because we'll never use them anyway.
    std_ws.set_require_optional_deps(false);
    // `test` is not in the default set because it is optional, but it needs
    // to be part of the resolve in case we do need it.
    let mut spec_pkgs = Vec::from(crates);
    spec_pkgs.push("test".to_string());
    let spec = Packages::Packages(spec_pkgs);
    let specs = spec.to_package_id_specs(&std_ws)?;
    let features = match &config.cli_unstable().build_std_features {
        Some(list) => list.clone(),
        None => vec![
            "panic-unwind".to_string(),
            "backtrace".to_string(),
            "default".to_string(),
        ],
    };
    let cli_features = CliFeatures::from_command_line(
        &features, /*all_features*/ false, /*uses_default_features*/ false,
    )?;
    let resolve = ops::resolve_ws_with_opts(
        &std_ws,
        target_data,
        requested_targets,
        &cli_features,
        &specs,
        HasDevUnits::No,
        crate::core::resolver::features::ForceAllTargets::No,
    )?;
    Ok((
        resolve.pkg_set,
        resolve.targeted_resolve,
        resolve.resolved_features,
    ))
}

/// Generate a list of root `Unit`s for the standard library.
///
/// The given slice of crate names is the root set.
pub fn generate_std_roots(
    crates: &[String],
    std_resolve: &Resolve,
    std_features: &ResolvedFeatures,
    kinds: &[CompileKind],
    package_set: &PackageSet<'_>,
    interner: &UnitInterner,
    profiles: &Profiles,
) -> CargoResult<HashMap<CompileKind, Vec<Unit>>> {
    // Generate the root Units for the standard library.
    let std_ids = crates
        .iter()
        .map(|crate_name| std_resolve.query(crate_name))
        .collect::<CargoResult<Vec<PackageId>>>()?;
    // Convert PackageId to Package.
    let std_pkgs = package_set.get_many(std_ids)?;
    // Generate a map of Units for each kind requested.
    let mut ret = HashMap::new();
    for pkg in std_pkgs {
        let lib = pkg
            .targets()
            .iter()
            .find(|t| t.is_lib())
            .expect("std has a lib");
        let unit_for = UnitFor::new_normal();
        // I don't think we need to bother with Check here, the difference
        // in time is minimal, and the difference in caching is
        // significant.
        let mode = CompileMode::Build;
        let features = std_features.activated_features(pkg.package_id(), FeaturesFor::NormalOrDev);

        for kind in kinds {
            let list = ret.entry(*kind).or_insert_with(Vec::new);
            let profile = profiles.get_profile(
                pkg.package_id(),
                /*is_member*/ false,
                /*is_local*/ false,
                unit_for,
                mode,
                *kind,
            );
            list.push(interner.intern(
                pkg,
                lib,
                profile,
                *kind,
                mode,
                features.clone(),
                /*is_std*/ true,
                /*dep_hash*/ 0,
            ));
        }
    }
    Ok(ret)
}

fn detect_sysroot_src_path(target_data: &RustcTargetData<'_>) -> CargoResult<PathBuf> {
    if let Some(s) = env::var_os("__CARGO_TESTS_ONLY_SRC_ROOT") {
        return Ok(s.into());
    }

    // NOTE: This is temporary until we figure out how to acquire the source.
    let src_path = target_data
        .info(CompileKind::Host)
        .sysroot
        .join("lib")
        .join("rustlib")
        .join("src")
        .join("rust");
    let lock = src_path.join("Cargo.lock");
    if !lock.exists() {
        anyhow::bail!(
            "{:?} does not exist, unable to build with the standard \
             library, try:\n        rustup component add rust-src",
            lock
        );
    }
    Ok(src_path)
}
<commit_msg>Auto merge of #9625 - hi-rustin:rustin-msg, r=ehuss<commit_after>//! Code for building the standard library.

use crate::core::compiler::UnitInterner;
use crate::core::compiler::{CompileKind, CompileMode, RustcTargetData, Unit};
use crate::core::profiles::{Profiles, UnitFor};
use crate::core::resolver::features::{CliFeatures, FeaturesFor, ResolvedFeatures};
use crate::core::resolver::HasDevUnits;
use crate::core::{Dependency, PackageId, PackageSet, Resolve, SourceId, Workspace};
use crate::ops::{self, Packages};
use crate::util::errors::CargoResult;
use std::collections::{HashMap, HashSet};
use std::env;
use std::path::PathBuf;

/// Parse the `-Zbuild-std` flag.
pub fn parse_unstable_flag(value: Option<&str>) -> Vec<String> {
    // This is a temporary hack until there is a more principled way to
    // declare dependencies in Cargo.toml.
    let value = value.unwrap_or("std");
    let mut crates: HashSet<&str> = value.split(',').collect();
    if crates.contains("std") {
        crates.insert("core");
        crates.insert("alloc");
        crates.insert("proc_macro");
        crates.insert("panic_unwind");
        crates.insert("compiler_builtins");
    } else if crates.contains("core") {
        crates.insert("compiler_builtins");
    }
    crates.into_iter().map(|s| s.to_string()).collect()
}

/// Resolve the standard library dependencies.
pub fn resolve_std<'cfg>(
    ws: &Workspace<'cfg>,
    target_data: &RustcTargetData<'cfg>,
    requested_targets: &[CompileKind],
    crates: &[String],
) -> CargoResult<(PackageSet<'cfg>, Resolve, ResolvedFeatures)> {
    let src_path = detect_sysroot_src_path(target_data)?;
    let to_patch = [
        "rustc-std-workspace-core",
        "rustc-std-workspace-alloc",
        "rustc-std-workspace-std",
    ];
    let patches = to_patch
        .iter()
        .map(|&name| {
            let source_path = SourceId::for_path(&src_path.join("library").join(name))?;
            let dep = Dependency::parse(name, None, source_path)?;
            Ok(dep)
        })
        .collect::<CargoResult<Vec<_>>>()?;
    let crates_io_url = crate::sources::CRATES_IO_INDEX.parse().unwrap();
    let mut patch = HashMap::new();
    patch.insert(crates_io_url, patches);
    let members = vec![
        String::from("library/std"),
        String::from("library/core"),
        String::from("library/alloc"),
        String::from("library/test"),
    ];
    let ws_config = crate::core::WorkspaceConfig::Root(crate::core::WorkspaceRootConfig::new(
        &src_path,
        &Some(members),
        /*default_members*/ &None,
        /*exclude*/ &None,
        /*custom_metadata*/ &None,
    ));
    let virtual_manifest = crate::core::VirtualManifest::new(
        /*replace*/ Vec::new(),
        patch,
        ws_config,
        /*profiles*/ None,
        crate::core::Features::default(),
        None,
    );

    let config = ws.config();
    // This is a delicate hack. In order for features to resolve correctly,
    // the resolver needs to run a specific "current" member of the workspace.
    // Thus, in order to set the features for `std`, we need to set `libtest`
    // to be the "current" member. `libtest` is the root, and all other
    // standard library crates are dependencies from there. Since none of the
    // other crates need to alter their features, this should be fine, for
    // now. Perhaps in the future features will be decoupled from the resolver
    // and it will be easier to control feature selection.
    let current_manifest = src_path.join("library/test/Cargo.toml");
    // TODO: Consider doing something to enforce --locked? Or to prevent the
    // lock file from being written, such as setting ephemeral.
    let mut std_ws = Workspace::new_virtual(src_path, current_manifest, virtual_manifest, config)?;
    // Don't require optional dependencies in this workspace, aka std's own
    // `[dev-dependencies]`. No need for us to generate a `Resolve` which has
    // those included because we'll never use them anyway.
    std_ws.set_require_optional_deps(false);
    // `test` is not in the default set because it is optional, but it needs
    // to be part of the resolve in case we do need it.
    let mut spec_pkgs = Vec::from(crates);
    spec_pkgs.push("test".to_string());
    let spec = Packages::Packages(spec_pkgs);
    let specs = spec.to_package_id_specs(&std_ws)?;
    let features = match &config.cli_unstable().build_std_features {
        Some(list) => list.clone(),
        None => vec![
            "panic-unwind".to_string(),
            "backtrace".to_string(),
            "default".to_string(),
        ],
    };
    let cli_features = CliFeatures::from_command_line(
        &features, /*all_features*/ false, /*uses_default_features*/ false,
    )?;
    let resolve = ops::resolve_ws_with_opts(
        &std_ws,
        target_data,
        requested_targets,
        &cli_features,
        &specs,
        HasDevUnits::No,
        crate::core::resolver::features::ForceAllTargets::No,
    )?;
    Ok((
        resolve.pkg_set,
        resolve.targeted_resolve,
        resolve.resolved_features,
    ))
}

/// Generate a list of root `Unit`s for the standard library.
///
/// The given slice of crate names is the root set.
pub fn generate_std_roots(
    crates: &[String],
    std_resolve: &Resolve,
    std_features: &ResolvedFeatures,
    kinds: &[CompileKind],
    package_set: &PackageSet<'_>,
    interner: &UnitInterner,
    profiles: &Profiles,
) -> CargoResult<HashMap<CompileKind, Vec<Unit>>> {
    // Generate the root Units for the standard library.
    let std_ids = crates
        .iter()
        .map(|crate_name| std_resolve.query(crate_name))
        .collect::<CargoResult<Vec<PackageId>>>()?;
    // Convert PackageId to Package.
    let std_pkgs = package_set.get_many(std_ids)?;
    // Generate a map of Units for each kind requested.
    let mut ret = HashMap::new();
    for pkg in std_pkgs {
        let lib = pkg
            .targets()
            .iter()
            .find(|t| t.is_lib())
            .expect("std has a lib");
        let unit_for = UnitFor::new_normal();
        // I don't think we need to bother with Check here, the difference
        // in time is minimal, and the difference in caching is
        // significant.
        let mode = CompileMode::Build;
        let features = std_features.activated_features(pkg.package_id(), FeaturesFor::NormalOrDev);

        for kind in kinds {
            let list = ret.entry(*kind).or_insert_with(Vec::new);
            let profile = profiles.get_profile(
                pkg.package_id(),
                /*is_member*/ false,
                /*is_local*/ false,
                unit_for,
                mode,
                *kind,
            );
            list.push(interner.intern(
                pkg,
                lib,
                profile,
                *kind,
                mode,
                features.clone(),
                /*is_std*/ true,
                /*dep_hash*/ 0,
            ));
        }
    }
    Ok(ret)
}

fn detect_sysroot_src_path(target_data: &RustcTargetData<'_>) -> CargoResult<PathBuf> {
    if let Some(s) = env::var_os("__CARGO_TESTS_ONLY_SRC_ROOT") {
        return Ok(s.into());
    }

    // NOTE: This is temporary until we figure out how to acquire the source.
    let src_path = target_data
        .info(CompileKind::Host)
        .sysroot
        .join("lib")
        .join("rustlib")
        .join("src")
        .join("rust");
    let lock = src_path.join("Cargo.lock");
    if !lock.exists() {
        let msg = format!(
            "{:?} does not exist, unable to build with the standard \
             library, try:\n        rustup component add rust-src",
            lock
        );
        match env::var("RUSTUP_TOOLCHAIN") {
            Ok(rustup_toolchain) => {
                anyhow::bail!("{} --toolchain {}", msg, rustup_toolchain);
            }
            Err(_) => {
                anyhow::bail!(msg);
            }
        }
    }
    Ok(src_path)
}
<|endoftext|>


<commit_before>use std::path::Path;

/// Shortcut to path->filename conversion.
///
/// Returns the unchanged `path` if there is a character encoding error or something.
pub fn filename<'a> (path: &'a str) -> &'a str {
  match Path::new (path) .file_name() {
    Some (filename) => match filename.to_str() {
      Some (fstr) => if fstr.ends_with (".rs") {&fstr[0 .. fstr.len() - 3]} else {fstr},
      None => path},
    None => path}}

// A trick to use the `try_s` macro from both the outside and inside of the crate.
//mod gstuff {pub fn filename<'a> (path: &'a str) -> &'a str {::filename (path)}}

/// Returns on error, converting the `Err` value to `String` and prepending the current location.
///
/// cf. http://www.reddit.com/r/rust/comments/29wwzw/error_handling_and_result_types/cipcm9a
#[macro_export] macro_rules! try_s {
  ($e: expr) => {match $e {Ok (ok) => ok, Err (err) => {return Err (format! ("{}:{}] {}", ::gstuff::filename (file!()), line!(), err));}}}}

// Like `try_s`, but takes a reference.
#[macro_export] macro_rules! try_sp {
  ($e: expr) => {match $e {&Ok (ref ok) => ok,
    &Err (ref err) => {return Err (format! ("{}:{}] {:?}", ::gstuff::filename (file!()), line!(), err));}}}}

/// Returns a `Err(String)`, prepending the current location (file name and line number) to the string.
///
/// Examples: `ERR! ("too bad")`; `ERR! ("{}", foo)`;
#[macro_export] macro_rules! ERR {
  ($format: expr, $($args: tt)+) => {Err (format! (concat! ("{}:{}] ", $format), ::gstuff::filename (file!()), line!(), $($args)+))};
  ($format: expr) => {Err (format! (concat! ("{}:{}] ", $format), ::gstuff::filename (file!()), line!()))}}

/// A helper to build a string on the stack.
///
/// Given an array it makes a writeable cursor available to the passed code block.
///
/// Returns a &str slice pointing at memory between the array start and the cursor.
///
/// Example:
///
///     use std::io::Write;
///     let mut foobar: [u8; 128] = unsafe {std::mem::uninitialized()};
///     let foobar = gstring! (foobar, {
///         write! (foobar, "foo") .unwrap();
///         write! (foobar, "bar") .unwrap();
///     });
///
/// Alternatives: https://crates.io/crates/string-wrapper; https://crates.io/crates/stack.
#[macro_export] macro_rules! gstring {($array: ident, $code: block) => {{
  let end = {
    let mut $array = ::std::io::Cursor::new (&mut $array[..]);
    let $array = &mut $array;
    $code;
    $array.position() as usize};
  unsafe {::std::str::from_utf8_unchecked (&$array[0..end])}}}}
<commit_msg>Checked out `string-wrapper` and found no alternatives in it.<commit_after>use std::path::Path;

/// Shortcut to path->filename conversion.
///
/// Returns the unchanged `path` if there is a character encoding error or something.
pub fn filename<'a> (path: &'a str) -> &'a str {
  match Path::new (path) .file_name() {
    Some (filename) => match filename.to_str() {
      Some (fstr) => if fstr.ends_with (".rs") {&fstr[0 .. fstr.len() - 3]} else {fstr},
      None => path},
    None => path}}

// A trick to use the `try_s` macro from both the outside and inside of the crate.
//mod gstuff {pub fn filename<'a> (path: &'a str) -> &'a str {::filename (path)}}

/// Returns on error, converting the `Err` value to `String` and prepending the current location.
///
/// cf. http://www.reddit.com/r/rust/comments/29wwzw/error_handling_and_result_types/cipcm9a
#[macro_export] macro_rules! try_s {
  ($e: expr) => {match $e {Ok (ok) => ok, Err (err) => {return Err (format! ("{}:{}] {}", ::gstuff::filename (file!()), line!(), err));}}}}

// Like `try_s`, but takes a reference.
#[macro_export] macro_rules! try_sp {
  ($e: expr) => {match $e {&Ok (ref ok) => ok,
    &Err (ref err) => {return Err (format! ("{}:{}] {:?}", ::gstuff::filename (file!()), line!(), err));}}}}

/// Returns a `Err(String)`, prepending the current location (file name and line number) to the string.
///
/// Examples: `ERR! ("too bad")`; `ERR! ("{}", foo)`;
#[macro_export] macro_rules! ERR {
  ($format: expr, $($args: tt)+) => {Err (format! (concat! ("{}:{}] ", $format), ::gstuff::filename (file!()), line!(), $($args)+))};
  ($format: expr) => {Err (format! (concat! ("{}:{}] ", $format), ::gstuff::filename (file!()), line!()))}}

/// A helper to build a string on the stack.
///
/// Given an array it makes a writeable cursor available to the passed code block.
///
/// Returns a &str slice pointing at memory between the array start and the cursor.
///
/// Example:
///
///     use std::io::Write;
///     let mut foobar: [u8; 128] = unsafe {std::mem::uninitialized()};
///     let foobar = gstring! (foobar, {
///         write! (foobar, "foo") .unwrap();
///         write! (foobar, "bar") .unwrap();
///     });
///
/// Alternatives: https://crates.io/crates/stack.
#[macro_export] macro_rules! gstring {($array: ident, $code: block) => {{
  let end = {
    let mut $array = ::std::io::Cursor::new (&mut $array[..]);
    let $array = &mut $array;
    $code;
    $array.position() as usize};
  unsafe {::std::str::from_utf8_unchecked (&$array[0..end])}}}}
<|endoftext|>