<commit_before>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern crate toml;
extern crate rustc_serialize;

use std::collections::{BTreeMap, HashMap};
use std::env;
use std::fs::File;
use std::io::{self, Read, Write};
use std::path::{PathBuf, Path};
use std::process::{Command, Stdio};

static HOSTS: &'static [&'static str] = &[
    "aarch64-unknown-linux-gnu",
    "arm-unknown-linux-gnueabi",
    "arm-unknown-linux-gnueabihf",
    "armv7-unknown-linux-gnueabihf",
    "i686-apple-darwin",
    "i686-pc-windows-gnu",
    "i686-pc-windows-msvc",
    "i686-unknown-linux-gnu",
    "mips-unknown-linux-gnu",
    "mips64-unknown-linux-gnuabi64",
    "mips64el-unknown-linux-gnuabi64",
    "mipsel-unknown-linux-gnu",
    "powerpc-unknown-linux-gnu",
    "powerpc64-unknown-linux-gnu",
    "powerpc64le-unknown-linux-gnu",
    "s390x-unknown-linux-gnu",
    "x86_64-apple-darwin",
    "x86_64-pc-windows-gnu",
    "x86_64-pc-windows-msvc",
    "x86_64-unknown-freebsd",
    "x86_64-unknown-linux-gnu",
    "x86_64-unknown-netbsd",
];

static TARGETS: &'static [&'static str] = &[
    "aarch64-apple-ios",
    "aarch64-linux-android",
    "aarch64-unknown-linux-gnu",
    "arm-linux-androideabi",
    "arm-unknown-linux-gnueabi",
    "arm-unknown-linux-gnueabihf",
    "arm-unknown-linux-musleabi",
    "arm-unknown-linux-musleabihf",
    "armv7-apple-ios",
    "armv7-linux-androideabi",
    "armv7-unknown-linux-gnueabihf",
    "armv7-unknown-linux-musleabihf",
    "armv7s-apple-ios",
    "asmjs-unknown-emscripten",
    "i386-apple-ios",
    "i586-pc-windows-msvc",
    "i586-unknown-linux-gnu",
    "i686-apple-darwin",
    "i686-linux-android",
    "i686-pc-windows-gnu",
    "i686-pc-windows-msvc",
    "i686-unknown-freebsd",
    "i686-unknown-linux-gnu",
    "i686-unknown-linux-musl",
    "mips-unknown-linux-gnu",
    "mips-unknown-linux-musl",
    "mips64-unknown-linux-gnuabi64",
    "mips64el-unknown-linux-gnuabi64",
    "mipsel-unknown-linux-gnu",
    "mipsel-unknown-linux-musl",
    "powerpc-unknown-linux-gnu",
    "powerpc64-unknown-linux-gnu",
    "powerpc64le-unknown-linux-gnu",
    "s390x-unknown-linux-gnu",
    "sparc64-unknown-linux-gnu",
    "wasm32-unknown-emscripten",
    "x86_64-apple-darwin",
    "x86_64-apple-ios",
    "x86_64-pc-windows-gnu",
    "x86_64-pc-windows-msvc",
    "x86_64-rumprun-netbsd",
    "x86_64-unknown-freebsd",
    "x86_64-unknown-linux-gnu",
    "x86_64-unknown-linux-musl",
    "x86_64-unknown-netbsd",
];

static MINGW: &'static [&'static str] = &[
    "i686-pc-windows-gnu",
    "x86_64-pc-windows-gnu",
];

struct Manifest {
    manifest_version: String,
    date: String,
    pkg: HashMap<String, Package>,
}

#[derive(RustcEncodable)]
struct Package {
    version: String,
    target: HashMap<String, Target>,
}

#[derive(RustcEncodable)]
struct Target {
    available: bool,
    url: Option<String>,
    hash: Option<String>,
    components: Option<Vec<Component>>,
    extensions: Option<Vec<Component>>,
}

#[derive(RustcEncodable)]
struct Component {
    pkg: String,
    target: String,
}

macro_rules! t {
    ($e:expr) => (match $e {
        Ok(e) => e,
        Err(e) => panic!("{} failed with {}", stringify!($e), e),
    })
}

struct Builder {
    channel: String,
    input: PathBuf,
    output: PathBuf,
    gpg_passphrase: String,
    digests: HashMap<String, String>,
    s3_address: String,
    date: String,
    rust_version: String,
    cargo_version: String,
}

fn main() {
    let mut args = env::args().skip(1);
    let input = PathBuf::from(args.next().unwrap());
    let output = PathBuf::from(args.next().unwrap());
    let date = args.next().unwrap();
    let channel = args.next().unwrap();
    let s3_address = args.next().unwrap();
    let mut passphrase = String::new();
    t!(io::stdin().read_to_string(&mut passphrase));

    Builder {
        channel: channel,
        input: input,
        output: output,
        gpg_passphrase: passphrase,
        digests: HashMap::new(),
        s3_address: s3_address,
        date: date,
        rust_version: String::new(),
        cargo_version: String::new(),
    }.build();
}

impl Builder {
    fn build(&mut self) {
        self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
        self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");

        self.digest_and_sign();
        let Manifest { manifest_version, date, pkg } = self.build_manifest();

        // Unfortunately we can't use derive(RustcEncodable) here because the
        // version field is called `manifest-version`, not `manifest_version`.
        // In lieu of that just create the table directly here with a `BTreeMap`
        // and wrap it up in a `Value::Table`.
        let mut manifest = BTreeMap::new();
        manifest.insert("manifest-version".to_string(),
                        toml::encode(&manifest_version));
        manifest.insert("date".to_string(), toml::encode(&date));
        manifest.insert("pkg".to_string(), toml::encode(&pkg));
        let manifest = toml::Value::Table(manifest).to_string();

        let filename = format!("channel-rust-{}.toml", self.channel);
        self.write_manifest(&manifest, &filename);

        if self.channel != "beta" && self.channel != "nightly" {
            self.write_manifest(&manifest, "channel-rust-stable.toml");
        }
    }

    fn digest_and_sign(&mut self) {
        for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
            let filename = file.file_name().unwrap().to_str().unwrap();
            let digest = self.hash(&file);
            self.sign(&file);
            assert!(self.digests.insert(filename.to_string(), digest).is_none());
        }
    }

    fn build_manifest(&mut self) -> Manifest {
        let mut manifest = Manifest {
            manifest_version: "2".to_string(),
            date: self.date.to_string(),
            pkg: HashMap::new(),
        };

        self.package("rustc", &mut manifest.pkg, HOSTS);
        self.package("cargo", &mut manifest.pkg, HOSTS);
        self.package("rust-mingw", &mut manifest.pkg, MINGW);
        self.package("rust-std", &mut manifest.pkg, TARGETS);
        self.package("rust-docs", &mut manifest.pkg, TARGETS);
        self.package("rust-src", &mut manifest.pkg, &["*"]);

        let mut pkg = Package {
            version: self.cached_version("rust").to_string(),
            target: HashMap::new(),
        };
        for host in HOSTS {
            let filename = self.filename("rust", host);
            let digest = match self.digests.remove(&filename) {
                Some(digest) => digest,
                None => {
                    pkg.target.insert(host.to_string(), Target {
                        available: false,
                        url: None,
                        hash: None,
                        components: None,
                        extensions: None,
                    });
                    continue
                }
            };
            let mut components = Vec::new();
            let mut extensions = Vec::new();

            // rustc/rust-std/cargo are all required, and so is rust-mingw if it's
            // available for the target.
            components.extend(vec![
                Component { pkg: "rustc".to_string(), target: host.to_string() },
                Component { pkg: "rust-std".to_string(), target: host.to_string() },
                Component { pkg: "cargo".to_string(), target: host.to_string() },
            ]);
            if host.contains("pc-windows-gnu") {
                components.push(Component {
                    pkg: "rust-mingw".to_string(),
                    target: host.to_string(),
                });
            }

            // Docs, other standard libraries, and the source package are all
            // optional.
            extensions.push(Component {
                pkg: "rust-docs".to_string(),
                target: host.to_string(),
            });
            for target in TARGETS {
                if target != host {
                    extensions.push(Component {
                        pkg: "rust-std".to_string(),
                        target: target.to_string(),
                    });
                }
            }
            extensions.push(Component {
                pkg: "rust-src".to_string(),
                target: "*".to_string(),
            });

            pkg.target.insert(host.to_string(), Target {
                available: true,
                url: Some(self.url("rust", host)),
                hash: Some(to_hex(digest.as_ref())),
                components: Some(components),
                extensions: Some(extensions),
            });
        }
        manifest.pkg.insert("rust".to_string(), pkg);

        return manifest
    }

    fn package(&mut self,
               pkgname: &str,
               dst: &mut HashMap<String, Package>,
               targets: &[&str]) {
        let targets = targets.iter().map(|name| {
            let filename = self.filename(pkgname, name);
            let digest = match self.digests.remove(&filename) {
                Some(digest) => digest,
                None => {
                    return (name.to_string(), Target {
                        available: false,
                        url: None,
                        hash: None,
                        components: None,
                        extensions: None,
                    })
                }
            };

            (name.to_string(), Target {
                available: true,
                url: Some(self.url(pkgname, name)),
                hash: Some(digest),
                components: None,
                extensions: None,
            })
        }).collect();

        dst.insert(pkgname.to_string(), Package {
            version: self.cached_version(pkgname).to_string(),
            target: targets,
        });
    }

    fn url(&self, component: &str, target: &str) -> String {
        format!("{}/{}/{}",
                self.s3_address,
                self.date,
                self.filename(component, target))
    }

    fn filename(&self, component: &str, target: &str) -> String {
        if component == "rust-src" {
            format!("rust-src-{}.tar.gz", self.channel)
        } else if component == "cargo" {
            format!("cargo-nightly-{}.tar.gz", target)
        } else {
            format!("{}-{}-{}.tar.gz", component, self.channel, target)
        }
    }

    fn cached_version(&self, component: &str) -> &str {
        if component == "cargo" {
            &self.cargo_version
        } else {
            &self.rust_version
        }
    }

    fn version(&self, component: &str, target: &str) -> String {
        let mut cmd = Command::new("tar");
        let filename = self.filename(component, target);
        cmd.arg("xf")
           .arg(self.input.join(&filename))
           .arg(format!("{}/version", filename.replace(".tar.gz", "")))
           .arg("-O");
        let version = t!(cmd.output());
        if !version.status.success() {
            panic!("failed to learn version:\n\n{:?}\n\n{}\n\n{}",
                   cmd,
                   String::from_utf8_lossy(&version.stdout),
                   String::from_utf8_lossy(&version.stderr));
        }
        String::from_utf8_lossy(&version.stdout).trim().to_string()
    }

    fn hash(&self, path: &Path) -> String {
        let sha = t!(Command::new("shasum")
                        .arg("-a").arg("256")
                        .arg(path.file_name().unwrap())
                        .current_dir(path.parent().unwrap())
                        .output());
        assert!(sha.status.success());

        let filename = path.file_name().unwrap().to_str().unwrap();
        let sha256 = self.output.join(format!("{}.sha256", filename));
        t!(t!(File::create(&sha256)).write_all(&sha.stdout));

        let stdout = String::from_utf8_lossy(&sha.stdout);
        stdout.split_whitespace().next().unwrap().to_string()
    }

    fn sign(&self, path: &Path) {
        let filename = path.file_name().unwrap().to_str().unwrap();
        let asc = self.output.join(format!("{}.asc", filename));
        println!("signing: {:?}", path);
        let mut cmd = Command::new("gpg");
        cmd.arg("--no-tty")
            .arg("--yes")
            .arg("--passphrase-fd").arg("0")
            .arg("--armor")
            .arg("--output").arg(&asc)
            .arg("--detach-sign").arg(path)
            .stdin(Stdio::piped());
        let mut child = t!(cmd.spawn());
        t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
        assert!(t!(child.wait()).success());
    }

    fn write_manifest(&self, manifest: &str, name: &str) {
        let dst = self.output.join(name);
        t!(t!(File::create(&dst)).write_all(manifest.as_bytes()));
        self.hash(&dst);
        self.sign(&dst);
    }
}

fn to_hex(digest: &[u8]) -> String {
    let mut ret = String::new();
    for byte in digest {
        ret.push(hex((byte & 0xf0) >> 4));
        ret.push(hex(byte & 0xf));
    }
    return ret;

    fn hex(b: u8) -> char {
        match b {
            0...9 => (b'0' + b) as char,
            _ => (b'a' + b - 10) as char,
        }
    }
}
<commit_msg>Rollup merge of #39676 - alexcrichton:fix-again, r=brson<commit_after>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern crate toml;
extern crate rustc_serialize;

use std::collections::{BTreeMap, HashMap};
use std::env;
use std::fs::File;
use std::io::{self, Read, Write};
use std::path::{PathBuf, Path};
use std::process::{Command, Stdio};

static HOSTS: &'static [&'static str] = &[
    "aarch64-unknown-linux-gnu",
    "arm-unknown-linux-gnueabi",
    "arm-unknown-linux-gnueabihf",
    "armv7-unknown-linux-gnueabihf",
    "i686-apple-darwin",
    "i686-pc-windows-gnu",
    "i686-pc-windows-msvc",
    "i686-unknown-linux-gnu",
    "mips-unknown-linux-gnu",
    "mips64-unknown-linux-gnuabi64",
    "mips64el-unknown-linux-gnuabi64",
    "mipsel-unknown-linux-gnu",
    "powerpc-unknown-linux-gnu",
    "powerpc64-unknown-linux-gnu",
    "powerpc64le-unknown-linux-gnu",
    "s390x-unknown-linux-gnu",
    "x86_64-apple-darwin",
    "x86_64-pc-windows-gnu",
    "x86_64-pc-windows-msvc",
    "x86_64-unknown-freebsd",
    "x86_64-unknown-linux-gnu",
    "x86_64-unknown-netbsd",
];

static TARGETS: &'static [&'static str] = &[
    "aarch64-apple-ios",
    "aarch64-linux-android",
    "aarch64-unknown-linux-gnu",
    "arm-linux-androideabi",
    "arm-unknown-linux-gnueabi",
    "arm-unknown-linux-gnueabihf",
    "arm-unknown-linux-musleabi",
    "arm-unknown-linux-musleabihf",
    "armv7-apple-ios",
    "armv7-linux-androideabi",
    "armv7-unknown-linux-gnueabihf",
    "armv7-unknown-linux-musleabihf",
    "armv7s-apple-ios",
    "asmjs-unknown-emscripten",
    "i386-apple-ios",
    "i586-pc-windows-msvc",
    "i586-unknown-linux-gnu",
    "i686-apple-darwin",
    "i686-linux-android",
    "i686-pc-windows-gnu",
    "i686-pc-windows-msvc",
    "i686-unknown-freebsd",
    "i686-unknown-linux-gnu",
    "i686-unknown-linux-musl",
    "mips-unknown-linux-gnu",
    "mips-unknown-linux-musl",
    "mips64-unknown-linux-gnuabi64",
    "mips64el-unknown-linux-gnuabi64",
    "mipsel-unknown-linux-gnu",
    "mipsel-unknown-linux-musl",
    "powerpc-unknown-linux-gnu",
    "powerpc64-unknown-linux-gnu",
    "powerpc64le-unknown-linux-gnu",
    "s390x-unknown-linux-gnu",
    "sparc64-unknown-linux-gnu",
    "wasm32-unknown-emscripten",
    "x86_64-apple-darwin",
    "x86_64-apple-ios",
    "x86_64-pc-windows-gnu",
    "x86_64-pc-windows-msvc",
    "x86_64-rumprun-netbsd",
    "x86_64-unknown-freebsd",
    "x86_64-unknown-linux-gnu",
    "x86_64-unknown-linux-musl",
    "x86_64-unknown-netbsd",
];

static MINGW: &'static [&'static str] = &[
    "i686-pc-windows-gnu",
    "x86_64-pc-windows-gnu",
];

struct Manifest {
    manifest_version: String,
    date: String,
    pkg: HashMap<String, Package>,
}

#[derive(RustcEncodable)]
struct Package {
    version: String,
    target: HashMap<String, Target>,
}

#[derive(RustcEncodable)]
struct Target {
    available: bool,
    url: Option<String>,
    hash: Option<String>,
    components: Option<Vec<Component>>,
    extensions: Option<Vec<Component>>,
}

#[derive(RustcEncodable)]
struct Component {
    pkg: String,
    target: String,
}

macro_rules! t {
    ($e:expr) => (match $e {
        Ok(e) => e,
        Err(e) => panic!("{} failed with {}", stringify!($e), e),
    })
}

struct Builder {
    channel: String,
    input: PathBuf,
    output: PathBuf,
    gpg_passphrase: String,
    digests: HashMap<String, String>,
    s3_address: String,
    date: String,
    rust_version: String,
    cargo_version: String,
}

fn main() {
    let mut args = env::args().skip(1);
    let input = PathBuf::from(args.next().unwrap());
    let output = PathBuf::from(args.next().unwrap());
    let date = args.next().unwrap();
    let channel = args.next().unwrap();
    let s3_address = args.next().unwrap();
    let mut passphrase = String::new();
    t!(io::stdin().read_to_string(&mut passphrase));

    Builder {
        channel: channel,
        input: input,
        output: output,
        gpg_passphrase: passphrase,
        digests: HashMap::new(),
        s3_address: s3_address,
        date: date,
        rust_version: String::new(),
        cargo_version: String::new(),
    }.build();
}

impl Builder {
    fn build(&mut self) {
        self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
        self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");

        self.digest_and_sign();
        let Manifest { manifest_version, date, pkg } = self.build_manifest();

        // Unfortunately we can't use derive(RustcEncodable) here because the
        // version field is called `manifest-version`, not `manifest_version`.
        // In lieu of that just create the table directly here with a `BTreeMap`
        // and wrap it up in a `Value::Table`.
        let mut manifest = BTreeMap::new();
        manifest.insert("manifest-version".to_string(),
                        toml::Value::String(manifest_version));
        manifest.insert("date".to_string(), toml::Value::String(date));
        manifest.insert("pkg".to_string(), toml::encode(&pkg));
        let manifest = toml::Value::Table(manifest).to_string();

        let filename = format!("channel-rust-{}.toml", self.channel);
        self.write_manifest(&manifest, &filename);

        if self.channel != "beta" && self.channel != "nightly" {
            self.write_manifest(&manifest, "channel-rust-stable.toml");
        }
    }

    fn digest_and_sign(&mut self) {
        for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
            let filename = file.file_name().unwrap().to_str().unwrap();
            let digest = self.hash(&file);
            self.sign(&file);
            assert!(self.digests.insert(filename.to_string(), digest).is_none());
        }
    }

    fn build_manifest(&mut self) -> Manifest {
        let mut manifest = Manifest {
            manifest_version: "2".to_string(),
            date: self.date.to_string(),
            pkg: HashMap::new(),
        };

        self.package("rustc", &mut manifest.pkg, HOSTS);
        self.package("cargo", &mut manifest.pkg, HOSTS);
        self.package("rust-mingw", &mut manifest.pkg, MINGW);
        self.package("rust-std", &mut manifest.pkg, TARGETS);
        self.package("rust-docs", &mut manifest.pkg, TARGETS);
        self.package("rust-src", &mut manifest.pkg, &["*"]);

        let mut pkg = Package {
            version: self.cached_version("rust").to_string(),
            target: HashMap::new(),
        };
        for host in HOSTS {
            let filename = self.filename("rust", host);
            let digest = match self.digests.remove(&filename) {
                Some(digest) => digest,
                None => {
                    pkg.target.insert(host.to_string(), Target {
                        available: false,
                        url: None,
                        hash: None,
                        components: None,
                        extensions: None,
                    });
                    continue
                }
            };
            let mut components = Vec::new();
            let mut extensions = Vec::new();

            // rustc/rust-std/cargo are all required, and so is rust-mingw if it's
            // available for the target.
            components.extend(vec![
                Component { pkg: "rustc".to_string(), target: host.to_string() },
                Component { pkg: "rust-std".to_string(), target: host.to_string() },
                Component { pkg: "cargo".to_string(), target: host.to_string() },
            ]);
            if host.contains("pc-windows-gnu") {
                components.push(Component {
                    pkg: "rust-mingw".to_string(),
                    target: host.to_string(),
                });
            }

            // Docs, other standard libraries, and the source package are all
            // optional.
            extensions.push(Component {
                pkg: "rust-docs".to_string(),
                target: host.to_string(),
            });
            for target in TARGETS {
                if target != host {
                    extensions.push(Component {
                        pkg: "rust-std".to_string(),
                        target: target.to_string(),
                    });
                }
            }
            extensions.push(Component {
                pkg: "rust-src".to_string(),
                target: "*".to_string(),
            });

            pkg.target.insert(host.to_string(), Target {
                available: true,
                url: Some(self.url("rust", host)),
                hash: Some(to_hex(digest.as_ref())),
                components: Some(components),
                extensions: Some(extensions),
            });
        }
        manifest.pkg.insert("rust".to_string(), pkg);

        return manifest
    }

    fn package(&mut self,
               pkgname: &str,
               dst: &mut HashMap<String, Package>,
               targets: &[&str]) {
        let targets = targets.iter().map(|name| {
            let filename = self.filename(pkgname, name);
            let digest = match self.digests.remove(&filename) {
                Some(digest) => digest,
                None => {
                    return (name.to_string(), Target {
                        available: false,
                        url: None,
                        hash: None,
                        components: None,
                        extensions: None,
                    })
                }
            };

            (name.to_string(), Target {
                available: true,
                url: Some(self.url(pkgname, name)),
                hash: Some(digest),
                components: None,
                extensions: None,
            })
        }).collect();

        dst.insert(pkgname.to_string(), Package {
            version: self.cached_version(pkgname).to_string(),
            target: targets,
        });
    }

    fn url(&self, component: &str, target: &str) -> String {
        format!("{}/{}/{}",
                self.s3_address,
                self.date,
                self.filename(component, target))
    }

    fn filename(&self, component: &str, target: &str) -> String {
        if component == "rust-src" {
            format!("rust-src-{}.tar.gz", self.channel)
        } else if component == "cargo" {
            format!("cargo-nightly-{}.tar.gz", target)
        } else {
            format!("{}-{}-{}.tar.gz", component, self.channel, target)
        }
    }

    fn cached_version(&self, component: &str) -> &str {
        if component == "cargo" {
            &self.cargo_version
        } else {
            &self.rust_version
        }
    }

    fn version(&self, component: &str, target: &str) -> String {
        let mut cmd = Command::new("tar");
        let filename = self.filename(component, target);
        cmd.arg("xf")
           .arg(self.input.join(&filename))
           .arg(format!("{}/version", filename.replace(".tar.gz", "")))
           .arg("-O");
        let version = t!(cmd.output());
        if !version.status.success() {
            panic!("failed to learn version:\n\n{:?}\n\n{}\n\n{}",
                   cmd,
                   String::from_utf8_lossy(&version.stdout),
                   String::from_utf8_lossy(&version.stderr));
        }
        String::from_utf8_lossy(&version.stdout).trim().to_string()
    }

    fn hash(&self, path: &Path) -> String {
        let sha = t!(Command::new("shasum")
                        .arg("-a").arg("256")
                        .arg(path.file_name().unwrap())
                        .current_dir(path.parent().unwrap())
                        .output());
        assert!(sha.status.success());

        let filename = path.file_name().unwrap().to_str().unwrap();
        let sha256 = self.output.join(format!("{}.sha256", filename));
        t!(t!(File::create(&sha256)).write_all(&sha.stdout));

        let stdout = String::from_utf8_lossy(&sha.stdout);
        stdout.split_whitespace().next().unwrap().to_string()
    }

    fn sign(&self, path: &Path) {
        let filename = path.file_name().unwrap().to_str().unwrap();
        let asc = self.output.join(format!("{}.asc", filename));
        println!("signing: {:?}", path);
        let mut cmd = Command::new("gpg");
        cmd.arg("--no-tty")
            .arg("--yes")
            .arg("--passphrase-fd").arg("0")
            .arg("--armor")
            .arg("--output").arg(&asc)
            .arg("--detach-sign").arg(path)
            .stdin(Stdio::piped());
        let mut child = t!(cmd.spawn());
        t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
        assert!(t!(child.wait()).success());
    }

    fn write_manifest(&self, manifest: &str, name: &str) {
        let dst = self.output.join(name);
        t!(t!(File::create(&dst)).write_all(manifest.as_bytes()));
        self.hash(&dst);
        self.sign(&dst);
    }
}

fn to_hex(digest: &[u8]) -> String {
    let mut ret = String::new();
    for byte in digest {
        ret.push(hex((byte & 0xf0) >> 4));
        ret.push(hex(byte & 0xf));
    }
    return ret;

    fn hex(b: u8) -> char {
        match b {
            0...9 => (b'0' + b) as char,
            _ => (b'a' + b - 10) as char,
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern crate toml;
extern crate rustc_serialize;

use std::collections::{BTreeMap, HashMap};
use std::env;
use std::fs::File;
use std::io::{self, Read, Write};
use std::path::{PathBuf, Path};
use std::process::{Command, Stdio};

static HOSTS: &'static [&'static str] = &[
    "aarch64-unknown-linux-gnu",
    "arm-unknown-linux-gnueabi",
    "arm-unknown-linux-gnueabihf",
    "armv7-unknown-linux-gnueabihf",
    "i686-apple-darwin",
    "i686-pc-windows-gnu",
    "i686-pc-windows-msvc",
    "i686-unknown-linux-gnu",
    "mips-unknown-linux-gnu",
    "mips64-unknown-linux-gnuabi64",
    "mips64el-unknown-linux-gnuabi64",
    "mipsel-unknown-linux-gnu",
    "powerpc-unknown-linux-gnu",
    "powerpc64-unknown-linux-gnu",
    "powerpc64le-unknown-linux-gnu",
    "s390x-unknown-linux-gnu",
    "x86_64-apple-darwin",
    "x86_64-pc-windows-gnu",
    "x86_64-pc-windows-msvc",
    "x86_64-unknown-freebsd",
    "x86_64-unknown-linux-gnu",
    "x86_64-unknown-netbsd",
];

static TARGETS: &'static [&'static str] = &[
    "aarch64-apple-ios",
    "aarch64-linux-android",
    "aarch64-unknown-linux-gnu",
    "arm-linux-androideabi",
    "arm-unknown-linux-gnueabi",
    "arm-unknown-linux-gnueabihf",
    "arm-unknown-linux-musleabi",
    "arm-unknown-linux-musleabihf",
    "armv7-apple-ios",
    "armv7-linux-androideabi",
    "armv7-unknown-linux-gnueabihf",
    "armv7-unknown-linux-musleabihf",
    "armv7s-apple-ios",
    "asmjs-unknown-emscripten",
    "i386-apple-ios",
    "i586-pc-windows-msvc",
    "i586-unknown-linux-gnu",
    "i686-apple-darwin",
    "i686-linux-android",
    "i686-pc-windows-gnu",
    "i686-pc-windows-msvc",
    "i686-unknown-freebsd",
    "i686-unknown-linux-gnu",
    "i686-unknown-linux-musl",
    "mips-unknown-linux-gnu",
    "mips-unknown-linux-musl",
    "mips64-unknown-linux-gnuabi64",
    "mips64el-unknown-linux-gnuabi64",
    "mipsel-unknown-linux-gnu",
    "mipsel-unknown-linux-musl",
    "powerpc-unknown-linux-gnu",
    "powerpc64-unknown-linux-gnu",
    "powerpc64le-unknown-linux-gnu",
    "s390x-unknown-linux-gnu",
    "sparc64-unknown-linux-gnu",
    "wasm32-unknown-emscripten",
    "x86_64-apple-darwin",
    "x86_64-apple-ios",
    "x86_64-pc-windows-gnu",
    "x86_64-pc-windows-msvc",
    "x86_64-rumprun-netbsd",
    "x86_64-unknown-freebsd",
    "x86_64-unknown-linux-gnu",
    "x86_64-unknown-linux-musl",
    "x86_64-unknown-netbsd",
];

static MINGW: &'static [&'static str] = &[
    "i686-pc-windows-gnu",
    "x86_64-pc-windows-gnu",
];

struct Manifest {
    manifest_version: String,
    date: String,
    pkg: HashMap<String, Package>,
}

#[derive(RustcEncodable)]
struct Package {
    version: String,
    target: HashMap<String, Target>,
}

#[derive(RustcEncodable)]
struct Target {
    available: bool,
    url: Option<String>,
    hash: Option<String>,
    components: Option<Vec<Component>>,
    extensions: Option<Vec<Component>>,
}

#[derive(RustcEncodable)]
struct Component {
    pkg: String,
    target: String,
}

macro_rules! t {
    ($e:expr) => (match $e {
        Ok(e) => e,
        Err(e) => panic!("{} failed with {}", stringify!($e), e),
    })
}

struct Builder {
    rust_release: String,
    cargo_release: String,
    input: PathBuf,
    output: PathBuf,
    gpg_passphrase: String,
    digests: HashMap<String, String>,
    s3_address: String,
    date: String,
    rust_version: String,
    cargo_version: String,
}

fn main() {
    let mut args = env::args().skip(1);
    let input = PathBuf::from(args.next().unwrap());
    let output = PathBuf::from(args.next().unwrap());
    let date = args.next().unwrap();
    let rust_release = args.next().unwrap();
    let cargo_release = args.next().unwrap();
    let s3_address = args.next().unwrap();
    let mut passphrase = String::new();
    t!(io::stdin().read_to_string(&mut passphrase));

    Builder {
        rust_release: rust_release,
        cargo_release: cargo_release,
        input: input,
        output: output,
        gpg_passphrase: passphrase,
        digests: HashMap::new(),
        s3_address: s3_address,
        date: date,
        rust_version: String::new(),
        cargo_version: String::new(),
    }.build();
}

impl Builder {
    fn build(&mut self) {
        self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
        self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");

        self.digest_and_sign();
        let Manifest { manifest_version, date, pkg } = self.build_manifest();

        // Unfortunately we can't use derive(RustcEncodable) here because the
        // version field is called `manifest-version`, not `manifest_version`.
        // In lieu of that just create the table directly here with a `BTreeMap`
        // and wrap it up in a `Value::Table`.
        let mut manifest = BTreeMap::new();
        manifest.insert("manifest-version".to_string(),
                        toml::Value::String(manifest_version));
        manifest.insert("date".to_string(), toml::Value::String(date));
        manifest.insert("pkg".to_string(), toml::encode(&pkg));
        let manifest = toml::Value::Table(manifest).to_string();

        let filename = format!("channel-rust-{}.toml", self.rust_release);
        self.write_manifest(&manifest, &filename);

        if self.rust_release != "beta" && self.rust_release != "nightly" {
            self.write_manifest(&manifest, "channel-rust-stable.toml");
        }
    }

    fn digest_and_sign(&mut self) {
        for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
            let filename = file.file_name().unwrap().to_str().unwrap();
            let digest = self.hash(&file);
            self.sign(&file);
            assert!(self.digests.insert(filename.to_string(), digest).is_none());
        }
    }

    fn build_manifest(&mut self) -> Manifest {
        let mut manifest = Manifest {
            manifest_version: "2".to_string(),
            date: self.date.to_string(),
            pkg: HashMap::new(),
        };

        self.package("rustc", &mut manifest.pkg, HOSTS);
        self.package("cargo", &mut manifest.pkg, HOSTS);
        self.package("rust-mingw", &mut manifest.pkg, MINGW);
        self.package("rust-std", &mut manifest.pkg, TARGETS);
        self.package("rust-docs", &mut manifest.pkg, TARGETS);
        self.package("rust-src", &mut manifest.pkg, &["*"]);

        if self.rust_release == "nightly" {
            self.package("rust-analysis", &mut manifest.pkg, TARGETS);
        }

        let mut pkg = Package {
            version: self.cached_version("rust").to_string(),
            target: HashMap::new(),
        };
        for host in HOSTS {
            let filename = self.filename("rust", host);
            let digest = match self.digests.remove(&filename) {
                Some(digest) => digest,
                None => {
                    pkg.target.insert(host.to_string(), Target {
                        available: false,
                        url: None,
                        hash: None,
                        components: None,
                        extensions: None,
                    });
                    continue
                }
            };
            let mut components = Vec::new();
            let mut extensions = Vec::new();

            // rustc/rust-std/cargo are all required, and so is rust-mingw if it's
            // available for the target.
            components.extend(vec![
                Component { pkg: "rustc".to_string(), target: host.to_string() },
                Component { pkg: "rust-std".to_string(), target: host.to_string() },
                Component { pkg: "cargo".to_string(), target: host.to_string() },
            ]);
            if host.contains("pc-windows-gnu") {
                components.push(Component {
                    pkg: "rust-mingw".to_string(),
                    target: host.to_string(),
                });
            }

            // Docs, other standard libraries, and the source package are all
            // optional.
            extensions.push(Component {
                pkg: "rust-docs".to_string(),
                target: host.to_string(),
            });
            for target in TARGETS {
                if target != host {
                    extensions.push(Component {
                        pkg: "rust-std".to_string(),
                        target: target.to_string(),
                    });
                }
                if self.rust_release == "nightly" {
                    extensions.push(Component {
                        pkg: "rust-analysis".to_string(),
                        target: target.to_string(),
                    });
                }
            }
            extensions.push(Component {
                pkg: "rust-src".to_string(),
                target: "*".to_string(),
            });

            pkg.target.insert(host.to_string(), Target {
                available: true,
                url: Some(self.url("rust", host)),
                hash: Some(digest),
                components: Some(components),
                extensions: Some(extensions),
            });
        }
        manifest.pkg.insert("rust".to_string(), pkg);

        return manifest
    }

    fn package(&mut self,
               pkgname: &str,
               dst: &mut HashMap<String, Package>,
               targets: &[&str]) {
        let targets = targets.iter().map(|name| {
            let filename = self.filename(pkgname, name);
            let digest = match self.digests.remove(&filename) {
                Some(digest) => digest,
                None => {
                    return (name.to_string(), Target {
                        available: false,
                        url: None,
                        hash: None,
                        components: None,
                        extensions: None,
                    })
                }
            };

            (name.to_string(), Target {
                available: true,
                url: Some(self.url(pkgname, name)),
                hash: Some(digest),
                components: None,
                extensions: None,
            })
        }).collect();

        dst.insert(pkgname.to_string(), Package {
            version: self.cached_version(pkgname).to_string(),
            target: targets,
        });
    }

    fn url(&self, component: &str, target: &str) -> String {
        format!("{}/{}/{}",
                self.s3_address,
                self.date,
                self.filename(component, target))
    }

    fn filename(&self, component: &str, target: &str) -> String {
        if component == "rust-src" {
            format!("rust-src-{}.tar.gz", self.rust_release)
        } else if component == "cargo" {
            format!("cargo-{}-{}.tar.gz", self.cargo_release, target)
        } else {
            format!("{}-{}-{}.tar.gz", component, self.rust_release, target)
        }
    }

    fn cached_version(&self, component: &str) -> &str {
        if component == "cargo" {
            &self.cargo_version
        } else {
            &self.rust_version
        }
    }

    fn version(&self, component: &str, target: &str) -> String {
        let mut cmd = Command::new("tar");
        let filename = self.filename(component, target);
        cmd.arg("xf")
           .arg(self.input.join(&filename))
           .arg(format!("{}/version", filename.replace(".tar.gz", "")))
           .arg("-O");
        let version = t!(cmd.output());
        if !version.status.success() {
            panic!("failed to learn version:\n\n{:?}\n\n{}\n\n{}",
                   cmd,
                   String::from_utf8_lossy(&version.stdout),
                   String::from_utf8_lossy(&version.stderr));
        }
        String::from_utf8_lossy(&version.stdout).trim().to_string()
    }

    fn hash(&self, path: &Path) -> String {
        let sha = t!(Command::new("shasum")
                        .arg("-a").arg("256")
                        .arg(path.file_name().unwrap())
                        .current_dir(path.parent().unwrap())
                        .output());
        assert!(sha.status.success());

        let filename = path.file_name().unwrap().to_str().unwrap();
        let sha256 = self.output.join(format!("{}.sha256", filename));
        t!(t!(File::create(&sha256)).write_all(&sha.stdout));

        let stdout = String::from_utf8_lossy(&sha.stdout);
        stdout.split_whitespace().next().unwrap().to_string()
    }

    fn sign(&self, path: &Path) {
        let filename = path.file_name().unwrap().to_str().unwrap();
        let asc = self.output.join(format!("{}.asc", filename));
        println!("signing: {:?}", path);
        let mut cmd = Command::new("gpg");
        cmd.arg("--no-tty")
            .arg("--yes")
            .arg("--passphrase-fd").arg("0")
            .arg("--armor")
            .arg("--output").arg(&asc)
            .arg("--detach-sign").arg(path)
            .stdin(Stdio::piped());
        let mut child = t!(cmd.spawn());
        t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
        assert!(t!(child.wait()).success());
    }

    fn write_manifest(&self, manifest: &str, name: &str) {
        let dst = self.output.join(name);
        t!(t!(File::create(&dst)).write_all(manifest.as_bytes()));
        self.hash(&dst);
        self.sign(&dst);
    }
}
<commit_msg>Restore creating the channel-rust-$channel-date.txt files<commit_after>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern crate toml;
extern crate rustc_serialize;

use std::collections::{BTreeMap, HashMap};
use std::env;
use std::fs::File;
use std::io::{self, Read, Write};
use std::path::{PathBuf, Path};
use std::process::{Command, Stdio};

static HOSTS: &'static [&'static str] = &[
    "aarch64-unknown-linux-gnu",
    "arm-unknown-linux-gnueabi",
    "arm-unknown-linux-gnueabihf",
    "armv7-unknown-linux-gnueabihf",
    "i686-apple-darwin",
    "i686-pc-windows-gnu",
    "i686-pc-windows-msvc",
    "i686-unknown-linux-gnu",
    "mips-unknown-linux-gnu",
    "mips64-unknown-linux-gnuabi64",
    "mips64el-unknown-linux-gnuabi64",
    "mipsel-unknown-linux-gnu",
    "powerpc-unknown-linux-gnu",
    "powerpc64-unknown-linux-gnu",
    "powerpc64le-unknown-linux-gnu",
    "s390x-unknown-linux-gnu",
    "x86_64-apple-darwin",
    "x86_64-pc-windows-gnu",
    "x86_64-pc-windows-msvc",
    "x86_64-unknown-freebsd",
    "x86_64-unknown-linux-gnu",
    "x86_64-unknown-netbsd",
];

static TARGETS: &'static [&'static str] = &[
    "aarch64-apple-ios",
    "aarch64-linux-android",
    "aarch64-unknown-linux-gnu",
    "arm-linux-androideabi",
    "arm-unknown-linux-gnueabi",
    "arm-unknown-linux-gnueabihf",
    "arm-unknown-linux-musleabi",
    "arm-unknown-linux-musleabihf",
    "armv7-apple-ios",
    "armv7-linux-androideabi",
    "armv7-unknown-linux-gnueabihf",
    "armv7-unknown-linux-musleabihf",
    "armv7s-apple-ios",
    "asmjs-unknown-emscripten",
    "i386-apple-ios",
    "i586-pc-windows-msvc",
    "i586-unknown-linux-gnu",
    "i686-apple-darwin",
    "i686-linux-android",
    "i686-pc-windows-gnu",
    "i686-pc-windows-msvc",
    "i686-unknown-freebsd",
    "i686-unknown-linux-gnu",
    "i686-unknown-linux-musl",
    "mips-unknown-linux-gnu",
    "mips-unknown-linux-musl",
    "mips64-unknown-linux-gnuabi64",
    "mips64el-unknown-linux-gnuabi64",
    "mipsel-unknown-linux-gnu",
    "mipsel-unknown-linux-musl",
    "powerpc-unknown-linux-gnu",
    "powerpc64-unknown-linux-gnu",
    "powerpc64le-unknown-linux-gnu",
    "s390x-unknown-linux-gnu",
    "sparc64-unknown-linux-gnu",
    "wasm32-unknown-emscripten",
    "x86_64-apple-darwin",
    "x86_64-apple-ios",
    "x86_64-pc-windows-gnu",
    "x86_64-pc-windows-msvc",
    "x86_64-rumprun-netbsd",
    "x86_64-unknown-freebsd",
    "x86_64-unknown-linux-gnu",
    "x86_64-unknown-linux-musl",
    "x86_64-unknown-netbsd",
];

static MINGW: &'static [&'static str] = &[
    "i686-pc-windows-gnu",
    "x86_64-pc-windows-gnu",
];

struct Manifest {
    manifest_version: String,
    date: String,
    pkg: HashMap<String, Package>,
}

#[derive(RustcEncodable)]
struct Package {
    version: String,
    target: HashMap<String, Target>,
}

#[derive(RustcEncodable)]
struct Target {
    available: bool,
    url: Option<String>,
    hash: Option<String>,
    components: Option<Vec<Component>>,
    extensions: Option<Vec<Component>>,
}

#[derive(RustcEncodable)]
struct Component {
    pkg: String,
    target: String,
}

macro_rules! t {
    ($e:expr) => (match $e {
        Ok(e) => e,
        Err(e) => panic!("{} failed with {}", stringify!($e), e),
    })
}

struct Builder {
    rust_release: String,
    cargo_release: String,
    input: PathBuf,
    output: PathBuf,
    gpg_passphrase: String,
    digests: HashMap<String, String>,
    s3_address: String,
    date: String,
    rust_version: String,
    cargo_version: String,
}

fn main() {
    let mut args = env::args().skip(1);
    let input = PathBuf::from(args.next().unwrap());
    let output = PathBuf::from(args.next().unwrap());
    let date = args.next().unwrap();
    let rust_release = args.next().unwrap();
    let cargo_release = args.next().unwrap();
    let s3_address = args.next().unwrap();
    let mut passphrase = String::new();
    t!(io::stdin().read_to_string(&mut passphrase));

    Builder {
        rust_release: rust_release,
        cargo_release: cargo_release,
        input: input,
        output: output,
        gpg_passphrase: passphrase,
        digests: HashMap::new(),
        s3_address: s3_address,
        date: date,
        rust_version: String::new(),
        cargo_version: String::new(),
    }.build();
}

impl Builder {
    fn build(&mut self) {
        self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
        self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");

        self.digest_and_sign();
        let Manifest { manifest_version, date, pkg } = self.build_manifest();

        // Unfortunately we can't use derive(RustcEncodable) here because the
        // version field is called `manifest-version`, not `manifest_version`.
        // In lieu of that just create the table directly here with a `BTreeMap`
        // and wrap it up in a `Value::Table`.
        let mut manifest = BTreeMap::new();
        manifest.insert("manifest-version".to_string(),
                        toml::Value::String(manifest_version));
        manifest.insert("date".to_string(), toml::Value::String(date.clone()));
        manifest.insert("pkg".to_string(), toml::encode(&pkg));
        let manifest = toml::Value::Table(manifest).to_string();

        let filename = format!("channel-rust-{}.toml", self.rust_release);
        self.write_manifest(&manifest, &filename);

        let filename = format!("channel-rust-{}-date.txt", self.rust_release);
        self.write_date_stamp(&date, &filename);

        if self.rust_release != "beta" && self.rust_release != "nightly" {
            self.write_manifest(&manifest, "channel-rust-stable.toml");
            self.write_date_stamp(&date, "channel-rust-stable-date.txt");
        }
    }

    fn digest_and_sign(&mut self) {
        for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
            let filename = file.file_name().unwrap().to_str().unwrap();
            let digest = self.hash(&file);
            self.sign(&file);
            assert!(self.digests.insert(filename.to_string(), digest).is_none());
        }
    }

    fn build_manifest(&mut self) -> Manifest {
        let mut manifest = Manifest {
            manifest_version: "2".to_string(),
            date: self.date.to_string(),
            pkg: HashMap::new(),
        };

        self.package("rustc", &mut manifest.pkg, HOSTS);
        self.package("cargo", &mut manifest.pkg, HOSTS);
        self.package("rust-mingw", &mut manifest.pkg, MINGW);
        self.package("rust-std", &mut manifest.pkg, TARGETS);
        self.package("rust-docs", &mut manifest.pkg, TARGETS);
        self.package("rust-src", &mut manifest.pkg, &["*"]);

        if self.rust_release == "nightly" {
            self.package("rust-analysis", &mut manifest.pkg, TARGETS);
        }

        let mut pkg = Package {
            version: self.cached_version("rust").to_string(),
            target: HashMap::new(),
        };
        for host in HOSTS {
            let filename = self.filename("rust", host);
            let digest = match self.digests.remove(&filename) {
                Some(digest) => digest,
                None => {
                    pkg.target.insert(host.to_string(), Target {
                        available: false,
                        url: None,
                        hash: None,
                        components: None,
                        extensions: None,
                    });
                    continue
                }
            };
            let mut components = Vec::new();
            let mut extensions = Vec::new();

            // rustc/rust-std/cargo are all required, and so is rust-mingw if it's
            // available for the target.
            components.extend(vec![
                Component { pkg: "rustc".to_string(), target: host.to_string() },
                Component { pkg: "rust-std".to_string(), target: host.to_string() },
                Component { pkg: "cargo".to_string(), target: host.to_string() },
            ]);
            if host.contains("pc-windows-gnu") {
                components.push(Component {
                    pkg: "rust-mingw".to_string(),
                    target: host.to_string(),
                });
            }

            // Docs, other standard libraries, and the source package are all
            // optional.
            extensions.push(Component {
                pkg: "rust-docs".to_string(),
                target: host.to_string(),
            });
            for target in TARGETS {
                if target != host {
                    extensions.push(Component {
                        pkg: "rust-std".to_string(),
                        target: target.to_string(),
                    });
                }
                if self.rust_release == "nightly" {
                    extensions.push(Component {
                        pkg: "rust-analysis".to_string(),
                        target: target.to_string(),
                    });
                }
            }
            extensions.push(Component {
                pkg: "rust-src".to_string(),
                target: "*".to_string(),
            });

            pkg.target.insert(host.to_string(), Target {
                available: true,
                url: Some(self.url("rust", host)),
                hash: Some(digest),
                components: Some(components),
                extensions: Some(extensions),
            });
        }
        manifest.pkg.insert("rust".to_string(), pkg);

        return manifest
    }

    fn package(&mut self,
               pkgname: &str,
               dst: &mut HashMap<String, Package>,
               targets: &[&str]) {
        let targets = targets.iter().map(|name| {
            let filename = self.filename(pkgname, name);
            let digest = match self.digests.remove(&filename) {
                Some(digest) => digest,
                None => {
                    return (name.to_string(), Target {
                        available: false,
                        url: None,
                        hash: None,
                        components: None,
                        extensions: None,
                    })
                }
            };

            (name.to_string(), Target {
                available: true,
                url: Some(self.url(pkgname, name)),
                hash: Some(digest),
                components: None,
                extensions: None,
            })
        }).collect();

        dst.insert(pkgname.to_string(), Package {
            version: self.cached_version(pkgname).to_string(),
            target: targets,
        });
    }

    fn url(&self, component: &str, target: &str) -> String {
        format!("{}/{}/{}",
                self.s3_address,
                self.date,
                self.filename(component, target))
    }

    fn filename(&self, component: &str, target: &str) -> String {
        if component == "rust-src" {
            format!("rust-src-{}.tar.gz", self.rust_release)
        } else if component == "cargo" {
            format!("cargo-{}-{}.tar.gz", self.cargo_release, target)
        } else {
            format!("{}-{}-{}.tar.gz", component, self.rust_release, target)
        }
    }

    fn cached_version(&self, component: &str) -> &str {
        if component == "cargo" {
            &self.cargo_version
        } else {
            &self.rust_version
        }
    }

    fn version(&self, component: &str, target: &str) -> String {
        let mut cmd = Command::new("tar");
        let filename = self.filename(component, target);
        cmd.arg("xf")
           .arg(self.input.join(&filename))
           .arg(format!("{}/version", filename.replace(".tar.gz", "")))
           .arg("-O");
        let version = t!(cmd.output());
        if !version.status.success() {
            panic!("failed to learn version:\n\n{:?}\n\n{}\n\n{}",
                   cmd,
                   String::from_utf8_lossy(&version.stdout),
                   String::from_utf8_lossy(&version.stderr));
        }
        String::from_utf8_lossy(&version.stdout).trim().to_string()
    }

    fn hash(&self, path: &Path) -> String {
        let sha = t!(Command::new("shasum")
                        .arg("-a").arg("256")
                        .arg(path.file_name().unwrap())
                        .current_dir(path.parent().unwrap())
                        .output());
        assert!(sha.status.success());

        let filename = path.file_name().unwrap().to_str().unwrap();
        let sha256 = self.output.join(format!("{}.sha256", filename));
        t!(t!(File::create(&sha256)).write_all(&sha.stdout));

        let stdout = String::from_utf8_lossy(&sha.stdout);
        stdout.split_whitespace().next().unwrap().to_string()
    }

    fn sign(&self, path: &Path) {
        let filename = path.file_name().unwrap().to_str().unwrap();
        let asc = self.output.join(format!("{}.asc", filename));
        println!("signing: {:?}", path);
        let mut cmd = Command::new("gpg");
        cmd.arg("--no-tty")
            .arg("--yes")
            .arg("--passphrase-fd").arg("0")
            .arg("--armor")
            .arg("--output").arg(&asc)
            .arg("--detach-sign").arg(path)
            .stdin(Stdio::piped());
        let mut child = t!(cmd.spawn());
        t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
        assert!(t!(child.wait()).success());
    }

    fn write_manifest(&self, manifest: &str, name: &str) {
        let dst = self.output.join(name);
        t!(t!(File::create(&dst)).write_all(manifest.as_bytes()));
        self.hash(&dst);
        self.sign(&dst);
    }

    fn write_date_stamp(&self, date: &str, name: &str) {
        let dst = self.output.join(name);
        t!(t!(File::create(&dst)).write_all(date.as_bytes()));
        self.hash(&dst);
        self.sign(&dst);
    }
}
<|endoftext|>


<commit_before>// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

use std::os::unix::io::AsRawFd;
use std::path::PathBuf;
use std::sync::mpsc::{channel, Receiver, Sender, TryRecvError};
use std::sync::{Arc, Mutex, RwLock};
use std::thread;

use api_server::{ApiRequest, ApiResponse, ApiServer};
use mmds::MMDS;
use polly::event_manager::{EventManager, Subscriber};
use seccomp::BpfProgram;
use utils::epoll::{EpollEvent, EventSet};
use utils::eventfd::EventFd;
use vmm::controller::VmmController;
use vmm::rpc_interface::{PrebootApiController, RuntimeApiController};
use vmm::vmm_config::instance_info::InstanceInfo;

use super::FIRECRACKER_VERSION;

struct ApiServerAdapter {
    api_event_fd: EventFd,
    from_api: Receiver<ApiRequest>,
    to_api: Sender<ApiResponse>,
    controller: RuntimeApiController,
}

impl ApiServerAdapter {
    /// Runs the vmm to completion, while any arising control events are deferred
    /// to a `RuntimeApiController`.
    fn run_microvm(
        api_event_fd: EventFd,
        from_api: Receiver<ApiRequest>,
        to_api: Sender<ApiResponse>,
        vmm_controller: VmmController,
        event_manager: &mut EventManager,
    ) {
        let api_adapter = Arc::new(Mutex::new(Self {
            api_event_fd,
            from_api,
            to_api,
            controller: RuntimeApiController(vmm_controller),
        }));
        event_manager
            .add_subscriber(api_adapter.clone())
            .expect("Cannot register the api event to the event manager.");
        loop {
            event_manager
                .run()
                .expect("EventManager events driver fatal error");
        }
    }
}
impl Subscriber for ApiServerAdapter {
    /// Handle a read event (EPOLLIN).
    fn process(&mut self, event: &EpollEvent, _: &mut EventManager) {
        let source = event.fd();
        let event_set = event.event_set();

        if source == self.api_event_fd.as_raw_fd() && event_set == EventSet::IN {
            let _ = self.api_event_fd.read();
            match self.from_api.try_recv() {
                Ok(api_request) => {
                    let response = self.controller.handle_request(*api_request);
                    // Send back the result.
                    self.to_api
                        .send(Box::new(response))
                        .map_err(|_| ())
                        .expect("one-shot channel closed");
                }
                Err(TryRecvError::Empty) => {
                    warn!("Got a spurious notification from api thread");
                }
                Err(TryRecvError::Disconnected) => {
                    panic!("The channel's sending half was disconnected. Cannot receive data.");
                }
            };
        } else {
            error!("Spurious EventManager event for handler: ApiServerAdapter");
        }
    }

    fn interest_list(&self) -> Vec<EpollEvent> {
        vec![EpollEvent::new(
            EventSet::IN,
            self.api_event_fd.as_raw_fd() as u64,
        )]
    }
}

pub fn run_with_api(
    seccomp_filter: BpfProgram,
    config_json: Option<String>,
    bind_path: PathBuf,
    instance_info: InstanceInfo,
    start_time_us: Option<u64>,
    start_time_cpu_us: Option<u64>,
) {
    // FD to notify of API events.
    let api_event_fd = EventFd::new(libc::EFD_NONBLOCK).expect("Cannot create API Eventfd.");
    // Channels for both directions between Vmm and Api threads.
    let (to_vmm, from_api) = channel();
    let (to_api, from_vmm) = channel();

    // MMDS only supported with API.
    let mmds_info = MMDS.clone();
    let api_shared_info = Arc::new(RwLock::new(instance_info));
    let vmm_shared_info = api_shared_info.clone();
    let to_vmm_event_fd = api_event_fd.try_clone().unwrap();

    let api_seccomp_filter = seccomp_filter.clone();
    // Start the separate API thread.
    thread::Builder::new()
        .name("fc_api".to_owned())
        .spawn(move || {
            match ApiServer::new(
                mmds_info,
                vmm_shared_info,
                to_vmm,
                from_vmm,
                to_vmm_event_fd,
            )
            .expect("Cannot create API server")
            .bind_and_run(
                bind_path,
                start_time_us,
                start_time_cpu_us,
                api_seccomp_filter,
            ) {
                Ok(_) => (),
                Err(api_server::Error::Io(inner)) => match inner.kind() {
                    std::io::ErrorKind::AddrInUse => panic!(
                        "Failed to open the API socket: {:?}",
                        api_server::Error::Io(inner)
                    ),
                    _ => panic!(
                        "Failed to communicate with the API socket: {:?}",
                        api_server::Error::Io(inner)
                    ),
                },
                Err(eventfd_err @ api_server::Error::Eventfd(_)) => {
                    panic!("Failed to open the API socket: {:?}", eventfd_err)
                }
            }
        })
        .expect("API thread spawn failed.");

    let mut event_manager = EventManager::new().expect("Unable to create EventManager");

    // Create the firecracker metrics object responsible for periodically printing metrics.
    let firecracker_metrics = Arc::new(Mutex::new(super::metrics::PeriodicMetrics::new()));
    event_manager
        .add_subscriber(firecracker_metrics.clone())
        .expect("Cannot register the metrics event to the event manager.");

    // Configure, build and start the microVM.
    let (vm_resources, vmm) = match config_json {
        Some(json) => super::build_microvm_from_json(seccomp_filter, &mut event_manager, json),
        None => PrebootApiController::build_microvm_from_requests(
            seccomp_filter,
            &mut event_manager,
            FIRECRACKER_VERSION.to_string(),
            || {
                let req = from_api
                    .recv()
                    .expect("The channel's sending half was disconnected. Cannot receive data.");
                // Also consume the API event along with the message. It is safe to unwrap()
                // since communication between this thread and the API thread is synchronous.
                api_event_fd
                    .read()
                    .expect("VMM: Failed to read the API event_fd");
                *req
            },
            |response| {
                to_api
                    .send(Box::new(response))
                    .expect("one-shot channel closed")
            },
        ),
    };

    // Start the metrics.
    firecracker_metrics
        .lock()
        .expect("Metrics lock poisoned.")
        .start(super::metrics::WRITE_METRICS_PERIOD_MS);

    // Update the api shared instance info.
    api_shared_info.write().unwrap().started = true;

    ApiServerAdapter::run_microvm(
        api_event_fd,
        from_api,
        to_api,
        VmmController::new(vm_resources, vmm),
        &mut event_manager,
    );
}
<commit_msg>api_server_adapter: make the api eventfd blocking<commit_after>// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

use std::os::unix::io::AsRawFd;
use std::path::PathBuf;
use std::sync::mpsc::{channel, Receiver, Sender, TryRecvError};
use std::sync::{Arc, Mutex, RwLock};
use std::thread;

use api_server::{ApiRequest, ApiResponse, ApiServer};
use mmds::MMDS;
use polly::event_manager::{EventManager, Subscriber};
use seccomp::BpfProgram;
use utils::epoll::{EpollEvent, EventSet};
use utils::eventfd::EventFd;
use vmm::controller::VmmController;
use vmm::rpc_interface::{PrebootApiController, RuntimeApiController};
use vmm::vmm_config::instance_info::InstanceInfo;

use super::FIRECRACKER_VERSION;

struct ApiServerAdapter {
    api_event_fd: EventFd,
    from_api: Receiver<ApiRequest>,
    to_api: Sender<ApiResponse>,
    controller: RuntimeApiController,
}

impl ApiServerAdapter {
    /// Runs the vmm to completion, while any arising control events are deferred
    /// to a `RuntimeApiController`.
    fn run_microvm(
        api_event_fd: EventFd,
        from_api: Receiver<ApiRequest>,
        to_api: Sender<ApiResponse>,
        vmm_controller: VmmController,
        event_manager: &mut EventManager,
    ) {
        let api_adapter = Arc::new(Mutex::new(Self {
            api_event_fd,
            from_api,
            to_api,
            controller: RuntimeApiController(vmm_controller),
        }));
        event_manager
            .add_subscriber(api_adapter.clone())
            .expect("Cannot register the api event to the event manager.");
        loop {
            event_manager
                .run()
                .expect("EventManager events driver fatal error");
        }
    }
}
impl Subscriber for ApiServerAdapter {
    /// Handle a read event (EPOLLIN).
    fn process(&mut self, event: &EpollEvent, _: &mut EventManager) {
        let source = event.fd();
        let event_set = event.event_set();

        if source == self.api_event_fd.as_raw_fd() && event_set == EventSet::IN {
            match self.from_api.try_recv() {
                Ok(api_request) => {
                    let response = self.controller.handle_request(*api_request);
                    // Send back the result.
                    self.to_api
                        .send(Box::new(response))
                        .map_err(|_| ())
                        .expect("one-shot channel closed");
                }
                Err(TryRecvError::Empty) => {
                    warn!("Got a spurious notification from api thread");
                }
                Err(TryRecvError::Disconnected) => {
                    panic!("The channel's sending half was disconnected. Cannot receive data.");
                }
            };
            let _ = self.api_event_fd.read();
        } else {
            error!("Spurious EventManager event for handler: ApiServerAdapter");
        }
    }

    fn interest_list(&self) -> Vec<EpollEvent> {
        vec![EpollEvent::new(
            EventSet::IN,
            self.api_event_fd.as_raw_fd() as u64,
        )]
    }
}

pub fn run_with_api(
    seccomp_filter: BpfProgram,
    config_json: Option<String>,
    bind_path: PathBuf,
    instance_info: InstanceInfo,
    start_time_us: Option<u64>,
    start_time_cpu_us: Option<u64>,
) {
    // FD to notify of API events. This is a blocking eventfd by design.
    // It is used in the config/pre-boot loop which is a simple blocking loop
    // which only consumes API events.
    let api_event_fd = EventFd::new(0).expect("Cannot create API Eventfd.");
    // Channels for both directions between Vmm and Api threads.
    let (to_vmm, from_api) = channel();
    let (to_api, from_vmm) = channel();

    // MMDS only supported with API.
    let mmds_info = MMDS.clone();
    let api_shared_info = Arc::new(RwLock::new(instance_info));
    let vmm_shared_info = api_shared_info.clone();
    let to_vmm_event_fd = api_event_fd.try_clone().unwrap();

    let api_seccomp_filter = seccomp_filter.clone();
    // Start the separate API thread.
    thread::Builder::new()
        .name("fc_api".to_owned())
        .spawn(move || {
            match ApiServer::new(
                mmds_info,
                vmm_shared_info,
                to_vmm,
                from_vmm,
                to_vmm_event_fd,
            )
            .expect("Cannot create API server")
            .bind_and_run(
                bind_path,
                start_time_us,
                start_time_cpu_us,
                api_seccomp_filter,
            ) {
                Ok(_) => (),
                Err(api_server::Error::Io(inner)) => match inner.kind() {
                    std::io::ErrorKind::AddrInUse => panic!(
                        "Failed to open the API socket: {:?}",
                        api_server::Error::Io(inner)
                    ),
                    _ => panic!(
                        "Failed to communicate with the API socket: {:?}",
                        api_server::Error::Io(inner)
                    ),
                },
                Err(eventfd_err @ api_server::Error::Eventfd(_)) => {
                    panic!("Failed to open the API socket: {:?}", eventfd_err)
                }
            }
        })
        .expect("API thread spawn failed.");

    let mut event_manager = EventManager::new().expect("Unable to create EventManager");

    // Create the firecracker metrics object responsible for periodically printing metrics.
    let firecracker_metrics = Arc::new(Mutex::new(super::metrics::PeriodicMetrics::new()));
    event_manager
        .add_subscriber(firecracker_metrics.clone())
        .expect("Cannot register the metrics event to the event manager.");

    // Configure, build and start the microVM.
    let (vm_resources, vmm) = match config_json {
        Some(json) => super::build_microvm_from_json(seccomp_filter, &mut event_manager, json),
        None => PrebootApiController::build_microvm_from_requests(
            seccomp_filter,
            &mut event_manager,
            FIRECRACKER_VERSION.to_string(),
            || {
                let req = from_api
                    .recv()
                    .expect("The channel's sending half was disconnected. Cannot receive data.");
                // Also consume the API event along with the message. It is safe to unwrap()
                // because this event_fd is blocking.
                api_event_fd
                    .read()
                    .expect("VMM: Failed to read the API event_fd");
                *req
            },
            |response| {
                to_api
                    .send(Box::new(response))
                    .expect("one-shot channel closed")
            },
        ),
    };

    // Start the metrics.
    firecracker_metrics
        .lock()
        .expect("Metrics lock poisoned.")
        .start(super::metrics::WRITE_METRICS_PERIOD_MS);

    // Update the api shared instance info.
    api_shared_info.write().unwrap().started = true;

    ApiServerAdapter::run_microvm(
        api_event_fd,
        from_api,
        to_api,
        VmmController::new(vm_resources, vmm),
        &mut event_manager,
    );
}
<|endoftext|>


<commit_before>use crate::arch::cpu;
use crate::arch::isr::{self, DropIrqHandler};
use crate::io::{AsyncRead, Pipe, Read, Write};
use crate::kobj::KObj;
use crate::spin::Mutex;
use alloc::sync::Arc;
use core::mem;

unsafe fn write_keyboard(port: u16, data: u8) {
    loop {
        let stat = cpu::inb(0x64);

        if (stat & 0x02) == 0 {
            break;
        }
    }

    cpu::outb(port, data);
}

unsafe fn read_keyboard() -> u8 {
    loop {
        let stat = cpu::inb(0x64);

        if (stat & 0x01) != 0 {
            let data = cpu::inb(0x60);

            if (stat & 0xc0) == 0 {
                return data;
            }
        }
    }
}

unsafe fn write_ps2_aux(data: u8) {
    write_keyboard(0x64, 0xd4);
    write_keyboard(0x60, data);

    let response = read_keyboard();
    assert_eq!(response, 0xfa);
}

unsafe fn read_ps2_aux() -> u8 {
    loop {
        let stat = cpu::inb(0x64);
        if (stat & 0x01) != 0 {
            let data = cpu::inb(0x60);

            if (stat & 0xc0) == 0 {
                return data;
            }
        }
    }
}

struct Ps2MouseState {
    has_wheel: bool,
    data: [u8; 4],
    len: usize,
}

impl Ps2MouseState {
    fn new(has_wheel: bool) -> Self {
        Self {
            has_wheel,
            data: [0; 4],
            len: 0,
        }
    }

    fn translate(&mut self, code: u8) -> Option<[u8; 6]> {
        self.data[self.len] = code;
        self.len += 1;

        if (self.len < 3 && !self.has_wheel) || (self.len < 4 && self.has_wheel) {
            return None;
        }

        let data = mem::replace(&mut self.data, [0; 4]);
        self.len = 0;

        let buttons = (data[0] & 0x04) >> 1 | /* middle */
            (data[0] & 0x02) >> 1 | /* right */
            (data[0] & 0x01) << 2; /* left */

        let dx = if (data[0] & 0x10) != 0 {
            data[1] as i16 - 256
        } else {
            data[1] as i16
        };

        let dy = if (data[0] & 0x20) != 0 {
            -(data[2] as i16 - 256)
        } else {
            -(data[2] as i16)
        };

        let dw = data[3] as i8;

        #[allow(dead_code)]
        struct MouseEvent {
            dx: i16,
            dy: i16,
            dw: i8,
            buttons: u8,
        }

        let response = MouseEvent { dx, dy, dw, buttons };
        Some(unsafe { mem::transmute::<MouseEvent, [u8; 6]>(response) })
    }
}

pub struct Ps2Mouse {
    _drop_irq_handler: DropIrqHandler,
    device: Arc<Pipe>,
}

impl Ps2Mouse {
    pub fn new() -> Self {
        let has_wheel = unsafe {
            write_keyboard(0x64, 0x20); // read controller RAM (controller command byte)
            let status = read_keyboard() & 0xfc; // clear interrupt bits
            write_keyboard(0x64, 0x60); // write controller RAM (controller command byte)
            write_keyboard(0x60, status);

            write_keyboard(0x64, 0xa8); // enable aux device

            for &data in &[
                0xf6, // reset
                0xf3, 200, // set sample rate = 200
                0xf3, 0x64, // set sample rate = 100
                0xf3, 0x50, // set sample rate = 80
            ] {
                write_ps2_aux(data);
            }

            write_ps2_aux(0xf2); // get device id

            let id = read_keyboard();

            for &data in &[
                0xf3, 0xff, // set sample rate = 255
            ] {
                write_ps2_aux(data);
            }

            write_ps2_aux(0xf4); // enable data reporting

            write_keyboard(0x64, 0x20); // read controller RAM (controller command byte)
            let status = read_keyboard() | 3; // set interrupt bits
            write_keyboard(0x64, 0x60); // write controller RAM (controller command byte)
            write_keyboard(0x60, status);

            id == 3
        };

        let device = Arc::new(Pipe::new());

        let handler = {
            let state = Mutex::new(Ps2MouseState::new(has_wheel));
            let device = device.clone();
            move || {
                let mut state = lock!(state);
                let code = unsafe { read_ps2_aux() };
                if let Some(bytes) = state.translate(code) {
                    let _ = Write::write(&*device, &bytes[..]);
                }
            }
        };

        Self {
            _drop_irq_handler: isr::register_irq_handler(12, handler),
            device,
        }
    }
}

impl KObj for Ps2Mouse {
    fn async_read(&self) -> Option<&dyn AsyncRead> {
        Some(&*self.device)
    }

    fn read(&self) -> Option<&dyn Read> {
        Some(&*self.device)
    }
}
<commit_msg>Remove duplicate function<commit_after>use crate::arch::cpu;
use crate::arch::isr::{self, DropIrqHandler};
use crate::io::{AsyncRead, Pipe, Read, Write};
use crate::kobj::KObj;
use crate::spin::Mutex;
use alloc::sync::Arc;
use core::mem;

unsafe fn write_keyboard(port: u16, data: u8) {
    loop {
        let stat = cpu::inb(0x64);

        if (stat & 0x02) == 0 {
            break;
        }
    }

    cpu::outb(port, data);
}

unsafe fn read_keyboard() -> u8 {
    loop {
        let stat = cpu::inb(0x64);

        if (stat & 0x01) != 0 {
            let data = cpu::inb(0x60);

            if (stat & 0xc0) == 0 {
                return data;
            }
        }
    }
}

unsafe fn write_ps2_aux(data: u8) {
    write_keyboard(0x64, 0xd4);
    write_keyboard(0x60, data);

    let response = read_keyboard();
    assert_eq!(response, 0xfa);
}

struct Ps2MouseState {
    has_wheel: bool,
    data: [u8; 4],
    len: usize,
}

impl Ps2MouseState {
    fn new(has_wheel: bool) -> Self {
        Self {
            has_wheel,
            data: [0; 4],
            len: 0,
        }
    }

    fn translate(&mut self, code: u8) -> Option<[u8; 6]> {
        self.data[self.len] = code;
        self.len += 1;

        if (self.len < 3 && !self.has_wheel) || (self.len < 4 && self.has_wheel) {
            return None;
        }

        let data = mem::replace(&mut self.data, [0; 4]);
        self.len = 0;

        let buttons = (data[0] & 0x04) >> 1 | /* middle */
            (data[0] & 0x02) >> 1 | /* right */
            (data[0] & 0x01) << 2; /* left */

        let dx = if (data[0] & 0x10) != 0 {
            data[1] as i16 - 256
        } else {
            data[1] as i16
        };

        let dy = if (data[0] & 0x20) != 0 {
            -(data[2] as i16 - 256)
        } else {
            -(data[2] as i16)
        };

        let dw = data[3] as i8;

        #[allow(dead_code)]
        struct MouseEvent {
            dx: i16,
            dy: i16,
            dw: i8,
            buttons: u8,
        }

        let response = MouseEvent { dx, dy, dw, buttons };
        Some(unsafe { mem::transmute::<MouseEvent, [u8; 6]>(response) })
    }
}

pub struct Ps2Mouse {
    _drop_irq_handler: DropIrqHandler,
    device: Arc<Pipe>,
}

impl Ps2Mouse {
    pub fn new() -> Self {
        let has_wheel = unsafe {
            write_keyboard(0x64, 0x20); // read controller RAM (controller command byte)
            let status = read_keyboard() & 0xfc; // clear interrupt bits
            write_keyboard(0x64, 0x60); // write controller RAM (controller command byte)
            write_keyboard(0x60, status);

            write_keyboard(0x64, 0xa8); // enable aux device

            for &data in &[
                0xf6, // reset
                0xf3, 200, // set sample rate = 200
                0xf3, 0x64, // set sample rate = 100
                0xf3, 0x50, // set sample rate = 80
            ] {
                write_ps2_aux(data);
            }

            write_ps2_aux(0xf2); // get device id

            let id = read_keyboard();

            for &data in &[
                0xf3, 0xff, // set sample rate = 255
            ] {
                write_ps2_aux(data);
            }

            write_ps2_aux(0xf4); // enable data reporting

            write_keyboard(0x64, 0x20); // read controller RAM (controller command byte)
            let status = read_keyboard() | 3; // set interrupt bits
            write_keyboard(0x64, 0x60); // write controller RAM (controller command byte)
            write_keyboard(0x60, status);

            id == 3
        };

        let device = Arc::new(Pipe::new());

        let handler = {
            let state = Mutex::new(Ps2MouseState::new(has_wheel));
            let device = device.clone();
            move || {
                let mut state = lock!(state);
                let code = unsafe { read_keyboard() };
                if let Some(bytes) = state.translate(code) {
                    let _ = Write::write(&*device, &bytes[..]);
                }
            }
        };

        Self {
            _drop_irq_handler: isr::register_irq_handler(12, handler),
            device,
        }
    }
}

impl KObj for Ps2Mouse {
    fn async_read(&self) -> Option<&dyn AsyncRead> {
        Some(&*self.device)
    }

    fn read(&self) -> Option<&dyn Read> {
        Some(&*self.device)
    }
}
<|endoftext|>


<commit_before>use std;
use std::net::{IpAddr};
use std::time::Duration;
use std::marker::PhantomData;
use std::ops::Deref;
use std::sync::Arc;
use std::io;
use std::fs::File;
use std::io::{Read};
use linked_hash_map::Entry as BtEntry;
use std::str::FromStr;

use serde_yaml;
use serde::{Deserialize, Deserializer};
use serde::de::Error as DesError;
use yaml_rust::{Yaml, YamlLoader, YamlEmitter};

use utils::IpSet;

#[derive(Debug)]
pub struct ProxiedValue<TStorage, TValue>(TValue, PhantomData<TStorage>);
impl<TStorage, TValue> Deref for ProxiedValue<TStorage, TValue> {
    type Target = TValue;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}
impl<TStorage, TValue> From<TValue> for ProxiedValue<TStorage, TValue> {
    fn from(val: TValue) -> Self {
        ProxiedValue(val, PhantomData)
    }
}
pub trait FromStorage<'de, TStorage> {
    fn from_storage<TDes: ?Sized + Deserializer<'de>>(storage: TStorage) -> Result<Self, TDes::Error> where Self: Sized;
}
impl<'de, TStorage, TValue> Deserialize<'de> for ProxiedValue<TStorage, TValue>
    where TStorage: Deserialize<'de>,
          TValue: FromStorage<'de, TStorage>,
{
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
        where D: Deserializer<'de>,
    {
        let val = try!(TStorage::deserialize(deserializer));
        let result = try!(TValue::from_storage::<D>(val));
        Ok(ProxiedValue(result, PhantomData))
    }
}
impl<'de> FromStorage<'de, u32> for Duration {
    fn from_storage<TDes: ?Sized + Deserializer<'de>>(storage: u32) -> Result<Self, TDes::Error> {
        Ok(Duration::from_secs(storage as u64))
    }
}
impl<'de> FromStorage<'de, String> for Arc<IpSet> {
    fn from_storage<TDes: ?Sized + Deserializer<'de>>(storage: String) -> Result<Self, TDes::Error> {
        IpSet::from_file(&storage)
        .map(Arc::new)
        .map_err(|e| TDes::Error::custom(format!(
            "Failed to load IP list from {}: {}", storage, e,
        )))
    }
}
macro_attr! {
    #[derive(Clone, Copy, Debug, NewtypeDisplay!, NewtypeFrom!, NewtypeDeref!)]
    pub struct PermissionBits(u32);
}
impl<'de> FromStorage<'de, String> for PermissionBits {
    fn from_storage<TDes: ?Sized + Deserializer<'de>>(storage: String) -> Result<Self, TDes::Error> {
        u32::from_str_radix(&storage, 8)
        .map(Into::into)
        .map_err(|_| TDes::Error::custom(&format!(
            "Invalid permission bits: {}", storage,
        )))
    }
}

#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct CacheConfig {
    pub min_cache_ttl: u32,
    pub min_response_ttl: u32,
    pub neg_cache_ttl: u32,
    pub cache_retention_time: ProxiedValue<u32, Duration>,
    pub cache_limit: usize,
    pub gc_aggressive_threshold: usize,
    pub gc_interval: ProxiedValue<u32, Duration>,
}
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct ServeConfig {
    pub ip: IpAddr,
    pub port: u16,
    pub tcp_timeout: ProxiedValue<u32, Duration>,
    pub setuid: Option<String>,
    pub setgid: Option<String>,
}
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct ForwarderConfig {
    pub servers: Vec<IpAddr>,
    pub accepted_ip_list: Option<ProxiedValue<String, Arc<IpSet>>>,
}
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct QueryConfig {
    pub timeout: ProxiedValue<u32, Duration>,
    pub trust_cname_hinting: bool,
    pub enable_dnssec: bool,
}
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct ControlConfig {
    pub sock_path: String,
    pub sock_permission: ProxiedValue<String, PermissionBits>,
}
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct InternalConfig {
    pub threads: usize,
    pub mio_notify_capacity: usize,
}
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct Config {
    pub serve: ServeConfig,
    pub root_servers: Vec<IpAddr>,
    pub cache: CacheConfig,
    pub forwarders: Vec<ForwarderConfig>,
    pub query: QueryConfig,
    pub control: ControlConfig,
    pub internal: InternalConfig,
}

const DEFAULT_CONFIG: &'static str = include_str!("../extra/config-default.yaml");
impl Default for Config {
    fn default() -> Self {
        serde_yaml::from_str(DEFAULT_CONFIG).expect("Default config should never fail")
    }
}
fn merge_yaml(base: Yaml, input: Yaml) -> Yaml {
    let input = if let Yaml::Hash(hash) = input {
        hash
    } else {
        return input
    };
    let mut target = if let Yaml::Hash(base_hash) = base {
        base_hash
    } else {
        return Yaml::Hash(input)
    };
    for (k, v) in input {
        match target.entry(k) {
            BtEntry::Vacant(e) => { e.insert(v); },
            BtEntry::Occupied(mut e) => {
                let old = e.insert(Yaml::Null);
                e.insert(merge_yaml(old, v));
            },
        };
    }
    Yaml::Hash(target)
}
fn merge_config_values(user_values: Yaml) -> Yaml {
    merge_yaml(
        YamlLoader::load_from_str(DEFAULT_CONFIG).unwrap().into_iter().next().unwrap(),
        user_values,
    )
}
impl Config {
    pub fn from_file(path: &str) -> io::Result<Self> {
        let mut file = try!(File::open(path));
        let mut buffer = String::new();
        try!(file.read_to_string(&mut buffer));
        Self::from_str(&buffer)
    }
}
impl FromStr for Config {
    type Err = io::Error;
    fn from_str(content: &str) -> std::result::Result<Self, Self::Err> {
        let parsed_yaml_docs = try!(
            YamlLoader::load_from_str(content)
            .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
        );
        if parsed_yaml_docs.len() != 1 {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                "Config file must contain exactly one YAML dictionary",
            ));
        }
        let merged = merge_config_values(parsed_yaml_docs.into_iter().next().unwrap());
        let mut out_str = String::new();
        {
            let mut emitter = YamlEmitter::new(&mut out_str);
            emitter.dump(&merged).unwrap(); // dump the YAML object to a String
        }
        serde_yaml::from_str(&out_str)
        .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("{}", e)))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::net::IpAddr;

    #[test]
    fn test_default_config() {
        Config::default();
    }
    #[test]
    fn test_merged_config() {
        let config_file = r#"---
serve:
    ip: "1.1.1.1"

forwarders:
    - servers:
        - "1.2.3.4"
        "#;
        let config = Config::from_str(config_file).unwrap();
        let default_config = Config::default();
        assert_eq!(config.serve.ip, "1.1.1.1".parse::<IpAddr>().unwrap());
        assert_eq!(config.serve.port, default_config.serve.port);
        assert_eq!(config.forwarders.len(), 1);
        assert_eq!(config.forwarders[0].servers[0], "1.2.3.4".parse::<IpAddr>().unwrap());
    }
}
<commit_msg>Allow including external config file<commit_after>use std;
use std::net::{IpAddr};
use std::time::Duration;
use std::marker::PhantomData;
use std::ops::Deref;
use std::sync::Arc;
use std::io;
use std::fs::File;
use std::io::{Read};
use linked_hash_map::Entry as BtEntry;
use std::str::FromStr;

use serde_yaml;
use serde::{Deserialize, Deserializer};
use serde::de::Error as DesError;
use yaml_rust::{Yaml, YamlLoader, YamlEmitter};

use utils::IpSet;

#[derive(Debug)]
pub struct ProxiedValue<TStorage, TValue>(TValue, PhantomData<TStorage>);
impl<TStorage, TValue> Deref for ProxiedValue<TStorage, TValue> {
    type Target = TValue;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}
impl<TStorage, TValue> From<TValue> for ProxiedValue<TStorage, TValue> {
    fn from(val: TValue) -> Self {
        ProxiedValue(val, PhantomData)
    }
}
pub trait FromStorage<'de, TStorage> {
    fn from_storage<TDes: ?Sized + Deserializer<'de>>(storage: TStorage) -> Result<Self, TDes::Error> where Self: Sized;
}
impl<'de, TStorage, TValue> Deserialize<'de> for ProxiedValue<TStorage, TValue>
    where TStorage: Deserialize<'de>,
          TValue: FromStorage<'de, TStorage>,
{
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
        where D: Deserializer<'de>,
    {
        let val = try!(TStorage::deserialize(deserializer));
        let result = try!(TValue::from_storage::<D>(val));
        Ok(ProxiedValue(result, PhantomData))
    }
}
impl<'de> FromStorage<'de, u32> for Duration {
    fn from_storage<TDes: ?Sized + Deserializer<'de>>(storage: u32) -> Result<Self, TDes::Error> {
        Ok(Duration::from_secs(storage as u64))
    }
}
impl<'de> FromStorage<'de, String> for Arc<IpSet> {
    fn from_storage<TDes: ?Sized + Deserializer<'de>>(storage: String) -> Result<Self, TDes::Error> {
        IpSet::from_file(&storage)
        .map(Arc::new)
        .map_err(|e| TDes::Error::custom(format!(
            "Failed to load IP list from {}: {}", storage, e,
        )))
    }
}
macro_attr! {
    #[derive(Clone, Copy, Debug, NewtypeDisplay!, NewtypeFrom!, NewtypeDeref!)]
    pub struct PermissionBits(u32);
}
impl<'de> FromStorage<'de, String> for PermissionBits {
    fn from_storage<TDes: ?Sized + Deserializer<'de>>(storage: String) -> Result<Self, TDes::Error> {
        u32::from_str_radix(&storage, 8)
        .map(Into::into)
        .map_err(|_| TDes::Error::custom(&format!(
            "Invalid permission bits: {}", storage,
        )))
    }
}

#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct CacheConfig {
    pub min_cache_ttl: u32,
    pub min_response_ttl: u32,
    pub neg_cache_ttl: u32,
    pub cache_retention_time: ProxiedValue<u32, Duration>,
    pub cache_limit: usize,
    pub gc_aggressive_threshold: usize,
    pub gc_interval: ProxiedValue<u32, Duration>,
}
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct ServeConfig {
    pub ip: IpAddr,
    pub port: u16,
    pub tcp_timeout: ProxiedValue<u32, Duration>,
    pub setuid: Option<String>,
    pub setgid: Option<String>,
}
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct ForwarderConfig {
    pub servers: Vec<IpAddr>,
    pub accepted_ip_list: Option<ProxiedValue<String, Arc<IpSet>>>,
}
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct QueryConfig {
    pub timeout: ProxiedValue<u32, Duration>,
    pub trust_cname_hinting: bool,
    pub enable_dnssec: bool,
}
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct ControlConfig {
    pub sock_path: String,
    pub sock_permission: ProxiedValue<String, PermissionBits>,
}
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct InternalConfig {
    pub threads: usize,
    pub mio_notify_capacity: usize,
}
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct Config {
    pub serve: ServeConfig,
    pub root_servers: Vec<IpAddr>,
    pub cache: CacheConfig,
    pub forwarders: Vec<ForwarderConfig>,
    pub query: QueryConfig,
    pub control: ControlConfig,
    pub internal: InternalConfig,
    include: Option<Vec<String>>,
}

const DEFAULT_CONFIG: &'static str = include_str!("../extra/config-default.yaml");
impl Default for Config {
    fn default() -> Self {
        serde_yaml::from_str(DEFAULT_CONFIG).expect("Default config should never fail")
    }
}
fn merge_yaml(base: Yaml, input: Yaml) -> Yaml {
    if let Yaml::Array(array_input) = input {
        if let Yaml::Array(array_base) = base {
            let mut new_array: Vec<Yaml> = Default::default();
            if array_input[0] == Yaml::String("-- APPEND --".into()) {
                new_array.extend(array_base);
                new_array.extend(array_input.into_iter().skip(1));
            } else {
                new_array.extend(array_input);
            }
            return Yaml::Array(new_array);
        } else {
            return Yaml::Array(array_input);
        }
    }
    let input = if let Yaml::Hash(hash) = input {
        hash
    } else {
        return input
    };
    let mut target = if let Yaml::Hash(base_hash) = base {
        base_hash
    } else {
        return Yaml::Hash(input)
    };
    for (k, v) in input {
        match target.entry(k) {
            BtEntry::Vacant(e) => { e.insert(v); },
            BtEntry::Occupied(mut e) => {
                let old = e.insert(Yaml::Null);
                e.insert(merge_yaml(old, v));
            },
        };
    }
    Yaml::Hash(target)
}
fn merge_config_values(user_values: Yaml) -> Yaml {
    merge_yaml(
        YamlLoader::load_from_str(DEFAULT_CONFIG).unwrap().into_iter().next().unwrap(),
        user_values,
    )
}
fn read_file(path: &str) -> io::Result<String> {
    let mut file = File::open(path)?;
    let mut buffer = String::new();
    file.read_to_string(&mut buffer)?;
    Ok(buffer)
}
fn parse_yaml(content: &str) -> io::Result<Yaml> {
    let parsed_yaml_docs = try!(
        YamlLoader::load_from_str(content)
        .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
    );
    if parsed_yaml_docs.len() != 1 {
        return Err(io::Error::new(
            io::ErrorKind::InvalidData,
            "Config file must contain exactly one YAML dictionary",
        ));
    }
    Ok(parsed_yaml_docs[0].clone())
}
impl Config {
    pub fn from_file(path: &str) -> io::Result<Self> {
        Self::from_str(&read_file(path)?)
    }
}
impl FromStr for Config {
    type Err = io::Error;
    fn from_str(content: &str) -> std::result::Result<Self, Self::Err> {
        let parsed_yaml_doc = parse_yaml(content)?;
        let mut merged = merge_config_values(parsed_yaml_doc);
        let err = |msg| (io::Error::new(io::ErrorKind::InvalidData, msg));
        if let Some(vec) = merged["include"].as_vec().map(|x| (*x).clone()) {
            for elem in &vec {
                if let Some(s) = elem.as_str() {
                    let content = read_file(s).map_err(|e| err(format!("Failed to read included file {}: {}", s, e)))?;
                    let doc = parse_yaml(&content).map_err(|e| err(format!("Failed to parse included file {}: {}", s, e)))?;
                    merged = merge_yaml(merged, doc);
                } else {
                    return Err(err(format!("Invalid data in include list: {:?}", elem)));
                }
            }
        }
        let mut out_str = String::new();
        {
            let mut emitter = YamlEmitter::new(&mut out_str);
            emitter.dump(&merged).unwrap(); // dump the YAML object to a String
        }
        serde_yaml::from_str(&out_str)
        .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("{}", e)))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::net::IpAddr;
    use std::fs::{File, remove_file};
    use std::io::Write;

    #[test]
    fn test_default_config() {
        Config::default();
    }
    #[test]
    fn test_merged_config() {
        let config_file = r#"---
serve:
    ip: "1.1.1.1"

forwarders:
    - servers:
        - "1.2.3.4"
        "#;
        let config = Config::from_str(config_file).unwrap();
        let default_config = Config::default();
        assert_eq!(config.serve.ip, "1.1.1.1".parse::<IpAddr>().unwrap());
        assert_eq!(config.serve.port, default_config.serve.port);
        assert_eq!(config.forwarders.len(), 1);
        assert_eq!(config.forwarders[0].servers[0], "1.2.3.4".parse::<IpAddr>().unwrap());
    }
    #[test]
    fn test_included_config() {
        const INCLUDE_PATH: &str = "/tmp/7f326c2f-9743-4c77-8c5f-136d8d6738d4";
        let included_file = r#"---
serve:
    ip: "1.1.1.1"

forwarders:
    - servers:
        - "1.2.3.4"
        "#;
        { File::create(INCLUDE_PATH).unwrap().write_all(included_file.as_bytes()).unwrap(); }
        let config = Config::from_str(&format!("---\n\ninclude: [\"{}\"]", INCLUDE_PATH)).unwrap();
        remove_file(INCLUDE_PATH).is_ok();
        let default_config = Config::default();
        assert_eq!(config.serve.ip, "1.1.1.1".parse::<IpAddr>().unwrap());
        assert_eq!(config.serve.port, default_config.serve.port);
        assert_eq!(config.forwarders.len(), 1);
        assert_eq!(config.forwarders[0].servers[0], "1.2.3.4".parse::<IpAddr>().unwrap());
    }
}
<|endoftext|>