<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

extern crate cmake;

use std::env;
use std::path::{Path, PathBuf};
use std::process::Command;

static SHADERC_REPO: &'static str = "https://github.com/google/shaderc";
static GLSLANG_REPO: &'static str = "https://github.com/google/glslang";
static SPIRV_TOOLS_REPO: &'static str = "https://github.com/KhronosGroup/SPIRV-Tools";
static SPIRV_HEADERS_REPO: &'static str = "https://github.com/KhronosGroup/SPIRV-Headers";

fn git_clone_or_update(project: &str, url: &str, dir: &PathBuf) {
    if dir.as_path().exists() {
        let status = Command::new("git")
            .arg("pull")
            .current_dir(dir)
            .status()
            .expect("failed to execute git pull");
        if !status.success() {
            panic!("git pull {} failed", project)
        }
    } else {
        let status = Command::new("git")
            .args(&["clone", url, dir.to_str().unwrap()])
            .status()
            .expect("failed to execute git clone");
        if !status.success() {
            panic!("git clone {} failed", project)
        }
    }
}

fn build_shaderc(shaderc_dir: &PathBuf) -> PathBuf {
        cmake::Config::new(shaderc_dir)
            .profile("Release")
            .define("CMAKE_POSITION_INDEPENDENT_CODE", "ON")
            .define("SPIRV_SKIP_EXECUTABLES", "ON")
            .define("SHADERC_SKIP_TESTS", "ON")
            .build()
}

fn main() {
    if env::var("CARGO_FEATURE_BUILD_NATIVE_SHADERC").is_err() {
        println!("cargo:warning=requested to skip building native C++ shaderc");
        return
    }

    let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
    let target_dir = Path::new(&manifest_dir).join("target");
    let shaderc_dir = Path::new(&target_dir).join("native-shaderc");
    let third_party_dir = Path::new(&shaderc_dir).join("third_party");
    let glslang_dir = Path::new(&third_party_dir).join("glslang");
    let spirv_tools_dir = Path::new(&third_party_dir).join("spirv-tools");
    let external_dir = Path::new(&spirv_tools_dir).join("external");
    let spirv_headers_dir = Path::new(&external_dir).join("spirv-headers");

    git_clone_or_update("shaderc", SHADERC_REPO, &shaderc_dir);
    git_clone_or_update("glslang", GLSLANG_REPO, &glslang_dir);
    git_clone_or_update("spirv-tools", SPIRV_TOOLS_REPO, &spirv_tools_dir);
    git_clone_or_update("spirv-headers", SPIRV_HEADERS_REPO, &spirv_headers_dir);

    let mut lib_path = build_shaderc(&shaderc_dir);
    lib_path.push("lib");

    println!("cargo:rustc-link-lib=static=shaderc_combined");
    println!("cargo:rustc-link-search=native={}", lib_path.display());
}
<commit_msg>Checkout specific commits for shaderc and dependency projects<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

extern crate cmake;

use std::env;
use std::path::{Path, PathBuf};
use std::process::Command;

static SHADERC_REPO: &'static str = "https://github.com/google/shaderc";
static GLSLANG_REPO: &'static str = "https://github.com/google/glslang";
static SPIRV_TOOLS_REPO: &'static str = "https://github.com/KhronosGroup/SPIRV-Tools";
static SPIRV_HEADERS_REPO: &'static str = "https://github.com/KhronosGroup/SPIRV-Headers";

// Updated on 2017-09-30
static SHADERC_COMMIT: &'static str = "c8ba3e4694ff554f95000ec35e6e55f2ba637fb3";
static GLSLANG_COMMIT: &'static str = "44dd6a00c388ba09f7ac64a3921b4717a89f949c";
static SPIRV_TOOLS_COMMIT: &'static str = "17a843c6b0ac39edce3ca45246b78c8f47c7ebee";
static SPIRV_HEADERS_COMMIT: &'static str = "77240d9e86c6ff135f6de8c7b89a0099a2d90e16";

fn git_clone_or_update(project: &str, url: &str, commit: &str, dir: &PathBuf) {
    if dir.as_path().exists() {
        let status = Command::new("git")
            .arg("pull")
            .current_dir(dir)
            .status()
            .expect("failed to execute git pull");
        if !status.success() {
            panic!("git pull {} failed", project)
        }
    } else {
        let status = Command::new("git")
            .args(&["clone", url, dir.to_str().unwrap()])
            .status()
            .expect("failed to execute git clone");
        if !status.success() {
            panic!("git clone {} failed", project)
        }
    }

    let status = Command::new("git")
        .args(&["checkout", commit])
        .current_dir(dir)
        .status()
        .expect("failed to execute git checkout");
    if !status.success() {
        panic!("git checkout {} failed", commit)
    }
}

fn build_shaderc(shaderc_dir: &PathBuf) -> PathBuf {
        cmake::Config::new(shaderc_dir)
            .profile("Release")
            .define("CMAKE_POSITION_INDEPENDENT_CODE", "ON")
            .define("SPIRV_SKIP_EXECUTABLES", "ON")
            .define("SHADERC_SKIP_TESTS", "ON")
            .build()
}

fn main() {
    if env::var("CARGO_FEATURE_BUILD_NATIVE_SHADERC").is_err() {
        println!("cargo:warning=requested to skip building native C++ shaderc");
        return
    }

    let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
    let target_dir = Path::new(&manifest_dir).join("target");
    let shaderc_dir = Path::new(&target_dir).join("native-shaderc");
    let third_party_dir = Path::new(&shaderc_dir).join("third_party");
    let glslang_dir = Path::new(&third_party_dir).join("glslang");
    let spirv_tools_dir = Path::new(&third_party_dir).join("spirv-tools");
    let external_dir = Path::new(&spirv_tools_dir).join("external");
    let spirv_headers_dir = Path::new(&external_dir).join("spirv-headers");

    git_clone_or_update("shaderc", SHADERC_REPO, SHADERC_COMMIT, &shaderc_dir);
    git_clone_or_update("glslang", GLSLANG_REPO, GLSLANG_COMMIT, &glslang_dir);
    git_clone_or_update("spirv-tools", SPIRV_TOOLS_REPO, SPIRV_TOOLS_COMMIT, &spirv_tools_dir);
    git_clone_or_update("spirv-headers", SPIRV_HEADERS_REPO, SPIRV_HEADERS_COMMIT, &spirv_headers_dir);

    let mut lib_path = build_shaderc(&shaderc_dir);
    lib_path.push("lib");

    println!("cargo:rustc-link-lib=static=shaderc_combined");
    println!("cargo:rustc-link-search=native={}", lib_path.display());
}
<|endoftext|>


<commit_before>use conduit::{Request, Response};
use conduit_middleware::Middleware;
use curl;
use curl::easy::{Easy, List};
use oauth2::*;
use app::App;
use util::{CargoResult, internal, ChainError, human};
use serde_json;
use serde::Deserialize;
use std::str;
use std::error::Error;

/// Does all the nonsense for sending a GET to Github. Doesn't handle parsing
/// because custom error-code handling may be desirable. Use
/// `parse_github_response` to handle the "common" processing of responses.
pub fn github(app: &App, url: &str, auth: &Token) -> Result<(Easy, Vec<u8>), curl::Error> {
    let url = format!("{}://api.github.com{}", app.config.api_protocol, url);
    info!("GITHUB HTTP: {}", url);

    let mut headers = List::new();
    headers
        .append("Accept: application/vnd.github.v3+json")
        .unwrap();
    headers.append("User-Agent: hello!").unwrap();
    headers
        .append(&format!("Authorization: token {}", auth.access_token))
        .unwrap();

    let mut handle = app.handle();
    handle.url(&url).unwrap();
    handle.get(true).unwrap();
    handle.http_headers(headers).unwrap();

    let mut data = Vec::new();
    {
        let mut transfer = handle.transfer();
        transfer
            .write_function(|buf| {
                data.extend_from_slice(buf);
                Ok(buf.len())
            })
            .unwrap();
        transfer.perform()?;
    }
    Ok((handle, data))
}

/// Checks for normal responses
pub fn parse_github_response<'de, 'a: 'de, T: Deserialize<'de>>(
    mut resp: Easy,
    data: &'a [u8],
) -> CargoResult<T> {
    match resp.response_code().unwrap() {
        200 => {} // Ok!
        403 => {
            return Err(human(
                "It looks like you don't have permission \
                 to query a necessary property from Github \
                 to complete this request. \
                 You may need to re-authenticate on \
                 crates.io to grant permission to read \
                 github org memberships. Just go to \
                 https://crates.io/login",
            ));
        }
        n => {
            let resp = String::from_utf8_lossy(data);
            return Err(internal(&format_args!(
                "didn't get a 200 result from \
                 github, got {} with: {}",
                n,
                resp
            )));
        }
    }

    let json = str::from_utf8(data).ok().chain_error(|| {
        internal("github didn't send a utf8-response")
    })?;

    serde_json::from_str(json).chain_error(|| internal("github didn't send a valid json response"))
}

/// Gets a token with the given string as the access token, but all
/// other info null'd out. Generally, just to be fed to the `github` fn.
pub fn token(token: String) -> Token {
    Token {
        access_token: token,
        scopes: Vec::new(),
        token_type: String::new(),
    }
}

pub struct SecurityHeadersMiddleware;

impl Middleware for SecurityHeadersMiddleware {
    fn after(&self, _: &mut Request, mut res: Result<Response, Box<Error+Send>>)
        -> Result<Response, Box<Error+Send>> {
        if let Ok(ref mut response) = res {
            response.headers.insert(
                "Content-Security-Policy".into(),
                vec!["default-src 'self'; \
                      script-src 'self' https://www.google-analytics.com https://www.google.com; \
                      style-src 'self' https://www.google.com; \
                      img-src *; \
                      object-src 'none'".into()],
            );
            response.headers.insert(
                "X-Content-Type-Options".into(),
                vec!["nosniff".into()],
            );
            response.headers.insert(
                "X-Frame-Options".into(),
                vec!["SAMEORIGIN".into()],
            );
            response.headers.insert(
                "X-XSS-Protection".into(),
                vec!["1; mode=block".into()],
            );
        }
        res
    }
}
<commit_msg>Add missing derives that we lint for now<commit_after>use conduit::{Request, Response};
use conduit_middleware::Middleware;
use curl;
use curl::easy::{Easy, List};
use oauth2::*;
use app::App;
use util::{CargoResult, internal, ChainError, human};
use serde_json;
use serde::Deserialize;
use std::str;
use std::error::Error;

/// Does all the nonsense for sending a GET to Github. Doesn't handle parsing
/// because custom error-code handling may be desirable. Use
/// `parse_github_response` to handle the "common" processing of responses.
pub fn github(app: &App, url: &str, auth: &Token) -> Result<(Easy, Vec<u8>), curl::Error> {
    let url = format!("{}://api.github.com{}", app.config.api_protocol, url);
    info!("GITHUB HTTP: {}", url);

    let mut headers = List::new();
    headers
        .append("Accept: application/vnd.github.v3+json")
        .unwrap();
    headers.append("User-Agent: hello!").unwrap();
    headers
        .append(&format!("Authorization: token {}", auth.access_token))
        .unwrap();

    let mut handle = app.handle();
    handle.url(&url).unwrap();
    handle.get(true).unwrap();
    handle.http_headers(headers).unwrap();

    let mut data = Vec::new();
    {
        let mut transfer = handle.transfer();
        transfer
            .write_function(|buf| {
                data.extend_from_slice(buf);
                Ok(buf.len())
            })
            .unwrap();
        transfer.perform()?;
    }
    Ok((handle, data))
}

/// Checks for normal responses
pub fn parse_github_response<'de, 'a: 'de, T: Deserialize<'de>>(
    mut resp: Easy,
    data: &'a [u8],
) -> CargoResult<T> {
    match resp.response_code().unwrap() {
        200 => {} // Ok!
        403 => {
            return Err(human(
                "It looks like you don't have permission \
                 to query a necessary property from Github \
                 to complete this request. \
                 You may need to re-authenticate on \
                 crates.io to grant permission to read \
                 github org memberships. Just go to \
                 https://crates.io/login",
            ));
        }
        n => {
            let resp = String::from_utf8_lossy(data);
            return Err(internal(&format_args!(
                "didn't get a 200 result from \
                 github, got {} with: {}",
                n,
                resp
            )));
        }
    }

    let json = str::from_utf8(data).ok().chain_error(|| {
        internal("github didn't send a utf8-response")
    })?;

    serde_json::from_str(json).chain_error(|| internal("github didn't send a valid json response"))
}

/// Gets a token with the given string as the access token, but all
/// other info null'd out. Generally, just to be fed to the `github` fn.
pub fn token(token: String) -> Token {
    Token {
        access_token: token,
        scopes: Vec::new(),
        token_type: String::new(),
    }
}

#[derive(Clone, Copy, Debug)]
pub struct SecurityHeadersMiddleware;

impl Middleware for SecurityHeadersMiddleware {
    fn after(&self, _: &mut Request, mut res: Result<Response, Box<Error+Send>>)
        -> Result<Response, Box<Error+Send>> {
        if let Ok(ref mut response) = res {
            response.headers.insert(
                "Content-Security-Policy".into(),
                vec!["default-src 'self'; \
                      script-src 'self' https://www.google-analytics.com https://www.google.com; \
                      style-src 'self' https://www.google.com; \
                      img-src *; \
                      object-src 'none'".into()],
            );
            response.headers.insert(
                "X-Content-Type-Options".into(),
                vec!["nosniff".into()],
            );
            response.headers.insert(
                "X-Frame-Options".into(),
                vec!["SAMEORIGIN".into()],
            );
            response.headers.insert(
                "X-XSS-Protection".into(),
                vec!["1; mode=block".into()],
            );
        }
        res
    }
}
<|endoftext|>


<commit_before>use std::fmt::{self, Display, Formatter};
use std::mem;
use std::pin::Pin;
use std::task::{Context, Poll};

use bytes::{Buf, Bytes, BytesMut};
use futures_util::ready;
use futures_util::stream::{Fuse, IntoStream, Stream, StreamExt, TryStream, TryStreamExt};
use http_body::Body;
use pin_project_lite::pin_project;

use crate::error::Error;

/// Creates an enum with `AsRef<str>` impl.
macro_rules! str_enum {
    (
        $(#[$attr:meta])*
        pub enum $E:ident {
            $(
                $(#[$v_attr:meta])*
                $V:ident = $by:expr
            ),*$(,)?
        }
    ) => {
        $(#[$attr])*
        pub enum $E {
            $(
                $(#[$v_attr])*
                $V,
            )*
        }

        impl std::convert::AsRef<str> for $E {
            fn as_ref(&self) -> &str {
                match *self {
                    $($E::$V => $by,)*
                }
            }
        }
    }
}

pin_project! {
    pub struct Lines<S> {
        #[pin]
        stream: Fuse<IntoStream<S>>,
        buf: BytesMut,
    }
}

pin_project! {
    /// Wraps `http_body::Body` to make it a `Stream`.
    pub struct HttpBodyAsStream<B> {
        #[pin]
        pub inner: B,
    }
}

impl<S: TryStream> Lines<S> {
    pub fn new(stream: S) -> Self {
        Lines {
            stream: stream.into_stream().fuse(),
            buf: BytesMut::new(),
        }
    }
}

impl<S: TryStream<Ok = Bytes, Error = Error<E>>, E> Stream for Lines<S> {
    type Item = Result<Bytes, Error<E>>;

    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
        let mut this = self.project();

        if let Some(line) = remove_first_line(&mut this.buf) {
            return Poll::Ready(Some(Ok(line.freeze())));
        }

        // Now `self.buf` does not have a CRLF.
        // Extend the buffer until a CRLF is found.

        loop {
            let mut chunk: BytesMut = loop {
                if let Some(c) = ready!(this.stream.as_mut().poll_next(cx)) {
                    let c = c?;
                    if !c.is_empty() {
                        // XXX: Copying data to a newly created `BytesMut` because
                        // `impl From<Bytes> for BytesMut` was removed in `bytes` 0.5.
                        break c[..].into();
                    }
                } else if this.buf.is_empty() {
                    return Poll::Ready(None);
                } else {
                    let ret = mem::replace(this.buf, BytesMut::new()).freeze();
                    return Poll::Ready(Some(Ok(ret)));
                }
            };

            if chunk[0] == b'\n' && this.buf.last() == Some(&b'\r') {
                // Drop the CRLF
                chunk.advance(1);
                let line_len = this.buf.len() - 1;
                this.buf.truncate(line_len);
                return Poll::Ready(Some(Ok(mem::replace(this.buf, chunk).freeze())));
            } else if let Some(line) = remove_first_line(&mut chunk) {
                this.buf.unsplit(line);
                return Poll::Ready(Some(Ok(mem::replace(this.buf, chunk).freeze())));
            } else {
                this.buf.unsplit(chunk);
            }
        }
    }
}

impl<B: Body> HttpBodyAsStream<B> {
    pub fn new(inner: B) -> Self {
        HttpBodyAsStream { inner }
    }
}

impl<B: Body> Stream for HttpBodyAsStream<B> {
    type Item = Result<Bytes, Error<B::Error>>;

    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
        self.project().inner.poll_data(cx).map(|opt| {
            opt.map(|result| result.map(|mut buf| buf.to_bytes()).map_err(Error::Service))
        })
    }
}

pub fn fmt_join<T: Display>(t: &[T], sep: &str, f: &mut Formatter<'_>) -> fmt::Result {
    let mut iter = t.iter();
    if let Some(t) = iter.next() {
        Display::fmt(t, f)?;
        for t in iter {
            write!(f, "{}{}", sep, t)?;
        }
    }
    Ok(())
}

fn remove_first_line(buf: &mut BytesMut) -> Option<BytesMut> {
    if buf.len() < 2 {
        return None;
    }

    if let Some(i) = memchr::memchr(b'\n', &buf[1..]) {
        if buf[i] == b'\r' {
            let mut line = buf.split_to(i + 2);
            line.truncate(i); // Drop the CRLF
            return Some(line);
        }
    }

    None
}

#[cfg(test)]
mod test {
    use super::*;
    use bytes::Bytes;
    use futures::executor::block_on_stream;
    use futures_util::stream;

    #[test]
    fn lines() {
        let body = [
            "abc\r\n",
            "d\r\nefg\r\n",
            "hi",
            "jk",
            "",
            "\r\n",
            "\r\n",
            "lmn\r\nop",
            "q\rrs\r",
            "\n\n\rtuv\r\r\n",
            "wxyz\n",
        ];

        let concat = body.concat();
        let expected = concat.split("\r\n");
        let lines = Lines::new(stream::iter(&body).map(|&c| Ok(Bytes::from_static(c.as_bytes()))));
        let lines = block_on_stream(lines)
            .map(|s: Result<_, Error>| String::from_utf8(s.unwrap().to_vec()).unwrap());

        assert_eq!(lines.collect::<Vec<_>>(), expected.collect::<Vec<_>>());
    }
}
<commit_msg>Refactor `util::Lines`<commit_after>use std::fmt::{self, Display, Formatter};
use std::mem;
use std::pin::Pin;
use std::task::{Context, Poll};

use bytes::{Buf, Bytes, BytesMut};
use futures_util::ready;
use futures_util::stream::{Fuse, IntoStream, Stream, StreamExt, TryStream, TryStreamExt};
use http_body::Body;
use pin_project_lite::pin_project;

use crate::error::Error;

/// Creates an enum with `AsRef<str>` impl.
macro_rules! str_enum {
    (
        $(#[$attr:meta])*
        pub enum $E:ident {
            $(
                $(#[$v_attr:meta])*
                $V:ident = $by:expr
            ),*$(,)?
        }
    ) => {
        $(#[$attr])*
        pub enum $E {
            $(
                $(#[$v_attr])*
                $V,
            )*
        }

        impl std::convert::AsRef<str> for $E {
            fn as_ref(&self) -> &str {
                match *self {
                    $($E::$V => $by,)*
                }
            }
        }
    }
}

pin_project! {
    pub struct Lines<S> {
        #[pin]
        stream: Fuse<IntoStream<S>>,
        buf: Bytes,
    }
}

pin_project! {
    /// Wraps `http_body::Body` to make it a `Stream`.
    pub struct HttpBodyAsStream<B> {
        #[pin]
        pub inner: B,
    }
}

impl<S: TryStream> Lines<S> {
    pub fn new(stream: S) -> Self {
        Lines {
            stream: stream.into_stream().fuse(),
            buf: Bytes::new(),
        }
    }
}

impl<S: TryStream<Ok = Bytes, Error = Error<E>>, E> Stream for Lines<S> {
    type Item = Result<Bytes, Error<E>>;

    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
        let mut this = self.project();

        if let Some(line) = remove_first_line(&mut this.buf) {
            return Poll::Ready(Some(Ok(line)));
        }

        // Now `self.buf` does not have a CRLF.
        // Extend the buffer until a CRLF is found.

        loop {
            let mut chunk = loop {
                if let Some(c) = ready!(this.stream.as_mut().poll_next(cx)?) {
                    if !c.is_empty() {
                        break c;
                    }
                } else if this.buf.is_empty() {
                    return Poll::Ready(None);
                } else {
                    let ret = mem::replace(this.buf, Bytes::new());
                    return Poll::Ready(Some(Ok(ret)));
                }
            };

            if chunk[0] == b'\n' && this.buf.last() == Some(&b'\r') {
                // Drop the CRLF
                this.buf.truncate(this.buf.len() - 1);
                chunk.advance(1);
                return Poll::Ready(Some(Ok(mem::replace(this.buf, chunk))));
            } else if let Some(line) = remove_first_line(&mut chunk) {
                let ret = if this.buf.is_empty() {
                    line
                } else {
                    let mut ret = BytesMut::with_capacity(this.buf.len() + line.len());
                    ret.extend_from_slice(&this.buf);
                    ret.extend_from_slice(&line);
                    ret.freeze()
                };
                *this.buf = chunk;
                return Poll::Ready(Some(Ok(ret)));
            } else {
                *this.buf = if this.buf.is_empty() {
                    chunk
                } else {
                    let mut buf = BytesMut::with_capacity(this.buf.len() + chunk.len());
                    buf.extend_from_slice(&this.buf);
                    buf.extend_from_slice(&chunk);
                    buf.freeze()
                }
            }
        }
    }
}

impl<B: Body> HttpBodyAsStream<B> {
    pub fn new(inner: B) -> Self {
        HttpBodyAsStream { inner }
    }
}

impl<B: Body> Stream for HttpBodyAsStream<B> {
    type Item = Result<Bytes, Error<B::Error>>;

    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
        self.project().inner.poll_data(cx).map(|opt| {
            opt.map(|result| result.map(|mut buf| buf.to_bytes()).map_err(Error::Service))
        })
    }
}

pub fn fmt_join<T: Display>(t: &[T], sep: &str, f: &mut Formatter<'_>) -> fmt::Result {
    let mut iter = t.iter();
    if let Some(t) = iter.next() {
        Display::fmt(t, f)?;
        for t in iter {
            write!(f, "{}{}", sep, t)?;
        }
    }
    Ok(())
}

fn remove_first_line(buf: &mut Bytes) -> Option<Bytes> {
    if buf.len() < 2 {
        return None;
    }

    if let Some(i) = memchr::memchr(b'\n', &buf[1..]) {
        if buf[i] == b'\r' {
            let mut line = buf.split_to(i + 2);
            line.truncate(i); // Drop the CRLF
            return Some(line);
        }
    }

    None
}

#[cfg(test)]
mod test {
    use super::*;
    use bytes::Bytes;
    use futures::executor::block_on_stream;
    use futures_util::stream;

    #[test]
    fn lines() {
        let body = [
            "abc\r\n",
            "d\r\nefg\r\n",
            "hi",
            "jk",
            "",
            "\r\n",
            "\r\n",
            "lmn\r\nop",
            "q\rrs\r",
            "\n\n\rtuv\r\r\n",
            "wxyz\n",
        ];

        let concat = body.concat();
        let expected = concat.split("\r\n");
        let lines = Lines::new(stream::iter(&body).map(|&c| Ok(Bytes::from_static(c.as_bytes()))));
        let lines = block_on_stream(lines)
            .map(|s: Result<_, Error>| String::from_utf8(s.unwrap().to_vec()).unwrap());

        assert_eq!(lines.collect::<Vec<_>>(), expected.collect::<Vec<_>>());
    }
}
<|endoftext|>


<commit_before>const FIXSTR_SIZE   : u8 = 0x1f;
const FIXARRAY_SIZE : u8 = 0x0f;
const FIXMAP_SIZE   : u8 = 0x0f;

#[derive(Clone, Copy, PartialEq, Debug)]
pub enum Marker {
    FixPos(u8),
    FixNeg(i8),
    Null,
    True,
    False,
    U8,
    U16,
    U32,
    U64,
    I8,
    I16,
    I32,
    I64,
    F32,
    F64,
    FixStr(u8),
    Str8,
    Str16,
    Str32,
    Bin8,
    Bin16,
    Bin32,
    FixArray(u8),
    Array16,
    Array32,
    FixMap(u8),
    Map16,
    Map32,
    FixExt1,
    FixExt2,
    FixExt4,
    FixExt8,
    FixExt16,
    Ext8,
    Ext16,
    Ext32,
    Reserved,
}

impl Marker {
    pub fn from_u8(n: u8) -> Marker {
        match n {
            0x00 ... 0x7f => Marker::FixPos(n),
            0xe0 ... 0xff => Marker::FixNeg(n as i8),
            0x80 ... 0x8f => Marker::FixMap(n & FIXMAP_SIZE),
            0x90 ... 0x9f => Marker::FixArray(n & FIXARRAY_SIZE),
            0xa0 ... 0xbf => Marker::FixStr(n & FIXSTR_SIZE),
            0xc0 => Marker::Null,
            /// Marked in MessagePack spec as never used.
            0xc1 => Marker::Reserved,
            0xc2 => Marker::False,
            0xc3 => Marker::True,
            0xc4 => Marker::Bin8,
            0xc5 => Marker::Bin16,
            0xc6 => Marker::Bin32,
            0xc7 => Marker::Ext8,
            0xc8 => Marker::Ext16,
            0xc9 => Marker::Ext32,
            0xca => Marker::F32,
            0xcb => Marker::F64,
            0xcc => Marker::U8,
            0xcd => Marker::U16,
            0xce => Marker::U32,
            0xcf => Marker::U64,
            0xd0 => Marker::I8,
            0xd1 => Marker::I16,
            0xd2 => Marker::I32,
            0xd3 => Marker::I64,
            0xd4 => Marker::FixExt1,
            0xd5 => Marker::FixExt2,
            0xd6 => Marker::FixExt4,
            0xd7 => Marker::FixExt8,
            0xd8 => Marker::FixExt16,
            0xd9 => Marker::Str8,
            0xda => Marker::Str16,
            0xdb => Marker::Str32,
            0xdc => Marker::Array16,
            0xdd => Marker::Array32,
            0xde => Marker::Map16,
            0xdf => Marker::Map32,
            _ => Marker::Reserved,
        }
    }

    pub fn to_u8(&self) -> u8 {
        match *self {
            Marker::FixPos(val)   => val,
            Marker::FixNeg(val)   => val as u8,

            Marker::Null          => 0xc0,

            Marker::True          => 0xc3,
            Marker::False         => 0xc2,

            Marker::U8            => 0xcc,
            Marker::U16           => 0xcd,
            Marker::U32           => 0xce,
            Marker::U64           => 0xcf,

            Marker::I8            => 0xd0,
            Marker::I16           => 0xd1,
            Marker::I32           => 0xd2,
            Marker::I64           => 0xd3,

            Marker::F32           => 0xca,
            Marker::F64           => 0xcb,

            Marker::FixStr(len)   => 0xa0 | (len & FIXSTR_SIZE),
            Marker::Str8          => 0xd9,
            Marker::Str16         => 0xda,
            Marker::Str32         => 0xdb,

            Marker::Bin8          => 0xc4,
            Marker::Bin16         => 0xc5,
            Marker::Bin32         => 0xc6,

            Marker::FixArray(len) => 0x90 | (len & FIXARRAY_SIZE),
            Marker::Array16       => 0xdc,
            Marker::Array32       => 0xdd,

            Marker::FixMap(len)   => 0x80 | (len & FIXMAP_SIZE),
            Marker::Map16         => 0xde,
            Marker::Map32         => 0xdf,

            Marker::FixExt1       => 0xd4,
            Marker::FixExt2       => 0xd5,
            Marker::FixExt4       => 0xd6,
            Marker::FixExt8       => 0xd7,
            Marker::FixExt16      => 0xd8,
            Marker::Ext8          => 0xc7,
            Marker::Ext16         => 0xc8,
            Marker::Ext32         => 0xc9,

            Marker::Reserved      => 0xc1,
        }
    }
}
<commit_msg>Misc: marked unreachable branch.<commit_after>const FIXSTR_SIZE   : u8 = 0x1f;
const FIXARRAY_SIZE : u8 = 0x0f;
const FIXMAP_SIZE   : u8 = 0x0f;

#[derive(Clone, Copy, PartialEq, Debug)]
pub enum Marker {
    FixPos(u8),
    FixNeg(i8),
    Null,
    True,
    False,
    U8,
    U16,
    U32,
    U64,
    I8,
    I16,
    I32,
    I64,
    F32,
    F64,
    FixStr(u8),
    Str8,
    Str16,
    Str32,
    Bin8,
    Bin16,
    Bin32,
    FixArray(u8),
    Array16,
    Array32,
    FixMap(u8),
    Map16,
    Map32,
    FixExt1,
    FixExt2,
    FixExt4,
    FixExt8,
    FixExt16,
    Ext8,
    Ext16,
    Ext32,
    Reserved,
}

impl Marker {
    pub fn from_u8(n: u8) -> Marker {
        match n {
            0x00 ... 0x7f => Marker::FixPos(n),
            0xe0 ... 0xff => Marker::FixNeg(n as i8),
            0x80 ... 0x8f => Marker::FixMap(n & FIXMAP_SIZE),
            0x90 ... 0x9f => Marker::FixArray(n & FIXARRAY_SIZE),
            0xa0 ... 0xbf => Marker::FixStr(n & FIXSTR_SIZE),
            0xc0 => Marker::Null,
            /// Marked in MessagePack spec as never used.
            0xc1 => Marker::Reserved,
            0xc2 => Marker::False,
            0xc3 => Marker::True,
            0xc4 => Marker::Bin8,
            0xc5 => Marker::Bin16,
            0xc6 => Marker::Bin32,
            0xc7 => Marker::Ext8,
            0xc8 => Marker::Ext16,
            0xc9 => Marker::Ext32,
            0xca => Marker::F32,
            0xcb => Marker::F64,
            0xcc => Marker::U8,
            0xcd => Marker::U16,
            0xce => Marker::U32,
            0xcf => Marker::U64,
            0xd0 => Marker::I8,
            0xd1 => Marker::I16,
            0xd2 => Marker::I32,
            0xd3 => Marker::I64,
            0xd4 => Marker::FixExt1,
            0xd5 => Marker::FixExt2,
            0xd6 => Marker::FixExt4,
            0xd7 => Marker::FixExt8,
            0xd8 => Marker::FixExt16,
            0xd9 => Marker::Str8,
            0xda => Marker::Str16,
            0xdb => Marker::Str32,
            0xdc => Marker::Array16,
            0xdd => Marker::Array32,
            0xde => Marker::Map16,
            0xdf => Marker::Map32,
            _ => unreachable!(),
        }
    }

    pub fn to_u8(&self) -> u8 {
        match *self {
            Marker::FixPos(val)   => val,
            Marker::FixNeg(val)   => val as u8,

            Marker::Null          => 0xc0,

            Marker::True          => 0xc3,
            Marker::False         => 0xc2,

            Marker::U8            => 0xcc,
            Marker::U16           => 0xcd,
            Marker::U32           => 0xce,
            Marker::U64           => 0xcf,

            Marker::I8            => 0xd0,
            Marker::I16           => 0xd1,
            Marker::I32           => 0xd2,
            Marker::I64           => 0xd3,

            Marker::F32           => 0xca,
            Marker::F64           => 0xcb,

            Marker::FixStr(len)   => 0xa0 | (len & FIXSTR_SIZE),
            Marker::Str8          => 0xd9,
            Marker::Str16         => 0xda,
            Marker::Str32         => 0xdb,

            Marker::Bin8          => 0xc4,
            Marker::Bin16         => 0xc5,
            Marker::Bin32         => 0xc6,

            Marker::FixArray(len) => 0x90 | (len & FIXARRAY_SIZE),
            Marker::Array16       => 0xdc,
            Marker::Array32       => 0xdd,

            Marker::FixMap(len)   => 0x80 | (len & FIXMAP_SIZE),
            Marker::Map16         => 0xde,
            Marker::Map32         => 0xdf,

            Marker::FixExt1       => 0xd4,
            Marker::FixExt2       => 0xd5,
            Marker::FixExt4       => 0xd6,
            Marker::FixExt8       => 0xd7,
            Marker::FixExt16      => 0xd8,
            Marker::Ext8          => 0xc7,
            Marker::Ext16         => 0xc8,
            Marker::Ext32         => 0xc9,

            Marker::Reserved      => 0xc1,
        }
    }
}
<|endoftext|>


<commit_before>//! Read and unfold a line from a `BufRead`.
//!
//! Individual lines within vCard are delimited by the [RFC5322] line
//! break, which is a CRLF sequence (U+000D followed by U+000A).  Long
//! logical lines of text can be split into a multiple-physical-line
//! representation using the following folding technique.  Content lines
//! SHOULD be folded to a maximum width of 75 octets, excluding the line
//! break.  Multi-octet characters MUST remain contiguous.  The rationale
//! for this folding process can be found in [RFC5322], Section 2.1.1.
//!
//! A logical line MAY be continued on the next physical line anywhere
//! between two characters by inserting a CRLF immediately followed by a
//! single white space character (space (U+0020) or horizontal tab
//! (U+0009)).  The folded line MUST contain at least one character.  Any
//! sequence of CRLF followed immediately by a single white space
//! character is ignored (removed) when processing the content type.
//!
//! # Examples
//!
//! ```toml
//! [dependencies.ical]
//! version = "0.3.*"
//! default-features = false
//! features = ["line-reader"]
//! ```
//!
//! ```rust
//! extern crate ical;
//!
//! use std::io::BufReader;
//! use std::fs::File;
//!
//! let buf = BufReader::new(File::open("./tests/ressources/vcard_input.vcf").unwrap());
//!
//! let reader = ical::LineReader::new(buf);
//!
//! for line in reader {
//!     println!("{}", line);
//! }
//! ```

use std::fmt;
use std::io::BufRead;
use std::iter::Iterator;

/// An unfolded raw line.
///
/// Its inner is only a raw line from the file. No parsing or checking have
/// been made yet.
#[derive(Debug, Clone, Default)]
pub struct Line {
    inner: String,
    number: usize,
}

impl Line {
    /// Return a new `Line` object.
    pub fn new(line: String, line_number: usize) -> Line {
        Line {
            inner: line,
            number: line_number,
        }
    }

    /// Return a `&str`
    pub fn as_str(&self) -> &str {
        self.inner.as_str()
    }

    /// Return the line number.
    pub fn number(&self) -> usize {
        self.number
    }
}

impl fmt::Display for Line {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Line {}: {}", self.number, self.inner)
    }
}

/// A trait generic for implementing line reading use crate::by `PropertyParser`.
pub trait LineRead {
    /// Return the next line unwrapped and formated.
    fn next_line(&mut self) -> Option<Line>;
}

#[derive(Debug, Clone, Default)]
/// Take a `BufRead` and return the unfolded `Line`.
pub struct LineReader<B> {
    reader: B,
    saved: Option<String>,
    number: usize,
}

impl<B: BufRead> LineReader<B> {
    /// Return a new `LineReader` from a `Reader`.
    pub fn new(reader: B) -> LineReader<B> {
        LineReader {
            reader,
            saved: None,
            number: 0,
        }
    }
}

impl<B: BufRead> LineRead for LineReader<B> {
    fn next_line(&mut self) -> Option<Line> {
        let mut next_line = String::new();
        let mut line_number: usize = 0;

        if let Some(start) = self.saved.take() {
            // If during the last iteration a new line have been saved, start with.
            next_line.push_str(start.as_str());
            self.number += 1;
            line_number = self.number;
        } else {
            // This is the first iteration, next_start isn't been filled yet.
            for line in self.reader.by_ref().lines() {
                let line = line.unwrap();
                self.number += 1;

                if !line.is_empty() {
                    next_line = line.trim_end().to_string();
                    line_number = self.number;
                    break;
                }
            }
        }

        for line in self.reader.by_ref().lines() {
            let mut line = line.unwrap();

            if line.is_empty() {
                self.number += 1;
            } else if line.starts_with(' ') || line.starts_with("  ") {
                // This is a multi-lines attribute.

                // Remove the ' ' charactere and join with the current line.
                line.remove(0);
                next_line.push_str(line.trim_end());
                self.number += 1;
            } else {
                // This is a new attribute so it need to be saved it for
                // the next iteration.
                self.saved = Some(line.trim().to_string());
                break;
            }
        }

        if next_line.is_empty() {
            None
        } else {
            Some(Line::new(next_line, line_number))
        }
    }
}

impl<B: BufRead> Iterator for LineReader<B> {
    type Item = Line;

    fn next(&mut self) -> Option<Line> {
        self.next_line()
    }
}
<commit_msg>Parse multiline attributes with tabs correctly<commit_after>//! Read and unfold a line from a `BufRead`.
//!
//! Individual lines within vCard are delimited by the [RFC5322] line
//! break, which is a CRLF sequence (U+000D followed by U+000A).  Long
//! logical lines of text can be split into a multiple-physical-line
//! representation using the following folding technique.  Content lines
//! SHOULD be folded to a maximum width of 75 octets, excluding the line
//! break.  Multi-octet characters MUST remain contiguous.  The rationale
//! for this folding process can be found in [RFC5322], Section 2.1.1.
//!
//! A logical line MAY be continued on the next physical line anywhere
//! between two characters by inserting a CRLF immediately followed by a
//! single white space character (space (U+0020) or horizontal tab
//! (U+0009)).  The folded line MUST contain at least one character.  Any
//! sequence of CRLF followed immediately by a single white space
//! character is ignored (removed) when processing the content type.
//!
//! # Examples
//!
//! ```toml
//! [dependencies.ical]
//! version = "0.3.*"
//! default-features = false
//! features = ["line-reader"]
//! ```
//!
//! ```rust
//! extern crate ical;
//!
//! use std::io::BufReader;
//! use std::fs::File;
//!
//! let buf = BufReader::new(File::open("./tests/ressources/vcard_input.vcf").unwrap());
//!
//! let reader = ical::LineReader::new(buf);
//!
//! for line in reader {
//!     println!("{}", line);
//! }
//! ```

use std::fmt;
use std::io::BufRead;
use std::iter::Iterator;

/// An unfolded raw line.
///
/// Its inner is only a raw line from the file. No parsing or checking have
/// been made yet.
#[derive(Debug, Clone, Default)]
pub struct Line {
    inner: String,
    number: usize,
}

impl Line {
    /// Return a new `Line` object.
    pub fn new(line: String, line_number: usize) -> Line {
        Line {
            inner: line,
            number: line_number,
        }
    }

    /// Return a `&str`
    pub fn as_str(&self) -> &str {
        self.inner.as_str()
    }

    /// Return the line number.
    pub fn number(&self) -> usize {
        self.number
    }
}

impl fmt::Display for Line {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Line {}: {}", self.number, self.inner)
    }
}

/// A trait generic for implementing line reading use crate::by `PropertyParser`.
pub trait LineRead {
    /// Return the next line unwrapped and formated.
    fn next_line(&mut self) -> Option<Line>;
}

#[derive(Debug, Clone, Default)]
/// Take a `BufRead` and return the unfolded `Line`.
pub struct LineReader<B> {
    reader: B,
    saved: Option<String>,
    number: usize,
}

impl<B: BufRead> LineReader<B> {
    /// Return a new `LineReader` from a `Reader`.
    pub fn new(reader: B) -> LineReader<B> {
        LineReader {
            reader,
            saved: None,
            number: 0,
        }
    }
}

impl<B: BufRead> LineRead for LineReader<B> {
    fn next_line(&mut self) -> Option<Line> {
        let mut next_line = String::new();
        let mut line_number: usize = 0;

        if let Some(start) = self.saved.take() {
            // If during the last iteration a new line have been saved, start with.
            next_line.push_str(start.as_str());
            self.number += 1;
            line_number = self.number;
        } else {
            // This is the first iteration, next_start isn't been filled yet.
            for line in self.reader.by_ref().lines() {
                let line = line.unwrap();
                self.number += 1;

                if !line.is_empty() {
                    next_line = line.trim_end().to_string();
                    line_number = self.number;
                    break;
                }
            }
        }

        for line in self.reader.by_ref().lines() {
            let mut line = line.unwrap();

            if line.is_empty() {
                self.number += 1;
            } else if line.starts_with(' ') || line.starts_with('\t') {
                // This is a multi-lines attribute.

                // Remove the whitespace character and join with the current line.
                line.remove(0);
                next_line.push_str(line.trim_end());
                self.number += 1;
            } else {
                // This is a new attribute so it need to be saved it for
                // the next iteration.
                self.saved = Some(line.trim().to_string());
                break;
            }
        }

        if next_line.is_empty() {
            None
        } else {
            Some(Line::new(next_line, line_number))
        }
    }
}

impl<B: BufRead> Iterator for LineReader<B> {
    type Item = Line;

    fn next(&mut self) -> Option<Line> {
        self.next_line()
    }
}
<|endoftext|>


<commit_before>use std::fs::File;
use std::io::BufReader;
use std::collections::HashMap;

use zip::read::{ZipFile, ZipArchive};
use zip::result::ZipError;
use quick_xml::{XmlReader, Event, AsStr};

use {DataType, ExcelReader, Range};
use vba::VbaProject;
use utils;
use errors::*;

/// A struct representing xml zipped excel file
/// Xlsx, Xlsm, Xlam
pub struct Xlsx {
    zip: ZipArchive<File>,
}

impl ExcelReader for Xlsx {

    fn new(f: File) -> Result<Self> {
        Ok(Xlsx { zip: try!(ZipArchive::new(f)) })
    }

    fn has_vba(&mut self) -> bool {
        self.zip.by_name("xl/vbaProject.bin").is_ok()
    }

    fn vba_project(&mut self) -> Result<VbaProject> {
        let mut f = try!(self.zip.by_name("xl/vbaProject.bin"));
        let len = f.size() as usize;
        VbaProject::new(&mut f, len)
    }

    fn read_shared_strings(&mut self) -> Result<Vec<String>> {
        let mut strings = Vec::new();
        match self.zip.by_name("xl/sharedStrings.xml") {
            Ok(f) => {
                let mut xml = XmlReader::from_reader(BufReader::new(f))
                    .with_check(false)
                    .trim_text(false);

                let mut rich_buffer: Option<String> = None;
                while let Some(res_event) = xml.next() {
                    match res_event {
                        Ok(Event::Start(ref e)) if e.name() == b"r" => {
                            if let None = rich_buffer {
                                // use a buffer since richtext has multiples <r> and <t> for the same cell
                                rich_buffer = Some(String::new());
                            }
                        },
                        Ok(Event::End(ref e)) if e.name() == b"si" => {
                            if let Some(s) = rich_buffer {
                                strings.push(s);
                                rich_buffer = None;
                            }
                        },
                        Ok(Event::Start(ref e)) if e.name() == b"t" => {
                            let value = try!(xml.read_text(b"t"));
                            if let Some(ref mut s) = rich_buffer {
                                s.push_str(&value);
                            } else {
                                strings.push(value);
                            }
                        }
                        Err(e) => return Err(e.into()),
                        _ => (),
                    }
                }
            },
            Err(ZipError::FileNotFound) => (),
            Err(e) => return Err(e.into()),
        }
        Ok(strings)
    }

    fn read_sheets_names(&mut self, relationships: &HashMap<Vec<u8>, String>)
        -> Result<HashMap<String, String>>
    {
        let mut sheets = HashMap::new();
        match self.zip.by_name("xl/workbook.xml") {
            Ok(f) => {
                let xml = XmlReader::from_reader(BufReader::new(f))
                    .with_check(false)
                    .trim_text(false);

                for res_event in xml {
                    match res_event {
                        Ok(Event::Start(ref e)) if e.name() == b"sheet" => {
                            let mut name = String::new();
                            let mut path = String::new();
                            for a in e.attributes() {
                                match try!(a) {
                                    (b"name", v) => name = try!(v.as_str()).to_string(),
                                    (b"r:id", v) => path = format!("xl/{}", relationships[v]),
                                    _ => (),
                                }
                            }
                            sheets.insert(name, path);
                        }
                        Err(e) => return Err(e.into()),
                        _ => (),
                    }
                }
            },
            Err(ZipError::FileNotFound) => (),
            Err(e) => return Err(e.into()),
        }
        Ok(sheets)
    }

    fn read_relationships(&mut self) -> Result<HashMap<Vec<u8>, String>> {
        let mut relationships = HashMap::new();
        match self.zip.by_name("xl/_rels/workbook.xml.rels") {
            Ok(f) => {
                let xml = XmlReader::from_reader(BufReader::new(f))
                    .with_check(false)
                    .trim_text(false);

                for res_event in xml {
                    match res_event {
                        Ok(Event::Start(ref e)) if e.name() == b"Relationship" => {
                            let mut id = Vec::new();
                            let mut target = String::new();
                            for a in e.attributes() {
                                match try!(a) {
                                    (b"Id", v) => id.extend_from_slice(v),
                                    (b"Target", v) => target = try!(v.as_str()).to_string(),
                                    _ => (),
                                }
                            }
                            relationships.insert(id, target);
                        }
                        Err(e) => return Err(e.into()),
                        _ => (),
                    }
                }
            },
            Err(ZipError::FileNotFound) => (),
            Err(e) => return Err(e.into()),
        }
        Ok(relationships)
    }

    fn read_worksheet_range(&mut self, path: &str, strings: &[String]) -> Result<Range> {
        let xml = try!(self.zip.by_name(path));
        let mut xml = XmlReader::from_reader(BufReader::new(xml))
            .with_check(false)
            .trim_text(false);
        let mut data = Range::default();
        while let Some(res_event) = xml.next() {
            match res_event {
                Err(e) => return Err(e.into()),
                Ok(Event::Start(ref e)) => {
                    match e.name() {
                        b"dimension" => {
                            let mut dim = None;
                            for a in e.attributes() {
                                if let (b"ref", rdim) = try!(a) {
                                    dim = Some(rdim);
                                    break;
                                }
                            }
                            match dim {
                                None => return Err(format!("Expecting dimension, got {:?}", e).into()),
                                Some(dim) => {
                                    let (position, size) = try!(utils::get_dimension(dim));
                                    data.position = position;
                                    data.size = (size.0 as usize, size.1 as usize);
                                }
                            }
                        },
                        b"sheetData" => {
                            data.inner = try!(read_sheet_data(&mut xml, strings,
                                                              data.size.0 * data.size.1))
                        },
                        _ => (),
                    }
                },
                _ => (),
            }
        }
        data.inner.shrink_to_fit();
        Ok(data)
    }
}

/// read sheetData node
fn read_sheet_data(xml: &mut XmlReader<BufReader<ZipFile>>, strings: &[String], len: usize)
    -> Result<Vec<DataType>>
{
    let mut data = Vec::with_capacity(len);
    while let Some(res_event) = xml.next() {
        match res_event {
            Err(e) => return Err(e.into()),
            Ok(Event::Start(ref c_element)) => {
                if c_element.name() == b"c" {
                    loop {
                        match xml.next() {
                            Some(Err(e)) => return Err(e.into()),
                            Some(Ok(Event::Start(ref e))) => match e.name() {
                                b"v" => {
                                    // value
                                    let v = try!(xml.read_text(b"v"));
                                    let value = match c_element.attributes()
                                        .filter_map(|a| a.ok())
                                        .find(|&(k, _)| k == b"t") {
                                            Some((_, b"s")) => {
                                                // shared string
                                                let idx: usize = try!(v.parse());
                                                DataType::String(strings[idx].clone())
                                            },
                                            Some((_, b"str")) => {
                                                // regular string
                                                DataType::String(v)
                                            },
                                            Some((_, b"b")) => {
                                                // boolean
                                                DataType::Bool(v != "0")
                                            },
                                            Some((_, b"e")) => {
                                                // error
                                                DataType::Error(try!(v.parse()))
                                            },
                                            _ => try!(v.parse().map(DataType::Float)),
                                        };
                                    data.push(value);
                                    break;
                                },
                                b"f" => (), // formula, ignore
                                _name => return Err("not v or f node".into()),
                            },
                            Some(Ok(Event::End(ref e))) => {
                                if e.name() == b"c" {
                                    data.push(DataType::Empty);
                                    break;
                                }
                            }
                            None => return Err("End of xml".into()),
                            _ => (),
                        }
                    }
                }
            },
            Ok(Event::End(ref e)) if e.name() == b"sheetData" => return Ok(data),
            _ => (),
        }
    }
    Err("Could not find </sheetData>".into())
}
<commit_msg>use unescaped attributes for xlsx sheet names<commit_after>use std::fs::File;
use std::io::BufReader;
use std::collections::HashMap;

use zip::read::{ZipFile, ZipArchive};
use zip::result::ZipError;
use quick_xml::{XmlReader, Event, AsStr};

use {DataType, ExcelReader, Range};
use vba::VbaProject;
use utils;
use errors::*;

/// A struct representing xml zipped excel file
/// Xlsx, Xlsm, Xlam
pub struct Xlsx {
    zip: ZipArchive<File>,
}

impl ExcelReader for Xlsx {

    fn new(f: File) -> Result<Self> {
        Ok(Xlsx { zip: try!(ZipArchive::new(f)) })
    }

    fn has_vba(&mut self) -> bool {
        self.zip.by_name("xl/vbaProject.bin").is_ok()
    }

    fn vba_project(&mut self) -> Result<VbaProject> {
        let mut f = try!(self.zip.by_name("xl/vbaProject.bin"));
        let len = f.size() as usize;
        VbaProject::new(&mut f, len)
    }

    fn read_shared_strings(&mut self) -> Result<Vec<String>> {
        let mut strings = Vec::new();
        match self.zip.by_name("xl/sharedStrings.xml") {
            Ok(f) => {
                let mut xml = XmlReader::from_reader(BufReader::new(f))
                    .with_check(false)
                    .trim_text(false);

                let mut rich_buffer: Option<String> = None;
                while let Some(res_event) = xml.next() {
                    match res_event {
                        Ok(Event::Start(ref e)) if e.name() == b"r" => {
                            if let None = rich_buffer {
                                // use a buffer since richtext has multiples <r> and <t> for the same cell
                                rich_buffer = Some(String::new());
                            }
                        },
                        Ok(Event::End(ref e)) if e.name() == b"si" => {
                            if let Some(s) = rich_buffer {
                                strings.push(s);
                                rich_buffer = None;
                            }
                        },
                        Ok(Event::Start(ref e)) if e.name() == b"t" => {
                            let value = try!(xml.read_text(b"t"));
                            if let Some(ref mut s) = rich_buffer {
                                s.push_str(&value);
                            } else {
                                strings.push(value);
                            }
                        }
                        Err(e) => return Err(e.into()),
                        _ => (),
                    }
                }
            },
            Err(ZipError::FileNotFound) => (),
            Err(e) => return Err(e.into()),
        }
        Ok(strings)
    }

    fn read_sheets_names(&mut self, relationships: &HashMap<Vec<u8>, String>)
        -> Result<HashMap<String, String>>
    {
        let mut sheets = HashMap::new();
        match self.zip.by_name("xl/workbook.xml") {
            Ok(f) => {
                let xml = XmlReader::from_reader(BufReader::new(f))
                    .with_check(false)
                    .trim_text(false);

                for res_event in xml {
                    match res_event {
                        Ok(Event::Start(ref e)) if e.name() == b"sheet" => {
                            let mut name = String::new();
                            let mut path = String::new();
                            for a in e.unescaped_attributes() {
                                match try!(a) {
                                    (b"name", v) => name = try!(v.as_str()).to_string(),
                                    (b"r:id", v) => path = format!("xl/{}", relationships[&*v]),
                                    _ => (),
                                }
                            }
                            sheets.insert(name, path);
                        }
                        Err(e) => return Err(e.into()),
                        _ => (),
                    }
                }
            },
            Err(ZipError::FileNotFound) => (),
            Err(e) => return Err(e.into()),
        }
        Ok(sheets)
    }

    fn read_relationships(&mut self) -> Result<HashMap<Vec<u8>, String>> {
        let mut relationships = HashMap::new();
        match self.zip.by_name("xl/_rels/workbook.xml.rels") {
            Ok(f) => {
                let xml = XmlReader::from_reader(BufReader::new(f))
                    .with_check(false)
                    .trim_text(false);

                for res_event in xml {
                    match res_event {
                        Ok(Event::Start(ref e)) if e.name() == b"Relationship" => {
                            let mut id = Vec::new();
                            let mut target = String::new();
                            for a in e.attributes() {
                                match try!(a) {
                                    (b"Id", v) => id.extend_from_slice(v),
                                    (b"Target", v) => target = try!(v.as_str()).to_string(),
                                    _ => (),
                                }
                            }
                            relationships.insert(id, target);
                        }
                        Err(e) => return Err(e.into()),
                        _ => (),
                    }
                }
            },
            Err(ZipError::FileNotFound) => (),
            Err(e) => return Err(e.into()),
        }
        Ok(relationships)
    }

    fn read_worksheet_range(&mut self, path: &str, strings: &[String]) -> Result<Range> {
        let xml = try!(self.zip.by_name(path));
        let mut xml = XmlReader::from_reader(BufReader::new(xml))
            .with_check(false)
            .trim_text(false);
        let mut data = Range::default();
        while let Some(res_event) = xml.next() {
            match res_event {
                Err(e) => return Err(e.into()),
                Ok(Event::Start(ref e)) => {
                    match e.name() {
                        b"dimension" => {
                            let mut dim = None;
                            for a in e.attributes() {
                                if let (b"ref", rdim) = try!(a) {
                                    dim = Some(rdim);
                                    break;
                                }
                            }
                            match dim {
                                None => return Err(format!("Expecting dimension, got {:?}", e).into()),
                                Some(dim) => {
                                    let (position, size) = try!(utils::get_dimension(dim));
                                    data.position = position;
                                    data.size = (size.0 as usize, size.1 as usize);
                                }
                            }
                        },
                        b"sheetData" => {
                            data.inner = try!(read_sheet_data(&mut xml, strings,
                                                              data.size.0 * data.size.1))
                        },
                        _ => (),
                    }
                },
                _ => (),
            }
        }
        data.inner.shrink_to_fit();
        Ok(data)
    }
}

/// read sheetData node
fn read_sheet_data(xml: &mut XmlReader<BufReader<ZipFile>>, strings: &[String], len: usize)
    -> Result<Vec<DataType>>
{
    let mut data = Vec::with_capacity(len);
    while let Some(res_event) = xml.next() {
        match res_event {
            Err(e) => return Err(e.into()),
            Ok(Event::Start(ref c_element)) => {
                if c_element.name() == b"c" {
                    loop {
                        match xml.next() {
                            Some(Err(e)) => return Err(e.into()),
                            Some(Ok(Event::Start(ref e))) => match e.name() {
                                b"v" => {
                                    // value
                                    let v = try!(xml.read_text(b"v"));
                                    let value = match c_element.attributes()
                                        .filter_map(|a| a.ok())
                                        .find(|&(k, _)| k == b"t") {
                                            Some((_, b"s")) => {
                                                // shared string
                                                let idx: usize = try!(v.parse());
                                                DataType::String(strings[idx].clone())
                                            },
                                            Some((_, b"str")) => {
                                                // regular string
                                                DataType::String(v)
                                            },
                                            Some((_, b"b")) => {
                                                // boolean
                                                DataType::Bool(v != "0")
                                            },
                                            Some((_, b"e")) => {
                                                // error
                                                DataType::Error(try!(v.parse()))
                                            },
                                            _ => try!(v.parse().map(DataType::Float)),
                                        };
                                    data.push(value);
                                    break;
                                },
                                b"f" => (), // formula, ignore
                                _name => return Err("not v or f node".into()),
                            },
                            Some(Ok(Event::End(ref e))) => {
                                if e.name() == b"c" {
                                    data.push(DataType::Empty);
                                    break;
                                }
                            }
                            None => return Err("End of xml".into()),
                            _ => (),
                        }
                    }
                }
            },
            Ok(Event::End(ref e)) if e.name() == b"sheetData" => return Ok(data),
            _ => (),
        }
    }
    Err("Could not find </sheetData>".into())
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use compositor_task::{CompositorEventListener, CompositorReceiver};
use compositor_task::{InitialCompositorState, Msg};
use euclid::scale_factor::ScaleFactor;
use euclid::{Point2D, Size2D};
use msg::constellation_msg::AnimationState;
use msg::constellation_msg::Msg as ConstellationMsg;
use msg::constellation_msg::{ConstellationChan, WindowSizeData};
use profile_traits::mem;
use profile_traits::time;
use windowing::WindowEvent;

/// Starts the compositor, which listens for messages on the specified port.
///
/// This is the null compositor which doesn't draw anything to the screen.
/// It's intended for headless testing.
pub struct NullCompositor {
    /// The port on which we receive messages.
    pub port: Box<CompositorReceiver>,
    /// A channel to the constellation.
    constellation_chan: ConstellationChan,
    /// A channel to the time profiler.
    time_profiler_chan: time::ProfilerChan,
    /// A channel to the memory profiler.
    mem_profiler_chan: mem::ProfilerChan,
}

impl NullCompositor {
    fn new(state: InitialCompositorState) -> NullCompositor {
        NullCompositor {
            port: state.receiver,
            constellation_chan: state.constellation_chan,
            time_profiler_chan: state.time_profiler_chan,
            mem_profiler_chan: state.mem_profiler_chan,
        }
    }

    pub fn create(state: InitialCompositorState) -> NullCompositor {
        let compositor = NullCompositor::new(state);

        // Tell the constellation about the initial fake size.
        {
            let ConstellationChan(ref chan) = compositor.constellation_chan;
            chan.send(ConstellationMsg::ResizedWindow(WindowSizeData {
                initial_viewport: Size2D::typed(640_f32, 480_f32),
                visible_viewport: Size2D::typed(640_f32, 480_f32),
                device_pixel_ratio: ScaleFactor::new(1.0),
            })).unwrap();
        }

        compositor
    }
}

impl CompositorEventListener for NullCompositor {
    fn handle_events(&mut self, _: Vec<WindowEvent>) -> bool {
        match self.port.recv_compositor_msg() {
            Msg::Exit(chan) => {
                debug!("shutting down the constellation");
                let ConstellationChan(ref con_chan) = self.constellation_chan;
                con_chan.send(ConstellationMsg::Exit).unwrap();
                chan.send(()).unwrap();
            }

            Msg::ShutdownComplete => {
                debug!("constellation completed shutdown");
                return false
            }

            Msg::GetNativeDisplay(chan) => {
                chan.send(None).unwrap();
            }

            Msg::SetFrameTree(_, response_chan, _) => {
                response_chan.send(()).unwrap();
            }

            Msg::GetClientWindow(send) => {
               let rect = (Size2D::zero(), Point2D::zero());
                send.send(rect).unwrap();
            }

            Msg::ChangeRunningAnimationsState(pipeline_id, animation_state) => {
                match animation_state {
                    AnimationState::AnimationsPresent |
                    AnimationState::NoAnimationsPresent |
                    AnimationState::NoAnimationCallbacksPresent => {}
                    AnimationState::AnimationCallbacksPresent => {
                        let msg = ConstellationMsg::TickAnimation(pipeline_id);
                        self.constellation_chan.0.send(msg).unwrap()
                    }
                }
            }

            // Explicitly list ignored messages so that when we add a new one,
            // we'll notice and think about whether it needs a response, like
            // SetFrameTree.

            Msg::InitializeLayersForPipeline(..) |
            Msg::AssignPaintedBuffers(..) |
            Msg::ScrollFragmentPoint(..) |
            Msg::Status(..) |
            Msg::LoadStart(..) |
            Msg::LoadComplete(..) |
            Msg::ScrollTimeout(..) |
            Msg::RecompositeAfterScroll |
            Msg::ChangePageTitle(..) |
            Msg::ChangePageUrl(..) |
            Msg::KeyEvent(..) |
            Msg::SetCursor(..) |
            Msg::ViewportConstrained(..) => {}
            Msg::CreatePng(..) |
            Msg::PaintTaskExited(..) |
            Msg::MoveTo(..) |
            Msg::ResizeTo(..) |
            Msg::IsReadyToSaveImageReply(..) => {}
            Msg::NewFavicon(..) => {}
            Msg::HeadParsed => {}
            Msg::ReturnUnusedNativeSurfaces(..) => {}
            Msg::CollectMemoryReports(..) => {}
            Msg::PipelineExited(..) => {}
            Msg::CreateLayerForSubpage(..) => {}
        }
        true
    }

    fn repaint_synchronously(&mut self) {}

    fn shutdown(&mut self) {
        // Drain compositor port, sometimes messages contain channels that are blocking
        // another task from finishing (i.e. SetIds)
        while self.port.try_recv_compositor_msg().is_some() {}

        self.time_profiler_chan.send(time::ProfilerMsg::Exit);
        self.mem_profiler_chan.send(mem::ProfilerMsg::Exit);
    }

    fn pinch_zoom_level(&self) -> f32 {
        1.0
    }

    fn title_for_main_frame(&self) {}
}
<commit_msg>Use same default window size in headless compositor<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use compositor_task::{CompositorEventListener, CompositorReceiver};
use compositor_task::{InitialCompositorState, Msg};
use euclid::scale_factor::ScaleFactor;
use euclid::{Point2D, Size2D};
use msg::constellation_msg::AnimationState;
use msg::constellation_msg::Msg as ConstellationMsg;
use msg::constellation_msg::{ConstellationChan, WindowSizeData};
use profile_traits::mem;
use profile_traits::time;
use util::opts;
use windowing::WindowEvent;

/// Starts the compositor, which listens for messages on the specified port.
///
/// This is the null compositor which doesn't draw anything to the screen.
/// It's intended for headless testing.
pub struct NullCompositor {
    /// The port on which we receive messages.
    pub port: Box<CompositorReceiver>,
    /// A channel to the constellation.
    constellation_chan: ConstellationChan,
    /// A channel to the time profiler.
    time_profiler_chan: time::ProfilerChan,
    /// A channel to the memory profiler.
    mem_profiler_chan: mem::ProfilerChan,
}

impl NullCompositor {
    fn new(state: InitialCompositorState) -> NullCompositor {
        NullCompositor {
            port: state.receiver,
            constellation_chan: state.constellation_chan,
            time_profiler_chan: state.time_profiler_chan,
            mem_profiler_chan: state.mem_profiler_chan,
        }
    }

    pub fn create(state: InitialCompositorState) -> NullCompositor {
        let compositor = NullCompositor::new(state);

        // Tell the constellation about the initial fake size.
        {
            let ConstellationChan(ref chan) = compositor.constellation_chan;
            chan.send(ConstellationMsg::ResizedWindow(WindowSizeData {
                initial_viewport: Size2D::typed(800_f32, 600_f32),
                visible_viewport: Size2D::typed(800_f32, 600_f32),
                device_pixel_ratio:
                    ScaleFactor::new(opts::get().device_pixels_per_px.unwrap_or(1.0)),
            })).unwrap();
        }

        compositor
    }
}

impl CompositorEventListener for NullCompositor {
    fn handle_events(&mut self, _: Vec<WindowEvent>) -> bool {
        match self.port.recv_compositor_msg() {
            Msg::Exit(chan) => {
                debug!("shutting down the constellation");
                let ConstellationChan(ref con_chan) = self.constellation_chan;
                con_chan.send(ConstellationMsg::Exit).unwrap();
                chan.send(()).unwrap();
            }

            Msg::ShutdownComplete => {
                debug!("constellation completed shutdown");
                return false
            }

            Msg::GetNativeDisplay(chan) => {
                chan.send(None).unwrap();
            }

            Msg::SetFrameTree(_, response_chan, _) => {
                response_chan.send(()).unwrap();
            }

            Msg::GetClientWindow(send) => {
               let rect = (Size2D::zero(), Point2D::zero());
                send.send(rect).unwrap();
            }

            Msg::ChangeRunningAnimationsState(pipeline_id, animation_state) => {
                match animation_state {
                    AnimationState::AnimationsPresent |
                    AnimationState::NoAnimationsPresent |
                    AnimationState::NoAnimationCallbacksPresent => {}
                    AnimationState::AnimationCallbacksPresent => {
                        let msg = ConstellationMsg::TickAnimation(pipeline_id);
                        self.constellation_chan.0.send(msg).unwrap()
                    }
                }
            }

            // Explicitly list ignored messages so that when we add a new one,
            // we'll notice and think about whether it needs a response, like
            // SetFrameTree.

            Msg::InitializeLayersForPipeline(..) |
            Msg::AssignPaintedBuffers(..) |
            Msg::ScrollFragmentPoint(..) |
            Msg::Status(..) |
            Msg::LoadStart(..) |
            Msg::LoadComplete(..) |
            Msg::ScrollTimeout(..) |
            Msg::RecompositeAfterScroll |
            Msg::ChangePageTitle(..) |
            Msg::ChangePageUrl(..) |
            Msg::KeyEvent(..) |
            Msg::SetCursor(..) |
            Msg::ViewportConstrained(..) => {}
            Msg::CreatePng(..) |
            Msg::PaintTaskExited(..) |
            Msg::MoveTo(..) |
            Msg::ResizeTo(..) |
            Msg::IsReadyToSaveImageReply(..) => {}
            Msg::NewFavicon(..) => {}
            Msg::HeadParsed => {}
            Msg::ReturnUnusedNativeSurfaces(..) => {}
            Msg::CollectMemoryReports(..) => {}
            Msg::PipelineExited(..) => {}
            Msg::CreateLayerForSubpage(..) => {}
        }
        true
    }

    fn repaint_synchronously(&mut self) {}

    fn shutdown(&mut self) {
        // Drain compositor port, sometimes messages contain channels that are blocking
        // another task from finishing (i.e. SetIds)
        while self.port.try_recv_compositor_msg().is_some() {}

        self.time_profiler_chan.send(time::ProfilerMsg::Exit);
        self.mem_profiler_chan.send(mem::ProfilerMsg::Exit);
    }

    fn pinch_zoom_level(&self) -> f32 {
        1.0
    }

    fn title_for_main_frame(&self) {}
}
<|endoftext|>


<commit_before>// Copyright:: Copyright (c) 2015-2016 The Habitat Maintainers
//
// The terms of the Evaluation Agreement (Habitat) between Chef Software Inc.
// and the party accessing this file ("Licensee") apply to Licensee's use of
// the Software until such time that the Software is made available under an
// open source license such as the Apache 2.0 License.

use std::fs::File;
use std::io::{BufReader, Read};
use std::mem;
use std::path::Path;
use std::ptr;

use libsodium_sys;
use rustc_serialize::hex::ToHex;

use error::Result;

const BUF_SIZE: usize = 1024;

/// Calculate the BLAKE2b hash of a file, return as a hex string
/// digest size = 32 BYTES
/// NOTE: the hashing is keyless
pub fn hash_file<P: AsRef<Path>>(filename: &P) -> Result<String> {
    let file = try!(File::open(filename.as_ref()));
    let mut reader = BufReader::new(file);
    hash_reader(&mut reader)
}

pub fn hash_string(data: &str) -> Result<String> {
    let mut out = [0u8; libsodium_sys::crypto_generichash_BYTES];
    let mut st = vec![0u8; (unsafe { libsodium_sys::crypto_generichash_statebytes() })];
    let pst = unsafe {
        mem::transmute::<*mut u8, *mut libsodium_sys::crypto_generichash_state>(st.as_mut_ptr())
    };
    unsafe {
        libsodium_sys::crypto_generichash_init(pst, ptr::null_mut(), 0, out.len());
        libsodium_sys::crypto_generichash_update(pst, data[..].as_ptr(), data.len() as u64);
        libsodium_sys::crypto_generichash_final(pst, out.as_mut_ptr(), out.len());
    }
    Ok(out.to_hex())
}

pub fn hash_reader(reader: &mut BufReader<File>) -> Result<String> {
    let mut out = [0u8; libsodium_sys::crypto_generichash_BYTES];
    let mut st = vec![0u8; (unsafe { libsodium_sys::crypto_generichash_statebytes() })];
    let pst = unsafe {
        mem::transmute::<*mut u8, *mut libsodium_sys::crypto_generichash_state>(st.as_mut_ptr())
    };

    unsafe {
        libsodium_sys::crypto_generichash_init(pst, ptr::null_mut(), 0, out.len());
    }

    let mut buf = [0u8; BUF_SIZE];
    loop {
        let bytes_read = try!(reader.read(&mut buf));
        if bytes_read == 0 {
            break;
        }
        let chunk = &buf[0..bytes_read];
        unsafe {
            libsodium_sys::crypto_generichash_update(pst, chunk.as_ptr(), chunk.len() as u64);
        }
    }
    unsafe {
        libsodium_sys::crypto_generichash_final(pst, out.as_mut_ptr(), out.len());
    }
    Ok(out.to_hex())
}

#[cfg(test)]
mod test {
    use std::env;
    #[allow(unused_imports)]
    use std::fs::{self, File};
    #[allow(unused_imports)]
    use std::io;
    use std::path::PathBuf;

    #[cfg(feature = "functional")]
    use hyper::{header, Client, Url};
    use super::*;
    use super::super::test_support::*;

    #[allow(dead_code)]
    fn mk_local_tmpdir() -> PathBuf {
        let dir = env::current_exe()
            .unwrap()
            .parent()
            .unwrap()
            .parent()
            .unwrap()
            .parent()
            .unwrap()
            .parent()
            .unwrap()
            .parent()
            .unwrap()
            .join("tmp");
        fs::create_dir_all(&dir).unwrap();
        dir
    }

    #[test]
    fn hash_file_working() {
        // The expected values were computed using the `b2sum` program from
        // https://github.com/dchest/b2sum using the the `-s=32` option. For example:
        //      b2sum -s=32 signme.dat

        let computed = hash_file(&fixture("signme.dat")).unwrap();
        let expected = "20590a52c4f00588c500328b16d466c982a26fabaa5fa4dcc83052dd0a84f233";
        assert_eq!(computed, expected);

        let computed = hash_file(&fixture("happyhumans-20160424223347.sig.key")).unwrap();
        let expected = "e966844bbc50b7a3a6d81e94dd38e27b92814b944095a8e55f1780921bfcfbe1";
        assert_eq!(computed, expected);

        let computed = hash_file(&fixture("happyhumans-20160424223347.pub")).unwrap();
        let expected = "b80c4f412f9a0a7727b6e6f115e1b5fa3bae79ad2fcf47f769ed4e42cfb12265";
        assert_eq!(computed, expected);
    }

    #[test]
    #[cfg(feature = "functional")]
    fn hash_file_large_binary() {
        let url = "http://ftp.kernel.org/pub/linux/kernel/v4.x/linux-4.3.tar.gz";
        let dst: PathBuf = {
            let file = mk_local_tmpdir().join(url.rsplit('/').next().unwrap());
            if !file.is_file() {
                let client = match env::var("http_proxy") {
                    Ok(url) => {
                        let url = Url::parse(&url).unwrap();
                        Client::with_http_proxy(url.host_str().unwrap().to_string(),
                                                url.port_or_known_default().unwrap())
                    }
                    _ => Client::new(),
                };
                let mut response = client.get(url)
                    .header(header::Connection::close())
                    .send()
                    .unwrap();
                let mut f = File::create(&file).unwrap();
                io::copy(&mut response, &mut f).unwrap();
            }
            file
        };
        let computed = hash_file(&dst).unwrap();
        let expected = "ba640dc063f0ed27e60b38dbb7cf19778cf7805d9fc91eb129fb68b409d46414";
        assert_eq!(computed, expected);
    }
}
<commit_msg>[rustfmt] Spacing fixes.<commit_after>// Copyright:: Copyright (c) 2015-2016 The Habitat Maintainers
//
// The terms of the Evaluation Agreement (Habitat) between Chef Software Inc.
// and the party accessing this file ("Licensee") apply to Licensee's use of
// the Software until such time that the Software is made available under an
// open source license such as the Apache 2.0 License.

use std::fs::File;
use std::io::{BufReader, Read};
use std::mem;
use std::path::Path;
use std::ptr;

use libsodium_sys;
use rustc_serialize::hex::ToHex;

use error::Result;

const BUF_SIZE: usize = 1024;

/// Calculate the BLAKE2b hash of a file, return as a hex string
/// digest size = 32 BYTES
/// NOTE: the hashing is keyless
pub fn hash_file<P: AsRef<Path>>(filename: &P) -> Result<String> {
    let file = try!(File::open(filename.as_ref()));
    let mut reader = BufReader::new(file);
    hash_reader(&mut reader)
}

pub fn hash_string(data: &str) -> Result<String> {
    let mut out = [0u8; libsodium_sys::crypto_generichash_BYTES];
    let mut st = vec![0u8; (unsafe { libsodium_sys::crypto_generichash_statebytes() })];
    let pst = unsafe {
        mem::transmute::<*mut u8, *mut libsodium_sys::crypto_generichash_state>(st.as_mut_ptr())
    };
    unsafe {
        libsodium_sys::crypto_generichash_init(pst, ptr::null_mut(), 0, out.len());
        libsodium_sys::crypto_generichash_update(pst, data[..].as_ptr(), data.len() as u64);
        libsodium_sys::crypto_generichash_final(pst, out.as_mut_ptr(), out.len());
    }
    Ok(out.to_hex())
}

pub fn hash_reader(reader: &mut BufReader<File>) -> Result<String> {
    let mut out = [0u8; libsodium_sys::crypto_generichash_BYTES];
    let mut st = vec![0u8; (unsafe { libsodium_sys::crypto_generichash_statebytes() })];
    let pst = unsafe {
        mem::transmute::<*mut u8, *mut libsodium_sys::crypto_generichash_state>(st.as_mut_ptr())
    };

    unsafe {
        libsodium_sys::crypto_generichash_init(pst, ptr::null_mut(), 0, out.len());
    }

    let mut buf = [0u8; BUF_SIZE];
    loop {
        let bytes_read = try!(reader.read(&mut buf));
        if bytes_read == 0 {
            break;
        }
        let chunk = &buf[0..bytes_read];
        unsafe {
            libsodium_sys::crypto_generichash_update(pst, chunk.as_ptr(), chunk.len() as u64);
        }
    }
    unsafe {
        libsodium_sys::crypto_generichash_final(pst, out.as_mut_ptr(), out.len());
    }
    Ok(out.to_hex())
}

#[cfg(test)]
mod test {
    use std::env;
    #[allow(unused_imports)]
    use std::fs::{self, File};
    #[allow(unused_imports)]
    use std::io;
    use std::path::PathBuf;

    #[cfg(feature = "functional")]
    use hyper::{header, Client, Url};
    use super::*;
    use super::super::test_support::*;

    #[allow(dead_code)]
    fn mk_local_tmpdir() -> PathBuf {
        let dir = env::current_exe()
                      .unwrap()
                      .parent()
                      .unwrap()
                      .parent()
                      .unwrap()
                      .parent()
                      .unwrap()
                      .parent()
                      .unwrap()
                      .parent()
                      .unwrap()
                      .join("tmp");
        fs::create_dir_all(&dir).unwrap();
        dir
    }

    #[test]
    fn hash_file_working() {
        // The expected values were computed using the `b2sum` program from
        // https://github.com/dchest/b2sum using the the `-s=32` option. For example:
        //      b2sum -s=32 signme.dat

        let computed = hash_file(&fixture("signme.dat")).unwrap();
        let expected = "20590a52c4f00588c500328b16d466c982a26fabaa5fa4dcc83052dd0a84f233";
        assert_eq!(computed, expected);

        let computed = hash_file(&fixture("happyhumans-20160424223347.sig.key")).unwrap();
        let expected = "e966844bbc50b7a3a6d81e94dd38e27b92814b944095a8e55f1780921bfcfbe1";
        assert_eq!(computed, expected);

        let computed = hash_file(&fixture("happyhumans-20160424223347.pub")).unwrap();
        let expected = "b80c4f412f9a0a7727b6e6f115e1b5fa3bae79ad2fcf47f769ed4e42cfb12265";
        assert_eq!(computed, expected);
    }

    #[test]
    #[cfg(feature = "functional")]
    fn hash_file_large_binary() {
        let url = "http://ftp.kernel.org/pub/linux/kernel/v4.x/linux-4.3.tar.gz";
        let dst: PathBuf = {
            let file = mk_local_tmpdir().join(url.rsplit('/').next().unwrap());
            if !file.is_file() {
                let client = match env::var("http_proxy") {
                    Ok(url) => {
                        let url = Url::parse(&url).unwrap();
                        Client::with_http_proxy(url.host_str().unwrap().to_string(),
                                                url.port_or_known_default().unwrap())
                    }
                    _ => Client::new(),
                };
                let mut response = client.get(url)
                                         .header(header::Connection::close())
                                         .send()
                                         .unwrap();
                let mut f = File::create(&file).unwrap();
                io::copy(&mut response, &mut f).unwrap();
            }
            file
        };
        let computed = hash_file(&dst).unwrap();
        let expected = "ba640dc063f0ed27e60b38dbb7cf19778cf7805d9fc91eb129fb68b409d46414";
        assert_eq!(computed, expected);
    }
}
<|endoftext|>


<commit_before>use crate::cli::valid_fully_qualified_ident;
use configopt::{self,
                ConfigOpt};
use habitat_common::types::ListenCtlAddr;
use habitat_core::{crypto::CACHE_KEY_PATH_ENV_VAR,
                   fs as hab_core_fs,
                   package::PackageIdent};
use lazy_static::lazy_static;
use std::{fmt,
          io,
          net::{SocketAddr,
                ToSocketAddrs},
          num::ParseIntError,
          path::PathBuf,
          str::FromStr,
          time::Duration};
use structopt::StructOpt;
use url::Url;

#[derive(ConfigOpt, StructOpt)]
#[structopt(no_version)]
#[allow(dead_code)]
pub struct AuthToken {
    /// Authentication token for Builder
    #[structopt(name = "AUTH_TOKEN", short = "z", long = "auth")]
    auth_token: Option<String>,
}

#[derive(ConfigOpt, StructOpt, Deserialize)]
#[structopt(no_version)]
#[allow(dead_code)]
pub struct BldrUrl {
    /// Specify an alternate Builder endpoint. If not specified, the value will be taken from
    /// the HAB_BLDR_URL environment variable if defined. (default: https://bldr.habitat.sh)
    // TODO (DM): This should probably use `env` and `default_value`
    #[structopt(name = "BLDR_URL", short = "u", long = "url")]
    bldr_url: Option<Url>,
}

lazy_static! {
    pub static ref CACHE_KEY_PATH_DEFAULT: String =
        hab_core_fs::CACHE_KEY_PATH.to_string_lossy().to_string();
}

#[derive(ConfigOpt, StructOpt, Debug, Deserialize)]
#[configopt(derive(Debug), attrs(serde))]
#[serde(deny_unknown_fields)]
#[structopt(no_version, rename_all = "screamingsnake")]
pub struct CacheKeyPath {
    /// Cache for creating and searching for encryption keys
    #[structopt(long = "cache-key-path",
                env = CACHE_KEY_PATH_ENV_VAR,
                default_value = &*CACHE_KEY_PATH_DEFAULT)]
    pub cache_key_path: PathBuf,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(try_from = "&str", into = "String")]
struct PkgIdentStringySerde(PackageIdent);

impl FromStr for PkgIdentStringySerde {
    type Err = habitat_core::Error;

    fn from_str(s: &str) -> Result<Self, Self::Err> { Ok(Self(s.parse()?)) }
}

impl std::convert::TryFrom<&str> for PkgIdentStringySerde {
    type Error = habitat_core::Error;

    fn try_from(s: &str) -> Result<Self, Self::Error> { Self::from_str(s) }
}

impl std::fmt::Display for PkgIdentStringySerde {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{}", self.0) }
}

impl From<PkgIdentStringySerde> for String {
    fn from(pkg_ident: PkgIdentStringySerde) -> Self { pkg_ident.to_string() }
}

#[derive(Clone, ConfigOpt, Debug, StructOpt, Deserialize, Serialize)]
#[configopt(derive(Clone, Serialize, Debug), attrs(serde))]
#[structopt(no_version)]
#[serde(transparent)]
pub struct PkgIdent {
    /// A package identifier (ex: core/redis, core/busybox-static/1.42.2)
    #[structopt(name = "PKG_IDENT")]
    pkg_ident: PkgIdentStringySerde,
}

impl PkgIdent {
    pub fn pkg_ident(self) -> PackageIdent { self.pkg_ident.0 }
}

#[derive(ConfigOpt, StructOpt)]
#[structopt(no_version)]
#[allow(dead_code)]
pub struct FullyQualifiedPkgIdent {
    /// A fully qualified package identifier (ex: core/busybox-static/1.42.2/20170513215502)
    #[structopt(name = "PKG_IDENT", validator = valid_fully_qualified_ident)]
    pkg_ident: PackageIdent,
}

#[derive(ConfigOpt, StructOpt, Deserialize, Debug)]
#[configopt(derive(Clone, Debug))]
#[structopt(no_version)]
pub struct RemoteSup {
    /// Address to a remote Supervisor's Control Gateway
    #[structopt(name = "REMOTE_SUP",
                long = "remote-sup",
                short = "r",
                default_value = ListenCtlAddr::default_as_str(),
                parse(try_from_str = ListenCtlAddr::resolve_listen_ctl_addr))]
    #[serde(default)]
    remote_sup: ListenCtlAddr,
}

impl RemoteSup {
    pub fn to_listen_ctl_addr(&self) -> ListenCtlAddr { self.remote_sup }
}

pub fn socket_addr_with_default_port<S: AsRef<str>>(addr: S,
                                                    default_port: u16)
                                                    -> io::Result<SocketAddr> {
    let addr = addr.as_ref();
    let mut iter = if addr.find(':').is_some() {
        addr.to_socket_addrs()
    } else {
        (addr, default_port).to_socket_addrs()
    }?;
    // We expect exactly one address
    iter.next().ok_or_else(|| {
                   io::Error::new(io::ErrorKind::InvalidInput,
                                  "input did not resolve to SocketAddr or error")
               })
}

pub fn socket_addrs_with_default_port<I>(addrs: I, default_port: u16) -> io::Result<Vec<SocketAddr>>
    where I: IntoIterator,
          I::Item: AsRef<str>
{
    addrs.into_iter()
         .map(|a| socket_addr_with_default_port(a, default_port))
         .collect()
}

#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(from = "u64", into = "u64")]
pub struct DurationProxy(Duration);

impl From<DurationProxy> for u64 {
    fn from(d: DurationProxy) -> Self { d.0.as_secs() }
}

impl From<u64> for DurationProxy {
    fn from(n: u64) -> Self { Self(Duration::from_secs(n)) }
}

impl From<DurationProxy> for Duration {
    fn from(d: DurationProxy) -> Self { d.0 }
}

impl From<Duration> for DurationProxy {
    fn from(d: Duration) -> Self { Self(d) }
}

impl FromStr for DurationProxy {
    type Err = ParseIntError;

    fn from_str(s: &str) -> Result<Self, Self::Err> { Ok(s.parse::<u64>()?.into()) }
}

impl fmt::Display for DurationProxy {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", u64::from(*self)) }
}

#[cfg(test)]
mod test {
    use super::{socket_addr_with_default_port,
                socket_addrs_with_default_port};

    #[test]
    fn test_socket_addrs_with_default_port() {
        assert_eq!(socket_addr_with_default_port("127.0.0.1", 89).unwrap(),
                   "127.0.0.1:89".parse().expect(""));
        assert_eq!(socket_addr_with_default_port("1.2.3.4:1500", 89).unwrap(),
                   "1.2.3.4:1500".parse().expect(""));
        assert!(socket_addr_with_default_port("an_invalid_address", 89).is_err());

        let expected = vec!["1.2.3.4:1500".parse().expect(""),
                            "0.0.0.0:5567".parse().expect(""),
                            "127.0.0.1:5567".parse().expect("")];
        assert_eq!(socket_addrs_with_default_port(&["1.2.3.4:1500", "0.0.0.0", "127.0.0.1"], 5567).unwrap(),
                   expected);
        assert!(socket_addrs_with_default_port(&["1.2.3.4:1500",
                                                 "0.0.0.0",
                                                 "an_error",
                                                 "127.0.0.1"],
                                               5567).is_err(),);
    }
}
<commit_msg>[REFACTOR] Remove unused socket_addrs_with_default_port function<commit_after>use crate::cli::valid_fully_qualified_ident;
use configopt::{self,
                ConfigOpt};
use habitat_common::types::ListenCtlAddr;
use habitat_core::{crypto::CACHE_KEY_PATH_ENV_VAR,
                   fs as hab_core_fs,
                   package::PackageIdent};
use lazy_static::lazy_static;
use std::{fmt,
          io,
          net::{SocketAddr,
                ToSocketAddrs},
          num::ParseIntError,
          path::PathBuf,
          str::FromStr,
          time::Duration};
use structopt::StructOpt;
use url::Url;

#[derive(ConfigOpt, StructOpt)]
#[structopt(no_version)]
#[allow(dead_code)]
pub struct AuthToken {
    /// Authentication token for Builder
    #[structopt(name = "AUTH_TOKEN", short = "z", long = "auth")]
    auth_token: Option<String>,
}

#[derive(ConfigOpt, StructOpt, Deserialize)]
#[structopt(no_version)]
#[allow(dead_code)]
pub struct BldrUrl {
    /// Specify an alternate Builder endpoint. If not specified, the value will be taken from
    /// the HAB_BLDR_URL environment variable if defined. (default: https://bldr.habitat.sh)
    // TODO (DM): This should probably use `env` and `default_value`
    #[structopt(name = "BLDR_URL", short = "u", long = "url")]
    bldr_url: Option<Url>,
}

lazy_static! {
    pub static ref CACHE_KEY_PATH_DEFAULT: String =
        hab_core_fs::CACHE_KEY_PATH.to_string_lossy().to_string();
}

#[derive(ConfigOpt, StructOpt, Debug, Deserialize)]
#[configopt(derive(Debug), attrs(serde))]
#[serde(deny_unknown_fields)]
#[structopt(no_version, rename_all = "screamingsnake")]
pub struct CacheKeyPath {
    /// Cache for creating and searching for encryption keys
    #[structopt(long = "cache-key-path",
                env = CACHE_KEY_PATH_ENV_VAR,
                default_value = &*CACHE_KEY_PATH_DEFAULT)]
    pub cache_key_path: PathBuf,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(try_from = "&str", into = "String")]
struct PkgIdentStringySerde(PackageIdent);

impl FromStr for PkgIdentStringySerde {
    type Err = habitat_core::Error;

    fn from_str(s: &str) -> Result<Self, Self::Err> { Ok(Self(s.parse()?)) }
}

impl std::convert::TryFrom<&str> for PkgIdentStringySerde {
    type Error = habitat_core::Error;

    fn try_from(s: &str) -> Result<Self, Self::Error> { Self::from_str(s) }
}

impl std::fmt::Display for PkgIdentStringySerde {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{}", self.0) }
}

impl From<PkgIdentStringySerde> for String {
    fn from(pkg_ident: PkgIdentStringySerde) -> Self { pkg_ident.to_string() }
}

#[derive(Clone, ConfigOpt, Debug, StructOpt, Deserialize, Serialize)]
#[configopt(derive(Clone, Serialize, Debug), attrs(serde))]
#[structopt(no_version)]
#[serde(transparent)]
pub struct PkgIdent {
    /// A package identifier (ex: core/redis, core/busybox-static/1.42.2)
    #[structopt(name = "PKG_IDENT")]
    pkg_ident: PkgIdentStringySerde,
}

impl PkgIdent {
    pub fn pkg_ident(self) -> PackageIdent { self.pkg_ident.0 }
}

#[derive(ConfigOpt, StructOpt)]
#[structopt(no_version)]
#[allow(dead_code)]
pub struct FullyQualifiedPkgIdent {
    /// A fully qualified package identifier (ex: core/busybox-static/1.42.2/20170513215502)
    #[structopt(name = "PKG_IDENT", validator = valid_fully_qualified_ident)]
    pkg_ident: PackageIdent,
}

#[derive(ConfigOpt, StructOpt, Deserialize, Debug)]
#[configopt(derive(Clone, Debug))]
#[structopt(no_version)]
pub struct RemoteSup {
    /// Address to a remote Supervisor's Control Gateway
    #[structopt(name = "REMOTE_SUP",
                long = "remote-sup",
                short = "r",
                default_value = ListenCtlAddr::default_as_str(),
                parse(try_from_str = ListenCtlAddr::resolve_listen_ctl_addr))]
    #[serde(default)]
    remote_sup: ListenCtlAddr,
}

impl RemoteSup {
    pub fn to_listen_ctl_addr(&self) -> ListenCtlAddr { self.remote_sup }
}

pub fn socket_addr_with_default_port<S: AsRef<str>>(addr: S,
                                                    default_port: u16)
                                                    -> io::Result<SocketAddr> {
    let addr = addr.as_ref();
    let mut iter = if addr.find(':').is_some() {
        addr.to_socket_addrs()
    } else {
        (addr, default_port).to_socket_addrs()
    }?;
    // We expect exactly one address
    iter.next().ok_or_else(|| {
                   io::Error::new(io::ErrorKind::InvalidInput,
                                  "input did not resolve to SocketAddr or error")
               })
}

#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(from = "u64", into = "u64")]
pub struct DurationProxy(Duration);

impl From<DurationProxy> for u64 {
    fn from(d: DurationProxy) -> Self { d.0.as_secs() }
}

impl From<u64> for DurationProxy {
    fn from(n: u64) -> Self { Self(Duration::from_secs(n)) }
}

impl From<DurationProxy> for Duration {
    fn from(d: DurationProxy) -> Self { d.0 }
}

impl From<Duration> for DurationProxy {
    fn from(d: Duration) -> Self { Self(d) }
}

impl FromStr for DurationProxy {
    type Err = ParseIntError;

    fn from_str(s: &str) -> Result<Self, Self::Err> { Ok(s.parse::<u64>()?.into()) }
}

impl fmt::Display for DurationProxy {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", u64::from(*self)) }
}

#[cfg(test)]
mod test {
    use super::socket_addr_with_default_port;

    #[test]
    fn test_socket_addr_with_default_port() {
        assert_eq!(socket_addr_with_default_port("127.0.0.1", 89).unwrap(),
                   "127.0.0.1:89".parse().expect(""));
        assert_eq!(socket_addr_with_default_port("1.2.3.4:1500", 89).unwrap(),
                   "1.2.3.4:1500".parse().expect(""));
        assert!(socket_addr_with_default_port("an_invalid_address", 89).is_err());
    }
}
<|endoftext|>