<commit_before>use super::headers;
use crate::event::SpecVersion;
use crate::message::{
    BinaryDeserializer, BinarySerializer, Encoding, MessageAttributeValue, MessageDeserializer,
    Result, StructuredDeserializer, StructuredSerializer,
};
use crate::{message, Event};
use actix_web::http::HeaderName;
use actix_web::web::{Bytes, BytesMut};
use actix_web::{web, HttpMessage, HttpRequest};
use async_trait::async_trait;
use futures::future::LocalBoxFuture;
use futures::{FutureExt, StreamExt};
use std::convert::TryFrom;

/// Wrapper for [`HttpRequest`] that implements [`MessageDeserializer`] trait.
pub struct HttpRequestDeserializer<'a> {
    req: &'a HttpRequest,
    body: Bytes,
}

impl HttpRequestDeserializer<'_> {
    pub fn new(req: &HttpRequest, body: Bytes) -> HttpRequestDeserializer {
        HttpRequestDeserializer { req, body }
    }
}

impl<'a> BinaryDeserializer for HttpRequestDeserializer<'a> {
    fn deserialize_binary<R: Sized, V: BinarySerializer<R>>(self, mut visitor: V) -> Result<R> {
        if self.encoding() != Encoding::BINARY {
            return Err(message::Error::WrongEncoding {});
        }

        let spec_version = SpecVersion::try_from(
            unwrap_optional_header!(self.req.headers(), headers::SPEC_VERSION_HEADER).unwrap()?,
        )?;

        visitor = visitor.set_spec_version(spec_version.clone())?;

        let attributes = spec_version.attribute_names();

        for (hn, hv) in
            self.req.headers().iter().filter(|(hn, _)| {
                headers::SPEC_VERSION_HEADER.ne(hn) && hn.as_str().starts_with("ce-")
            })
        {
            let name = &hn.as_str()["ce-".len()..];

            if attributes.contains(&name) {
                visitor = visitor.set_attribute(
                    name,
                    MessageAttributeValue::String(String::from(header_value_to_str!(hv)?)),
                )?
            } else {
                visitor = visitor.set_extension(
                    name,
                    MessageAttributeValue::String(String::from(header_value_to_str!(hv)?)),
                )?
            }
        }

        if let Some(hv) = self.req.headers().get("content-type") {
            visitor = visitor.set_attribute(
                "datacontenttype",
                MessageAttributeValue::String(String::from(header_value_to_str!(hv)?)),
            )?
        }

        if !self.body.is_empty() {
            visitor.end_with_data(self.body.to_vec())
        } else {
            visitor.end()
        }
    }
}

impl<'a> StructuredDeserializer for HttpRequestDeserializer<'a> {
    fn deserialize_structured<R: Sized, V: StructuredSerializer<R>>(self, visitor: V) -> Result<R> {
        if self.encoding() != Encoding::STRUCTURED {
            return Err(message::Error::WrongEncoding {});
        }
        visitor.set_structured_event(self.body.to_vec())
    }
}

impl<'a> MessageDeserializer for HttpRequestDeserializer<'a> {
    fn encoding(&self) -> Encoding {
        if self.req.content_type() == "application/cloudevents+json" {
            Encoding::STRUCTURED
        } else if self
            .req
            .headers()
            .get::<&'static HeaderName>(&super::headers::SPEC_VERSION_HEADER)
            .is_some()
        {
            Encoding::BINARY
        } else {
            Encoding::UNKNOWN
        }
    }
}

/// Method to transform an incoming [`HttpRequest`] to [`Event`].
pub async fn request_to_event(
    req: &HttpRequest,
    mut payload: web::Payload,
) -> std::result::Result<Event, actix_web::error::Error> {
    let mut bytes = BytesMut::new();
    while let Some(item) = payload.next().await {
        bytes.extend_from_slice(&item?);
    }
    MessageDeserializer::into_event(HttpRequestDeserializer::new(req, bytes.freeze()))
        .map_err(actix_web::error::ErrorBadRequest)
}

/// So that an actix-web handler may take an Event parameter
impl actix_web::FromRequest for Event {
    type Config = ();
    type Error = actix_web::Error;
    type Future = LocalBoxFuture<'static, std::result::Result<Self, Self::Error>>;

    fn from_request(r: &HttpRequest, p: &mut actix_web::dev::Payload) -> Self::Future {
        let payload = web::Payload(p.take());
        let request = r.to_owned();
        async move { request_to_event(&request, payload).await }.boxed_local()
    }
}

/// Extention Trait for [`HttpRequest`] which acts as a wrapper for the function [`request_to_event()`].
///
/// This trait is sealed and cannot be implemented for types outside of this crate.
#[async_trait(?Send)]
pub trait HttpRequestExt: private::Sealed {
    /// Convert this [`HttpRequest`] into an [`Event`].
    async fn to_event(
        &self,
        mut payload: web::Payload,
    ) -> std::result::Result<Event, actix_web::error::Error>;
}

#[async_trait(?Send)]
impl HttpRequestExt for HttpRequest {
    async fn to_event(
        &self,
        payload: web::Payload,
    ) -> std::result::Result<Event, actix_web::error::Error> {
        request_to_event(self, payload).await
    }
}

mod private {
    // Sealing the RequestExt
    pub trait Sealed {}
    impl Sealed for actix_web::HttpRequest {}
}

#[cfg(test)]
mod tests {
    use super::*;
    use actix_web::test;

    use crate::{EventBuilder, EventBuilderV10};
    use chrono::Utc;
    use serde_json::json;

    #[actix_rt::test]
    async fn test_request() {
        let time = Utc::now();
        let expected = EventBuilderV10::new()
            .id("0001")
            .ty("example.test")
            .source("http://localhost/")
            //TODO this is required now because the message deserializer implictly set default values
            // As soon as this defaulting doesn't happen anymore, we can remove it (Issues #40/#41)
            .time(time)
            .extension("someint", "10")
            .build()
            .unwrap();

        let (req, payload) = test::TestRequest::post()
            .header("ce-specversion", "1.0")
            .header("ce-id", "0001")
            .header("ce-type", "example.test")
            .header("ce-source", "http://localhost/")
            .header("ce-someint", "10")
            .header("ce-time", time.to_rfc3339())
            .to_http_parts();

        let resp = req.to_event(web::Payload(payload)).await.unwrap();
        assert_eq!(expected, resp);
    }

    #[actix_rt::test]
    async fn test_request_with_full_data() {
        let time = Utc::now();
        let j = json!({"hello": "world"});

        let expected = EventBuilderV10::new()
            .id("0001")
            .ty("example.test")
            .source("http://localhost")
            //TODO this is required now because the message deserializer implictly set default values
            // As soon as this defaulting doesn't happen anymore, we can remove it (Issues #40/#41)
            .time(time)
            .data("application/json", j.to_string().into_bytes())
            .extension("someint", "10")
            .build()
            .unwrap();

        let (req, payload) = test::TestRequest::post()
            .header("ce-specversion", "1.0")
            .header("ce-id", "0001")
            .header("ce-type", "example.test")
            .header("ce-source", "http://localhost")
            .header("ce-someint", "10")
            .header("ce-time", time.to_rfc3339())
            .header("content-type", "application/json")
            .set_json(&j)
            .to_http_parts();

        let resp = req.to_event(web::Payload(payload)).await.unwrap();
        assert_eq!(expected, resp);
    }
}
<commit_msg>Add Actix structured test and clean up a bit (#141)<commit_after>use super::headers;
use crate::event::SpecVersion;
use crate::message::{
    BinaryDeserializer, BinarySerializer, Encoding, MessageAttributeValue, MessageDeserializer,
    Result, StructuredDeserializer, StructuredSerializer,
};
use crate::{message, Event};
use actix_web::http::HeaderName;
use actix_web::web::{Bytes, BytesMut};
use actix_web::{web, HttpMessage, HttpRequest};
use async_trait::async_trait;
use futures::future::LocalBoxFuture;
use futures::{FutureExt, StreamExt};
use std::convert::TryFrom;

/// Wrapper for [`HttpRequest`] that implements [`MessageDeserializer`] trait.
pub struct HttpRequestDeserializer<'a> {
    req: &'a HttpRequest,
    body: Bytes,
}

impl HttpRequestDeserializer<'_> {
    pub fn new(req: &HttpRequest, body: Bytes) -> HttpRequestDeserializer {
        HttpRequestDeserializer { req, body }
    }
}

impl<'a> BinaryDeserializer for HttpRequestDeserializer<'a> {
    fn deserialize_binary<R: Sized, V: BinarySerializer<R>>(self, mut visitor: V) -> Result<R> {
        if self.encoding() != Encoding::BINARY {
            return Err(message::Error::WrongEncoding {});
        }

        let spec_version = SpecVersion::try_from(
            unwrap_optional_header!(self.req.headers(), headers::SPEC_VERSION_HEADER).unwrap()?,
        )?;

        visitor = visitor.set_spec_version(spec_version.clone())?;

        let attributes = spec_version.attribute_names();

        for (hn, hv) in
            self.req.headers().iter().filter(|(hn, _)| {
                headers::SPEC_VERSION_HEADER.ne(hn) && hn.as_str().starts_with("ce-")
            })
        {
            let name = &hn.as_str()["ce-".len()..];

            if attributes.contains(&name) {
                visitor = visitor.set_attribute(
                    name,
                    MessageAttributeValue::String(String::from(header_value_to_str!(hv)?)),
                )?
            } else {
                visitor = visitor.set_extension(
                    name,
                    MessageAttributeValue::String(String::from(header_value_to_str!(hv)?)),
                )?
            }
        }

        if let Some(hv) = self.req.headers().get("content-type") {
            visitor = visitor.set_attribute(
                "datacontenttype",
                MessageAttributeValue::String(String::from(header_value_to_str!(hv)?)),
            )?
        }

        if !self.body.is_empty() {
            visitor.end_with_data(self.body.to_vec())
        } else {
            visitor.end()
        }
    }
}

impl<'a> StructuredDeserializer for HttpRequestDeserializer<'a> {
    fn deserialize_structured<R: Sized, V: StructuredSerializer<R>>(self, visitor: V) -> Result<R> {
        if self.encoding() != Encoding::STRUCTURED {
            return Err(message::Error::WrongEncoding {});
        }
        visitor.set_structured_event(self.body.to_vec())
    }
}

impl<'a> MessageDeserializer for HttpRequestDeserializer<'a> {
    fn encoding(&self) -> Encoding {
        if self.req.content_type() == "application/cloudevents+json" {
            Encoding::STRUCTURED
        } else if self
            .req
            .headers()
            .get::<&'static HeaderName>(&super::headers::SPEC_VERSION_HEADER)
            .is_some()
        {
            Encoding::BINARY
        } else {
            Encoding::UNKNOWN
        }
    }
}

/// Method to transform an incoming [`HttpRequest`] to [`Event`].
pub async fn request_to_event(
    req: &HttpRequest,
    mut payload: web::Payload,
) -> std::result::Result<Event, actix_web::error::Error> {
    let mut bytes = BytesMut::new();
    while let Some(item) = payload.next().await {
        bytes.extend_from_slice(&item?);
    }
    MessageDeserializer::into_event(HttpRequestDeserializer::new(req, bytes.freeze()))
        .map_err(actix_web::error::ErrorBadRequest)
}

/// So that an actix-web handler may take an Event parameter
impl actix_web::FromRequest for Event {
    type Config = ();
    type Error = actix_web::Error;
    type Future = LocalBoxFuture<'static, std::result::Result<Self, Self::Error>>;

    fn from_request(r: &HttpRequest, p: &mut actix_web::dev::Payload) -> Self::Future {
        let payload = web::Payload(p.take());
        let request = r.to_owned();
        async move { request_to_event(&request, payload).await }.boxed_local()
    }
}

/// Extension Trait for [`HttpRequest`] which acts as a wrapper for the function [`request_to_event()`].
///
/// This trait is sealed and cannot be implemented for types outside of this crate.
#[async_trait(?Send)]
pub trait HttpRequestExt: private::Sealed {
    /// Convert this [`HttpRequest`] into an [`Event`].
    async fn to_event(
        &self,
        mut payload: web::Payload,
    ) -> std::result::Result<Event, actix_web::error::Error>;
}

#[async_trait(?Send)]
impl HttpRequestExt for HttpRequest {
    async fn to_event(
        &self,
        payload: web::Payload,
    ) -> std::result::Result<Event, actix_web::error::Error> {
        request_to_event(self, payload).await
    }
}

mod private {
    // Sealing the RequestExt
    pub trait Sealed {}
    impl Sealed for actix_web::HttpRequest {}
}

#[cfg(test)]
mod tests {
    use super::*;
    use actix_web::test;

    use crate::{EventBuilder, EventBuilderV10};
    use serde_json::json;

    #[actix_rt::test]
    async fn test_request() {
        let expected = EventBuilderV10::new()
            .id("0001")
            .ty("example.test")
            .source("http://localhost/")
            .extension("someint", "10")
            .build()
            .unwrap();

        let (req, payload) = test::TestRequest::post()
            .header("ce-specversion", "1.0")
            .header("ce-id", "0001")
            .header("ce-type", "example.test")
            .header("ce-source", "http://localhost/")
            .header("ce-someint", "10")
            .to_http_parts();

        let resp = req.to_event(web::Payload(payload)).await.unwrap();
        assert_eq!(expected, resp);
    }

    #[actix_rt::test]
    async fn test_request_with_full_data() {
        let j = json!({"hello": "world"});

        let expected = EventBuilderV10::new()
            .id("0001")
            .ty("example.test")
            .source("http://localhost")
            .data("application/json", j.to_string().into_bytes())
            .extension("someint", "10")
            .build()
            .unwrap();

        let (req, payload) = test::TestRequest::post()
            .header("ce-specversion", "1.0")
            .header("ce-id", "0001")
            .header("ce-type", "example.test")
            .header("ce-source", "http://localhost")
            .header("ce-someint", "10")
            .header("content-type", "application/json")
            .set_json(&j)
            .to_http_parts();

        let resp = req.to_event(web::Payload(payload)).await.unwrap();
        assert_eq!(expected, resp);
    }

    #[actix_rt::test]
    async fn test_structured_request_with_full_data() {
        let j = json!({"hello": "world"});
        let payload = json!({
            "specversion": "1.0",
            "id": "0001",
            "type": "example.test",
            "source": "http://localhost",
            "someint": "10",
            "datacontenttype": "application/json",
            "data": j
        });
        let bytes = serde_json::to_string(&payload).expect("Failed to serialize test data to json");

        let expected = EventBuilderV10::new()
            .id("0001")
            .ty("example.test")
            .source("http://localhost")
            .data("application/json", j)
            .extension("someint", "10")
            .build()
            .unwrap();

        let (req, payload) = test::TestRequest::post()
            .header("content-type", "application/cloudevents+json")
            .set_payload(bytes)
            .to_http_parts();

        let resp = req.to_event(web::Payload(payload)).await.unwrap();
        assert_eq!(expected, resp);
    }
}
<|endoftext|>


<commit_before>use crate::{
    config, db,
    models::Version,
    schema::{crates, readme_renderings, versions},
    uploaders::Uploader,
};
use std::{io::Read, path::Path, sync::Arc, thread};

use cargo_registry_markdown::text_to_html;
use chrono::{TimeZone, Utc};
use clap::Clap;
use diesel::{dsl::any, prelude::*};
use flate2::read::GzDecoder;
use reqwest::{blocking::Client, header};
use tar::{self, Archive};

const CACHE_CONTROL_README: &str = "public,max-age=604800";
const USER_AGENT: &str = "crates-admin";

#[derive(Clap, Debug)]
#[clap(
    name = "render-readmes",
    about = "Iterates over every crate versions ever uploaded and (re-)renders their \
        readme using the readme renderer from the cargo_registry crate.",
    after_help = "Warning: this can take a lot of time."
)]
pub struct Opts {
    /// How many versions should be queried and processed at a time.
    #[clap(long, default_value = "25")]
    page_size: usize,

    /// Only rerender readmes that are older than this date.
    #[clap(long)]
    older_than: Option<String>,

    /// Only rerender readmes for the specified crate.
    #[clap(long = "crate")]
    crate_name: Option<String>,
}

pub fn run(opts: Opts) {
    let base_config = Arc::new(config::Base::from_environment());
    let conn = db::connect_now().unwrap();

    let start_time = Utc::now();

    let older_than = if let Some(ref time) = opts.older_than {
        Utc.datetime_from_str(time, "%Y-%m-%d %H:%M:%S")
            .expect("Could not parse --older-than argument as a time")
    } else {
        start_time
    };
    let older_than = older_than.naive_utc();

    println!("Start time:                   {}", start_time);
    println!("Rendering readmes older than: {}", older_than);

    let mut query = versions::table
        .inner_join(crates::table)
        .left_outer_join(readme_renderings::table)
        .filter(
            readme_renderings::rendered_at
                .lt(older_than)
                .or(readme_renderings::version_id.is_null()),
        )
        .select(versions::id)
        .into_boxed();

    if let Some(crate_name) = opts.crate_name {
        println!("Rendering readmes for {}", crate_name);
        query = query.filter(crates::name.eq(crate_name));
    }

    let version_ids: Vec<i32> = query.load(&conn).expect("error loading version ids");

    let total_versions = version_ids.len();
    println!("Rendering {} versions", total_versions);

    let page_size = opts.page_size;

    let total_pages = total_versions / page_size;
    let total_pages = if total_versions % page_size == 0 {
        total_pages
    } else {
        total_pages + 1
    };

    let client = Client::new();

    for (page_num, version_ids_chunk) in version_ids.chunks(page_size).enumerate() {
        println!(
            "= Page {} of {} ==================================",
            page_num + 1,
            total_pages
        );

        let versions: Vec<(Version, String)> = versions::table
            .inner_join(crates::table)
            .filter(versions::id.eq(any(version_ids_chunk)))
            .select((versions::all_columns, crates::name))
            .load(&conn)
            .expect("error loading versions");

        let mut tasks = Vec::with_capacity(page_size as usize);
        for (version, krate_name) in versions {
            Version::record_readme_rendering(version.id, &conn).unwrap_or_else(|_| {
                panic!(
                    "[{}-{}] Couldn't record rendering time",
                    krate_name, version.num
                )
            });
            let client = client.clone();
            let base_config = base_config.clone();
            let handle = thread::spawn(move || {
                println!("[{}-{}] Rendering README...", krate_name, version.num);
                let readme = get_readme(base_config.uploader(), &client, &version, &krate_name);
                if readme.is_none() {
                    return;
                }
                let readme = readme.unwrap();
                let content_length = readme.len() as u64;
                let content = std::io::Cursor::new(readme);
                let readme_path = format!("readmes/{0}/{0}-{1}.html", krate_name, version.num);
                let mut extra_headers = header::HeaderMap::new();
                extra_headers.insert(
                    header::CACHE_CONTROL,
                    header::HeaderValue::from_static(CACHE_CONTROL_README),
                );
                base_config
                    .uploader()
                    .upload(
                        &client,
                        &readme_path,
                        content,
                        content_length,
                        "text/html",
                        extra_headers,
                    )
                    .unwrap_or_else(|_| {
                        panic!(
                            "[{}-{}] Couldn't upload file to S3",
                            krate_name, version.num
                        )
                    });
            });
            tasks.push(handle);
        }
        for handle in tasks {
            if let Err(err) = handle.join() {
                println!("Thread panicked: {:?}", err);
            }
        }
    }
}

/// Renders the readme of an uploaded crate version.
fn get_readme(
    uploader: &Uploader,
    client: &Client,
    version: &Version,
    krate_name: &str,
) -> Option<String> {
    let pkg_name = format!("{}-{}", krate_name, version.num);

    let location = uploader.crate_location(krate_name, &version.num.to_string());

    let location = match uploader {
        Uploader::S3 { .. } => location,
        Uploader::Local => format!("http://localhost:8888/{}", location),
    };

    let mut extra_headers = header::HeaderMap::new();
    extra_headers.insert(
        header::USER_AGENT,
        header::HeaderValue::from_static(USER_AGENT),
    );
    let response = match client.get(&location).headers(extra_headers).send() {
        Ok(r) => r,
        Err(err) => {
            println!("[{}] Unable to fetch crate: {}", pkg_name, err);
            return None;
        }
    };

    if !response.status().is_success() {
        println!(
            "[{}] Failed to get a 200 response: {}",
            pkg_name,
            response.text().unwrap()
        );
        return None;
    }

    let reader = GzDecoder::new(response);
    let archive = Archive::new(reader);
    render_pkg_readme(archive, &pkg_name)
}

fn render_pkg_readme<R: Read>(mut archive: Archive<R>, pkg_name: &str) -> Option<String> {
    let mut entries = archive
        .entries()
        .unwrap_or_else(|_| panic!("[{}] Invalid tar archive entries", pkg_name));

    let manifest: Manifest = {
        let path = format!("{}/Cargo.toml", pkg_name);
        let contents = find_file_by_path(&mut entries, Path::new(&path), pkg_name);
        toml::from_str(&contents)
            .unwrap_or_else(|_| panic!("[{}] Syntax error in manifest file", pkg_name))
    };

    let rendered = {
        let readme_path = manifest.package.readme.as_ref()?;
        let path = format!("{}/{}", pkg_name, readme_path);
        let contents = find_file_by_path(&mut entries, Path::new(&path), pkg_name);
        text_to_html(
            &contents,
            readme_path,
            manifest.package.repository.as_deref(),
        )
    };
    return Some(rendered);

    #[derive(Debug, Deserialize)]
    struct Package {
        readme: Option<String>,
        repository: Option<String>,
    }

    #[derive(Debug, Deserialize)]
    struct Manifest {
        package: Package,
    }
}

/// Search an entry by its path in a Tar archive.
fn find_file_by_path<R: Read>(
    entries: &mut tar::Entries<'_, R>,
    path: &Path,
    pkg_name: &str,
) -> String {
    let mut file = entries
        .find(|entry| match *entry {
            Err(_) => false,
            Ok(ref file) => {
                let filepath = match file.path() {
                    Ok(p) => p,
                    Err(_) => return false,
                };
                filepath == path
            }
        })
        .unwrap_or_else(|| panic!("[{}] couldn't open file: {}", pkg_name, path.display()))
        .unwrap_or_else(|_| panic!("[{}] file is not present: {}", pkg_name, path.display()));

    let mut contents = String::new();
    file.read_to_string(&mut contents)
        .unwrap_or_else(|_| panic!("[{}] Couldn't read file contents", pkg_name));
    contents
}

#[cfg(test)]
mod tests {
    use std::io::Write;
    use tar;

    use super::render_pkg_readme;

    fn add_file<W: Write>(pkg: &mut tar::Builder<W>, path: &str, content: &[u8]) {
        let mut header = tar::Header::new_gnu();
        header.set_size(content.len() as u64);
        header.set_cksum();
        pkg.append_data(&mut header, path, content).unwrap();
    }

    #[test]
    fn test_render_pkg_readme() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
readme = "README.md"
"#,
        );
        add_file(&mut pkg, "foo-0.0.1/README.md", b"readme");
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("readme"))
    }

    #[test]
    fn test_render_pkg_readme_w_link() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
readme = "README.md"
repository = "https://github.com/foo/foo"
"#,
        );
        add_file(
            &mut pkg,
            "foo-0.0.1/README.md",
            b"readme [link](./Other.md)",
        );
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("\"https://github.com/foo/foo/blob/HEAD/./Other.md\""))
    }

    #[test]
    fn test_render_pkg_readme_not_at_root() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
readme = "docs/README.md"
repository = "https://github.com/foo/foo"
"#,
        );
        add_file(
            &mut pkg,
            "foo-0.0.1/docs/README.md",
            b"docs/readme [link](./Other.md)",
        );
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("docs/readme"));
        assert!(result.contains("\"https://github.com/foo/foo/blob/HEAD/docs/./Other.md\""))
    }
}
<commit_msg>Auto merge of #3970 - nipunn1313:rerender_readme3, r=Turbo87<commit_after>use crate::{
    config, db,
    models::Version,
    schema::{crates, readme_renderings, versions},
    uploaders::Uploader,
};
use std::{io::Read, path::Path, sync::Arc, thread};

use cargo_registry_markdown::text_to_html;
use chrono::{TimeZone, Utc};
use clap::Clap;
use diesel::{dsl::any, prelude::*};
use flate2::read::GzDecoder;
use reqwest::{blocking::Client, header};
use tar::{self, Archive};

const CACHE_CONTROL_README: &str = "public,max-age=604800";
const USER_AGENT: &str = "crates-admin";

#[derive(Clap, Debug)]
#[clap(
    name = "render-readmes",
    about = "Iterates over every crate versions ever uploaded and (re-)renders their \
        readme using the readme renderer from the cargo_registry crate.",
    after_help = "Warning: this can take a lot of time."
)]
pub struct Opts {
    /// How many versions should be queried and processed at a time.
    #[clap(long, default_value = "25")]
    page_size: usize,

    /// Only rerender readmes that are older than this date.
    #[clap(long)]
    older_than: Option<String>,

    /// Only rerender readmes for the specified crate.
    #[clap(long = "crate")]
    crate_name: Option<String>,
}

pub fn run(opts: Opts) {
    let base_config = Arc::new(config::Base::from_environment());
    let conn = db::connect_now().unwrap();

    let start_time = Utc::now();

    let older_than = if let Some(ref time) = opts.older_than {
        Utc.datetime_from_str(time, "%Y-%m-%d %H:%M:%S")
            .expect("Could not parse --older-than argument as a time")
    } else {
        start_time
    };
    let older_than = older_than.naive_utc();

    println!("Start time:                   {}", start_time);
    println!("Rendering readmes older than: {}", older_than);

    let mut query = versions::table
        .inner_join(crates::table)
        .left_outer_join(readme_renderings::table)
        .filter(
            readme_renderings::rendered_at
                .lt(older_than)
                .or(readme_renderings::version_id.is_null()),
        )
        .select(versions::id)
        .into_boxed();

    if let Some(crate_name) = opts.crate_name {
        println!("Rendering readmes for {}", crate_name);
        query = query.filter(crates::name.eq(crate_name));
    }

    let version_ids: Vec<i32> = query.load(&conn).expect("error loading version ids");

    let total_versions = version_ids.len();
    println!("Rendering {} versions", total_versions);

    let page_size = opts.page_size;

    let total_pages = total_versions / page_size;
    let total_pages = if total_versions % page_size == 0 {
        total_pages
    } else {
        total_pages + 1
    };

    let client = Client::new();

    for (page_num, version_ids_chunk) in version_ids.chunks(page_size).enumerate() {
        println!(
            "= Page {} of {} ==================================",
            page_num + 1,
            total_pages
        );

        let versions: Vec<(Version, String)> = versions::table
            .inner_join(crates::table)
            .filter(versions::id.eq(any(version_ids_chunk)))
            .select((versions::all_columns, crates::name))
            .load(&conn)
            .expect("error loading versions");

        let mut tasks = Vec::with_capacity(page_size as usize);
        for (version, krate_name) in versions {
            Version::record_readme_rendering(version.id, &conn).unwrap_or_else(|_| {
                panic!(
                    "[{}-{}] Couldn't record rendering time",
                    krate_name, version.num
                )
            });
            let client = client.clone();
            let base_config = base_config.clone();
            let handle = thread::spawn(move || {
                println!("[{}-{}] Rendering README...", krate_name, version.num);
                let readme = get_readme(base_config.uploader(), &client, &version, &krate_name);
                if readme.is_none() {
                    return;
                }
                let readme = readme.unwrap();
                let content_length = readme.len() as u64;
                let content = std::io::Cursor::new(readme);
                let readme_path = format!("readmes/{0}/{0}-{1}.html", krate_name, version.num);
                let mut extra_headers = header::HeaderMap::new();
                extra_headers.insert(
                    header::CACHE_CONTROL,
                    header::HeaderValue::from_static(CACHE_CONTROL_README),
                );
                base_config
                    .uploader()
                    .upload(
                        &client,
                        &readme_path,
                        content,
                        content_length,
                        "text/html",
                        extra_headers,
                    )
                    .unwrap_or_else(|_| {
                        panic!(
                            "[{}-{}] Couldn't upload file to S3",
                            krate_name, version.num
                        )
                    });
            });
            tasks.push(handle);
        }
        for handle in tasks {
            if let Err(err) = handle.join() {
                println!("Thread panicked: {:?}", err);
            }
        }
    }
}

/// Renders the readme of an uploaded crate version.
fn get_readme(
    uploader: &Uploader,
    client: &Client,
    version: &Version,
    krate_name: &str,
) -> Option<String> {
    let pkg_name = format!("{}-{}", krate_name, version.num);

    let location = uploader.crate_location(krate_name, &version.num.to_string());

    let location = match uploader {
        Uploader::S3 { .. } => location,
        Uploader::Local => format!("http://localhost:8888/{}", location),
    };

    let mut extra_headers = header::HeaderMap::new();
    extra_headers.insert(
        header::USER_AGENT,
        header::HeaderValue::from_static(USER_AGENT),
    );
    let response = match client.get(&location).headers(extra_headers).send() {
        Ok(r) => r,
        Err(err) => {
            println!("[{}] Unable to fetch crate: {}", pkg_name, err);
            return None;
        }
    };

    if !response.status().is_success() {
        println!(
            "[{}] Failed to get a 200 response: {}",
            pkg_name,
            response.text().unwrap()
        );
        return None;
    }

    let reader = GzDecoder::new(response);
    let archive = Archive::new(reader);
    render_pkg_readme(archive, &pkg_name)
}

fn render_pkg_readme<R: Read>(mut archive: Archive<R>, pkg_name: &str) -> Option<String> {
    let mut entries = archive
        .entries()
        .unwrap_or_else(|_| panic!("[{}] Invalid tar archive entries", pkg_name));

    let manifest: Manifest = {
        let path = format!("{}/Cargo.toml", pkg_name);
        let contents = find_file_by_path(&mut entries, Path::new(&path), pkg_name)
            .unwrap_or_else(|| panic!("[{}] couldn't open file: Cargo.toml", pkg_name));
        toml::from_str(&contents)
            .unwrap_or_else(|_| panic!("[{}] Syntax error in manifest file", pkg_name))
    };

    let rendered = {
        let readme_path = manifest
            .package
            .readme
            .clone()
            .unwrap_or_else(|| "README.md".into());
        let path = format!("{}/{}", pkg_name, readme_path);
        let contents = find_file_by_path(&mut entries, Path::new(&path), pkg_name)?;
        text_to_html(
            &contents,
            &readme_path,
            manifest.package.repository.as_deref(),
        )
    };
    return Some(rendered);

    #[derive(Debug, Deserialize)]
    struct Package {
        readme: Option<String>,
        repository: Option<String>,
    }

    #[derive(Debug, Deserialize)]
    struct Manifest {
        package: Package,
    }
}

/// Search an entry by its path in a Tar archive.
fn find_file_by_path<R: Read>(
    entries: &mut tar::Entries<'_, R>,
    path: &Path,
    pkg_name: &str,
) -> Option<String> {
    let mut file = entries
        .find(|entry| match *entry {
            Err(_) => false,
            Ok(ref file) => {
                let filepath = match file.path() {
                    Ok(p) => p,
                    Err(_) => return false,
                };
                filepath == path
            }
        })?
        .unwrap_or_else(|_| panic!("[{}] file is not present: {}", pkg_name, path.display()));

    let mut contents = String::new();
    file.read_to_string(&mut contents)
        .unwrap_or_else(|_| panic!("[{}] Couldn't read file contents", pkg_name));
    Some(contents)
}

#[cfg(test)]
mod tests {
    use std::io::Write;
    use tar;

    use super::render_pkg_readme;

    fn add_file<W: Write>(pkg: &mut tar::Builder<W>, path: &str, content: &[u8]) {
        let mut header = tar::Header::new_gnu();
        header.set_size(content.len() as u64);
        header.set_cksum();
        pkg.append_data(&mut header, path, content).unwrap();
    }

    #[test]
    fn test_render_pkg_readme() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
readme = "README.md"
"#,
        );
        add_file(&mut pkg, "foo-0.0.1/README.md", b"readme");
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("readme"))
    }

    #[test]
    fn test_render_pkg_no_readme() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
"#,
        );
        let serialized_archive = pkg.into_inner().unwrap();
        assert!(render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").is_none());
    }

    #[test]
    fn test_render_pkg_implicit_readme() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
"#,
        );
        add_file(&mut pkg, "foo-0.0.1/README.md", b"readme");
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("readme"))
    }

    #[test]
    fn test_render_pkg_readme_w_link() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
readme = "README.md"
repository = "https://github.com/foo/foo"
"#,
        );
        add_file(
            &mut pkg,
            "foo-0.0.1/README.md",
            b"readme [link](./Other.md)",
        );
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("\"https://github.com/foo/foo/blob/HEAD/./Other.md\""))
    }

    #[test]
    fn test_render_pkg_readme_not_at_root() {
        let mut pkg = tar::Builder::new(vec![]);
        add_file(
            &mut pkg,
            "foo-0.0.1/Cargo.toml",
            br#"
[package]
readme = "docs/README.md"
repository = "https://github.com/foo/foo"
"#,
        );
        add_file(
            &mut pkg,
            "foo-0.0.1/docs/README.md",
            b"docs/readme [link](./Other.md)",
        );
        let serialized_archive = pkg.into_inner().unwrap();
        let result =
            render_pkg_readme(tar::Archive::new(&*serialized_archive), "foo-0.0.1").unwrap();
        assert!(result.contains("docs/readme"));
        assert!(result.contains("\"https://github.com/foo/foo/blob/HEAD/docs/./Other.md\""))
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2017-2018 Rene van der Meer
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.

macro_rules! parse_retval {
    ($retval:expr) => {{
        let retval = $retval;

        if retval == -1 {
            Err(std::io::Error::last_os_error())
        } else {
            Ok(retval)
        }
    }};
}
<commit_msg>Add macro to initialize arrays with non-Copy types<commit_after>// Copyright (c) 2017-2018 Rene van der Meer
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.

macro_rules! parse_retval {
    ($retval:expr) => {{
        let retval = $retval;

        if retval == -1 {
            Err(std::io::Error::last_os_error())
        } else {
            Ok(retval)
        }
    }};
}

// Initialize an array with a type that doesn't implement Copy
macro_rules! init_array {
    ($val:expr; $len:expr) => {{
        let mut array: [_; $len] = unsafe { std::mem::uninitialized() };
        for elem in array.iter_mut() {
            unsafe {
                std::ptr::write(elem, $val);
            }
        }
        array
    }};
}
<|endoftext|>


<commit_before>#[macro_export]
macro_rules! js {
	($j:tt) => ({
		let json_raw = stringify!($j);
		let mut json = String::with_capacity(json_raw.len());
		$crate::parse::sanitise(json_raw.as_bytes(), &mut json);

		json
	})
}
<commit_msg>work on stable support and geo types<commit_after><|endoftext|>


<commit_before>#![cfg(test)]
#![allow(non_upper_case_globals)]
#![allow(dead_code)]
extern crate core;
extern crate brotli_decompressor;
use brotli_decompressor::{CustomRead, CustomWrite};
use super::brotli::enc::BrotliEncoderParams;
use super::brotli::concat::{BroCatli, BroCatliResult};
use std::io::{Read, Write};
use super::integration_tests::UnlimitedBuffer;
static RANDOM_THEN_UNICODE : &'static [u8] = include_bytes!("../../testdata/random_then_unicode");
static ALICE: &'static[u8]  = include_bytes!("../../testdata/alice29.txt");
use super::Rebox;

#[test]
fn test_custom_dict() {
    let mut raw = UnlimitedBuffer::new(ALICE);
    let mut params = BrotliEncoderParams::default();
    params.quality = 10;
    let mut br = UnlimitedBuffer::new(&[]);
    let mut rt = UnlimitedBuffer::new(&[]);
    let dict = &ALICE[12515..23411];
    super::compress(&mut raw, &mut br, 4096, &params, dict).unwrap();
    raw.reset_read();
    let mut vec = Vec::<u8>::new();
    vec.extend(dict);
    super::decompress(&mut br, &mut rt, 4096, Rebox::from(vec)).unwrap();
    assert_eq!(rt.data(), raw.data());
    assert_eq!(br.data().len(), 43629);
}

#[test]
fn test_custom_wrong_dict_fails() {
    let mut raw = UnlimitedBuffer::new(ALICE);
    let mut params = BrotliEncoderParams::default();
    params.quality = 10;
    let mut br = UnlimitedBuffer::new(&[]);
    let mut rt = UnlimitedBuffer::new(&[]);
    let dict = &ALICE[12515..19515];
    super::compress(&mut raw, &mut br, 4096, &params, dict).unwrap();
    raw.reset_read();
    let mut vec = Vec::<u8>::new();
    vec.extend(&dict[1..]); // slightly offset dictionary to be wrong, and ensure the dict was being used above
    match super::decompress(&mut br, &mut rt, 4096, Rebox::from(vec)) {
        Ok(_) => panic!("Decompression should have failed"),
        Err(_) => {},
    }
    if rt.data() == raw.data() {
        panic!("they should be unequal");
    }
}

#[test]
fn test_custom_wrong_dict_fails_but_doesnt_disrupt_compression_strategy() {
    let mut raw = UnlimitedBuffer::new(ALICE);
    let mut params = BrotliEncoderParams::default();
    params.quality = 6;
    let mut br = UnlimitedBuffer::new(&[]);
    let mut rt = UnlimitedBuffer::new(&[]);
    let dict = &ALICE[12515..19515];
    super::compress(&mut raw, &mut br, 4096, &params, dict).unwrap();
    raw.reset_read();
    let mut vec = Vec::<u8>::new();
    vec.extend(&dict[1..]); // slightly offset dictionary to be wrong, and ensure the dict was being used above
    super::decompress(&mut br, &mut rt, 4096, Rebox::from(vec)).unwrap();
    if rt.data() == raw.data() {
        panic!("they should be unequal");
    }
}

#[test]
fn test_custom_dict_for_multithreading() {
    let mut raws = [UnlimitedBuffer::new(&ALICE[..ALICE.len()/3]),
                    UnlimitedBuffer::new(&ALICE[ALICE.len()/3..2*ALICE.len()/3]),
                    UnlimitedBuffer::new(&ALICE[2 * ALICE.len()/3..]),
    ];
    let mut params = BrotliEncoderParams::default();
    params.quality = 10;
    params.appendable = true;
    let mut brs = [
        UnlimitedBuffer::new(&[]),
        UnlimitedBuffer::new(&[]),
        UnlimitedBuffer::new(&[]),
    ];
    let mut rts = [
        UnlimitedBuffer::new(&[]),
        UnlimitedBuffer::new(&[]),
        UnlimitedBuffer::new(&[]),
    ];
    let dicts = [
        &[],
        &ALICE[..ALICE.len()/3],
        &ALICE[..2*ALICE.len()/3],
    ];
    for (raw, (br, (rt, dict))) in raws.iter_mut().zip(brs.iter_mut().zip(rts.iter_mut().zip(dicts.iter()))) {
        super::compress(raw, br, 4096, &params, dict).unwrap();
        raw.reset_read();
        let mut vec = Vec::<u8>::new();
        vec.extend(*dict);
        super::decompress(br, rt, 4096, Rebox::from(vec)).unwrap();
        assert_eq!(rt.data(), raw.data());
        params.catable = true;
    }
    let mut bro_cat_li = BroCatli::new();
    let mut output = UnlimitedBuffer::new(&[]);
    let mut ibuffer = vec![0u8; 1];
    let mut obuffer = vec![0u8; 1];
    let mut ooffset = 0usize;
    for brotli in brs.iter_mut() {
        brotli.reset_read();
        bro_cat_li.new_brotli_file();
        let mut input = brotli;
        loop {
            let mut ioffset = 0usize;
            match input.read(&mut ibuffer[..]) {
                Err(e) => panic!(e),
                Ok(cur_read) => {
                    if cur_read == 0 {
                        break;
                    }
                    loop {
                        match bro_cat_li.stream(&ibuffer[..cur_read], &mut ioffset,
                                                &mut obuffer[..], &mut ooffset) {
                            BroCatliResult::NeedsMoreOutput => {
                                match output.write(&obuffer[..ooffset]) {
                                    Err(why) => panic!("couldn't write: {:}", why),
                                    Ok(count) => {assert_eq!(count, ooffset);},
                                }
                                ooffset = 0;
                            },
                            BroCatliResult::NeedsMoreInput => {
                                break;
                            },
                            BroCatliResult::Success => {
                                panic!("Unexpected state: Success when streaming before finish");
                            },
                            failure => {
                                panic!(failure);
                            },
                        }
                    }
                }
            }
        }
    }
    loop {
        match bro_cat_li.finish(&mut obuffer[..], &mut ooffset) {
            BroCatliResult::NeedsMoreOutput => {
                match output.write(&obuffer[..ooffset]) {
                    Err(why) => panic!("couldn't write\n{:}", why),
                    Ok(count) => {assert_eq!(count, ooffset);},
                }
                ooffset = 0;
            },
            BroCatliResult::NeedsMoreInput => {
                panic!("Unexpected EOF");
            },
            BroCatliResult::Success => {
                if ooffset != 0 {
                    match output.write(&obuffer[..ooffset]) {
                        Err(why) => panic!("couldn't write\n{:}", why),
                        Ok(count) => {assert_eq!(count, ooffset);},
                    }
                }
                break;
            }
            failure => {
                panic!(failure)
            }
        }
    }
    let mut rt = UnlimitedBuffer::new(&[]);
    output.reset_read();
    super::decompress(&mut output, &mut rt, 4096, Rebox::default()).unwrap();
    assert_eq!(rt.data(), ALICE);
    assert_eq!(output.data().len(), 48564); // as opposed to 46487 with standard settings
}

<commit_msg>fix nostdlib run by allowing some grace for file sizes since nostd approximates some log functions<commit_after>#![cfg(test)]
#![allow(non_upper_case_globals)]
#![allow(dead_code)]
extern crate core;
extern crate brotli_decompressor;
use super::brotli::enc::BrotliEncoderParams;
use super::brotli::concat::{BroCatli, BroCatliResult};
use std::io::{Read, Write};
use super::integration_tests::UnlimitedBuffer;
static RANDOM_THEN_UNICODE : &'static [u8] = include_bytes!("../../testdata/random_then_unicode");
static ALICE: &'static[u8]  = include_bytes!("../../testdata/alice29.txt");
use super::Rebox;

#[test]
fn test_custom_dict() {
    let mut raw = UnlimitedBuffer::new(ALICE);
    let mut params = BrotliEncoderParams::default();
    params.quality = 10;
    let mut br = UnlimitedBuffer::new(&[]);
    let mut rt = UnlimitedBuffer::new(&[]);
    let dict = &ALICE[12515..23411];
    super::compress(&mut raw, &mut br, 4096, &params, dict).unwrap();
    raw.reset_read();
    let mut vec = Vec::<u8>::new();
    vec.extend(dict);
    super::decompress(&mut br, &mut rt, 4096, Rebox::from(vec)).unwrap();
    assert_eq!(rt.data(), raw.data());
    if br.data().len() != 43654 {
        assert_eq!(br.data().len(), 43629);
    }
}

#[test]
fn test_custom_wrong_dict_fails() {
    let mut raw = UnlimitedBuffer::new(ALICE);
    let mut params = BrotliEncoderParams::default();
    params.quality = 10;
    let mut br = UnlimitedBuffer::new(&[]);
    let mut rt = UnlimitedBuffer::new(&[]);
    let dict = &ALICE[12515..19515];
    super::compress(&mut raw, &mut br, 4096, &params, dict).unwrap();
    raw.reset_read();
    let mut vec = Vec::<u8>::new();
    vec.extend(&dict[1..]); // slightly offset dictionary to be wrong, and ensure the dict was being used above
    match super::decompress(&mut br, &mut rt, 4096, Rebox::from(vec)) {
        Ok(_) => panic!("Decompression should have failed"),
        Err(_) => {},
    }
    if rt.data() == raw.data() {
        panic!("they should be unequal");
    }
}

#[test]
fn test_custom_wrong_dict_fails_but_doesnt_disrupt_compression_strategy() {
    let mut raw = UnlimitedBuffer::new(ALICE);
    let mut params = BrotliEncoderParams::default();
    params.quality = 6;
    let mut br = UnlimitedBuffer::new(&[]);
    let mut rt = UnlimitedBuffer::new(&[]);
    let dict = &ALICE[12515..19515];
    super::compress(&mut raw, &mut br, 4096, &params, dict).unwrap();
    raw.reset_read();
    let mut vec = Vec::<u8>::new();
    vec.extend(&dict[1..]); // slightly offset dictionary to be wrong, and ensure the dict was being used above
    super::decompress(&mut br, &mut rt, 4096, Rebox::from(vec)).unwrap();
    if rt.data() == raw.data() {
        panic!("they should be unequal");
    }
}

#[test]
fn test_custom_dict_for_multithreading() {
    let mut raws = [UnlimitedBuffer::new(&ALICE[..ALICE.len()/3]),
                    UnlimitedBuffer::new(&ALICE[ALICE.len()/3..2*ALICE.len()/3]),
                    UnlimitedBuffer::new(&ALICE[2 * ALICE.len()/3..]),
    ];
    let mut params = BrotliEncoderParams::default();
    params.quality = 10;
    params.appendable = true;
    let mut brs = [
        UnlimitedBuffer::new(&[]),
        UnlimitedBuffer::new(&[]),
        UnlimitedBuffer::new(&[]),
    ];
    let mut rts = [
        UnlimitedBuffer::new(&[]),
        UnlimitedBuffer::new(&[]),
        UnlimitedBuffer::new(&[]),
    ];
    let dicts = [
        &[],
        &ALICE[..ALICE.len()/3],
        &ALICE[..2*ALICE.len()/3],
    ];
    for (raw, (br, (rt, dict))) in raws.iter_mut().zip(brs.iter_mut().zip(rts.iter_mut().zip(dicts.iter()))) {
        super::compress(raw, br, 4096, &params, dict).unwrap();
        raw.reset_read();
        let mut vec = Vec::<u8>::new();
        vec.extend(*dict);
        super::decompress(br, rt, 4096, Rebox::from(vec)).unwrap();
        assert_eq!(rt.data(), raw.data());
        params.catable = true;
    }
    let mut bro_cat_li = BroCatli::new();
    let mut output = UnlimitedBuffer::new(&[]);
    let mut ibuffer = vec![0u8; 1];
    let mut obuffer = vec![0u8; 1];
    let mut ooffset = 0usize;
    for brotli in brs.iter_mut() {
        brotli.reset_read();
        bro_cat_li.new_brotli_file();
        let mut input = brotli;
        loop {
            let mut ioffset = 0usize;
            match input.read(&mut ibuffer[..]) {
                Err(e) => panic!(e),
                Ok(cur_read) => {
                    if cur_read == 0 {
                        break;
                    }
                    loop {
                        match bro_cat_li.stream(&ibuffer[..cur_read], &mut ioffset,
                                                &mut obuffer[..], &mut ooffset) {
                            BroCatliResult::NeedsMoreOutput => {
                                match output.write(&obuffer[..ooffset]) {
                                    Err(why) => panic!("couldn't write: {:}", why),
                                    Ok(count) => {assert_eq!(count, ooffset);},
                                }
                                ooffset = 0;
                            },
                            BroCatliResult::NeedsMoreInput => {
                                break;
                            },
                            BroCatliResult::Success => {
                                panic!("Unexpected state: Success when streaming before finish");
                            },
                            failure => {
                                panic!(failure);
                            },
                        }
                    }
                }
            }
        }
    }
    loop {
        match bro_cat_li.finish(&mut obuffer[..], &mut ooffset) {
            BroCatliResult::NeedsMoreOutput => {
                match output.write(&obuffer[..ooffset]) {
                    Err(why) => panic!("couldn't write\n{:}", why),
                    Ok(count) => {assert_eq!(count, ooffset);},
                }
                ooffset = 0;
            },
            BroCatliResult::NeedsMoreInput => {
                panic!("Unexpected EOF");
            },
            BroCatliResult::Success => {
                if ooffset != 0 {
                    match output.write(&obuffer[..ooffset]) {
                        Err(why) => panic!("couldn't write\n{:}", why),
                        Ok(count) => {assert_eq!(count, ooffset);},
                    }
                }
                break;
            }
            failure => {
                panic!(failure)
            }
        }
    }
    let mut rt = UnlimitedBuffer::new(&[]);
    output.reset_read();
    super::decompress(&mut output, &mut rt, 4096, Rebox::default()).unwrap();
    assert_eq!(rt.data(), ALICE);
    // no-stdlib mode has some approximations that make it 4 bytes bigger
    if output.data().len() != 48568 {
       assert_eq!(output.data().len(), 48564); // as opposed to 46487 with standard settings
    }
}

<|endoftext|>


<commit_before>extern crate wm_daemons;
use wm_daemons::config::{load_config, load_config_path};
use wm_daemons::dbus_listen::{CallbackMap, SignalInfo, match_signal};
use wm_daemons::exec::run_config_program;

#[macro_use]
extern crate clap;
use clap::{Arg, App};

extern crate config;
use self::config::types::Config;

extern crate dbus;
use self::dbus::{Connection, BusType};

use std::error::Error;
use std::path::Path;

fn run_program(action: &str, conf: &Config) -> () {
    let path = format!("actions.{}", action);
    let res = run_config_program(conf, &path[..]);
    if res.is_err() {
        println!("failed to handle '{}' action: {}", action, res.err().unwrap());
    }
}

fn handle_signal(info: &SignalInfo, conf: &Config) -> () {
    if info.member == Some("Lock".to_string()) {
        run_program("lock", conf);
    } else if info.member == Some("Unlock".to_string()) {
        run_program("unlock", conf);
    }
}

fn try_main() -> Result<(), Box<Error>> {
    let matches = App::new("wm-session-agent")
        .version(&crate_version!()[..])
        .author("Ben Boeckel <mathstuf@gmail.com>")
        .about("Listens for logind's Lock and Unlock signals")
        .arg(Arg::with_name("CONFIG")
                .short("c")
                .long("config")
                .help("Path to the configuration value")
                .takes_value(true))
        .arg(Arg::with_name("SESSION")
                .short("s")
                .long("session")
                .required(true)
                .help("Session ID to listen on")
                .takes_value(true))
        .get_matches();

    let conf = try!(if matches.is_present("CONFIG") {
            load_config("wm-session-agent", "config")
        } else {
            load_config_path(Path::new(matches.value_of("CONFIG").unwrap()))
        });

    let sid = matches.value_of("SESSION").unwrap();
    let spath = format!("/org/freedesktop/login1/session/_{}", sid);

    let conn = try!(Connection::get_private(BusType::System));

    let cbs: CallbackMap<Config> = vec![
        (SignalInfo {
            path: None,
            object: None,
            member: Some("Lock".to_string()),
        }, handle_signal),
        (SignalInfo {
            path: None,
            object: None,
            member: Some("Unlock".to_string()),
        }, handle_signal),
    ];

    let match_str = format!("type='signal',interface='org.freedesktop.login1.Session',path='{}'", spath);
    try!(conn.add_match(&match_str[..]));

    for items in conn.iter(100) {
        match_signal(items, &cbs, &conf);
    }

    Ok(())
}

fn main() -> () {
    if let Err(err) = try_main() {
        panic!("{}", err.description());
    }
}
<commit_msg>wm-session-agent: fix config file reading<commit_after>extern crate wm_daemons;
use wm_daemons::config::{load_config, load_config_path};
use wm_daemons::dbus_listen::{CallbackMap, SignalInfo, match_signal};
use wm_daemons::exec::run_config_program;

#[macro_use]
extern crate clap;
use clap::{Arg, App};

extern crate config;
use self::config::types::Config;

extern crate dbus;
use self::dbus::{Connection, BusType};

use std::error::Error;
use std::path::Path;

fn run_program(action: &str, conf: &Config) -> () {
    let path = format!("actions.{}", action);
    let res = run_config_program(conf, &path[..]);
    if res.is_err() {
        println!("failed to handle '{}' action: {}", action, res.err().unwrap());
    }
}

fn handle_signal(info: &SignalInfo, conf: &Config) -> () {
    if info.member == Some("Lock".to_string()) {
        run_program("lock", conf);
    } else if info.member == Some("Unlock".to_string()) {
        run_program("unlock", conf);
    }
}

fn try_main() -> Result<(), Box<Error>> {
    let matches = App::new("wm-session-agent")
        .version(&crate_version!()[..])
        .author("Ben Boeckel <mathstuf@gmail.com>")
        .about("Listens for logind's Lock and Unlock signals")
        .arg(Arg::with_name("CONFIG")
                .short("c")
                .long("config")
                .help("Path to the configuration value")
                .takes_value(true))
        .arg(Arg::with_name("SESSION")
                .short("s")
                .long("session")
                .required(true)
                .help("Session ID to listen on")
                .takes_value(true))
        .get_matches();

    let conf = try!(if matches.is_present("CONFIG") {
            load_config_path(Path::new(matches.value_of("CONFIG").unwrap()))
        } else {
            load_config("wm-session-agent", "config")
        });

    let sid = matches.value_of("SESSION").unwrap();
    let spath = format!("/org/freedesktop/login1/session/_{}", sid);

    let conn = try!(Connection::get_private(BusType::System));

    let cbs: CallbackMap<Config> = vec![
        (SignalInfo {
            path: None,
            object: None,
            member: Some("Lock".to_string()),
        }, handle_signal),
        (SignalInfo {
            path: None,
            object: None,
            member: Some("Unlock".to_string()),
        }, handle_signal),
    ];

    let match_str = format!("type='signal',interface='org.freedesktop.login1.Session',path='{}'", spath);
    try!(conn.add_match(&match_str[..]));

    for items in conn.iter(100) {
        match_signal(items, &cbs, &conf);
    }

    Ok(())
}

fn main() -> () {
    if let Err(err) = try_main() {
        panic!("{}", err.description());
    }
}
<|endoftext|>


<commit_before>use std;

use options::*;

/// DHCP Packet Structure
pub struct Packet<'a> {
    pub reply: bool, // false = request, true = reply
    pub hops: u8,
    pub xid: [u8; 4], // Random identifier
    pub secs: u16,
    pub broadcast: bool,
    pub ciaddr: [u8; 4],
    pub yiaddr: [u8; 4],
    pub siaddr: [u8; 4],
    pub giaddr: [u8; 4],
    pub chaddr: [u8; 6],
    pub options: Vec<Option<'a>>,
}

/// Parses Packet from byte array
pub fn decode(p: &[u8]) -> Result<Packet, &'static str> {
    if p[236..240] != COOKIE {
        return Err("Invalid Cookie");
    }

    let reply = match p[0] {
        BOOT_REPLY => true,
        BOOT_REQUEST => false,
        _ => return Err("Invalid OpCode"),
    };
    // TODO hlen check
    let mut options = Vec::new();
    let mut i: usize = 240;
    loop {
        let l = p.len();
        if i < l {
            let code = p[i];
            if code == END {
                break;
            }
            if i + 2 < l {
                let opt_end = (p[i + 1]) as usize + i + 2;
                if opt_end < l {
                    options.push(Option {
                        code: code,
                        data: &p[i + 2..opt_end],
                    });
                    i = opt_end;
                    continue;
                }
            }
        }
        return Err("Options Problem");
    }
    Ok(Packet {
        reply: reply,
        hops: p[3],
        secs: ((p[8] as u16) << 8) + p[9] as u16,
        broadcast: p[10] & 128 == 128,
        ciaddr: [p[12], p[13], p[14], p[15]],
        yiaddr: [p[16], p[17], p[18], p[19]],
        siaddr: [p[20], p[21], p[22], p[23]],
        giaddr: [p[24], p[25], p[26], p[27]],
        options: options,
        chaddr: [p[28], p[29], p[30], p[31], p[32], p[33]],
        xid: [p[4], p[5], p[6], p[7]],
    })
}

impl<'a> Packet<'a> {
    /// Extracts requested option payload from packet if available
    pub fn option(&self, code: u8) -> std::option::Option<&'a [u8]> {
        for option in &self.options {
            if option.code == code {
                return Some(&option.data);
            }
        }
        None
    }

    /// Convenience function for extracting a packet's message type.
    pub fn message_type(&self) -> u8 {
        if let Some(x) = self.option(DHCP_MESSAGE_TYPE) {
            if x.len() > 0 {
                return x[0];
            }
        }
        0
    }

    /// Creates byte array DHCP packet
    pub fn encode<'c>(&'c self, p: &'c mut [u8]) -> &[u8] {
        p[..12].clone_from_slice(&[(if self.reply {
                                       BOOT_REPLY
                                   } else {
                                       BOOT_REQUEST
                                   }),
                                   1,
                                   6,
                                   self.hops,
                                   self.xid[0],
                                   self.xid[1],
                                   self.xid[2],
                                   self.xid[3],
                                   (self.secs >> 8) as u8,
                                   (self.secs & 255) as u8,
                                   (if self.broadcast {
                                       128
                                   } else {
                                       0
                                   }),
                                   0]);
        p[12..16].clone_from_slice(&self.ciaddr);
        p[16..20].clone_from_slice(&self.yiaddr);
        p[20..24].clone_from_slice(&self.siaddr);
        p[24..28].clone_from_slice(&self.giaddr);
        p[28..34].clone_from_slice(&self.chaddr);
        p[34..236].clone_from_slice(&[0; 202]);
        p[236..240].clone_from_slice(&COOKIE);

        let mut length: usize = 240;
        for option in &self.options {
            p[length] = option.code;
            p[length + 1] = option.data.len() as u8;
            p[length + 2..length + 2 + option.data.len()].clone_from_slice(option.data);
            length += 2 + option.data.len();
        }
        p[length] = END;
        length += 1;
        if length < 272 {
            // Pad to min size
            p[length..272].clone_from_slice(&[PAD; 32][..length - 240]);
            length = 272
        }
        &p[..length]
    }
}

const COOKIE: [u8; 4] = [99, 130, 83, 99];

const BOOT_REQUEST: u8 = 1; // From Client;
const BOOT_REPLY: u8 = 2; // From Server;

const END: u8 = 255;
const PAD: u8 = 0;
<commit_msg>Fixed the packet padding size calculation<commit_after>use std;

use options::*;

/// DHCP Packet Structure
pub struct Packet<'a> {
    pub reply: bool, // false = request, true = reply
    pub hops: u8,
    pub xid: [u8; 4], // Random identifier
    pub secs: u16,
    pub broadcast: bool,
    pub ciaddr: [u8; 4],
    pub yiaddr: [u8; 4],
    pub siaddr: [u8; 4],
    pub giaddr: [u8; 4],
    pub chaddr: [u8; 6],
    pub options: Vec<Option<'a>>,
}

/// Parses Packet from byte array
pub fn decode(p: &[u8]) -> Result<Packet, &'static str> {
    if p[236..240] != COOKIE {
        return Err("Invalid Cookie");
    }

    let reply = match p[0] {
        BOOT_REPLY => true,
        BOOT_REQUEST => false,
        _ => return Err("Invalid OpCode"),
    };
    // TODO hlen check
    let mut options = Vec::new();
    let mut i: usize = 240;
    loop {
        let l = p.len();
        if i < l {
            let code = p[i];
            if code == END {
                break;
            }
            if i + 2 < l {
                let opt_end = (p[i + 1]) as usize + i + 2;
                if opt_end < l {
                    options.push(Option {
                        code: code,
                        data: &p[i + 2..opt_end],
                    });
                    i = opt_end;
                    continue;
                }
            }
        }
        return Err("Options Problem");
    }
    Ok(Packet {
        reply: reply,
        hops: p[3],
        secs: ((p[8] as u16) << 8) + p[9] as u16,
        broadcast: p[10] & 128 == 128,
        ciaddr: [p[12], p[13], p[14], p[15]],
        yiaddr: [p[16], p[17], p[18], p[19]],
        siaddr: [p[20], p[21], p[22], p[23]],
        giaddr: [p[24], p[25], p[26], p[27]],
        options: options,
        chaddr: [p[28], p[29], p[30], p[31], p[32], p[33]],
        xid: [p[4], p[5], p[6], p[7]],
    })
}

impl<'a> Packet<'a> {
    /// Extracts requested option payload from packet if available
    pub fn option(&self, code: u8) -> std::option::Option<&'a [u8]> {
        for option in &self.options {
            if option.code == code {
                return Some(&option.data);
            }
        }
        None
    }

    /// Convenience function for extracting a packet's message type.
    pub fn message_type(&self) -> u8 {
        if let Some(x) = self.option(DHCP_MESSAGE_TYPE) {
            if x.len() > 0 {
                return x[0];
            }
        }
        0
    }

    /// Creates byte array DHCP packet
    pub fn encode<'c>(&'c self, p: &'c mut [u8]) -> &[u8] {
        p[..12].clone_from_slice(&[(if self.reply {
                                       BOOT_REPLY
                                   } else {
                                       BOOT_REQUEST
                                   }),
                                   1,
                                   6,
                                   self.hops,
                                   self.xid[0],
                                   self.xid[1],
                                   self.xid[2],
                                   self.xid[3],
                                   (self.secs >> 8) as u8,
                                   (self.secs & 255) as u8,
                                   (if self.broadcast {
                                       128
                                   } else {
                                       0
                                   }),
                                   0]);
        p[12..16].clone_from_slice(&self.ciaddr);
        p[16..20].clone_from_slice(&self.yiaddr);
        p[20..24].clone_from_slice(&self.siaddr);
        p[24..28].clone_from_slice(&self.giaddr);
        p[28..34].clone_from_slice(&self.chaddr);
        p[34..236].clone_from_slice(&[0; 202]);
        p[236..240].clone_from_slice(&COOKIE);

        let mut length: usize = 240;
        for option in &self.options {
            p[length] = option.code;
            p[length + 1] = option.data.len() as u8;
            p[length + 2..length + 2 + option.data.len()].clone_from_slice(option.data);
            length += 2 + option.data.len();
        }
        p[length] = END;
        length += 1;
        if length < 272 {
            // Pad to min size
            p[length..272].clone_from_slice(&[PAD; 32][..272 - length]);
            length = 272
        }
        &p[..length]
    }
}

const COOKIE: [u8; 4] = [99, 130, 83, 99];

const BOOT_REQUEST: u8 = 1; // From Client;
const BOOT_REPLY: u8 = 2; // From Server;

const END: u8 = 255;
const PAD: u8 = 0;
<|endoftext|>


<commit_before>use super::{Document,Root,RootChild};

struct Parser;

impl Parser {
    fn new() -> Parser {
        Parser
    }

    fn parse_preamble(&self, xml: &str) -> uint {
        // Parse the preamble
        let idx = xml.find_str("?>").expect("No preamble end");
        let end_of_preamble = idx + "?>".len();
        end_of_preamble
    }

    fn parse_element<'a>(&self, xml: &'a str) -> (uint, &'a str) {
        // Skip the opening
        // TODO: verify is a <
        let after_brace = "<".len();
        let ele = xml.slice_from(after_brace);
        let name = ele.slice_name().expect("failed to parse a name!");
        (after_brace + name.len(), name)
    }

    fn parse(&self, xml: &str) -> Document {
        let doc = Document::new();

        let end_of_preamble = self.parse_preamble(xml);
        let element = xml.slice_from(end_of_preamble);
        let (_end_of_element, name) = self.parse_element(element);

        let e = doc.new_element(name.to_string());
        doc.root().append_child(e);
        doc
    }
}

trait XmlStr<'a> {
    fn slice_name(&self) -> Option<&'a str>;
}

impl<'a> XmlStr<'a> for &'a str {
    fn slice_name(&self) -> Option<&'a str> {
        let mut positions = self.char_indices();

        let first_char = match positions.next() {
            Some((_, c)) if c.is_name_start_char() => c,
            Some((_, c)) => return None,
            None => return None,
        };

        // Skip past all the name chars
        let mut positions = positions.skip_while(|&(_, c)| c.is_name_char());

        match positions.next() {
            Some((offset, _)) => Some(self.slice_to(offset)),
            None => Some(self.clone()),
        }
    }
}

trait XmlChar {
    fn is_name_start_char(&self) -> bool;
    fn is_name_char(&self) -> bool;
}

impl XmlChar for char {
    fn is_name_start_char(&self) -> bool {
        match *self {
            ':'                        |
            'A'..'Z'                   |
            '_'                        |
            'a'..'z'                   |
            '\U000000C0'..'\U000000D6' |
            '\U000000D8'..'\U000000F6' |
            '\U000000F8'..'\U000002FF' |
            '\U00000370'..'\U0000037D' |
            '\U0000037F'..'\U00001FFF' |
            '\U0000200C'..'\U0000200D' |
            '\U00002070'..'\U0000218F' |
            '\U00002C00'..'\U00002FEF' |
            '\U00003001'..'\U0000D7FF' |
            '\U0000F900'..'\U0000FDCF' |
            '\U0000FDF0'..'\U0000FFFD' |
            '\U00010000'..'\U000EFFFF' => true,
            _ => false,
        }
    }

    fn is_name_char(&self) -> bool {
        if self.is_name_start_char() { return true; }
        match *self {
            '-'                |
            '.'                |
            '0'..'9'           |
            '\u00B7'           |
            '\u0300'..'\u036F' |
            '\u203F'..'\u2040' => true,
            _ => false
        }
    }
}

trait Hax {
    fn first_child(&self) -> Option<RootChild>;
}

impl Hax for Root {
    fn first_child(&self) -> Option<RootChild> {
        self.children().remove(0)
    }
}

#[test]
fn parses_a_document_with_a_single_element() {
    let parser = Parser::new();
    let doc = parser.parse("<?xml version='1.0' ?><hello />");
    let top = doc.root().first_child().unwrap().element().unwrap();

    assert_eq!(top.name().as_slice(), "hello");
}
<commit_msg>Add some more helpers<commit_after>use super::{Document,Root,RootChild,Element};

struct Parser;

impl Parser {
    fn new() -> Parser {
        Parser
    }

    fn parse_preamble<'a>(&self, xml: &'a str) -> &'a str {
        // Parse the preamble
        let idx = xml.find_str("?>").expect("No preamble end");
        let end_of_preamble = idx + "?>".len();
        xml.slice_from(end_of_preamble)
    }

    fn parse_element<'a>(&self, doc: Document, xml: &'a str) -> (Element, &'a str) {
        let (_, after_start_brace) = xml.expect_literal("<").expect("no start brace");

        let (name, after_name) = after_start_brace.slice_name().expect("failed to parse a name!");

        // skip_space
        let (_, after_end_brace) = after_name.expect_literal(" />").expect("no end brace");

        let e = doc.new_element(name.to_string());

        (e, after_end_brace)
    }

    fn parse(&self, xml: &str) -> Document {
        let doc = Document::new();

        let after_preamble = self.parse_preamble(xml);

        let (element, _tail) = self.parse_element(doc.clone(), after_preamble);
        doc.root().append_child(element);

        doc
    }
}

trait XmlStr<'a> {
    fn slice_at(&self, position: uint) -> (&'a str, &'a str);
    fn expect_literal(&self, expected: &str) -> Option<(&'a str, &'a str)>;
    fn slice_name(&self) -> Option<(&'a str, &'a str)>;
}

impl<'a> XmlStr<'a> for &'a str {
    fn slice_at(&self, position: uint) -> (&'a str, &'a str) {
        (self.slice_to(position), self.slice_from(position))
    }

    fn expect_literal(&self, expected: &str) -> Option<(&'a str, &'a str)> {
        if self.starts_with(expected) {
            Some(self.slice_at(expected.len()))
        } else {
            None
        }
    }

    fn slice_name(&self) -> Option<(&'a str, &'a str)> {
        let mut positions = self.char_indices();

        let first_char = match positions.next() {
            Some((_, c)) if c.is_name_start_char() => c,
            Some((_, c)) => return None,
            None => return None,
        };

        // Skip past all the name chars
        let mut positions = positions.skip_while(|&(_, c)| c.is_name_char());

        match positions.next() {
            Some((offset, _)) => Some(self.slice_at(offset)),
            None => Some((self.clone(), "")),
        }
    }
}

trait XmlChar {
    fn is_name_start_char(&self) -> bool;
    fn is_name_char(&self) -> bool;
}

impl XmlChar for char {
    fn is_name_start_char(&self) -> bool {
        match *self {
            ':'                        |
            'A'..'Z'                   |
            '_'                        |
            'a'..'z'                   |
            '\U000000C0'..'\U000000D6' |
            '\U000000D8'..'\U000000F6' |
            '\U000000F8'..'\U000002FF' |
            '\U00000370'..'\U0000037D' |
            '\U0000037F'..'\U00001FFF' |
            '\U0000200C'..'\U0000200D' |
            '\U00002070'..'\U0000218F' |
            '\U00002C00'..'\U00002FEF' |
            '\U00003001'..'\U0000D7FF' |
            '\U0000F900'..'\U0000FDCF' |
            '\U0000FDF0'..'\U0000FFFD' |
            '\U00010000'..'\U000EFFFF' => true,
            _ => false,
        }
    }

    fn is_name_char(&self) -> bool {
        if self.is_name_start_char() { return true; }
        match *self {
            '-'                |
            '.'                |
            '0'..'9'           |
            '\u00B7'           |
            '\u0300'..'\u036F' |
            '\u203F'..'\u2040' => true,
            _ => false
        }
    }
}

trait Hax {
    fn first_child(&self) -> Option<RootChild>;
}

impl Hax for Root {
    fn first_child(&self) -> Option<RootChild> {
        self.children().remove(0)
    }
}

#[test]
fn parses_a_document_with_a_single_element() {
    let parser = Parser::new();
    let doc = parser.parse("<?xml version='1.0' ?><hello />");
    let top = doc.root().first_child().unwrap().element().unwrap();

    assert_eq!(top.name().as_slice(), "hello");
}
<|endoftext|>


<commit_before>/// Doc-test test
/// ```rust
/// assert!(cargo_miri_test::make_true());
/// ```
/// ```rust,no_run
/// assert!(cargo_miri_test::make_true());
/// ```
/// ```rust,compile_fail
/// assert!(cargo_miri_test::make_true() == 5);
/// ```
pub fn make_true() -> bool {
    issue_1567::use_the_dependency();
    issue_1705::use_the_dependency();
    issue_1760::use_the_dependency!();
    issue_1691::use_me()
}
<commit_msg>Auto merge of #1770 - RalfJung:no_run, r=RalfJung<commit_after>/// Doc-test test
/// ```rust
/// assert!(cargo_miri_test::make_true());
/// ```
/// ```rust,no_run
/// assert!(!cargo_miri_test::make_true());
/// ```
/// ```rust,compile_fail
/// assert!(cargo_miri_test::make_true() == 5);
/// ```
pub fn make_true() -> bool {
    issue_1567::use_the_dependency();
    issue_1705::use_the_dependency();
    issue_1760::use_the_dependency!();
    issue_1691::use_me()
}
<|endoftext|>


<commit_before>use std::collections::VecDeque;
use std::iter::Peekable;
use lexer::Lexer;
use token::{Token, TokenType};

#[derive(Clone, PartialEq, Eq, Debug)]
enum MessageType {
    PGPMessage,
    PGPPublicKeyBlock,
    PGPPrivateKeyBlock,
    PGPSignature,
    PGPMessagePartXofY(usize, usize),
    PGPMessagePartX(usize),
}

#[derive(Clone, PartialEq, Eq, Debug)]
enum ParseError {
    CorruptHeader,
    EndOfFile,
    ParseError,
}

type ParseResult<T> = Result<T, ParseError>;

impl ParseError {
    fn eof<T>() -> ParseResult<T> {
        Err(ParseError::EndOfFile)
    }

    fn parse_error<T>() -> ParseResult<T> {
        Err(ParseError::ParseError)
    }

    fn corrupt_header<T>() -> ParseResult<T> {
        Err(ParseError::CorruptHeader)
    }
}

pub struct Parser<S> where S: Iterator<Item=char> {
    input:  Peekable<Lexer<S>>,
    lookahead: VecDeque<Token>,
    offset: usize
}

impl<S> Parser<S> where S: Iterator<Item=char> {
    pub fn new(mut input: Lexer<S>) -> Parser<S> {
        Parser {
            input:     input.peekable(),
            lookahead: VecDeque::with_capacity(20),
            offset:    0
        }
    }

    fn peek_token(&mut self) -> Option<Token> {
        if self.lookahead.is_empty() {
            self.offset = 0;
            let next_token = self.input.next();
            if next_token.is_some() {
                self.lookahead.push_back(next_token.clone().unwrap());
                Some(next_token.unwrap())
            } else {
                None
            }
        } else {
            self.sync();
            Some(self.lookahead[self.offset].clone())
        }
    }

    fn read_token(&mut self) -> Option<Token> {
        match self.peek_token() {
            Some(token) => {
                self.offset += 1;
                Some(token)
            }
            None => None
        }
    }

    fn sync(&mut self) {
        if self.offset > self.lookahead.len()-1 {
            let n = self.offset - (self.lookahead.len()-1);
            self.fill(n);
        }
    }

    fn fill(&mut self, amount: usize) {
        for _ in 0..amount {
            match self.input.next() {
                Some(token) => {
                    self.lookahead.push_back(token);
                }
                None => break
            }
        }
    }

    fn reset_offset(&mut self) {
        self.offset = 0;
    }

    fn consume(&mut self) {
        for _ in 0..self.offset {
            self.lookahead.pop_front();
        }
        self.reset_offset();
    }

    fn consume_char(&mut self) {
        if self.lookahead.is_empty() {
            self.reset_offset();
        } else {
            self.lookahead.pop_front();
            if self.offset > 0 {
                self.offset -= 1;
            }
        }
    }

    fn backtrack(&mut self, amount: usize) {
        if amount > self.offset {
            self.reset_offset();
        } else {
            self.offset -= amount;
        }
    }

    fn parse_number(&mut self) -> ParseResult<usize> {
        let mut result = String::new();
        loop {
            match self.peek_token() {
                Some(token) => {
                    match token.token_type() {
                        TokenType::Digit => {
                            result.push_str(token.as_str());
                            self.read_token();
                        }
                        _ => break
                    }
                }
                None => break
            }
        }

        if !result.is_empty() {
            let parse_result = result.parse::<usize>().unwrap();
            Ok(parse_result)
        } else if self.peek_token().is_none() {
            Err(ParseError::EndOfFile)
        } else {
            Err(ParseError::ParseError)
        }
    }

    fn read_or_else(&mut self, tt: TokenType, err: ParseError) -> ParseResult<Token> {
        match self.peek_token() {
            Some(token) => {
                if !token.has_token_type(tt) {
                    return Err(err);
                }
            } None => {
                return Err(ParseError::EndOfFile);
            }
        }

        Ok(self.read_token().unwrap())
    }

    fn parse_x_div_y(&mut self) -> ParseResult<(usize, usize)> {
        let num_x = self.parse_number();
        match num_x {
            Ok(_) => {}
            Err(e) => return Err(e)
        }

        match self.peek_token() {
            Some(token) => {
                match token.token_type() {
                    TokenType::ForwardSlash => {}
                    TokenType::FiveDashes => {}
                    _ => {
                        return Err(ParseError::CorruptHeader);
                    }
                }
            }
            None => return Err(ParseError::CorruptHeader)
        }

        let num_y = self.parse_number();
        match num_y {
            Ok(_) => {}
            Err(e) => return Err(e)
        }

        Ok((num_x.unwrap(), num_y.unwrap()))
    }

    fn parse_pgp_message_part(&mut self) -> ParseResult<MessageType> {
        match self.peek_token() {
            Some(token) => {
                match token.token_type() {
                    TokenType::PGPMessagePart => {
                        match self.parse_x_div_y() {
                            Ok((x,y)) => {
                                self.consume();
                                return Ok(MessageType::PGPMessagePartXofY(x,y))
                            }
                            Err(_)    => {}
                        }
                        match self.parse_number() {
                            Ok(x)  => {
                                self.consume();
                                return Ok(MessageType::PGPMessagePartX(x))
                            }
                            Err(e) => Err(ParseError::CorruptHeader)
                        }
                    }
                    _ => return Err(ParseError::CorruptHeader)
                }
            }
            None => return Err(ParseError::EndOfFile)
        }
    }

    fn parse_pgp_message(&mut self) -> ParseResult<MessageType> {
        match self.peek_token() {
            Some(token) => {
                match token.token_type() {
                    TokenType::PGPMessage => {
                        self.consume();
                        Ok(MessageType::PGPMessage)
                    }
                    _ => Err(ParseError::CorruptHeader)
                }
            }
            None => return Err(ParseError::EndOfFile)
        }
    }

    fn parse_pgp_publickey_block(&mut self) -> ParseResult<MessageType> {
        match self.peek_token() {
            Some(token) => {
                match token.token_type() {
                    TokenType::PGPPublicKeyBlock => {
                        self.consume();
                        Ok(MessageType::PGPPublicKeyBlock)
                    }
                    _ => Err(ParseError::CorruptHeader)
                }
            }
            None => return Err(ParseError::EndOfFile)
        }
    }

    fn parse_pgp_privatekey_block(&mut self) -> ParseResult<MessageType> {
        match self.peek_token() {
            Some(token) => {
                match token.token_type() {
                    TokenType::PGPPrivateKeyBlock => {
                        self.consume();
                        Ok(MessageType::PGPPrivateKeyBlock)
                    }
                    _ => Err(ParseError::CorruptHeader)
                }
            }
            None => return Err(ParseError::EndOfFile)
        }
    }

    fn parse_pgp_signature(&mut self) -> ParseResult<MessageType> {
        match self.peek_token() {
            Some(token) => {
                match token.token_type() {
                    TokenType::PGPSignature => {
                        self.consume();
                        Ok(MessageType::PGPSignature)
                    }
                    _ => Err(ParseError::CorruptHeader)
                }
            }
            None => return Err(ParseError::EndOfFile)
        }
    }

    fn parse_header_line(&mut self) -> ParseResult<MessageType> {
        match self.read_or_else(TokenType::FiveDashes, ParseError::CorruptHeader) {
            Ok(_)  => {}
            Err(e) => return Err(e)
        }
        match self.read_or_else(TokenType::Begin, ParseError::CorruptHeader) {
            Ok(_) => {}
            Err(e) => return Err(e)
        }

        let message_type = match self.peek_token() {
            Some(token) => {
                match token.token_type() {
                    TokenType::PGPMessagePart     => self.parse_pgp_message_part(),
                    TokenType::PGPMessage         => self.parse_pgp_message(),
                    TokenType::PGPPublicKeyBlock  => self.parse_pgp_publickey_block(),
                    TokenType::PGPPrivateKeyBlock => self.parse_pgp_privatekey_block(),
                    TokenType::PGPSignature       => self.parse_pgp_signature(),
                    _ => return Err(ParseError::CorruptHeader)
                }
            }
            None => return Err(ParseError::EndOfFile)
        };

        match message_type {
            Ok(_) => {}
            Err(e) => return Err(e)
        }

        match self.read_or_else(TokenType::FiveDashes, ParseError::CorruptHeader) {
            Ok(_)  => {}
            Err(e) => return Err(e)
        }

        self.consume();
        Ok(message_type.unwrap())
    }
}

#[cfg(test)]
mod tests {

}
<commit_msg>Refactor header and tail parser.<commit_after>use std::collections::VecDeque;
use std::iter::Peekable;
use lexer::Lexer;
use token::{Token, TokenType};

#[derive(Clone, PartialEq, Eq, Debug)]
enum MessageType {
    PGPMessage,
    PGPPublicKeyBlock,
    PGPPrivateKeyBlock,
    PGPSignature,
    PGPMessagePartXofY(usize, usize),
    PGPMessagePartX(usize),
}

#[derive(Clone, PartialEq, Eq, Debug)]
enum ParseError {
    CorruptHeader,
    EndOfFile,
    ParseError,
}

type ParseResult<T> = Result<T, ParseError>;

impl ParseError {
    fn eof<T>() -> ParseResult<T> {
        Err(ParseError::EndOfFile)
    }

    fn parse_error<T>() -> ParseResult<T> {
        Err(ParseError::ParseError)
    }

    fn corrupt_header<T>() -> ParseResult<T> {
        Err(ParseError::CorruptHeader)
    }
}

pub struct Parser<S> where S: Iterator<Item=char> {
    input:  Peekable<Lexer<S>>,
    lookahead: VecDeque<Token>,
    offset: usize
}

impl<S> Parser<S> where S: Iterator<Item=char> {
    pub fn new(mut input: Lexer<S>) -> Parser<S> {
        Parser {
            input:     input.peekable(),
            lookahead: VecDeque::with_capacity(20),
            offset:    0
        }
    }

    fn peek_token(&mut self) -> Option<Token> {
        if self.lookahead.is_empty() {
            self.offset = 0;
            let next_token = self.input.next();
            if next_token.is_some() {
                self.lookahead.push_back(next_token.clone().unwrap());
                Some(next_token.unwrap())
            } else {
                None
            }
        } else {
            self.sync();
            Some(self.lookahead[self.offset].clone())
        }
    }

    fn read_token(&mut self) -> Option<Token> {
        match self.peek_token() {
            Some(token) => {
                self.offset += 1;
                Some(token)
            }
            None => None
        }
    }

    fn sync(&mut self) {
        if self.offset > self.lookahead.len()-1 {
            let n = self.offset - (self.lookahead.len()-1);
            self.fill(n);
        }
    }

    fn fill(&mut self, amount: usize) {
        for _ in 0..amount {
            match self.input.next() {
                Some(token) => {
                    self.lookahead.push_back(token);
                }
                None => break
            }
        }
    }

    fn reset_offset(&mut self) {
        self.offset = 0;
    }

    fn consume(&mut self) {
        for _ in 0..self.offset {
            self.lookahead.pop_front();
        }
        self.reset_offset();
    }

    fn consume_char(&mut self) {
        if self.lookahead.is_empty() {
            self.reset_offset();
        } else {
            self.lookahead.pop_front();
            if self.offset > 0 {
                self.offset -= 1;
            }
        }
    }

    fn backtrack(&mut self, amount: usize) {
        if amount > self.offset {
            self.reset_offset();
        } else {
            self.offset -= amount;
        }
    }

    fn parse_number(&mut self) -> ParseResult<usize> {
        let mut result = String::new();
        loop {
            match self.peek_token() {
                Some(token) => {
                    match token.token_type() {
                        TokenType::Digit => {
                            result.push_str(token.as_str());
                            self.read_token();
                        }
                        _ => break
                    }
                }
                None => break
            }
        }

        if !result.is_empty() {
            let parse_result = result.parse::<usize>().unwrap();
            Ok(parse_result)
        } else if self.peek_token().is_none() {
            Err(ParseError::EndOfFile)
        } else {
            Err(ParseError::ParseError)
        }
    }

    fn read_or_else(&mut self, tt: TokenType, err: ParseError) -> ParseResult<Token> {
        match self.peek_token() {
            Some(token) => {
                if !token.has_token_type(tt) {
                    return Err(err);
                }
            } None => {
                return Err(ParseError::EndOfFile);
            }
        }

        Ok(self.read_token().unwrap())
    }

    fn parse_x_div_y(&mut self) -> ParseResult<(usize, usize)> {
        let num_x = self.parse_number();
        match num_x {
            Ok(_) => {}
            Err(e) => return Err(e)
        }

        match self.peek_token() {
            Some(token) => {
                match token.token_type() {
                    TokenType::ForwardSlash => {}
                    TokenType::FiveDashes => {}
                    _ => {
                        return Err(ParseError::CorruptHeader);
                    }
                }
            }
            None => return Err(ParseError::CorruptHeader)
        }

        let num_y = self.parse_number();
        match num_y {
            Ok(_) => {}
            Err(e) => return Err(e)
        }

        Ok((num_x.unwrap(), num_y.unwrap()))
    }

    fn parse_pgp_message_part(&mut self) -> ParseResult<MessageType> {
        match self.peek_token() {
            Some(token) => {
                match token.token_type() {
                    TokenType::PGPMessagePart => {
                        match self.parse_x_div_y() {
                            Ok((x,y)) => {
                                self.consume();
                                return Ok(MessageType::PGPMessagePartXofY(x,y))
                            }
                            Err(_)    => {}
                        }
                        match self.parse_number() {
                            Ok(x)  => {
                                self.consume();
                                return Ok(MessageType::PGPMessagePartX(x))
                            }
                            Err(e) => Err(ParseError::CorruptHeader)
                        }
                    }
                    _ => return Err(ParseError::CorruptHeader)
                }
            }
            None => return Err(ParseError::EndOfFile)
        }
    }

    fn parse_pgp_message(&mut self) -> ParseResult<MessageType> {
        match self.peek_token() {
            Some(token) => {
                match token.token_type() {
                    TokenType::PGPMessage => {
                        self.consume();
                        Ok(MessageType::PGPMessage)
                    }
                    _ => Err(ParseError::CorruptHeader)
                }
            }
            None => return Err(ParseError::EndOfFile)
        }
    }

    fn parse_pgp_publickey_block(&mut self) -> ParseResult<MessageType> {
        match self.peek_token() {
            Some(token) => {
                match token.token_type() {
                    TokenType::PGPPublicKeyBlock => {
                        self.consume();
                        Ok(MessageType::PGPPublicKeyBlock)
                    }
                    _ => Err(ParseError::CorruptHeader)
                }
            }
            None => return Err(ParseError::EndOfFile)
        }
    }

    fn parse_pgp_privatekey_block(&mut self) -> ParseResult<MessageType> {
        match self.peek_token() {
            Some(token) => {
                match token.token_type() {
                    TokenType::PGPPrivateKeyBlock => {
                        self.consume();
                        Ok(MessageType::PGPPrivateKeyBlock)
                    }
                    _ => Err(ParseError::CorruptHeader)
                }
            }
            None => return Err(ParseError::EndOfFile)
        }
    }

    fn parse_pgp_signature(&mut self) -> ParseResult<MessageType> {
        match self.peek_token() {
            Some(token) => {
                match token.token_type() {
                    TokenType::PGPSignature => {
                        self.consume();
                        Ok(MessageType::PGPSignature)
                    }
                    _ => Err(ParseError::CorruptHeader)
                }
            }
            None => return Err(ParseError::EndOfFile)
        }
    }

    fn __parse_header_tail_line(&mut self, tt: TokenType) -> ParseResult<MessageType> {
        match self.read_or_else(TokenType::FiveDashes, ParseError::CorruptHeader) {
            Ok(_)  => {}
            Err(e) => return Err(e)
        }
        match self.read_or_else(tt, ParseError::CorruptHeader) {
            Ok(_) => {}
            Err(e) => return Err(e)
        }

        let message_type = match self.peek_token() {
            Some(token) => {
                match token.token_type() {
                    TokenType::PGPMessagePart     => self.parse_pgp_message_part(),
                    TokenType::PGPMessage         => self.parse_pgp_message(),
                    TokenType::PGPPublicKeyBlock  => self.parse_pgp_publickey_block(),
                    TokenType::PGPPrivateKeyBlock => self.parse_pgp_privatekey_block(),
                    TokenType::PGPSignature       => self.parse_pgp_signature(),
                    _ => return Err(ParseError::CorruptHeader)
                }
            }
            None => return Err(ParseError::EndOfFile)
        };

        match message_type {
            Ok(_) => {}
            Err(e) => return Err(e)
        }

        match self.read_or_else(TokenType::FiveDashes, ParseError::CorruptHeader) {
            Ok(_)  => {}
            Err(e) => return Err(e)
        }

        self.consume();
        Ok(message_type.unwrap())
    }

    fn parse_header_line(&mut self) -> ParseResult<MessageType> {
        self.__parse_header_tail_line(TokenType::Begin)
    }

    fn parse_tail_line(&mut self) -> ParseResult<MessageType> {
        self.__parse_header_tail_line(TokenType::End)
    }
}

#[cfg(test)]
mod tests {

}
<|endoftext|>