<commit_before>//! `core_arch`

#[macro_use]
mod macros;

#[cfg(any(target_arch = "arm", target_arch = "aarch64", doc))]
mod arm_shared;

mod simd;

#[doc = include_str!("core_arch_docs.md")]
#[stable(feature = "simd_arch", since = "1.27.0")]
pub mod arch {
    /// Inline assembly.
    ///
    /// Read the [unstable book] for the usage.
    ///
    /// [unstable book]: ../unstable-book/library-features/asm.html
    #[unstable(
        feature = "asm",
        issue = "72016",
        reason = "inline assembly is not stable enough for use and is subject to change"
    )]
    #[rustc_builtin_macro]
    pub macro asm("assembly template", $(operands,)* $(options($(option),*))?) {
        /* compiler built-in */
    }

    /// Module-level inline assembly.
    #[unstable(
        feature = "global_asm",
        issue = "35119",
        reason = "`global_asm!` is not stable enough for use and is subject to change"
    )]
    #[rustc_builtin_macro]
    pub macro global_asm("assembly template", $(operands,)* $(options($(option),*))?) {
        /* compiler built-in */
    }

    /// Platform-specific intrinsics for the `x86` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "x86", doc))]
    #[doc(cfg(target_arch = "x86"))]
    #[stable(feature = "simd_x86", since = "1.27.0")]
    pub mod x86 {
        #[stable(feature = "simd_x86", since = "1.27.0")]
        pub use crate::core_arch::x86::*;
    }

    /// Platform-specific intrinsics for the `x86_64` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "x86_64", doc))]
    #[doc(cfg(target_arch = "x86_64"))]
    #[stable(feature = "simd_x86", since = "1.27.0")]
    pub mod x86_64 {
        #[stable(feature = "simd_x86", since = "1.27.0")]
        pub use crate::core_arch::x86::*;
        #[stable(feature = "simd_x86", since = "1.27.0")]
        pub use crate::core_arch::x86_64::*;
    }

    /// Platform-specific intrinsics for the `arm` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "arm", doc))]
    #[doc(cfg(target_arch = "arm"))]
    #[unstable(feature = "stdsimd", issue = "27731")]
    pub mod arm {
        pub use crate::core_arch::arm::*;
    }

    /// Platform-specific intrinsics for the `aarch64` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "aarch64", doc))]
    #[doc(cfg(target_arch = "aarch64"))]
    #[unstable(feature = "stdsimd", issue = "27731")]
    pub mod aarch64 {
        pub use crate::core_arch::aarch64::*;
    }

    /// Platform-specific intrinsics for the `wasm32` platform.
    ///
    /// This module provides intrinsics specific to the WebAssembly
    /// architecture. Here you'll find intrinsics specific to WebAssembly that
    /// aren't otherwise surfaced somewhere in a cross-platform abstraction of
    /// `std`, and you'll also find functions for leveraging WebAssembly
    /// proposals such as [atomics] and [simd].
    ///
    /// Intrinsics in the `wasm32` module are modeled after the WebAssembly
    /// instructions that they represent. Most functions are named after the
    /// instruction they intend to correspond to, and the arguments/results
    /// correspond to the type signature of the instruction itself. Stable
    /// WebAssembly instructions are [documented online][instrdoc].
    ///
    /// [instrdoc]: https://webassembly.github.io/spec/core/valid/instructions.html
    ///
    /// If a proposal is not yet stable in WebAssembly itself then the functions
    /// within this function may be unstable and require the nightly channel of
    /// Rust to use. As the proposal itself stabilizes the intrinsics in this
    /// module should stabilize as well.
    ///
    /// [atomics]: https://github.com/webassembly/threads
    /// [simd]: https://github.com/webassembly/simd
    ///
    /// See the [module documentation](../index.html) for general information
    /// about the `arch` module and platform intrinsics.
    ///
    /// ## Atomics
    ///
    /// The [threads proposal][atomics] for WebAssembly adds a number of
    /// instructions for dealing with multithreaded programs. Most instructions
    /// added in the [atomics] proposal are exposed in Rust through the
    /// `std::sync::atomic` module. Some instructions, however, don't have
    /// direct equivalents in Rust so they're exposed here instead.
    ///
    /// Note that the instructions added in the [atomics] proposal can work in
    /// either a context with a shared wasm memory and without. These intrinsics
    /// are always available in the standard library, but you likely won't be
    /// able to use them too productively unless you recompile the standard
    /// library (and all your code) with `-Ctarget-feature=+atomics`.
    ///
    /// It's also worth pointing out that multi-threaded WebAssembly and its
    /// story in Rust is still in a somewhat "early days" phase as of the time
    /// of this writing. Pieces should mostly work but it generally requires a
    /// good deal of manual setup. At this time it's not as simple as "just call
    /// `std::thread::spawn`", but it will hopefully get there one day!
    ///
    /// ## SIMD
    ///
    /// The [simd proposal][simd] for WebAssembly added a new `v128` type for a
    /// 128-bit SIMD register. It also added a large array of instructions to
    /// operate on the `v128` type to perform data processing. Using SIMD on
    /// wasm is intended to be similar to as you would on `x86_64`, for example.
    /// You'd write a function such as:
    ///
    /// ```rust,ignore
    /// #[cfg(target_arch = "wasm32")]
    /// #[target_feature(enable = "simd128")]
    /// unsafe fn uses_simd() {
    ///     use std::arch::wasm32::*;
    ///     // ...
    /// }
    /// ```
    ///
    /// Unlike `x86_64`, however, WebAssembly does not currently have dynamic
    /// detection at runtime as to whether SIMD is supported (this is one of the
    /// motivators for the [conditional sections][condsections] and [feature
    /// detection] proposals, but that is still pretty early days). This means
    /// that your binary will either have SIMD and can only run on engines
    /// which support SIMD, or it will not have SIMD at all. For compatibility
    /// the standard library itself does not use any SIMD internally.
    /// Determining how best to ship your WebAssembly binary with SIMD is
    /// largely left up to you as it can can be pretty nuanced depending on
    /// your situation.
    ///
    /// [condsections]: https://github.com/webassembly/conditional-sections
    /// [feature detection]: https://github.com/WebAssembly/feature-detection
    ///
    /// To enable SIMD support at compile time you need to do one of two things:
    ///
    /// * First you can annotate functions with `#[target_feature(enable =
    ///   "simd128")]`. This causes just that one function to have SIMD support
    ///   available to it, and intrinsics will get inlined as usual in this
    ///   situation.
    ///
    /// * Second you can compile your program with `-Ctarget-feature=+simd128`.
    ///   This compilation flag blanket enables SIMD support for your entire
    ///   compilation. Note that this does not include the standard library
    ///   unless you [recompile the standard library][buildstd].
    ///
    /// [buildstd]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#build-std
    ///
    /// If you enable SIMD via either of these routes then you'll have a
    /// WebAssembly binary that uses SIMD instructions, and you'll need to ship
    /// that accordingly. Also note that if you call SIMD intrinsics but don't
    /// enable SIMD via either of these mechanisms, you'll still have SIMD
    /// generated in your program. This means to generate a binary without SIMD
    /// you'll need to avoid both options above plus calling into any intrinsics
    /// in this module.
    #[cfg(any(target_arch = "wasm32", doc))]
    #[doc(cfg(target_arch = "wasm32"))]
    #[stable(feature = "simd_wasm32", since = "1.33.0")]
    pub mod wasm32 {
        #[stable(feature = "simd_wasm32", since = "1.33.0")]
        pub use crate::core_arch::wasm32::*;
    }

    /// Platform-specific intrinsics for the `mips` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "mips", doc))]
    #[doc(cfg(target_arch = "mips"))]
    #[unstable(feature = "stdsimd", issue = "27731")]
    pub mod mips {
        pub use crate::core_arch::mips::*;
    }

    /// Platform-specific intrinsics for the `mips64` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "mips64", doc))]
    #[doc(cfg(target_arch = "mips64"))]
    #[unstable(feature = "stdsimd", issue = "27731")]
    pub mod mips64 {
        pub use crate::core_arch::mips::*;
    }

    /// Platform-specific intrinsics for the `PowerPC` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "powerpc", doc))]
    #[doc(cfg(target_arch = "powerpc"))]
    #[unstable(feature = "stdsimd", issue = "27731")]
    pub mod powerpc {
        pub use crate::core_arch::powerpc::*;
    }

    /// Platform-specific intrinsics for the `PowerPC64` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "powerpc64", doc))]
    #[doc(cfg(target_arch = "powerpc64"))]
    #[unstable(feature = "stdsimd", issue = "27731")]
    pub mod powerpc64 {
        pub use crate::core_arch::powerpc64::*;
    }

    /// Platform-specific intrinsics for the `NVPTX` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "nvptx", target_arch = "nvptx64", doc))]
    #[doc(cfg(any(target_arch = "nvptx", target_arch = "nvptx64")))]
    #[unstable(feature = "stdsimd", issue = "27731")]
    pub mod nvptx {
        pub use crate::core_arch::nvptx::*;
    }
}

mod simd_llvm;

#[cfg(any(target_arch = "x86", target_arch = "x86_64", doc))]
#[doc(cfg(any(target_arch = "x86", target_arch = "x86_64")))]
mod x86;
#[cfg(any(target_arch = "x86_64", doc))]
#[doc(cfg(target_arch = "x86_64"))]
mod x86_64;

#[cfg(any(target_arch = "aarch64", doc))]
#[doc(cfg(target_arch = "aarch64"))]
mod aarch64;
#[cfg(any(target_arch = "arm", doc))]
#[doc(cfg(any(target_arch = "arm")))]
mod arm;

#[cfg(any(target_arch = "wasm32", doc))]
#[doc(cfg(target_arch = "wasm32"))]
mod wasm32;

#[cfg(any(target_arch = "mips", target_arch = "mips64", doc))]
#[doc(cfg(any(target_arch = "mips", target_arch = "mips64")))]
mod mips;

#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64", doc))]
#[doc(cfg(any(target_arch = "powerpc", target_arch = "powerpc64")))]
mod powerpc;

#[cfg(any(target_arch = "powerpc64", doc))]
#[doc(cfg(target_arch = "powerpc64"))]
mod powerpc64;

#[cfg(any(target_arch = "nvptx", target_arch = "nvptx64", doc))]
#[doc(cfg(any(target_arch = "nvptx", target_arch = "nvptx64")))]
mod nvptx;
<commit_msg>Revert "Move `asm!` and `global_asm!` to `core::arch` (#1183)" (#1185)<commit_after>//! `core_arch`

#[macro_use]
mod macros;

#[cfg(any(target_arch = "arm", target_arch = "aarch64", doc))]
mod arm_shared;

mod simd;

#[doc = include_str!("core_arch_docs.md")]
#[stable(feature = "simd_arch", since = "1.27.0")]
pub mod arch {
    /// Platform-specific intrinsics for the `x86` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "x86", doc))]
    #[doc(cfg(target_arch = "x86"))]
    #[stable(feature = "simd_x86", since = "1.27.0")]
    pub mod x86 {
        #[stable(feature = "simd_x86", since = "1.27.0")]
        pub use crate::core_arch::x86::*;
    }

    /// Platform-specific intrinsics for the `x86_64` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "x86_64", doc))]
    #[doc(cfg(target_arch = "x86_64"))]
    #[stable(feature = "simd_x86", since = "1.27.0")]
    pub mod x86_64 {
        #[stable(feature = "simd_x86", since = "1.27.0")]
        pub use crate::core_arch::x86::*;
        #[stable(feature = "simd_x86", since = "1.27.0")]
        pub use crate::core_arch::x86_64::*;
    }

    /// Platform-specific intrinsics for the `arm` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "arm", doc))]
    #[doc(cfg(target_arch = "arm"))]
    #[unstable(feature = "stdsimd", issue = "27731")]
    pub mod arm {
        pub use crate::core_arch::arm::*;
    }

    /// Platform-specific intrinsics for the `aarch64` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "aarch64", doc))]
    #[doc(cfg(target_arch = "aarch64"))]
    #[unstable(feature = "stdsimd", issue = "27731")]
    pub mod aarch64 {
        pub use crate::core_arch::aarch64::*;
    }

    /// Platform-specific intrinsics for the `wasm32` platform.
    ///
    /// This module provides intrinsics specific to the WebAssembly
    /// architecture. Here you'll find intrinsics specific to WebAssembly that
    /// aren't otherwise surfaced somewhere in a cross-platform abstraction of
    /// `std`, and you'll also find functions for leveraging WebAssembly
    /// proposals such as [atomics] and [simd].
    ///
    /// Intrinsics in the `wasm32` module are modeled after the WebAssembly
    /// instructions that they represent. Most functions are named after the
    /// instruction they intend to correspond to, and the arguments/results
    /// correspond to the type signature of the instruction itself. Stable
    /// WebAssembly instructions are [documented online][instrdoc].
    ///
    /// [instrdoc]: https://webassembly.github.io/spec/core/valid/instructions.html
    ///
    /// If a proposal is not yet stable in WebAssembly itself then the functions
    /// within this function may be unstable and require the nightly channel of
    /// Rust to use. As the proposal itself stabilizes the intrinsics in this
    /// module should stabilize as well.
    ///
    /// [atomics]: https://github.com/webassembly/threads
    /// [simd]: https://github.com/webassembly/simd
    ///
    /// See the [module documentation](../index.html) for general information
    /// about the `arch` module and platform intrinsics.
    ///
    /// ## Atomics
    ///
    /// The [threads proposal][atomics] for WebAssembly adds a number of
    /// instructions for dealing with multithreaded programs. Most instructions
    /// added in the [atomics] proposal are exposed in Rust through the
    /// `std::sync::atomic` module. Some instructions, however, don't have
    /// direct equivalents in Rust so they're exposed here instead.
    ///
    /// Note that the instructions added in the [atomics] proposal can work in
    /// either a context with a shared wasm memory and without. These intrinsics
    /// are always available in the standard library, but you likely won't be
    /// able to use them too productively unless you recompile the standard
    /// library (and all your code) with `-Ctarget-feature=+atomics`.
    ///
    /// It's also worth pointing out that multi-threaded WebAssembly and its
    /// story in Rust is still in a somewhat "early days" phase as of the time
    /// of this writing. Pieces should mostly work but it generally requires a
    /// good deal of manual setup. At this time it's not as simple as "just call
    /// `std::thread::spawn`", but it will hopefully get there one day!
    ///
    /// ## SIMD
    ///
    /// The [simd proposal][simd] for WebAssembly added a new `v128` type for a
    /// 128-bit SIMD register. It also added a large array of instructions to
    /// operate on the `v128` type to perform data processing. Using SIMD on
    /// wasm is intended to be similar to as you would on `x86_64`, for example.
    /// You'd write a function such as:
    ///
    /// ```rust,ignore
    /// #[cfg(target_arch = "wasm32")]
    /// #[target_feature(enable = "simd128")]
    /// unsafe fn uses_simd() {
    ///     use std::arch::wasm32::*;
    ///     // ...
    /// }
    /// ```
    ///
    /// Unlike `x86_64`, however, WebAssembly does not currently have dynamic
    /// detection at runtime as to whether SIMD is supported (this is one of the
    /// motivators for the [conditional sections][condsections] and [feature
    /// detection] proposals, but that is still pretty early days). This means
    /// that your binary will either have SIMD and can only run on engines
    /// which support SIMD, or it will not have SIMD at all. For compatibility
    /// the standard library itself does not use any SIMD internally.
    /// Determining how best to ship your WebAssembly binary with SIMD is
    /// largely left up to you as it can can be pretty nuanced depending on
    /// your situation.
    ///
    /// [condsections]: https://github.com/webassembly/conditional-sections
    /// [feature detection]: https://github.com/WebAssembly/feature-detection
    ///
    /// To enable SIMD support at compile time you need to do one of two things:
    ///
    /// * First you can annotate functions with `#[target_feature(enable =
    ///   "simd128")]`. This causes just that one function to have SIMD support
    ///   available to it, and intrinsics will get inlined as usual in this
    ///   situation.
    ///
    /// * Second you can compile your program with `-Ctarget-feature=+simd128`.
    ///   This compilation flag blanket enables SIMD support for your entire
    ///   compilation. Note that this does not include the standard library
    ///   unless you [recompile the standard library][buildstd].
    ///
    /// [buildstd]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#build-std
    ///
    /// If you enable SIMD via either of these routes then you'll have a
    /// WebAssembly binary that uses SIMD instructions, and you'll need to ship
    /// that accordingly. Also note that if you call SIMD intrinsics but don't
    /// enable SIMD via either of these mechanisms, you'll still have SIMD
    /// generated in your program. This means to generate a binary without SIMD
    /// you'll need to avoid both options above plus calling into any intrinsics
    /// in this module.
    #[cfg(any(target_arch = "wasm32", doc))]
    #[doc(cfg(target_arch = "wasm32"))]
    #[stable(feature = "simd_wasm32", since = "1.33.0")]
    pub mod wasm32 {
        #[stable(feature = "simd_wasm32", since = "1.33.0")]
        pub use crate::core_arch::wasm32::*;
    }

    /// Platform-specific intrinsics for the `mips` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "mips", doc))]
    #[doc(cfg(target_arch = "mips"))]
    #[unstable(feature = "stdsimd", issue = "27731")]
    pub mod mips {
        pub use crate::core_arch::mips::*;
    }

    /// Platform-specific intrinsics for the `mips64` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "mips64", doc))]
    #[doc(cfg(target_arch = "mips64"))]
    #[unstable(feature = "stdsimd", issue = "27731")]
    pub mod mips64 {
        pub use crate::core_arch::mips::*;
    }

    /// Platform-specific intrinsics for the `PowerPC` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "powerpc", doc))]
    #[doc(cfg(target_arch = "powerpc"))]
    #[unstable(feature = "stdsimd", issue = "27731")]
    pub mod powerpc {
        pub use crate::core_arch::powerpc::*;
    }

    /// Platform-specific intrinsics for the `PowerPC64` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "powerpc64", doc))]
    #[doc(cfg(target_arch = "powerpc64"))]
    #[unstable(feature = "stdsimd", issue = "27731")]
    pub mod powerpc64 {
        pub use crate::core_arch::powerpc64::*;
    }

    /// Platform-specific intrinsics for the `NVPTX` platform.
    ///
    /// See the [module documentation](../index.html) for more details.
    #[cfg(any(target_arch = "nvptx", target_arch = "nvptx64", doc))]
    #[doc(cfg(any(target_arch = "nvptx", target_arch = "nvptx64")))]
    #[unstable(feature = "stdsimd", issue = "27731")]
    pub mod nvptx {
        pub use crate::core_arch::nvptx::*;
    }
}

mod simd_llvm;

#[cfg(any(target_arch = "x86", target_arch = "x86_64", doc))]
#[doc(cfg(any(target_arch = "x86", target_arch = "x86_64")))]
mod x86;
#[cfg(any(target_arch = "x86_64", doc))]
#[doc(cfg(target_arch = "x86_64"))]
mod x86_64;

#[cfg(any(target_arch = "aarch64", doc))]
#[doc(cfg(target_arch = "aarch64"))]
mod aarch64;
#[cfg(any(target_arch = "arm", doc))]
#[doc(cfg(any(target_arch = "arm")))]
mod arm;

#[cfg(any(target_arch = "wasm32", doc))]
#[doc(cfg(target_arch = "wasm32"))]
mod wasm32;

#[cfg(any(target_arch = "mips", target_arch = "mips64", doc))]
#[doc(cfg(any(target_arch = "mips", target_arch = "mips64")))]
mod mips;

#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64", doc))]
#[doc(cfg(any(target_arch = "powerpc", target_arch = "powerpc64")))]
mod powerpc;

#[cfg(any(target_arch = "powerpc64", doc))]
#[doc(cfg(target_arch = "powerpc64"))]
mod powerpc64;

#[cfg(any(target_arch = "nvptx", target_arch = "nvptx64", doc))]
#[doc(cfg(any(target_arch = "nvptx", target_arch = "nvptx64")))]
mod nvptx;
<|endoftext|>


<commit_before>use chrono::DateTime;
use chrono::offset::Utc;
use diesel;
use diesel::prelude::*;
use diesel::pg::PgConnection;
use diesel::pg::upsert::*;
use futures::{Future, stream, Stream};
use futures_cpupool::CpuPool;
use hyper::{Body, Client, Uri};
use hyper::client::HttpConnector;
use hyper_tls::HttpsConnector;
use InsertError;
use kuchiki;
use kuchiki::traits::*;
use r2d2_diesel::ConnectionManager;
use r2d2::Pool;
use rusoto_core::{default_tls_client, Region};
use rusoto_credential::DefaultCredentialsProvider;
use rusoto_s3::{PutObjectRequest, S3Client, S3};
use schema::ads;
use server::AdPost;

const ENDPOINT: &'static str = "https://pp-facebook-ads.s3.amazonaws.com/";

fn get_title(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    document
        .select("h5 a, h6 a, strong")
        .map_err(InsertError::HTML)?
        .nth(0)
        .and_then(|a| Some(a.text_contents()))
        .ok_or(InsertError::HTML(()))
}

fn get_image(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    document
        .select("img")
        .map_err(InsertError::HTML)?
        .nth(0)
        .and_then(|a| {
            a.attributes.borrow().get("src").and_then(
                |src| Some(src.to_string()),
            )
        })
        .ok_or(InsertError::HTML(()))
}

fn get_message(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    let selectors = vec![".userContent p", "span"];
    let iters = selectors.iter().map(|s| document.select(s)).flat_map(|a| a);

    iters
        .map(|i| {
            i.fold(String::new(), |m, a| m + &a.as_node().to_string())
        })
        .filter(|i| !i.is_empty())
        .nth(0)
        .ok_or(InsertError::HTML(()))
}

fn get_images(document: &kuchiki::NodeRef) -> Result<Vec<String>, InsertError> {
    Ok(
        document
            .select("img")
            .map_err(InsertError::HTML)?
            .skip(1)
            .map(|a| {
                a.attributes.borrow().get("src").and_then(
                    |s| Some(s.to_string()),
                )
            })
            .filter(|s| s.is_some())
            .map(|s| s.unwrap())
            .collect::<Vec<String>>(),
    )
}

#[derive(AsChangeset, Debug)]
#[table_name = "ads"]
pub struct Images {
    thumbnail: Option<String>,
    images: Vec<String>,
    title: String,
    message: String,
    html: String,
}

impl Images {
    fn from_ad(ad: &Ad, images: Vec<Uri>) -> Result<Images, InsertError> {
        let thumb = images
            .iter()
            .filter(|i| ad.thumbnail.contains(i.path()))
            .map(|i| ENDPOINT.to_string() + i.path().trim_left_matches('/'))
            .nth(0);

        let mut rest = images.clone();
        if let Some(thumb) = thumb.clone() {
            rest.retain(|x| !thumb.contains(x.path()))
        };

        let collection = rest.iter()
            .filter(|i| ad.images.iter().any(|a| a.contains(i.path())))
            .map(|i| ENDPOINT.to_string() + i.path().trim_left_matches('/'))
            .collect::<Vec<String>>();

        let document = kuchiki::parse_html().one(ad.html.clone());
        for a in document.select("img").map_err(InsertError::HTML)? {
            if let Some(x) = a.attributes.borrow_mut().get_mut("src") {
                if let Ok(u) = x.parse::<Uri>() {
                    if let Some(i) = images.iter().find(|i| i.path() == u.path()) {
                        *x = ENDPOINT.to_string() + i.path().trim_left_matches('/');
                    } else {
                        *x = "".to_string();
                    }
                }
            };
        }

        let title = get_title(&document)?;
        let message = get_title(&document)?;
        Ok(Images {
            thumbnail: thumb,
            images: collection,
            title: title,
            html: document
                .select("div")
                .map_err(InsertError::HTML)?
                .nth(0)
                .ok_or(InsertError::HTML(()))?
                .as_node()
                .to_string(),
            message: message,
        })
    }
}

#[derive(Queryable, Debug, Clone)]
pub struct Ad {
    id: String,
    html: String,
    political: i32,
    not_political: i32,
    fuzzy_id: Option<i32>,
    title: String,
    message: String,
    thumbnail: String,
    created_at: DateTime<Utc>,
    updated_at: DateTime<Utc>,
    browser_lang: String,
    images: Vec<String>,
    impressions: i32,
    political_probability: f64,
    targeting: Option<String>,
}

impl Ad {
    // This will asynchronously save the images to s3 we may very well end up
    // dropping images, but I can't see any way around it right now. Also we
    // should think about splitting this up, but I'm fine -- if a little
    // embarassed about it -- right now. This function swallows errors, and
    // there's a chance we'll end up with no images at the end, but I think we
    // can handle that in the extension's UI.
    pub fn grab_and_store(
        &self,
        client: Client<HttpsConnector<HttpConnector>, Body>,
        db: &Pool<ConnectionManager<PgConnection>>,
        pool: CpuPool,
    ) -> Box<Future<Item = (), Error = ()>> {
        let ad = self.clone();
        let pool_s3 = pool.clone();
        let pool_db = pool.clone();
        let db = db.clone();
        let future = stream::iter_ok(self.image_urls())
            .map(|u| u.unwrap())
            // filter ones we already have in the db and ones we can verify as
            // coming from fb, we don't want to become a malware vector :)
            // currently we redownload images we already have, but ok.
            .filter(|u| {
                info!("testing {:?}", u.host());
                match u.host() {
                    Some(h) => (h == "pp-facebook-ads.s3.amazonaws.com" || h.ends_with("fbcdn.net")),
                    None => false
                }
            })
            // grab image
            .and_then(move |img| {
                info!("getting {:?}", img);
                let cloned = img.clone();
                client
                    .get(img)
                    .and_then(|res| {
                        res.body().concat2().and_then(|chunk| Ok((chunk, cloned)))
                    })
                    .map_err(InsertError::Hyper)
            })
            // upload them to s3
            .and_then(move |tuple| {
                let pool = pool_s3.clone();
                // we do this in a worker thread because rusoto isn't on
                // Hyper async yet.
                pool.spawn_fn(move || {
                    if tuple.1.host().unwrap() != "pp-facebook-ads.s3.amazonaws.com" {
                        let client =
                            S3Client::new(default_tls_client().map_err(InsertError::TLS)?,
                                          DefaultCredentialsProvider::new().map_err(InsertError::AWS)?,
                                          Region::UsEast1);
                        let req = PutObjectRequest {
                            bucket: "pp-facebook-ads".to_string(),
                            key: tuple.1.path().trim_left_matches('/').to_string(),
                            acl: Some("public-read".to_string()),
                            body: Some(tuple.0.to_vec()),
                            ..PutObjectRequest::default()
                        };
                        client.put_object(&req).map_err(InsertError::S3)?;
                    }
                    Ok(tuple.1)
                })
            })
            .collect()
            // save the new urls to the database. the images variable will
            // include only those that we've successfully saved to s3, so we
            // have to do a funky merge here.
            .and_then(move |images| {
                let imgs = images.clone();
                pool_db.spawn_fn(move || {
                    use schema::ads::dsl::*;
                    let update = Images::from_ad(&ad, imgs)?;
                    let connection = db.get().map_err(InsertError::Timeout)?;
                    diesel::update(ads.filter(id.eq(&ad.id)))
                        .set(&update)
                        .execute(&*connection)
                        .map_err(InsertError::DataBase)?;
                    info!("saved {:?}", ad.id);
                    Ok(())
                })
            })
            .map_err(|e| {
                warn!("{:?}", e);
                ()
            });
        Box::new(future)
    }

    pub fn image_urls(&self) -> Vec<Result<Uri, InsertError>> {
        let images = [vec![self.thumbnail.clone()], self.images.clone()];
        images
            .concat()
            .iter()
            .map(|a| {
                let a: Uri = a.parse().map_err(InsertError::Uri)?;
                Ok(a)
            })
            .collect()
    }
}

#[derive(Insertable)]
#[table_name = "ads"]
pub struct NewAd<'a> {
    id: &'a str,
    html: &'a str,
    political: i32,
    not_political: i32,

    title: String,
    message: String,
    thumbnail: String,

    browser_lang: &'a str,
    images: Vec<String>,
    impressions: i32,

    targeting: Option<&'a str>,
}

impl<'a> NewAd<'a> {
    pub fn new(ad: &'a AdPost) -> Result<NewAd<'a>, InsertError> {
        let document = kuchiki::parse_html().one(ad.html.clone());

        let thumb = get_image(&document)?;
        let images = get_images(&document)?;
        let message = get_message(&document)?;
        let title = get_title(&document)?;

        Ok(NewAd {
            id: &ad.id,
            html: &ad.html,
            // we try unwrapping or we chose the false branch in both of these
            // cases to count impressions
            political: if ad.political.unwrap_or(false) { 1 } else { 0 },
            not_political: if !ad.political.unwrap_or(true) { 1 } else { 0 },
            title: title,
            message: message,
            thumbnail: thumb,
            browser_lang: &ad.browser_lang,
            images: images,
            impressions: if !ad.political.is_some() { 1 } else { 0 },
            targeting: None,
        })
    }

    pub fn save(&self, pool: &Pool<ConnectionManager<PgConnection>>) -> Result<Ad, InsertError> {
        use schema::ads;
        use schema::ads::dsl::*;
        let connection = pool.get().map_err(InsertError::Timeout)?;
        // increment impressions if this is a background save,
        // otherwise increment political counters
        let ad: Ad = diesel::insert(&self.on_conflict(
            id,
            do_update().set((
                political.eq(political + self.political),
                not_political.eq(
                    not_political +
                        self.not_political,
                ),
                impressions.eq(
                    impressions + self.impressions,
                ),
                updated_at.eq(Utc::now()),
            )),
        )).into(ads::table)
            .get_result(&*connection)
            .map_err(InsertError::DataBase)?;
        Ok(ad)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn ad_parsing() {
        let ad = include_str!("./html-test.txt");
        let post = AdPost {
            id: "test".to_string(),
            html: ad.to_string(),
            political: None,
            browser_lang: "en-US".to_string(),
        };
        let new_ad = NewAd::new(&post).unwrap();
        assert!(new_ad.thumbnail.len() > 0);
        assert_eq!(new_ad.images.len(), 2);
        assert!(new_ad.title.len() > 0);

        assert_eq!(
            kuchiki::parse_html().one(new_ad.message).text_contents(),
            kuchiki::parse_html().one("
#Valerian is “the best experience since ‘Avatar.’” See it in 3D and RealD3D theaters this Friday. Get tickets now: ValerianTickets.com
").text_contents()
        );
    }

    #[test]
    fn image_parsing() {
        use chrono::prelude::*;
        let ad = include_str!("./html-test.txt");
        let document = kuchiki::parse_html().one(ad);

        let saved_ad = Ad {
            id: "test".to_string(),
            html: ad.to_string(),
            political: 1,
            not_political: 2,
            fuzzy_id: None,
            title: get_title(&document).unwrap(),
            message: get_message(&document).unwrap(),
            thumbnail: get_image(&document).unwrap(),
            created_at: Utc::now(),
            updated_at: Utc::now(),
            browser_lang: "en-US".to_string(),
            images: get_images(&document).unwrap(),
            impressions: 1,
        };
        let urls = saved_ad
            .image_urls()
            .into_iter()
            .map(|x| x.unwrap())
            .collect();
        let images = Images::from_ad(&saved_ad, urls).unwrap();
        assert!(images.html != saved_ad.html);
        assert!(!images.html.contains("fbcdn"));
        assert!(!images.html.contains("html"));
        assert!(images.images.len() == saved_ad.images.len());
        assert!(images.thumbnail.unwrap() != saved_ad.thumbnail);
    }
}
<commit_msg>tests work now<commit_after>use chrono::DateTime;
use chrono::offset::Utc;
use diesel;
use diesel::prelude::*;
use diesel::pg::PgConnection;
use diesel::pg::upsert::*;
use futures::{Future, stream, Stream};
use futures_cpupool::CpuPool;
use hyper::{Body, Client, Uri};
use hyper::client::HttpConnector;
use hyper_tls::HttpsConnector;
use InsertError;
use kuchiki;
use kuchiki::traits::*;
use r2d2_diesel::ConnectionManager;
use r2d2::Pool;
use rusoto_core::{default_tls_client, Region};
use rusoto_credential::DefaultCredentialsProvider;
use rusoto_s3::{PutObjectRequest, S3Client, S3};
use schema::ads;
use server::AdPost;

const ENDPOINT: &'static str = "https://pp-facebook-ads.s3.amazonaws.com/";

fn get_title(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    document
        .select("h5 a, h6 a, strong")
        .map_err(InsertError::HTML)?
        .nth(0)
        .and_then(|a| Some(a.text_contents()))
        .ok_or(InsertError::HTML(()))
}

fn get_image(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    document
        .select("img")
        .map_err(InsertError::HTML)?
        .nth(0)
        .and_then(|a| {
            a.attributes.borrow().get("src").and_then(
                |src| Some(src.to_string()),
            )
        })
        .ok_or(InsertError::HTML(()))
}

fn get_message(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    let selectors = vec![".userContent p", "span"];
    let iters = selectors.iter().map(|s| document.select(s)).flat_map(|a| a);

    iters
        .map(|i| {
            i.fold(String::new(), |m, a| m + &a.as_node().to_string())
        })
        .filter(|i| !i.is_empty())
        .nth(0)
        .ok_or(InsertError::HTML(()))
}

fn get_images(document: &kuchiki::NodeRef) -> Result<Vec<String>, InsertError> {
    Ok(
        document
            .select("img")
            .map_err(InsertError::HTML)?
            .skip(1)
            .map(|a| {
                a.attributes.borrow().get("src").and_then(
                    |s| Some(s.to_string()),
                )
            })
            .filter(|s| s.is_some())
            .map(|s| s.unwrap())
            .collect::<Vec<String>>(),
    )
}

#[derive(AsChangeset, Debug)]
#[table_name = "ads"]
pub struct Images {
    thumbnail: Option<String>,
    images: Vec<String>,
    title: String,
    message: String,
    html: String,
}

impl Images {
    fn from_ad(ad: &Ad, images: Vec<Uri>) -> Result<Images, InsertError> {
        let thumb = images
            .iter()
            .filter(|i| ad.thumbnail.contains(i.path()))
            .map(|i| ENDPOINT.to_string() + i.path().trim_left_matches('/'))
            .nth(0);

        let mut rest = images.clone();
        if let Some(thumb) = thumb.clone() {
            rest.retain(|x| !thumb.contains(x.path()))
        };

        let collection = rest.iter()
            .filter(|i| ad.images.iter().any(|a| a.contains(i.path())))
            .map(|i| ENDPOINT.to_string() + i.path().trim_left_matches('/'))
            .collect::<Vec<String>>();

        let document = kuchiki::parse_html().one(ad.html.clone());
        for a in document.select("img").map_err(InsertError::HTML)? {
            if let Some(x) = a.attributes.borrow_mut().get_mut("src") {
                if let Ok(u) = x.parse::<Uri>() {
                    if let Some(i) = images.iter().find(|i| i.path() == u.path()) {
                        *x = ENDPOINT.to_string() + i.path().trim_left_matches('/');
                    } else {
                        *x = "".to_string();
                    }
                }
            };
        }

        let title = get_title(&document)?;
        let message = get_title(&document)?;
        Ok(Images {
            thumbnail: thumb,
            images: collection,
            title: title,
            html: document
                .select("div")
                .map_err(InsertError::HTML)?
                .nth(0)
                .ok_or(InsertError::HTML(()))?
                .as_node()
                .to_string(),
            message: message,
        })
    }
}

#[derive(Queryable, Debug, Clone)]
pub struct Ad {
    id: String,
    html: String,
    political: i32,
    not_political: i32,
    fuzzy_id: Option<i32>,
    title: String,
    message: String,
    thumbnail: String,
    created_at: DateTime<Utc>,
    updated_at: DateTime<Utc>,
    browser_lang: String,
    images: Vec<String>,
    impressions: i32,
    political_probability: f64,
    targeting: Option<String>,
}

impl Ad {
    // This will asynchronously save the images to s3 we may very well end up
    // dropping images, but I can't see any way around it right now. Also we
    // should think about splitting this up, but I'm fine -- if a little
    // embarassed about it -- right now. This function swallows errors, and
    // there's a chance we'll end up with no images at the end, but I think we
    // can handle that in the extension's UI.
    pub fn grab_and_store(
        &self,
        client: Client<HttpsConnector<HttpConnector>, Body>,
        db: &Pool<ConnectionManager<PgConnection>>,
        pool: CpuPool,
    ) -> Box<Future<Item = (), Error = ()>> {
        let ad = self.clone();
        let pool_s3 = pool.clone();
        let pool_db = pool.clone();
        let db = db.clone();
        let future = stream::iter_ok(self.image_urls())
            .map(|u| u.unwrap())
            // filter ones we already have in the db and ones we can verify as
            // coming from fb, we don't want to become a malware vector :)
            // currently we redownload images we already have, but ok.
            .filter(|u| {
                info!("testing {:?}", u.host());
                match u.host() {
                    Some(h) => (h == "pp-facebook-ads.s3.amazonaws.com" || h.ends_with("fbcdn.net")),
                    None => false
                }
            })
            // grab image
            .and_then(move |img| {
                info!("getting {:?}", img);
                let cloned = img.clone();
                client
                    .get(img)
                    .and_then(|res| {
                        res.body().concat2().and_then(|chunk| Ok((chunk, cloned)))
                    })
                    .map_err(InsertError::Hyper)
            })
            // upload them to s3
            .and_then(move |tuple| {
                let pool = pool_s3.clone();
                // we do this in a worker thread because rusoto isn't on
                // Hyper async yet.
                pool.spawn_fn(move || {
                    if tuple.1.host().unwrap() != "pp-facebook-ads.s3.amazonaws.com" {
                        let client =
                            S3Client::new(default_tls_client().map_err(InsertError::TLS)?,
                                          DefaultCredentialsProvider::new().map_err(InsertError::AWS)?,
                                          Region::UsEast1);
                        let req = PutObjectRequest {
                            bucket: "pp-facebook-ads".to_string(),
                            key: tuple.1.path().trim_left_matches('/').to_string(),
                            acl: Some("public-read".to_string()),
                            body: Some(tuple.0.to_vec()),
                            ..PutObjectRequest::default()
                        };
                        client.put_object(&req).map_err(InsertError::S3)?;
                    }
                    Ok(tuple.1)
                })
            })
            .collect()
            // save the new urls to the database. the images variable will
            // include only those that we've successfully saved to s3, so we
            // have to do a funky merge here.
            .and_then(move |images| {
                let imgs = images.clone();
                pool_db.spawn_fn(move || {
                    use schema::ads::dsl::*;
                    let update = Images::from_ad(&ad, imgs)?;
                    let connection = db.get().map_err(InsertError::Timeout)?;
                    diesel::update(ads.filter(id.eq(&ad.id)))
                        .set(&update)
                        .execute(&*connection)
                        .map_err(InsertError::DataBase)?;
                    info!("saved {:?}", ad.id);
                    Ok(())
                })
            })
            .map_err(|e| {
                warn!("{:?}", e);
                ()
            });
        Box::new(future)
    }

    pub fn image_urls(&self) -> Vec<Result<Uri, InsertError>> {
        let images = [vec![self.thumbnail.clone()], self.images.clone()];
        images
            .concat()
            .iter()
            .map(|a| {
                let a: Uri = a.parse().map_err(InsertError::Uri)?;
                Ok(a)
            })
            .collect()
    }
}

#[derive(Insertable)]
#[table_name = "ads"]
pub struct NewAd<'a> {
    id: &'a str,
    html: &'a str,
    political: i32,
    not_political: i32,

    title: String,
    message: String,
    thumbnail: String,

    browser_lang: &'a str,
    images: Vec<String>,
    impressions: i32,

    targeting: Option<&'a str>,
}

impl<'a> NewAd<'a> {
    pub fn new(ad: &'a AdPost) -> Result<NewAd<'a>, InsertError> {
        let document = kuchiki::parse_html().one(ad.html.clone());

        let thumb = get_image(&document)?;
        let images = get_images(&document)?;
        let message = get_message(&document)?;
        let title = get_title(&document)?;

        Ok(NewAd {
            id: &ad.id,
            html: &ad.html,
            // we try unwrapping or we chose the false branch in both of these
            // cases to count impressions
            political: if ad.political.unwrap_or(false) { 1 } else { 0 },
            not_political: if !ad.political.unwrap_or(true) { 1 } else { 0 },
            title: title,
            message: message,
            thumbnail: thumb,
            browser_lang: &ad.browser_lang,
            images: images,
            impressions: if !ad.political.is_some() { 1 } else { 0 },
            targeting: None,
        })
    }

    pub fn save(&self, pool: &Pool<ConnectionManager<PgConnection>>) -> Result<Ad, InsertError> {
        use schema::ads;
        use schema::ads::dsl::*;
        let connection = pool.get().map_err(InsertError::Timeout)?;
        // increment impressions if this is a background save,
        // otherwise increment political counters
        let ad: Ad = diesel::insert(&self.on_conflict(
            id,
            do_update().set((
                political.eq(political + self.political),
                not_political.eq(
                    not_political +
                        self.not_political,
                ),
                impressions.eq(
                    impressions + self.impressions,
                ),
                updated_at.eq(Utc::now()),
            )),
        )).into(ads::table)
            .get_result(&*connection)
            .map_err(InsertError::DataBase)?;
        Ok(ad)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn ad_parsing() {
        let ad = include_str!("./html-test.txt");
        let post = AdPost {
            id: "test".to_string(),
            html: ad.to_string(),
            political: None,
            browser_lang: "en-US".to_string(),
            targeting: None,
        };
        let new_ad = NewAd::new(&post).unwrap();
        assert!(new_ad.thumbnail.len() > 0);
        assert_eq!(new_ad.images.len(), 2);
        assert!(new_ad.title.len() > 0);

        assert_eq!(
            kuchiki::parse_html().one(new_ad.message).text_contents(),
            kuchiki::parse_html().one("
#Valerian is “the best experience since ‘Avatar.’” See it in 3D and RealD3D theaters this Friday. Get tickets now: ValerianTickets.com
").text_contents()
        );
    }

    #[test]
    fn image_parsing() {
        use chrono::prelude::*;
        let ad = include_str!("./html-test.txt");
        let document = kuchiki::parse_html().one(ad);

        let saved_ad = Ad {
            id: "test".to_string(),
            html: ad.to_string(),
            political: 1,
            not_political: 2,
            fuzzy_id: None,
            title: get_title(&document).unwrap(),
            message: get_message(&document).unwrap(),
            thumbnail: get_image(&document).unwrap(),
            created_at: Utc::now(),
            updated_at: Utc::now(),
            browser_lang: "en-US".to_string(),
            images: get_images(&document).unwrap(),
            impressions: 1,
            targeting: None,
            political_probability: 0.0,
        };
        let urls = saved_ad
            .image_urls()
            .into_iter()
            .map(|x| x.unwrap())
            .collect();
        let images = Images::from_ad(&saved_ad, urls).unwrap();
        assert!(images.html != saved_ad.html);
        assert!(!images.html.contains("fbcdn"));
        assert!(!images.html.contains("html"));
        assert!(images.images.len() == saved_ad.images.len());
        assert!(images.thumbnail.unwrap() != saved_ad.thumbnail);
    }
}
<|endoftext|>


<commit_before>use std::fs::File;
use std::path::{ Path, PathBuf };

use download;
use extract;
use verify;
use find_package;
use find_package::Package;

fn _install_package(package: &Package, path: &Path) -> Result<bool, String> {
    let mut download_path = PathBuf::new();
        download_path.push(path);
        download_path.push(format!("download/{}", package.name));

    if !download_path.exists() {
        try!(download::download_file(&package.url, download_path.as_path()));
    }

    let mut tar_path = PathBuf::new();
        tar_path.push(path);
        tar_path.push(format!("download/{}.tar", package.name));

    if !tar_path.exists() {
        try!(extract::extract_gz(download_path.as_path(), tar_path.as_path()));
    }

    let mut install_path = PathBuf::new();
        tar_path.push(path);
        tar_path.push(format!("install/{}", package.name));

    if !install_path.exists() {
        try!(extract::extract_tar(tar_path.as_path(), install_path.as_path()));
    } else {
        return Ok(false)
    }

    return Ok(true)
}

pub fn install_package(name: &str, path: &Path) -> Result<bool, String> {
    let found_packages = try!(find_package::find_package(name, path));

    if !found_packages.is_empty() && found_packages.len() > 1 {
        for (i, package) in found_packages.iter().enumerate() {
            let mut install_path = PathBuf::new();
                install_path.push(path);
                install_path.push(format!("installed/{}", package.name));

            if !install_path.exists() {
                println!("Package {}: {}", i, package.name);
            }
        }

        //TODO: Accept user input and install user-chosen package.
        return _install_package(&found_packages[0], path)
    } else {
        return _install_package(&found_packages[0], path)
    }
}
<commit_msg>Ability to select package to install<commit_after>use std::fs::File;
use std::io;
use std::path::{ Path, PathBuf };

use download;
use extract;
use verify;
use find_package;
use find_package::Package;

fn _install_package(package: &Package, path: &Path) -> Result<bool, String> {
    let mut download_path = PathBuf::new();
        download_path.push(path);
        download_path.push(format!("download/{}", package.name));

    if !download_path.exists() {
        try!(download::download_file(&package.url, download_path.as_path()));
    }

    let mut tar_path = PathBuf::new();
        tar_path.push(path);
        tar_path.push(format!("download/{}.tar", package.name));

    if !tar_path.exists() {
        try!(extract::extract_gz(download_path.as_path(), tar_path.as_path()));
    }

    let mut install_path = PathBuf::new();
        tar_path.push(path);
        tar_path.push(format!("install/{}", package.name));

    if !install_path.exists() {
        try!(extract::extract_tar(tar_path.as_path(), install_path.as_path()));
    } else {
        return Ok(false)
    }

    return Ok(true)
}

pub fn install_package(name: &str, path: &Path) -> Result<bool, String> {
    let found_packages = find_package::find_package(name, path)?;

    if !found_packages.is_empty() && found_packages.len() > 1 {
        for (i, package) in found_packages.iter().enumerate() {
            let mut install_path = PathBuf::new();
                install_path.push(path);
                install_path.push(format!("installed/{}", package.name));

            if !install_path.exists() {
                println!("Package {}: {}", i, package.name);
            }
        }

        let mut input = String::new();
        io::stdin().read_line(&mut input)?;

        let input: u16 = input.trim().parse().ok().expect("");

        if input > found_packages.len() as u16 {
            return Err(String::from("Invalid input"))
        } else {
            return _install_package(&found_packages[input as usize], path)
        }

        return _install_package(&found_packages[0], path)
    } else {
        return _install_package(&found_packages[0], path)
    }
}
<|endoftext|>


<commit_before>extern crate petgraph;

pub mod indextype;
pub mod inner;

use super::traits::{InstructionType, NavigationInternal};

use self::indextype::IndexType;
use self::inner::{InnerGraph, InnerEdgeLight, InnerEdgeTrait, AuxQuery};
use self::petgraph::{Incoming, Outgoing};
use self::petgraph::graph::{Edge, EdgeIndex, Graph, NodeIndex};

type DefaultInnerEdge<Index/*: IndexType*/> = InnerEdgeLight<Index>;

trait BlockToReprPayload { type Type; }
trait UsedInnerEdgeType { type InnerEdgeType: InnerEdgeTrait; }

pub struct PhiIndex<Index>(Index);

pub struct BasicBlock<Index: IndexType, Instruction: InstructionType> {
	num_ext: Index,
	phis: Vec<(Instruction::PhiType, <DefaultInnerEdge<Index> as self::inner::InnerEdgeTrait>::NodeAux)>,
	inner_graph: InnerGraph<Instruction, DefaultInnerEdge<Index>>
}

enum LookupResult<Index: IndexType, Instruction: InstructionType> {
	FoundInBB(Instruction),
	Phi(Index),
	External(Index),
	NotFound(Index)
}

impl<Index: IndexType, Instruction: InstructionType> BlockToReprPayload for BasicBlock<Index, Instruction> {
	type Type = <InnerEdgeLight<Index> as InnerEdgeTrait>::NodeAux;
}

impl<Index: IndexType, Instruction: InstructionType> BasicBlock<Index, Instruction> {
	pub fn new() -> Self {
		BasicBlock::<Index, Instruction> {
			num_ext: Index::zero(),
			phis: Vec::new(),
			inner_graph: InnerGraph::<Instruction, InnerEdgeLight<Index>>::new()
		}
	}

	fn lookup(&self, i: Index) -> LookupResult<Index, Instruction> {
		if i >= Index::zero() {
			let instr = self.inner_graph.lookup(i);
			if instr.is_phi() {
				// TODO
			}
			LookupResult::NotFound(i)

		} else if i + self.num_ext >= Index::zero() {
			LookupResult::External(i)

		} else {
			LookupResult::NotFound(i)
		}
	}
}

type DefaultInnerIndex = i16;
type DefaultBasicBlock<Instruction> = BasicBlock<DefaultInnerIndex, Instruction>;

enum IRNode<Instruction: InstructionType> {
	// Represents an operation of a basic block that is used by non-phi nodes outside of that basic block
	Repr,
	// Represents a basic block
	BasicBlock(DefaultBasicBlock<Instruction>)
}

enum IREdge<Instruction: InstructionType> {
	// Points from a `IRNode::Repr` to the `IR::BasicBlock` that it belongs to
	ReprToBlock(DefaultInnerIndex),
	// Points from a `IR::BasicBlock`s to `IRNode::Repr` whose target BlockToReprPayloadis used by operations in the basic block
	// Contains a id of the first user in the basic block, and the id the `IRNode::Repr` is known by in the basic block
	BlockToRepr(DefaultInnerIndex, <DefaultBasicBlock<Instruction> as BlockToReprPayload>::Type),
	// Points from one `IR::BasicBlock` to another. Represents control flow.
	// It contains a vector of 'n' references to operations in the origin basic block
	// that are used by 'n' phi operations in the target basic block.
	Flow(Vec<DefaultInnerIndex>, u64)
}

type IRGraph<Instruction> = Graph<IRNode<Instruction>, IREdge<Instruction>>;

pub struct NodeRef<I>(NodeIndex, I);
struct AuxQueryImpl<'a, Instruction: InstructionType + 'a>(&'a mut IRGraph<Instruction>, NodeIndex);

// forward that type
impl<Index: IndexType, Instruction: InstructionType> UsedInnerEdgeType for BasicBlock<Index, Instruction> {
	type InnerEdgeType = DefaultInnerEdge<Index>;
}
impl<Instruction: InstructionType> UsedInnerEdgeType for IRGraph<Instruction> {
	type InnerEdgeType = <DefaultBasicBlock<Instruction> as UsedInnerEdgeType>::InnerEdgeType;
}
impl<'a, Instruction: InstructionType> UsedInnerEdgeType for AuxQueryImpl<'a, Instruction> {
	type InnerEdgeType = <IRGraph<Instruction> as UsedInnerEdgeType>::InnerEdgeType;
}

impl<'b, Instruction: InstructionType/*, InnerEdge: InnerEdgeTrait<Index=DefaultInnerIndex>*/>
	inner::AuxQuery<<AuxQueryImpl<'b, Instruction> as UsedInnerEdgeType>::InnerEdgeType>
	for AuxQueryImpl<'b, Instruction>
{
	//use AuxQueryImpl<'b, Instruction>::InnerEdgeType as InnerEdge;

	fn access_aux<'a>(&'a mut self, i: <<Self as UsedInnerEdgeType>::InnerEdgeType as InnerEdgeTrait>::Index)
		-> &'a mut <<Self as UsedInnerEdgeType>::InnerEdgeType as InnerEdgeTrait>::NodeAux
	{

		let mut fedge: Option<EdgeIndex> = Option::None;
		{
			let graph = &*self.0;
			let mut edges = graph.walk_edges_directed(self.1, Outgoing);
			while let Some(edge) = edges.next(graph) {
				match graph.edge_weight(edge) {
					Some(&IREdge::BlockToRepr(key, _)) => {
						if key == i {
							fedge = Option::Some(edge);
							break
						}
					},
					_ => {}
				}
			}
		}
		if let Some(edge) = fedge {
			if let Some(&mut IREdge::BlockToRepr(_, ref mut aux)) = self.0.edge_weight_mut(edge){
				return aux
			}
		}
		panic!();
	}
}

//fn deref<InstructionType>(graph: &IRGraph<Instruction>) {
//	//
//}

impl<Index: IndexType, Instruction: InstructionType> NavigationInternal<NodeRef<Index>> for IRGraph<Instruction> {
	fn add_uses_to(&self, noderef: NodeRef<Index>, r: &mut Vec<NodeRef<Index>>) {
		if let Option::Some(node) = self.node_weight(noderef.0) {
			// TODO
			panic!();
		} else {
			panic!();
		}
	}

	fn add_args_to(&self, noderef: NodeRef<Index>, r: &mut Vec<NodeRef<Index>>) {
		if let Option::Some(node) = self.node_weight(noderef.0) {
			let bb = match *node {
				IRNode::Repr => panic!(),
				IRNode::BasicBlock(ref bb) => {
					bb
				}
			};
			/* // TODO
			if (noderef.is_phi()) {
				let phiindex = !noderef.1;
				let mut edges = self.walk_edges_directed(noderef.0, Incoming);
				while let Some(edge) = edges.next(&self) {
					match self.edge_weight(edge) {
						None => {},
						Some(&IREdge::ReprToBlock(_)) => {},
						Some(&IREdge::BlockToRepr(_, _)) => {},
						Some(&IREdge::Flow(phisource, id)) => {
							// r.push(deref_noderef(self, noderef.0, i));
						}
					}
				}
			} else {
				let inner_r = Vec::<Index>::new();
				bb.inner_graph.add_args_to(noderef.1, &mut r);
				for i in inner_r {
					// TODO
					// r.push(deref_noderef(self, noderef.0, i));
				}
			}*/
		} else {
			panic!();
		}
	}
}

mod test {
	use super::super::traits::InstructionType;
	use super::IRGraph;

	#[derive(Debug)]
	enum TestInstr { Phi, NotPhi }

	impl InstructionType for TestInstr {
		type PhiType = ();
		fn make_phi(_: ()) -> TestInstr { TestInstr::Phi }
		fn is_phi(&self) -> bool { if let TestInstr::Phi = *self { true } else { false } }
	}

	#[test]
	fn construct() {
		let graph: IRGraph<TestInstr>;
	}

	// more tests will follow
}
<commit_msg>ir/graph: Add macro for edge search<commit_after>extern crate petgraph;

pub mod indextype;
pub mod inner;

use super::traits::{InstructionType, NavigationInternal};

use self::indextype::IndexType;
use self::inner::{InnerGraph, InnerEdgeLight, InnerEdgeTrait, AuxQuery};
use self::petgraph::{Incoming, Outgoing};
use self::petgraph::graph::{Edge, EdgeIndex, Graph, NodeIndex};

type DefaultInnerEdge<Index/*: IndexType*/> = InnerEdgeLight<Index>;

trait BlockToReprPayload { type Type; }
trait UsedInnerEdgeType { type InnerEdgeType: InnerEdgeTrait; }

pub struct PhiIndex<Index>(Index);

pub struct BasicBlock<Index: IndexType, Instruction: InstructionType> {
	num_ext: Index,
	phis: Vec<(Instruction::PhiType, <DefaultInnerEdge<Index> as self::inner::InnerEdgeTrait>::NodeAux)>,
	inner_graph: InnerGraph<Instruction, DefaultInnerEdge<Index>>
}

enum LookupResult<Index: IndexType, Instruction: InstructionType> {
	FoundInBB(Instruction),
	Phi(Index),
	External(Index),
	NotFound(Index)
}

impl<Index: IndexType, Instruction: InstructionType> BlockToReprPayload for BasicBlock<Index, Instruction> {
	type Type = <InnerEdgeLight<Index> as InnerEdgeTrait>::NodeAux;
}

impl<Index: IndexType, Instruction: InstructionType> BasicBlock<Index, Instruction> {
	pub fn new() -> Self {
		BasicBlock::<Index, Instruction> {
			num_ext: Index::zero(),
			phis: Vec::new(),
			inner_graph: InnerGraph::<Instruction, InnerEdgeLight<Index>>::new()
		}
	}

	fn lookup(&self, i: Index) -> LookupResult<Index, Instruction> {
		if i >= Index::zero() {
			let instr = self.inner_graph.lookup(i);
			if instr.is_phi() {
				// TODO
			}
			LookupResult::NotFound(i)

		} else if i + self.num_ext >= Index::zero() {
			LookupResult::External(i)

		} else {
			LookupResult::NotFound(i)
		}
	}
}

type DefaultInnerIndex = i16;
type DefaultBasicBlock<Instruction> = BasicBlock<DefaultInnerIndex, Instruction>;

enum IRNode<Instruction: InstructionType> {
	// Represents an operation of a basic block that is used by non-phi nodes outside of that basic block
	Repr,
	// Represents a basic block
	BasicBlock(DefaultBasicBlock<Instruction>)
}

enum IREdge<Instruction: InstructionType> {
	// Points from a `IRNode::Repr` to the `IR::BasicBlock` that it belongs to
	ReprToBlock(DefaultInnerIndex),
	// Points from a `IR::BasicBlock`s to `IRNode::Repr` whose target BlockToReprPayloadis used by operations in the basic block
	// Contains a id of the first user in the basic block, and the id the `IRNode::Repr` is known by in the basic block
	BlockToRepr(DefaultInnerIndex, <DefaultBasicBlock<Instruction> as BlockToReprPayload>::Type),
	// Points from one `IR::BasicBlock` to another. Represents control flow.
	// It contains a vector of 'n' references to operations in the origin basic block
	// that are used by 'n' phi operations in the target basic block.
	Flow(Vec<DefaultInnerIndex>, u64)
}

type IRGraph<Instruction> = Graph<IRNode<Instruction>, IREdge<Instruction>>;

pub struct NodeRef<I>(NodeIndex, I);
struct AuxQueryImpl<'a, Instruction: InstructionType + 'a>(&'a mut IRGraph<Instruction>, NodeIndex);

// forward that type
impl<Index: IndexType, Instruction: InstructionType> UsedInnerEdgeType for BasicBlock<Index, Instruction> {
	type InnerEdgeType = DefaultInnerEdge<Index>;
}
impl<Instruction: InstructionType> UsedInnerEdgeType for IRGraph<Instruction> {
	type InnerEdgeType = <DefaultBasicBlock<Instruction> as UsedInnerEdgeType>::InnerEdgeType;
}
impl<'a, Instruction: InstructionType> UsedInnerEdgeType for AuxQueryImpl<'a, Instruction> {
	type InnerEdgeType = <IRGraph<Instruction> as UsedInnerEdgeType>::InnerEdgeType;
}

macro_rules! find_edge {
	( $graph:expr, $node:expr, $direction:expr, $( $pat:pat => $cond:expr ),* ) => {
		{
			let mut fedge: Option<EdgeIndex> = Option::None;
			let mut temp_edgewalk = $graph.walk_edges_directed($node, $direction);
			while let Some(edge) = temp_edgewalk.next($graph) {
				if match $graph.edge_weight(edge).expect("petgraph.walk_edges_directed gave us a non-existant edge") {
					$(
						$pat => $cond
					),*
					_ => { false }
				} {
					fedge = Option::Some(edge);
					break
				}
			}
			fedge
		}
	};
}

impl<'b, Instruction: InstructionType/*, InnerEdge: InnerEdgeTrait<Index=DefaultInnerIndex>*/>
	inner::AuxQuery<<AuxQueryImpl<'b, Instruction> as UsedInnerEdgeType>::InnerEdgeType>
	for AuxQueryImpl<'b, Instruction>
{
	//use AuxQueryImpl<'b, Instruction>::InnerEdgeType as InnerEdge;

	fn access_aux<'a>(&'a mut self, i: <<Self as UsedInnerEdgeType>::InnerEdgeType as InnerEdgeTrait>::Index)
		-> &'a mut <<Self as UsedInnerEdgeType>::InnerEdgeType as InnerEdgeTrait>::NodeAux
	{
		let fedge: Option<EdgeIndex> = find_edge!(self.0, self.1, Outgoing,
			&IREdge::BlockToRepr(key, _) => { key == i }
		);
		if let Some(edge) = fedge {
			if let Some(&mut IREdge::BlockToRepr(_, ref mut aux)) = self.0.edge_weight_mut(edge){
				return aux
			}
		}
		panic!();
	}
}

fn deref<Instruction: InstructionType>(graph: &IRGraph<Instruction>, noderef: NodeRef<DefaultInnerIndex>) {

	let fedge: Option<EdgeIndex> = find_edge!(graph, noderef.0, Outgoing,
		&IREdge::BlockToRepr(key, _) => { key == noderef.1 }
	);
	if let Some(edge) = fedge {
		// TODO
	}
}

impl<Index: IndexType, Instruction: InstructionType> NavigationInternal<NodeRef<Index>> for IRGraph<Instruction> {
	fn add_uses_to(&self, noderef: NodeRef<Index>, r: &mut Vec<NodeRef<Index>>) {
		if let Option::Some(node) = self.node_weight(noderef.0) {
			// TODO
			panic!();
		} else {
			panic!();
		}
	}

	fn add_args_to(&self, noderef: NodeRef<Index>, r: &mut Vec<NodeRef<Index>>) {
		if let Option::Some(node) = self.node_weight(noderef.0) {
			let bb = match *node {
				IRNode::Repr => panic!(),
				IRNode::BasicBlock(ref bb) => {
					bb
				}
			};
			/* // TODO
			if (noderef.is_phi()) {
				let phiindex = !noderef.1;
				let mut edges = self.walk_edges_directed(noderef.0, Incoming);
				while let Some(edge) = edges.next(&self) {
					match self.edge_weight(edge) {
						None => {},
						Some(&IREdge::ReprToBlock(_)) => {},
						Some(&IREdge::BlockToRepr(_, _)) => {},
						Some(&IREdge::Flow(phisource, id)) => {
							// r.push(deref_noderef(self, noderef.0, i));
						}
					}
				}
			} else {
				let inner_r = Vec::<Index>::new();
				bb.inner_graph.add_args_to(noderef.1, &mut r);
				for i in inner_r {
					// TODO
					// r.push(deref_noderef(self, noderef.0, i));
				}
			}*/
		} else {
			panic!();
		}
	}
}

mod test {
	use super::super::traits::InstructionType;
	use super::IRGraph;

	#[derive(Debug)]
	enum TestInstr { Phi, NotPhi }

	impl InstructionType for TestInstr {
		type PhiType = ();
		fn make_phi(_: ()) -> TestInstr { TestInstr::Phi }
		fn is_phi(&self) -> bool { if let TestInstr::Phi = *self { true } else { false } }
	}

	#[test]
	fn construct() {
		let graph: IRGraph<TestInstr>;
	}

	// more tests will follow
}
<|endoftext|>


<commit_before>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{mapping::SourceMapping, source_map::SourceMap};
use anyhow::{format_err, Result};
use codespan_reporting::{
    diagnostic::{Diagnostic, Label},
    files::SimpleFiles,
    term::{
        emit,
        termcolor::{ColorChoice, StandardStream},
        Config,
    },
};
use move_ir_types::location::Loc;
use std::{fs::File, io::Read, path::Path};

type FileId = usize;

pub type Error = (Loc, String);
pub type Errors = Vec<Error>;

pub fn source_map_from_file(file_path: &Path) -> Result<SourceMap> {
    let mut bytes = Vec::new();
    File::open(file_path)
        .ok()
        .and_then(|mut file| file.read_to_end(&mut bytes).ok())
        .ok_or_else(|| format_err!("Error while reading in source map information"))?;
    bcs::from_bytes::<SourceMap>(&bytes)
        .map_err(|_| format_err!("Error deserializing into source map"))
}

pub fn render_errors(source_mapper: &SourceMapping, errors: Errors) -> Result<()> {
    if let Some((source_file_name, source_string)) = &source_mapper.source_code {
        let mut codemap = SimpleFiles::new();
        let id = codemap.add(source_file_name, source_string.to_string());
        for err in errors {
            let diagnostic = create_diagnostic(id, err);
            let writer = &mut StandardStream::stderr(ColorChoice::Auto);
            emit(writer, &Config::default(), &codemap, &diagnostic).unwrap();
        }
        Ok(())
    } else {
        Err(format_err!(
            "Unable to render errors since source file information is not available"
        ))
    }
}

pub fn create_diagnostic(id: FileId, (loc, msg): Error) -> Diagnostic<FileId> {
    Diagnostic::error().with_labels(vec![Label::primary(id, loc.usize_range()).with_message(msg)])
}
<commit_msg>[move] Remove unused diagnostics<commit_after>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::source_map::SourceMap;
use anyhow::{format_err, Result};
use move_ir_types::location::Loc;
use std::{fs::File, io::Read, path::Path};

pub type Error = (Loc, String);
pub type Errors = Vec<Error>;

pub fn source_map_from_file(file_path: &Path) -> Result<SourceMap> {
    let mut bytes = Vec::new();
    File::open(file_path)
        .ok()
        .and_then(|mut file| file.read_to_end(&mut bytes).ok())
        .ok_or_else(|| format_err!("Error while reading in source map information"))?;
    bcs::from_bytes::<SourceMap>(&bytes)
        .map_err(|_| format_err!("Error deserializing into source map"))
}
<|endoftext|>


<commit_before>//@ignore-windows: Concurrency on Windows is not supported yet.
//@compile-flags: -Zmiri-ignore-leaks -Zmiri-preemption-rate=0

// Tests showing weak memory behaviours are exhibited. All tests
// return true when the desired behaviour is seen.
// This is scheduler and pseudo-RNG dependent, so each test is
// run multiple times until one try returns true.
// Spurious failure is possible, if you are really unlucky with
// the RNG and always read the latest value from the store buffer.

use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering::*;
use std::thread::spawn;

#[derive(Copy, Clone)]
struct EvilSend<T>(pub T);

unsafe impl<T> Send for EvilSend<T> {}
unsafe impl<T> Sync for EvilSend<T> {}

// We can't create static items because we need to run each test
// multiple times
fn static_atomic(val: usize) -> &'static AtomicUsize {
    let ret = Box::leak(Box::new(AtomicUsize::new(val)));
    ret
}

// Spins until it reads the given value
fn reads_value(loc: &AtomicUsize, val: usize) -> usize {
    while loc.load(Relaxed) != val {
        std::hint::spin_loop();
    }
    val
}

fn relaxed() -> bool {
    let x = static_atomic(0);
    let j1 = spawn(move || {
        x.store(1, Relaxed);
        x.store(2, Relaxed);
    });

    let j2 = spawn(move || x.load(Relaxed));

    j1.join().unwrap();
    let r2 = j2.join().unwrap();

    r2 == 1
}

// https://www.doc.ic.ac.uk/~afd/homepages/papers/pdfs/2017/POPL.pdf Figure 8
fn seq_cst() -> bool {
    let x = static_atomic(0);

    let j1 = spawn(move || {
        x.store(1, Relaxed);
    });

    let j2 = spawn(move || {
        x.store(2, SeqCst);
        x.store(3, SeqCst);
    });

    let j3 = spawn(move || x.load(SeqCst));

    j1.join().unwrap();
    j2.join().unwrap();
    let r3 = j3.join().unwrap();

    r3 == 1
}

fn initialization_write() -> bool {
    let x = static_atomic(11);
    assert_eq!(x.load(Relaxed), 11);

    let wait = static_atomic(0);

    let j1 = spawn(move || {
        x.store(22, Relaxed);
        // Relaxed is intentional. We want to test if the thread 2 reads the initialisation write
        // after a relaxed write
        wait.store(1, Relaxed);
    });

    let j2 = spawn(move || {
        reads_value(wait, 1);
        x.load(Relaxed)
    });

    j1.join().unwrap();
    let r2 = j2.join().unwrap();

    r2 == 11
}

// Asserts that the function returns true at least once in 100 runs
macro_rules! assert_once {
    ($f:ident) => {
        assert!(std::iter::repeat_with(|| $f()).take(100).any(|x| x));
    };
}

pub fn main() {
    assert_once!(relaxed);
    assert_once!(seq_cst);
    assert_once!(initialization_write);
}
<commit_msg>replace a macro by a function<commit_after>//@ignore-windows: Concurrency on Windows is not supported yet.
//@compile-flags: -Zmiri-ignore-leaks -Zmiri-preemption-rate=0

// Tests showing weak memory behaviours are exhibited. All tests
// return true when the desired behaviour is seen.
// This is scheduler and pseudo-RNG dependent, so each test is
// run multiple times until one try returns true.
// Spurious failure is possible, if you are really unlucky with
// the RNG and always read the latest value from the store buffer.

use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering::*;
use std::thread::spawn;

#[derive(Copy, Clone)]
struct EvilSend<T>(pub T);

unsafe impl<T> Send for EvilSend<T> {}
unsafe impl<T> Sync for EvilSend<T> {}

// We can't create static items because we need to run each test
// multiple times
fn static_atomic(val: usize) -> &'static AtomicUsize {
    let ret = Box::leak(Box::new(AtomicUsize::new(val)));
    ret
}

// Spins until it reads the given value
fn reads_value(loc: &AtomicUsize, val: usize) -> usize {
    while loc.load(Relaxed) != val {
        std::hint::spin_loop();
    }
    val
}

fn relaxed() -> bool {
    let x = static_atomic(0);
    let j1 = spawn(move || {
        x.store(1, Relaxed);
        x.store(2, Relaxed);
    });

    let j2 = spawn(move || x.load(Relaxed));

    j1.join().unwrap();
    let r2 = j2.join().unwrap();

    r2 == 1
}

// https://www.doc.ic.ac.uk/~afd/homepages/papers/pdfs/2017/POPL.pdf Figure 8
fn seq_cst() -> bool {
    let x = static_atomic(0);

    let j1 = spawn(move || {
        x.store(1, Relaxed);
    });

    let j2 = spawn(move || {
        x.store(2, SeqCst);
        x.store(3, SeqCst);
    });

    let j3 = spawn(move || x.load(SeqCst));

    j1.join().unwrap();
    j2.join().unwrap();
    let r3 = j3.join().unwrap();

    r3 == 1
}

fn initialization_write() -> bool {
    let x = static_atomic(11);
    assert_eq!(x.load(Relaxed), 11); // work around https://github.com/rust-lang/miri/issues/2164

    let wait = static_atomic(0);

    let j1 = spawn(move || {
        x.store(22, Relaxed);
        // Relaxed is intentional. We want to test if the thread 2 reads the initialisation write
        // after a relaxed write
        wait.store(1, Relaxed);
    });

    let j2 = spawn(move || {
        reads_value(wait, 1);
        x.load(Relaxed)
    });

    j1.join().unwrap();
    let r2 = j2.join().unwrap();

    r2 == 11
}

/// Asserts that the function returns true at least once in 100 runs
fn assert_once(f: fn() -> bool) {
    assert!(std::iter::repeat_with(|| f()).take(100).any(|x| x));
}

pub fn main() {
    assert_once(relaxed);
    assert_once(seq_cst);
    assert_once(initialization_write);
}
<|endoftext|>


<commit_before>use crate::runtime::task::RawTask;

use std::fmt;
use std::future::Future;
use std::marker::PhantomData;
use std::pin::Pin;
use std::task::{Context, Poll};

doc_rt_core! {
    /// An owned permission to join on a task (await its termination).
    ///
    /// This can be thought of as the equivalent of [`std::thread::JoinHandle`] for
    /// a task rather than a thread.
    ///
    /// A `JoinHandle` *detaches* the associated task when it is dropped, which
    /// means that there is no longer any handle to the task, and no way to `join`
    /// on it.
    ///
    /// This `struct` is created by the [`task::spawn`] and [`task::spawn_blocking`]
    /// functions.
    ///
    /// # Examples
    ///
    /// Creation from [`task::spawn`]:
    ///
    /// ```
    /// use tokio::task;
    ///
    /// # async fn doc() {
    /// let join_handle: task::JoinHandle<_> = task::spawn(async {
    ///     // some work here
    /// });
    /// # }
    /// ```
    ///
    /// Creation from [`task::spawn_blocking`]:
    ///
    /// ```
    /// use tokio::task;
    ///
    /// # async fn doc() {
    /// let join_handle: task::JoinHandle<_> = task::spawn_blocking(|| {
    ///     // some blocking work here
    /// });
    /// # }
    /// ```
    ///
    /// The generic parameter `T` in `JoinHandle<T>` is the return type of the spawned task.
    /// If the return value is an i32, the join handle has type `JoinHandle<i32>`:
    ///
    /// ```
    /// use tokio::task;
    ///
    /// # async fn doc() {
    /// let join_handle: task::JoinHandle<i32> = task::spawn(async {
    ///     5 + 3
    /// });
    /// # }
    ///
    /// ```
    ///
    /// If the task does not have a return value, the join handle has type `JoinHandle<()>`:
    ///
    /// ```
    /// use tokio::task;
    ///
    /// # async fn doc() {
    /// let join_handle: task::JoinHandle<()> = task::spawn(async {
    ///     println!("I return nothing.");
    /// });
    /// # }
    /// ```
    ///
    /// Note that `handle.await` doesn't give you the return type directly. It is wrapped in a
    /// `Result` because panics in the spawned task are caught by Tokio. The `?` operator has
    /// to be double chained to extract the returned value:
    ///
    /// ```
    /// use tokio::task;
    /// use std::io;
    ///
    /// #[tokio::main]
    /// async fn main() -> io::Result<()> {
    ///     let join_handle: task::JoinHandle<Result<i32, io::Error>> = tokio::spawn(async {
    ///         Ok(5 + 3)
    ///     });
    ///
    ///     let result = join_handle.await??;
    ///     assert_eq!(result, 8);
    ///     Ok(())
    /// }
    /// ```
    ///
    /// If the task panics, the error is a [`JoinError`] that contains the panic:
    ///
    /// ```
    /// use tokio::task;
    /// use std::io;
    /// use std::panic;
    ///
    /// #[tokio::main]
    /// async fn main() -> io::Result<()> {
    ///     let join_handle: task::JoinHandle<Result<i32, io::Error>> = tokio::spawn(async {
    ///         panic!("boom");
    ///     });
    ///
    ///     let err = join_handle.await.unwrap_err();
    ///     assert!(err.is_panic());
    ///     Ok(())
    /// }
    ///
    /// ```
    /// Child being detached and outliving its parent:
    ///
    /// ```no_run
    /// use tokio::task;
    /// use tokio::time;
    /// use std::time::Duration;
    ///
    /// # #[tokio::main] async fn main() {
    /// let original_task = task::spawn(async {
    ///     let _detached_task = task::spawn(async {
    ///         // Here we sleep to make sure that the first task returns before.
    ///         time::delay_for(Duration::from_millis(10)).await;
    ///         // This will be called, even though the JoinHandle is dropped.
    ///         println!("♫ Still alive ♫");
    ///     });
    /// });
    ///
    /// original_task.await.expect("The task being joined has panicked");
    /// println!("Original task is joined.");
    ///
    /// // We make sure that the new task has time to run, before the main
    /// // task returns.
    ///
    /// time::delay_for(Duration::from_millis(1000)).await;
    /// # }
    /// ```
    ///
    /// [`task::spawn`]: crate::task::spawn()
    /// [`task::spawn_blocking`]: crate::task::spawn_blocking
    /// [`std::thread::JoinHandle`]: std::thread::JoinHandle
    /// [`JoinError`]: crate::task::JoinError
    pub struct JoinHandle<T> {
        raw: Option<RawTask>,
        _p: PhantomData<T>,
    }
}

unsafe impl<T: Send> Send for JoinHandle<T> {}
unsafe impl<T: Send> Sync for JoinHandle<T> {}

impl<T> JoinHandle<T> {
    pub(super) fn new(raw: RawTask) -> JoinHandle<T> {
        JoinHandle {
            raw: Some(raw),
            _p: PhantomData,
        }
    }
}

impl<T> Unpin for JoinHandle<T> {}

impl<T> Future for JoinHandle<T> {
    type Output = super::Result<T>;

    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
        let mut ret = Poll::Pending;

        // Keep track of task budget
        let coop = ready!(crate::coop::poll_proceed(cx));

        // Raw should always be set. If it is not, this is due to polling after
        // completion
        let raw = self
            .raw
            .as_ref()
            .expect("polling after `JoinHandle` already completed");

        // Try to read the task output. If the task is not yet complete, the
        // waker is stored and is notified once the task does complete.
        //
        // The function must go via the vtable, which requires erasing generic
        // types. To do this, the function "return" is placed on the stack
        // **before** calling the function and is passed into the function using
        // `*mut ()`.
        //
        // Safety:
        //
        // The type of `T` must match the task's output type.
        unsafe {
            raw.try_read_output(&mut ret as *mut _ as *mut (), cx.waker());
        }

        if ret.is_ready() {
            coop.made_progress();
        }

        ret
    }
}

impl<T> Drop for JoinHandle<T> {
    fn drop(&mut self) {
        if let Some(raw) = self.raw.take() {
            if raw.header().state.drop_join_handle_fast().is_ok() {
                return;
            }

            raw.drop_join_handle_slow();
        }
    }
}

impl<T> fmt::Debug for JoinHandle<T>
where
    T: fmt::Debug,
{
    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
        fmt.debug_struct("JoinHandle").finish()
    }
}
<commit_msg>sync: add JoinHandle::abort (#2474)<commit_after>use crate::runtime::task::RawTask;

use std::fmt;
use std::future::Future;
use std::marker::PhantomData;
use std::pin::Pin;
use std::task::{Context, Poll};

doc_rt_core! {
    /// An owned permission to join on a task (await its termination).
    ///
    /// This can be thought of as the equivalent of [`std::thread::JoinHandle`] for
    /// a task rather than a thread.
    ///
    /// A `JoinHandle` *detaches* the associated task when it is dropped, which
    /// means that there is no longer any handle to the task, and no way to `join`
    /// on it.
    ///
    /// This `struct` is created by the [`task::spawn`] and [`task::spawn_blocking`]
    /// functions.
    ///
    /// # Examples
    ///
    /// Creation from [`task::spawn`]:
    ///
    /// ```
    /// use tokio::task;
    ///
    /// # async fn doc() {
    /// let join_handle: task::JoinHandle<_> = task::spawn(async {
    ///     // some work here
    /// });
    /// # }
    /// ```
    ///
    /// Creation from [`task::spawn_blocking`]:
    ///
    /// ```
    /// use tokio::task;
    ///
    /// # async fn doc() {
    /// let join_handle: task::JoinHandle<_> = task::spawn_blocking(|| {
    ///     // some blocking work here
    /// });
    /// # }
    /// ```
    ///
    /// The generic parameter `T` in `JoinHandle<T>` is the return type of the spawned task.
    /// If the return value is an i32, the join handle has type `JoinHandle<i32>`:
    ///
    /// ```
    /// use tokio::task;
    ///
    /// # async fn doc() {
    /// let join_handle: task::JoinHandle<i32> = task::spawn(async {
    ///     5 + 3
    /// });
    /// # }
    ///
    /// ```
    ///
    /// If the task does not have a return value, the join handle has type `JoinHandle<()>`:
    ///
    /// ```
    /// use tokio::task;
    ///
    /// # async fn doc() {
    /// let join_handle: task::JoinHandle<()> = task::spawn(async {
    ///     println!("I return nothing.");
    /// });
    /// # }
    /// ```
    ///
    /// Note that `handle.await` doesn't give you the return type directly. It is wrapped in a
    /// `Result` because panics in the spawned task are caught by Tokio. The `?` operator has
    /// to be double chained to extract the returned value:
    ///
    /// ```
    /// use tokio::task;
    /// use std::io;
    ///
    /// #[tokio::main]
    /// async fn main() -> io::Result<()> {
    ///     let join_handle: task::JoinHandle<Result<i32, io::Error>> = tokio::spawn(async {
    ///         Ok(5 + 3)
    ///     });
    ///
    ///     let result = join_handle.await??;
    ///     assert_eq!(result, 8);
    ///     Ok(())
    /// }
    /// ```
    ///
    /// If the task panics, the error is a [`JoinError`] that contains the panic:
    ///
    /// ```
    /// use tokio::task;
    /// use std::io;
    /// use std::panic;
    ///
    /// #[tokio::main]
    /// async fn main() -> io::Result<()> {
    ///     let join_handle: task::JoinHandle<Result<i32, io::Error>> = tokio::spawn(async {
    ///         panic!("boom");
    ///     });
    ///
    ///     let err = join_handle.await.unwrap_err();
    ///     assert!(err.is_panic());
    ///     Ok(())
    /// }
    ///
    /// ```
    /// Child being detached and outliving its parent:
    ///
    /// ```no_run
    /// use tokio::task;
    /// use tokio::time;
    /// use std::time::Duration;
    ///
    /// # #[tokio::main] async fn main() {
    /// let original_task = task::spawn(async {
    ///     let _detached_task = task::spawn(async {
    ///         // Here we sleep to make sure that the first task returns before.
    ///         time::delay_for(Duration::from_millis(10)).await;
    ///         // This will be called, even though the JoinHandle is dropped.
    ///         println!("♫ Still alive ♫");
    ///     });
    /// });
    ///
    /// original_task.await.expect("The task being joined has panicked");
    /// println!("Original task is joined.");
    ///
    /// // We make sure that the new task has time to run, before the main
    /// // task returns.
    ///
    /// time::delay_for(Duration::from_millis(1000)).await;
    /// # }
    /// ```
    ///
    /// [`task::spawn`]: crate::task::spawn()
    /// [`task::spawn_blocking`]: crate::task::spawn_blocking
    /// [`std::thread::JoinHandle`]: std::thread::JoinHandle
    /// [`JoinError`]: crate::task::JoinError
    pub struct JoinHandle<T> {
        raw: Option<RawTask>,
        _p: PhantomData<T>,
    }
}

unsafe impl<T: Send> Send for JoinHandle<T> {}
unsafe impl<T: Send> Sync for JoinHandle<T> {}

impl<T> JoinHandle<T> {
    pub(super) fn new(raw: RawTask) -> JoinHandle<T> {
        JoinHandle {
            raw: Some(raw),
            _p: PhantomData,
        }
    }

    /// Abort the task associated with the handle.
    ///
    /// Awaiting a cancelled task might complete as usual if the task was
    /// already completed at the time it was cancelled, but most likely it
    /// will complete with a `Err(JoinError::Cancelled)`.
    ///
    /// ```rust
    /// use tokio::time;
    ///
    /// #[tokio::main]
    /// async fn main() {
    ///    let mut handles = Vec::new();
    ///
    ///    handles.push(tokio::spawn(async {
    ///       time::delay_for(time::Duration::from_secs(10)).await;
    ///       true
    ///    }));
    ///
    ///    handles.push(tokio::spawn(async {
    ///       time::delay_for(time::Duration::from_secs(10)).await;
    ///       false
    ///    }));
    ///
    ///    for handle in &handles {
    ///        handle.abort();
    ///    }
    ///
    ///    for handle in handles {
    ///        assert!(handle.await.unwrap_err().is_cancelled());
    ///    }
    /// }
    /// ```
    pub fn abort(&self) {
        if let Some(raw) = self.raw {
            raw.shutdown();
        }
    }
}

impl<T> Unpin for JoinHandle<T> {}

impl<T> Future for JoinHandle<T> {
    type Output = super::Result<T>;

    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
        let mut ret = Poll::Pending;

        // Keep track of task budget
        let coop = ready!(crate::coop::poll_proceed(cx));

        // Raw should always be set. If it is not, this is due to polling after
        // completion
        let raw = self
            .raw
            .as_ref()
            .expect("polling after `JoinHandle` already completed");

        // Try to read the task output. If the task is not yet complete, the
        // waker is stored and is notified once the task does complete.
        //
        // The function must go via the vtable, which requires erasing generic
        // types. To do this, the function "return" is placed on the stack
        // **before** calling the function and is passed into the function using
        // `*mut ()`.
        //
        // Safety:
        //
        // The type of `T` must match the task's output type.
        unsafe {
            raw.try_read_output(&mut ret as *mut _ as *mut (), cx.waker());
        }

        if ret.is_ready() {
            coop.made_progress();
        }

        ret
    }
}

impl<T> Drop for JoinHandle<T> {
    fn drop(&mut self) {
        if let Some(raw) = self.raw.take() {
            if raw.header().state.drop_join_handle_fast().is_ok() {
                return;
            }

            raw.drop_join_handle_slow();
        }
    }
}

impl<T> fmt::Debug for JoinHandle<T>
where
    T: fmt::Debug,
{
    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
        fmt.debug_struct("JoinHandle").finish()
    }
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

use std::convert::TryInto;

use crate::action;
use crate::session;

/// Individual action response.
///
/// This type can be used both for action replies (for a particular session) and
/// responses sent to sinks.
pub struct Response<R: action::Response> {
    /// A server-issued identifier, usually corresponding to a flow.
    pub session_id: String,
    /// A server-issued identifier for the current action request.
    pub request_id: u64,
    /// An unique (to the current action request) identifier of the response.
    pub response_id: u64,
    /// An action-specific response.
    pub data: R,
}

/// Final session response indicating success or failure of action execution.
///
/// Each session is supposed to send this final response and communicate to the
/// server that it has finished working (whether it is because of all work that
/// was supposed to be done is done or because an error has occurred).
pub struct Status {
    /// A server-issued identifier, usually corresponding to a flow.
    pub session_id: String,
    /// A server-issued identifier for the current action request.
    pub request_id: u64,
    /// An unique (to the current action request) identifier of status response.
    pub response_id: u64,
    /// A result of action execution.
    pub result: session::Result<()>,
}

impl<R> TryInto<rrg_proto::jobs::GrrMessage> for Response<R>
where
    R: action::Response,
{
    type Error = protobuf::ProtobufError;

    fn try_into(self) -> Result<rrg_proto::jobs::GrrMessage, protobuf::ProtobufError> {
        let mut message = rrg_proto::jobs::GrrMessage::new();
        message.set_session_id(self.session_id);
        message.set_request_id(self.request_id);
        message.set_response_id(self.response_id);
        message.set_field_type(rrg_proto::jobs::GrrMessage_Type::MESSAGE);

        if let Some(rdf_name) = R::RDF_NAME {
            message.set_args_rdf_name(String::from(rdf_name));
        }
        message.set_args(protobuf::Message::write_to_bytes(&self.data.into_proto())?);

        Ok(message)
    }
}

impl TryInto<rrg_proto::jobs::GrrMessage> for Status {

    type Error = protobuf::ProtobufError;

    fn try_into(self) -> Result<rrg_proto::jobs::GrrMessage, protobuf::ProtobufError> {
        let mut status = rrg_proto::jobs::GrrStatus::new();
        match self.result {
            Ok(()) => {
                status.set_status(rrg_proto::jobs::GrrStatus_ReturnedStatus::OK);
            },
            Err(error) => {
                status.set_status(rrg_proto::jobs::GrrStatus_ReturnedStatus::GENERIC_ERROR);
                status.set_error_message(error.to_string());
            },
        }

        let mut message = rrg_proto::jobs::GrrMessage::new();
        message.set_session_id(self.session_id);
        message.set_request_id(self.request_id);
        message.set_response_id(self.response_id);
        message.set_field_type(rrg_proto::jobs::GrrMessage_Type::STATUS);

        message.set_args(protobuf::Message::write_to_bytes(&status)?);
        message.set_args_rdf_name(String::from("GrrStatus"));

        Ok(message)
    }
}
<commit_msg>Reduce number of top-level imports.<commit_after>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

use std::convert::TryInto;

/// Individual action response.
///
/// This type can be used both for action replies (for a particular session) and
/// responses sent to sinks.
pub struct Response<R: crate::action::Response> {
    /// A server-issued identifier, usually corresponding to a flow.
    pub session_id: String,
    /// A server-issued identifier for the current action request.
    pub request_id: u64,
    /// An unique (to the current action request) identifier of the response.
    pub response_id: u64,
    /// An action-specific response.
    pub data: R,
}

/// Final session response indicating success or failure of action execution.
///
/// Each session is supposed to send this final response and communicate to the
/// server that it has finished working (whether it is because of all work that
/// was supposed to be done is done or because an error has occurred).
pub struct Status {
    /// A server-issued identifier, usually corresponding to a flow.
    pub session_id: String,
    /// A server-issued identifier for the current action request.
    pub request_id: u64,
    /// An unique (to the current action request) identifier of status response.
    pub response_id: u64,
    /// A result of action execution.
    pub result: crate::session::Result<()>,
}

impl<R> TryInto<rrg_proto::jobs::GrrMessage> for Response<R>
where
    R: crate::action::Response,
{
    type Error = protobuf::ProtobufError;

    fn try_into(self) -> Result<rrg_proto::jobs::GrrMessage, protobuf::ProtobufError> {
        let mut message = rrg_proto::jobs::GrrMessage::new();
        message.set_session_id(self.session_id);
        message.set_request_id(self.request_id);
        message.set_response_id(self.response_id);
        message.set_field_type(rrg_proto::jobs::GrrMessage_Type::MESSAGE);

        if let Some(rdf_name) = R::RDF_NAME {
            message.set_args_rdf_name(String::from(rdf_name));
        }
        message.set_args(protobuf::Message::write_to_bytes(&self.data.into_proto())?);

        Ok(message)
    }
}

impl TryInto<rrg_proto::jobs::GrrMessage> for Status {

    type Error = protobuf::ProtobufError;

    fn try_into(self) -> Result<rrg_proto::jobs::GrrMessage, protobuf::ProtobufError> {
        let mut status = rrg_proto::jobs::GrrStatus::new();
        match self.result {
            Ok(()) => {
                status.set_status(rrg_proto::jobs::GrrStatus_ReturnedStatus::OK);
            },
            Err(error) => {
                status.set_status(rrg_proto::jobs::GrrStatus_ReturnedStatus::GENERIC_ERROR);
                status.set_error_message(error.to_string());
            },
        }

        let mut message = rrg_proto::jobs::GrrMessage::new();
        message.set_session_id(self.session_id);
        message.set_request_id(self.request_id);
        message.set_response_id(self.response_id);
        message.set_field_type(rrg_proto::jobs::GrrMessage_Type::STATUS);

        message.set_args(protobuf::Message::write_to_bytes(&status)?);
        message.set_args_rdf_name(String::from("GrrStatus"));

        Ok(message)
    }
}
<|endoftext|>