<commit_before>/* Christopher Piraino
 *
 *
 * Ferrous Threads
 *
 */
use std::mem;
use std::boxed::FnBox;
use std::thread::{self, spawn, JoinHandle};
use canal::mpmc::{Sender, Receiver, mpmc_channel};

const QUEUE_SIZE: usize = ((0 - 1) as u8) as usize;

/// The unit of work for a TaskRunner.
pub enum Task<'a> {
    Data(TaskData<'a>),
    Stop,
}

pub struct TaskData<'a> {
    task_func: Box<FnBox() + Send + 'a>,
}

impl<'a> TaskData<'a> {
    fn run(self) {
        self.task_func.call_box(())
    }
}

impl<'a> Task<'a> {
    fn new<F>(func: F) -> Task<'a> where F: FnOnce() + Send + 'a {
        Task::Data(TaskData { task_func: box func })
    }

    fn run(self) {
        match self {
            Task::Data(task) => task.run(),
            Task::Stop => (),
        }
    }
}

/// A TaskRunner is used to run short-lived tasks in parallel without having to
/// spin up a new thread each and every time.
///
/// The TaskRunner will immediately spin up the number of threads that was passed in
/// on creation.
///
/// Spins up a number of threads and distbutes the enqueued tasks through a
/// multi-producer/multi-consumer queue. This allows every worker to draw from the same queue,
/// ensuring that work will be efficiently distributed across the threads.
///
/// # Panics
/// The failure case of a task panicking and destroying the worker is not handled.
///
/// # Examples
/// ```
/// use ferrous_threads::task_runner::TaskRunner;
///
/// use std::sync::mpsc::channel;
///
/// let (sn, rc) = channel();
/// let taskpool = TaskRunner::new(1);
/// taskpool.enqueue(move || { sn.send(9u8).unwrap();}).ok().expect("Task not enqueued");
/// assert!(rc.recv().unwrap() == 9u8);
/// ```
pub struct TaskRunner<'a> {
    queue: Sender<Task<'a>>,
    workers: Vec<JoinHandle<()>>,
}

impl<'a> TaskRunner<'a> {
    pub fn new(num_threads: u8) -> TaskRunner<'a> {
        let (sn, rc): (Sender<Task<'a>>, Receiver<Task<'a>>) = mpmc_channel::<Task>(QUEUE_SIZE);
        let mut guards = Vec::new();
        for _i in 0..num_threads {
            // spawned threads cannot guarantee lifetimes, but we explicitly join on Drop.
            let rc: Receiver<Task<'static>> = unsafe { mem::transmute(rc.clone()) };
            let thr = spawn(move || { TaskRunner::worker(rc) });
            guards.push(thr);
        }
        TaskRunner { queue: sn, workers: guards }
    }

    /// Places the enqueued function on the worker queue.
    pub fn enqueue<F>(&self, func: F) -> Result<(), Task<'a>> where F: 'a + FnOnce() + Send {
        let task = Task::new(func);
        self.queue.send(task)
    }

    fn worker(rc: Receiver<Task>) {
        loop {
            let msg = rc.recv();
            match msg {
                Some(Task::Data(task)) => task.run(),
                Some(Task::Stop) => break,
                None  => thread::yield_now(),
            }
        }
    }
}

impl<'a> Drop for TaskRunner<'a> {
    fn drop(&mut self) {
        // Send stop message without blocking.
        for _thr in self.workers.iter() {
            self.queue.send(Task::Stop).ok().expect("Could not send a stop message.");
        }

        for thr in self.workers.drain(..) {
            thr.join().unwrap();
        }
    }
}

#[cfg(test)]
mod test {
    use super::{Task, TaskRunner};
    use std::sync::mpsc::{channel};

    #[test]
    fn test_task() {
        let (sn, rc) = channel::<u8>();
        let task_closure = move || {
            sn.send(0u8).unwrap();
        };
        let task = Task::new(task_closure);
        task.run();
        assert!(rc.recv().unwrap() == 0);
    }

    #[test]
    fn test_task_vector() {
        let (sn1, rc1) = channel::<isize>();
        let (sn2, rc2) = channel::<Option<u8>>();
        let task_closure = move || {
            sn1.send(10).unwrap();
        };
        let int_task = Task::new(task_closure);

        let task_closure = move || {
            sn2.send(Some(10u8)).unwrap();
        };
        let task = Task::new(task_closure);

        let vec = vec![int_task, task];
        for t in vec.into_iter() {
            t.run();
        }

        assert!(rc1.recv().unwrap() == 10);
        assert!(rc2.recv().unwrap().is_some());
    }

    #[test]
    fn test_task_pool() {
        let (sn1, rc1) = channel::<isize>();
        let task_closure = move || {
            sn1.send(10).unwrap();
        };
        let taskpool = TaskRunner::new(1);

        taskpool.enqueue(task_closure).ok().expect("Task not enqueued");

        assert_eq!(rc1.recv().unwrap(), 10);
    }

    #[test]
    fn test_task_pool_multi_workers() {
        let (sn1, rc1) = channel::<isize>();
        let sn2 = sn1.clone();
        let task_closure = move || {
            sn1.send(10).unwrap();
        };
        let task_closure2 = move || {
            sn2.send(10).unwrap();
        };

        let taskpool = TaskRunner::new(3);

        taskpool.enqueue(task_closure).ok().expect("Task not enqueued");
        taskpool.enqueue(task_closure2).ok().expect("Task not enqueued");

        assert_eq!(rc1.recv().unwrap(), 10);
        assert_eq!(rc1.recv().unwrap(), 10);
    }
}
<commit_msg>Fix canal 0.1.2 issue<commit_after>/* Christopher Piraino
 *
 *
 * Ferrous Threads
 *
 */
use std::mem;
use std::boxed::FnBox;
use std::thread::{self, spawn, JoinHandle};
use canal::mpmc::{Sender, Receiver, mpmc_channel};

const QUEUE_SIZE: usize = ((0 - 1) as u8) as usize;

/// The unit of work for a TaskRunner.
pub enum Task<'a> {
    Data(TaskData<'a>),
    Stop,
}

pub struct TaskData<'a> {
    task_func: Box<FnBox() + Send + 'a>,
}

impl<'a> TaskData<'a> {
    fn run(self) {
        self.task_func.call_box(())
    }
}

impl<'a> Task<'a> {
    fn new<F>(func: F) -> Task<'a> where F: FnOnce() + Send + 'a {
        Task::Data(TaskData { task_func: box func })
    }

    fn run(self) {
        match self {
            Task::Data(task) => task.run(),
            Task::Stop => (),
        }
    }
}

/// A TaskRunner is used to run short-lived tasks in parallel without having to
/// spin up a new thread each and every time.
///
/// The TaskRunner will immediately spin up the number of threads that was passed in
/// on creation.
///
/// Spins up a number of threads and distbutes the enqueued tasks through a
/// multi-producer/multi-consumer queue. This allows every worker to draw from the same queue,
/// ensuring that work will be efficiently distributed across the threads.
///
/// # Panics
/// The failure case of a task panicking and destroying the worker is not handled.
///
/// # Examples
/// ```
/// use ferrous_threads::task_runner::TaskRunner;
///
/// use std::sync::mpsc::channel;
///
/// let (sn, rc) = channel();
/// let taskpool = TaskRunner::new(1);
/// taskpool.enqueue(move || { sn.send(9u8).unwrap();}).ok().expect("Task not enqueued");
/// assert!(rc.recv().unwrap() == 9u8);
/// ```
pub struct TaskRunner<'a> {
    queue: Sender<Task<'a>>,
    workers: Vec<JoinHandle<()>>,
}

impl<'a> TaskRunner<'a> {
    pub fn new(num_threads: u8) -> TaskRunner<'a> {
        let (sn, rc): (Sender<Task<'a>>, Receiver<Task<'a>>) = mpmc_channel::<Task>(QUEUE_SIZE);
        let mut guards = Vec::new();
        for _i in 0..num_threads {
            // spawned threads cannot guarantee lifetimes, but we explicitly join on Drop.
            let rc: Receiver<Task<'static>> = unsafe { mem::transmute(rc.clone()) };
            let thr = spawn(move || { TaskRunner::worker(rc) });
            guards.push(thr);
        }
        TaskRunner { queue: sn, workers: guards }
    }

    /// Places the enqueued function on the worker queue.
    pub fn enqueue<F>(&self, func: F) -> Result<(), Task<'a>> where F: 'a + FnOnce() + Send {
        let task = Task::new(func);
        self.queue.send(task)
    }

    fn worker(rc: Receiver<Task>) {
        loop {
            let msg = rc.recv();
            match msg {
                Ok(Task::Data(task)) => task.run(),
                Ok(Task::Stop) => break,
                Err(e)  => panic!(e),
            }
        }
    }
}

impl<'a> Drop for TaskRunner<'a> {
    fn drop(&mut self) {
        // Send stop message without blocking.
        for _thr in self.workers.iter() {
            self.queue.send(Task::Stop).ok().expect("Could not send a stop message.");
        }

        for thr in self.workers.drain(..) {
            thr.join().unwrap();
        }
    }
}

#[cfg(test)]
mod test {
    use super::{Task, TaskRunner};
    use std::sync::mpsc::{channel};

    #[test]
    fn test_task() {
        let (sn, rc) = channel::<u8>();
        let task_closure = move || {
            sn.send(0u8).unwrap();
        };
        let task = Task::new(task_closure);
        task.run();
        assert!(rc.recv().unwrap() == 0);
    }

    #[test]
    fn test_task_vector() {
        let (sn1, rc1) = channel::<isize>();
        let (sn2, rc2) = channel::<Option<u8>>();
        let task_closure = move || {
            sn1.send(10).unwrap();
        };
        let int_task = Task::new(task_closure);

        let task_closure = move || {
            sn2.send(Some(10u8)).unwrap();
        };
        let task = Task::new(task_closure);

        let vec = vec![int_task, task];
        for t in vec.into_iter() {
            t.run();
        }

        assert!(rc1.recv().unwrap() == 10);
        assert!(rc2.recv().unwrap().is_some());
    }

    #[test]
    fn test_task_pool() {
        let (sn1, rc1) = channel::<isize>();
        let task_closure = move || {
            sn1.send(10).unwrap();
        };
        let taskpool = TaskRunner::new(1);

        taskpool.enqueue(task_closure).ok().expect("Task not enqueued");

        assert_eq!(rc1.recv().unwrap(), 10);
    }

    #[test]
    fn test_task_pool_multi_workers() {
        let (sn1, rc1) = channel::<isize>();
        let sn2 = sn1.clone();
        let task_closure = move || {
            sn1.send(10).unwrap();
        };
        let task_closure2 = move || {
            sn2.send(10).unwrap();
        };

        let taskpool = TaskRunner::new(3);

        taskpool.enqueue(task_closure).ok().expect("Task not enqueued");
        taskpool.enqueue(task_closure2).ok().expect("Task not enqueued");

        assert_eq!(rc1.recv().unwrap(), 10);
        assert_eq!(rc1.recv().unwrap(), 10);
    }
}
<|endoftext|>


<commit_before>use std::fs::File;

use rmps::{Deserializer, Serializer};
use serde::{Deserialize, Serialize};

use std::path::Path;
use std::io;
use std::io::{Write, Read, Error, ErrorKind};

use tensor::TensorKind;

pub fn save<P>(path: P, arg: &Vec<TensorKind>) -> io::Result<usize>
    where P: AsRef<Path>
{
    let mut buffer = File::create(path)?;
    let mut encoded = Vec::new();
    arg.serialize(&mut Serializer::new(&mut encoded)).unwrap();
    buffer.write(encoded.as_slice())
}

pub fn load<P>(path: P) -> io::Result<Vec<TensorKind>>
    where P: AsRef<Path>
{
    let mut buffer = File::open(path)?;
    let mut encoded = Vec::new();
    let size = buffer.read_to_end(&mut encoded)?;
    if size == 0 {
        return Err(Error::new(ErrorKind::UnexpectedEof, "Empty File"));
    };
    let mut de = Deserializer::new(&encoded[..]);
    let decoded = Deserialize::deserialize(&mut de).expect("decode failed");
    Ok(decoded)
}
<commit_msg>generalize save / load to a vector of anything<commit_after>use std::fs::File;

use rmps::{Deserializer, Serializer};
use serde::{Deserialize, Serialize};

use std::path::Path;
use std::io;
use std::io::{Write, Read, Error, ErrorKind};

use tensor::TensorKind;

pub fn save<P, T: Serialize>(path: P, arg: &Vec<T>) -> io::Result<usize>
    where P: AsRef<Path>
{
    let mut buffer = File::create(path)?;
    let mut encoded = Vec::new();
    arg.serialize(&mut Serializer::new(&mut encoded)).unwrap();
    buffer.write(encoded.as_slice())
}

pub fn load<'a, P, T: 'a + Deserialize<'a>>(path: P) -> io::Result<Vec<T>>
    where P: AsRef<Path>
{
    let mut buffer = File::open(path)?;
    let mut encoded = Vec::new();
    let size = buffer.read_to_end(&mut encoded)?;
    if size == 0 {
        return Err(Error::new(ErrorKind::UnexpectedEof, "Empty File"));
    };
    let mut de = Deserializer::new(&encoded[..]);
    let decoded = Deserialize::deserialize(&mut de).expect("decode failed");
    Ok(decoded)
}
<|endoftext|>


<commit_before>use crate::{layout, types};

use super::auto::{HasInner, Spawnable, Abstract};
use super::container::Container;
use super::drawable::{Drawable, OuterDrawable};
use super::has_layout::{HasLayout, HasLayoutInner};
use super::has_size::{HasSize, HasSizeInner, OnSize};
use super::has_visibility::{HasVisibility, HasVisibilityInner, OnVisibility};
use super::member::{AMember, Member, MemberBase, MemberInner};
use super::button::MaybeButton;
use super::layout_linear::MaybeLinearLayout;
use super::splitted::MaybeSplitted;
use super::frame::MaybeFrame;
use super::image::MaybeImage;
use super::list::MaybeList;
use super::progress_bar::MaybeProgressBar;
use super::text::MaybeText;
use super::seal::Sealed;

pub trait OuterControl: Sealed {
    fn on_added_to_container(&mut self, parent: &dyn Container, x: i32, y: i32, w: u16, h: u16);
    fn on_removed_from_container(&mut self, parent: &dyn Container);

    #[cfg(feature = "markup")]
    fn fill_from_markup(&mut self, markup: &crate::markup::Markup, registry: &mut crate::markup::MarkupRegistry);
}
impl<T: ControlInner> OuterControl for AMember<AControl<T>> {
    #[inline]
    fn on_added_to_container(&mut self, parent: &dyn Container, x: i32, y: i32, w: u16, h: u16) {
        #[cfg(feature = "type_check")]
        unsafe {
            use std::any::Any;
            if self.inner.inner.native_id().type_id() != parent.type_id() {
                panic!("Attempt to use the control from an incompatible backend!")
            }
        }
        self.inner.base.coords = Some((x, y));
        self.inner.inner.on_added_to_container(&mut self.base, &mut self.inner.base, parent, x, y, w, h)
    }
    #[inline]
    fn on_removed_from_container(&mut self, parent: &dyn Container) {
        self.inner.inner.on_removed_from_container(&mut self.base, &mut self.inner.base, parent);
        self.inner.base.coords = None;
    }

    #[cfg(feature = "markup")]
    default fn fill_from_markup(&mut self, markup: &crate::markup::Markup, registry: &mut crate::markup::MarkupRegistry) {
        self.inner.inner.fill_from_markup(&mut self.base, &mut self.inner.base, markup, registry)
    }
}

pub trait Control: Member + HasSize + HasVisibility + HasLayout + OuterDrawable + OuterControl
        + MaybeButton + MaybeLinearLayout + MaybeSplitted + MaybeFrame + MaybeImage + MaybeList + MaybeProgressBar + MaybeText {
    fn parent(&self) -> Option<&dyn Member>;
    fn parent_mut(&mut self) -> Option<&mut dyn Member>;
    fn root(&self) -> Option<&dyn Member>;
    fn root_mut(&mut self) -> Option<&mut dyn Member>;

    fn as_control(&self) -> &dyn Control;
    fn as_control_mut(&mut self) -> &mut dyn Control;
    fn into_control(self: Box<Self>) -> Box<dyn Control>;
}

pub trait ControlInner: MemberInner + HasSizeInner + HasVisibilityInner + HasLayoutInner + Drawable + Spawnable {
    fn on_added_to_container(&mut self, member: &mut MemberBase, control: &mut ControlBase, parent: &dyn Container, x: i32, y: i32, w: u16, h: u16);
    fn on_removed_from_container(&mut self, member: &mut MemberBase, control: &mut ControlBase, parent: &dyn Container);

    fn parent(&self) -> Option<&dyn Member>;
    fn parent_mut(&mut self) -> Option<&mut dyn Member>;
    fn root(&self) -> Option<&dyn Member>;
    fn root_mut(&mut self) -> Option<&mut dyn Member>;

    #[cfg(feature = "markup")]
    fn fill_from_markup(&mut self, member: &mut MemberBase, control: &mut ControlBase, mberarkup: &crate::markup::Markup, registry: &mut crate::markup::MarkupRegistry);
}

#[repr(C)]
pub struct ControlBase {
    pub layout: layout::Attributes,
    pub visibility: types::Visibility,
    pub measured: (u16, u16),
    pub coords: Option<(i32, i32)>,
    pub skip_draw: bool,

    pub on_size: Option<OnSize>,
    pub on_visibility: Option<OnVisibility>,
}
#[repr(C)]
pub struct AControl<T: ControlInner> {
    pub base: ControlBase,
    pub inner: T,
}

maybe!(Control);

impl Default for ControlBase {
    fn default() -> Self {
        ControlBase {
            layout: layout::Attributes::default(),
            visibility: types::Visibility::Visible,
            measured: (0, 0),
            coords: None,
            skip_draw: false,

            on_size: None,
            on_visibility: None,
        }
    }
}
impl<T: ControlInner> MemberInner for AControl<T> {}

impl<T: ControlInner> AControl<T> {
    #[inline]
    pub fn with_inner(inner: T) -> Self {
        AControl { inner: inner, base: Default::default() }
    }
}
impl<T: ControlInner> OuterDrawable for AMember<AControl<T>> {
    #[inline]
    fn draw(&mut self, coords: Option<(i32, i32)>) {
        if coords.is_some() {
            self.inner.base.coords = coords;
        }
        if !self.is_skip_draw() {
            self.inner.inner.draw(&mut self.base, &mut self.inner.base)
        }
    }
    #[inline]
    fn measure(&mut self, w: u16, h: u16) -> (u16, u16, bool) {
        self.inner.inner.measure(&mut self.base, &mut self.inner.base, w, h)
    }
    #[inline]
    fn invalidate(&mut self) {
        self.inner.inner.invalidate(&mut self.base, &mut self.inner.base)
    }
    #[inline]
    fn set_skip_draw(&mut self, skip: bool) {
        self.inner.base.skip_draw = skip;
    }
    #[inline]
    fn is_skip_draw(&self) -> bool {
        self.inner.base.skip_draw
    }
    #[inline]
    fn as_drawable(&self) -> &dyn OuterDrawable {
        self
    }
    #[inline]
    fn as_drawable_mut(&mut self) -> &mut dyn OuterDrawable {
        self
    }
    #[inline]
    fn into_drawable(self: Box<Self>) -> Box<dyn OuterDrawable> {
        self
    }
}
impl<T: ControlInner> HasLayout for AMember<AControl<T>> {
    fn layout(&self) -> (layout::Size, layout::Size) {
        (self.inner.base.layout.width, self.inner.base.layout.height)
    }
    fn set_layout(&mut self, width: layout::Size, height: layout::Size) {
        self.inner.base.layout.width = width;
        self.inner.base.layout.width = height;
    }

    #[inline]
    fn layout_width(&self) -> layout::Size {
        self.inner.base.layout.width
    }
    #[inline]
    fn layout_height(&self) -> layout::Size {
        self.inner.base.layout.height
    }
    #[inline]
    fn layout_margin(&self) -> layout::BoundarySize {
        self.inner.inner.layout_margin(&self.base)
    }

    #[inline]
    fn set_layout_width(&mut self, value: layout::Size) {
        self.inner.base.layout.width = value;
        self.inner.inner.on_layout_changed(&mut self.base);
    }
    #[inline]
    fn set_layout_height(&mut self, value: layout::Size) {
        self.inner.base.layout.height = value;
        self.inner.inner.on_layout_changed(&mut self.base);
    }

    #[inline]
    fn as_has_layout(&self) -> &dyn HasLayout {
        self
    }
    #[inline]
    fn as_has_layout_mut(&mut self) -> &mut dyn HasLayout {
        self
    }
    #[inline]
    fn into_has_layout(self: Box<Self>) -> Box<dyn HasLayout> {
        self
    }
}
impl<T: ControlInner> HasVisibility for AMember<AControl<T>> {
    #[inline]
    fn visibility(&self) -> types::Visibility {
        self.inner.base.visibility
    }
    #[inline]
    fn set_visibility(&mut self, visibility: types::Visibility) {
        if self.inner.inner.on_visibility_set(&mut self.base, visibility) {
            self.inner.base.visibility = visibility;
            self.call_on_visibility::<Self>(visibility);
        }
    }
    #[inline]
    fn on_visibility(&mut self, callback: Option<OnVisibility>) {
        self.inner.base.on_visibility = callback;
    }
    #[inline]
    fn as_has_visibility(&self) -> &dyn HasVisibility {
        self
    }
    #[inline]
    fn as_has_visibility_mut(&mut self) -> &mut dyn HasVisibility {
        self
    }
    #[inline]
    fn into_has_visibility(self: Box<Self>) -> Box<dyn HasVisibility> {
        self
    }
}
impl<T: ControlInner> HasSize for AMember<AControl<T>> {
    #[inline]
    fn size(&self) -> (u16, u16) {
        self.inner.base.measured
    }
    #[inline]
    fn set_size(&mut self, width: u16, height: u16) {
        if self.inner.inner.on_size_set(&mut self.base, (width, height)) {
            self.inner.base.measured = (width, height);
            self.call_on_size::<Self>(width, height);
        }
    }
    #[inline]
    fn on_size(&mut self, callback: Option<OnSize>) {
        self.inner.base.on_size = callback;
    }

    #[inline]
    fn as_has_size(&self) -> &dyn HasSize {
        self
    }
    #[inline]
    fn as_has_size_mut(&mut self) -> &mut dyn HasSize {
        self
    }
    #[inline]
    fn into_has_size(self: Box<Self>) -> Box<dyn HasSize> {
        self
    }
}
impl<T: ControlInner> Control for AMember<AControl<T>> {
    #[inline]
    fn parent(&self) -> Option<&dyn Member> {
        self.inner.inner.parent()
    }
    #[inline]
    fn parent_mut(&mut self) -> Option<&mut dyn Member> {
        self.inner.inner.parent_mut()
    }
    #[inline]
    fn root(&self) -> Option<&dyn Member> {
        self.inner.inner.root()
    }
    #[inline]
    fn root_mut(&mut self) -> Option<&mut dyn Member> {
        self.inner.inner.root_mut()
    }

    #[inline]
    fn as_control(&self) -> &dyn Control {
        self
    }
    #[inline]
    fn as_control_mut(&mut self) -> &mut dyn Control {
        self
    }
    #[inline]
    fn into_control(self: Box<Self>) -> Box<dyn Control> {
        self
    }
}
impl<T: ControlInner> AMember<AControl<T>> {
    #[inline]
    pub fn control(&self) -> &AControl<T> {
        &self.inner
    }
    #[inline]
    pub fn control_mut(&mut self) -> &mut AControl<T> {
        &mut self.inner
    }
    #[inline]
    pub fn call_on_size<O: Control>(&mut self, w: u16, h: u16) {
        let self2 = self as *mut _ as *mut O;
        if let Some(ref mut cb) = self.inner.base.on_size {
            (cb.as_mut())(unsafe { &mut *self2 }, w, h);
        }
    }
    #[inline]
    pub fn call_on_visibility<O: Control>(&mut self, v: types::Visibility) {
        let self2 = self as *mut _ as *mut O;
        if let Some(ref mut cb) = self.inner.base.on_visibility {
            (cb.as_mut())(unsafe { &mut *self2 }, v);
        }
    }
    #[inline]
    pub fn as_control_parts_mut(&mut self) -> (&mut MemberBase, &mut ControlBase, &mut T) {
        let self2 = self as *mut Self;
        let self3 = self as *mut Self;
        (&mut unsafe { &mut *self2 }.base, &mut unsafe { &mut *self3 }.inner.base, &mut self.inner.inner)
    }
    #[inline]
    pub fn spawn() -> Box<dyn Control> {
        T::spawn()
    }
}
impl<T: ControlInner> HasInner for AControl<T> {
    type I = T;

    #[inline]
    fn inner(&self) -> &Self::I {
        &self.inner
    }
    #[inline]
    fn inner_mut(&mut self) -> &mut Self::I {
        &mut self.inner
    }
    #[inline]
    fn into_inner(self) -> Self::I {
        self.inner
    }
}

impl<T: ControlInner> Abstract for AControl<T> {}
impl<II: ControlInner, T: HasInner<I = II> + Abstract + 'static> ControlInner for T {
    #[inline]
    fn on_added_to_container(&mut self, member: &mut MemberBase, control: &mut ControlBase, parent: &dyn Container, x: i32, y: i32, w: u16, h: u16) {
        self.inner_mut().on_added_to_container(member, control, parent, x, y, w, h)
    }
    #[inline]
    fn on_removed_from_container(&mut self, member: &mut MemberBase, control: &mut ControlBase, parent: &dyn Container) {
        self.inner_mut().on_removed_from_container(member, control, parent)
    }

    #[inline]
    fn parent(&self) -> Option<&dyn Member> {
        self.inner().parent()
    }
    #[inline]
    fn parent_mut(&mut self) -> Option<&mut dyn Member> {
        self.inner_mut().parent_mut()
    }
    #[inline]
    fn root(&self) -> Option<&dyn Member> {
        self.inner().root()
    }
    #[inline]
    fn root_mut(&mut self) -> Option<&mut dyn Member> {
        self.inner_mut().root_mut()
    }

    #[cfg(feature = "markup")]
    fn fill_from_markup(&mut self, member: &mut MemberBase, control: &mut ControlBase, markup: &crate::markup::Markup, registry: &mut crate::markup::MarkupRegistry) {
        self.inner_mut().fill_from_markup(member, control, markup, registry)
    }
}
<commit_msg>Minor control update<commit_after>use crate::{layout, types, utils};

use super::auto::{HasInner, Spawnable, Abstract};
use super::container::Container;
use super::drawable::{Drawable, OuterDrawable};
use super::has_layout::{HasLayout, HasLayoutInner};
use super::has_size::{HasSize, HasSizeInner, OnSize};
use super::has_visibility::{HasVisibility, HasVisibilityInner, OnVisibility};
use super::member::{AMember, Member, MemberBase, MemberInner};
use super::button::MaybeButton;
use super::layout_linear::MaybeLinearLayout;
use super::splitted::MaybeSplitted;
use super::frame::MaybeFrame;
use super::image::MaybeImage;
use super::list::MaybeList;
use super::progress_bar::MaybeProgressBar;
use super::text::MaybeText;
use super::seal::Sealed;

pub trait OuterControl: Sealed {
    fn on_added_to_container(&mut self, parent: &dyn Container, x: i32, y: i32, w: u16, h: u16);
    fn on_removed_from_container(&mut self, parent: &dyn Container);

    #[cfg(feature = "markup")]
    fn fill_from_markup(&mut self, markup: &crate::markup::Markup, registry: &mut crate::markup::MarkupRegistry);
}
impl<T: ControlInner> OuterControl for AMember<AControl<T>> {
    #[inline]
    fn on_added_to_container(&mut self, parent: &dyn Container, x: i32, y: i32, w: u16, h: u16) {
        #[cfg(feature = "type_check")]
        unsafe {
            use std::any::Any;
            if self.inner.inner.native_id().type_id() != parent.type_id() {
                panic!("Attempt to use the control from an incompatible backend!")
            }
        }
        self.inner.base.coords = Some((x, y));
        self.inner.inner.on_added_to_container(&mut self.base, &mut self.inner.base, parent, x, y, w, h)
    }
    #[inline]
    fn on_removed_from_container(&mut self, parent: &dyn Container) {
        self.inner.inner.on_removed_from_container(&mut self.base, &mut self.inner.base, parent);
        self.inner.base.coords = None;
    }

    #[cfg(feature = "markup")]
    default fn fill_from_markup(&mut self, markup: &crate::markup::Markup, registry: &mut crate::markup::MarkupRegistry) {
        self.inner.inner.fill_from_markup(&mut self.base, &mut self.inner.base, markup, registry)
    }
}

pub trait Control: Member + HasSize + HasVisibility + HasLayout + OuterDrawable + OuterControl
        + MaybeButton + MaybeLinearLayout + MaybeSplitted + MaybeFrame + MaybeImage + MaybeList + MaybeProgressBar + MaybeText {
    fn parent(&self) -> Option<&dyn Member>;
    fn parent_mut(&mut self) -> Option<&mut dyn Member>;
    fn root(&self) -> Option<&dyn Member>;
    fn root_mut(&mut self) -> Option<&mut dyn Member>;

    fn as_control(&self) -> &dyn Control;
    fn as_control_mut(&mut self) -> &mut dyn Control;
    fn into_control(self: Box<Self>) -> Box<dyn Control>;
}

pub trait ControlInner: MemberInner + HasSizeInner + HasVisibilityInner + HasLayoutInner + Drawable + Spawnable {
    fn on_added_to_container(&mut self, member: &mut MemberBase, control: &mut ControlBase, parent: &dyn Container, x: i32, y: i32, w: u16, h: u16);
    fn on_removed_from_container(&mut self, member: &mut MemberBase, control: &mut ControlBase, parent: &dyn Container);

    fn parent(&self) -> Option<&dyn Member>;
    fn parent_mut(&mut self) -> Option<&mut dyn Member>;
    fn root(&self) -> Option<&dyn Member>;
    fn root_mut(&mut self) -> Option<&mut dyn Member>;

    #[cfg(feature = "markup")]
    fn fill_from_markup(&mut self, member: &mut MemberBase, control: &mut ControlBase, mberarkup: &crate::markup::Markup, registry: &mut crate::markup::MarkupRegistry);
}

#[repr(C)]
pub struct ControlBase {
    pub layout: layout::Attributes,
    pub visibility: types::Visibility,
    pub measured: (u16, u16),
    pub coords: Option<(i32, i32)>,
    pub skip_draw: bool,

    pub on_size: Option<OnSize>,
    pub on_visibility: Option<OnVisibility>,
}
#[repr(C)]
pub struct AControl<T: ControlInner> {
    pub base: ControlBase,
    pub inner: T,
}

maybe!(Control);

impl Default for ControlBase {
    fn default() -> Self {
        ControlBase {
            layout: layout::Attributes::default(),
            visibility: types::Visibility::Visible,
            measured: (0, 0),
            coords: None,
            skip_draw: false,

            on_size: None,
            on_visibility: None,
        }
    }
}
impl<T: ControlInner> MemberInner for AControl<T> {}

impl<T: ControlInner> AControl<T> {
    #[inline]
    pub fn with_inner(inner: T) -> Self {
        AControl { inner: inner, base: Default::default() }
    }
}
impl<T: ControlInner> OuterDrawable for AMember<AControl<T>> {
    #[inline]
    fn draw(&mut self, coords: Option<(i32, i32)>) {
        if coords.is_some() {
            self.inner.base.coords = coords;
        }
        if !self.is_skip_draw() {
            self.inner.inner.draw(&mut self.base, &mut self.inner.base)
        }
    }
    #[inline]
    fn measure(&mut self, w: u16, h: u16) -> (u16, u16, bool) {
        self.inner.inner.measure(&mut self.base, &mut self.inner.base, w, h)
    }
    #[inline]
    fn invalidate(&mut self) {
        self.inner.inner.invalidate(&mut self.base, &mut self.inner.base)
    }
    #[inline]
    fn set_skip_draw(&mut self, skip: bool) {
        self.inner.base.skip_draw = skip;
    }
    #[inline]
    fn is_skip_draw(&self) -> bool {
        self.inner.base.skip_draw
    }
    #[inline]
    fn as_drawable(&self) -> &dyn OuterDrawable {
        self
    }
    #[inline]
    fn as_drawable_mut(&mut self) -> &mut dyn OuterDrawable {
        self
    }
    #[inline]
    fn into_drawable(self: Box<Self>) -> Box<dyn OuterDrawable> {
        self
    }
}
impl<T: ControlInner> HasLayout for AMember<AControl<T>> {
    fn layout(&self) -> (layout::Size, layout::Size) {
        (self.inner.base.layout.width, self.inner.base.layout.height)
    }
    fn set_layout(&mut self, width: layout::Size, height: layout::Size) {
        self.inner.base.layout.width = width;
        self.inner.base.layout.width = height;
    }

    #[inline]
    fn layout_width(&self) -> layout::Size {
        self.inner.base.layout.width
    }
    #[inline]
    fn layout_height(&self) -> layout::Size {
        self.inner.base.layout.height
    }
    #[inline]
    fn layout_margin(&self) -> layout::BoundarySize {
        self.inner.inner.layout_margin(&self.base)
    }

    #[inline]
    fn set_layout_width(&mut self, value: layout::Size) {
        self.inner.base.layout.width = value;
        self.inner.inner.on_layout_changed(&mut self.base);
    }
    #[inline]
    fn set_layout_height(&mut self, value: layout::Size) {
        self.inner.base.layout.height = value;
        self.inner.inner.on_layout_changed(&mut self.base);
    }

    #[inline]
    fn as_has_layout(&self) -> &dyn HasLayout {
        self
    }
    #[inline]
    fn as_has_layout_mut(&mut self) -> &mut dyn HasLayout {
        self
    }
    #[inline]
    fn into_has_layout(self: Box<Self>) -> Box<dyn HasLayout> {
        self
    }
}
impl<T: ControlInner> HasVisibility for AMember<AControl<T>> {
    #[inline]
    fn visibility(&self) -> types::Visibility {
        self.inner.base.visibility
    }
    #[inline]
    fn set_visibility(&mut self, visibility: types::Visibility) {
        if self.inner.inner.on_visibility_set(&mut self.base, visibility) {
            self.inner.base.visibility = visibility;
            self.call_on_visibility::<Self>(visibility);
        }
    }
    #[inline]
    fn on_visibility(&mut self, callback: Option<OnVisibility>) {
        self.inner.base.on_visibility = callback;
    }
    #[inline]
    fn as_has_visibility(&self) -> &dyn HasVisibility {
        self
    }
    #[inline]
    fn as_has_visibility_mut(&mut self) -> &mut dyn HasVisibility {
        self
    }
    #[inline]
    fn into_has_visibility(self: Box<Self>) -> Box<dyn HasVisibility> {
        self
    }
}
impl<T: ControlInner> HasSize for AMember<AControl<T>> {
    #[inline]
    fn size(&self) -> (u16, u16) {
        self.inner.base.measured
    }
    #[inline]
    fn set_size(&mut self, width: u16, height: u16) {
        if self.inner.inner.on_size_set(&mut self.base, (width, height)) {
            self.inner.base.measured = (width, height);
            self.call_on_size::<Self>(width, height);
        }
    }
    #[inline]
    fn on_size(&mut self, callback: Option<OnSize>) {
        self.inner.base.on_size = callback;
    }

    #[inline]
    fn as_has_size(&self) -> &dyn HasSize {
        self
    }
    #[inline]
    fn as_has_size_mut(&mut self) -> &mut dyn HasSize {
        self
    }
    #[inline]
    fn into_has_size(self: Box<Self>) -> Box<dyn HasSize> {
        self
    }
}
impl<T: ControlInner> Control for AMember<AControl<T>> {
    #[inline]
    fn parent(&self) -> Option<&dyn Member> {
        self.inner.inner.parent()
    }
    #[inline]
    fn parent_mut(&mut self) -> Option<&mut dyn Member> {
        self.inner.inner.parent_mut()
    }
    #[inline]
    fn root(&self) -> Option<&dyn Member> {
        self.inner.inner.root()
    }
    #[inline]
    fn root_mut(&mut self) -> Option<&mut dyn Member> {
        self.inner.inner.root_mut()
    }

    #[inline]
    fn as_control(&self) -> &dyn Control {
        self
    }
    #[inline]
    fn as_control_mut(&mut self) -> &mut dyn Control {
        self
    }
    #[inline]
    fn into_control(self: Box<Self>) -> Box<dyn Control> {
        self
    }
}
impl<T: ControlInner> AMember<AControl<T>> {
    #[inline]
    pub fn control(&self) -> &AControl<T> {
        &self.inner
    }
    #[inline]
    pub fn control_mut(&mut self) -> &mut AControl<T> {
        &mut self.inner
    }
    #[inline]
    pub fn call_on_size<O: Control>(&mut self, w: u16, h: u16) {
        let self2 = self as *mut _ as *mut O;
        if let Some(ref mut cb) = self.inner.base.on_size {
            (cb.as_mut())(unsafe { &mut *self2 }, w, h);
        }
    }
    #[inline]
    pub fn call_on_visibility<O: Control>(&mut self, v: types::Visibility) {
        let self2 = self as *mut _ as *mut O;
        if let Some(ref mut cb) = self.inner.base.on_visibility {
            (cb.as_mut())(unsafe { &mut *self2 }, v);
        }
    }
    #[inline]
    pub fn as_control_parts_mut(&mut self) -> (&mut MemberBase, &mut ControlBase, &mut T) {
        let self2 = self as *mut Self;
        let self3 = self as *mut Self;
        (&mut unsafe { &mut *self2 }.base, &mut unsafe { &mut *self3 }.inner.base, &mut self.inner.inner)
    }
    #[inline]
    pub unsafe fn control_base_parts_mut(base: &mut MemberBase) -> (&mut MemberBase, &mut ControlBase, &mut T) {
        utils::base_to_impl_mut::<Self>(base).as_control_parts_mut()
    }
    #[inline]
    pub fn spawn() -> Box<dyn Control> {
        T::spawn()
    }
}
impl<T: ControlInner> HasInner for AControl<T> {
    type I = T;

    #[inline]
    fn inner(&self) -> &Self::I {
        &self.inner
    }
    #[inline]
    fn inner_mut(&mut self) -> &mut Self::I {
        &mut self.inner
    }
    #[inline]
    fn into_inner(self) -> Self::I {
        self.inner
    }
}

impl<T: ControlInner> Abstract for AControl<T> {}
impl<II: ControlInner, T: HasInner<I = II> + Abstract + 'static> ControlInner for T {
    #[inline]
    fn on_added_to_container(&mut self, member: &mut MemberBase, control: &mut ControlBase, parent: &dyn Container, x: i32, y: i32, w: u16, h: u16) {
        self.inner_mut().on_added_to_container(member, control, parent, x, y, w, h)
    }
    #[inline]
    fn on_removed_from_container(&mut self, member: &mut MemberBase, control: &mut ControlBase, parent: &dyn Container) {
        self.inner_mut().on_removed_from_container(member, control, parent)
    }

    #[inline]
    fn parent(&self) -> Option<&dyn Member> {
        self.inner().parent()
    }
    #[inline]
    fn parent_mut(&mut self) -> Option<&mut dyn Member> {
        self.inner_mut().parent_mut()
    }
    #[inline]
    fn root(&self) -> Option<&dyn Member> {
        self.inner().root()
    }
    #[inline]
    fn root_mut(&mut self) -> Option<&mut dyn Member> {
        self.inner_mut().root_mut()
    }

    #[cfg(feature = "markup")]
    fn fill_from_markup(&mut self, member: &mut MemberBase, control: &mut ControlBase, markup: &crate::markup::Markup, registry: &mut crate::markup::MarkupRegistry) {
        self.inner_mut().fill_from_markup(member, control, markup, registry)
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Fabian Schuiki

//! Multi-type arena allocation

#![deny(missing_docs)]

/// Allocates objects into an arena.
pub trait Alloc<T> {
    /// Allocate an object of type `T` into this arena.
    fn alloc(&self, value: T) -> &mut T;
}

#[macro_export]
macro_rules! make_arenas {
    ($(#[$arena_attr:meta])* pub struct $arena_name:ident { $($name:ident: $type:ty,)* }) => (
        $(#[$arena_attr])*
        pub struct $arena_name {
            $(pub $name: Arena<$type>,)*
        }

        impl Default for $arena_name {
            fn default() -> $arena_name {
                $arena_name {
                    $($name: Arena::new(),)*
                }
            }
        }

        $(
            impl $crate::arenas::Alloc<$type> for $arena_name {
                fn alloc(&self, value: $type) -> &mut $type {
                    self.$name.alloc(value)
                }
            }
        )*
    );
}
<commit_msg>vhdl: add AllocInto trait<commit_after>// Copyright (c) 2018 Fabian Schuiki

//! Multi-type arena allocation

#![deny(missing_docs)]

/// Allocates objects into an arena.
pub trait Alloc<T> {
    /// Allocate an object of type `T` into this arena.
    fn alloc(&self, value: T) -> &mut T;
}

/// Allocates objects into a remote arena.
///
/// In contrast to `Alloc`, the lifetime of the references returned by this
/// trait are not bound to the trait itself. Rather, the lifetime is a parameter
/// of the trait itself. This allows context objects to be created that hold a
/// reference to an arena, and allow for direct allocation into that arena.
pub trait AllocInto<'t, T> {
    /// Allocate an object of type `T`.
    fn alloc(&self, value: T) -> &'t mut T;
}

impl<'t, T> AllocInto<'t, T> for &'t Alloc<T> {
    fn alloc(&self, value: T) -> &'t mut T {
        Alloc::alloc(*self, value)
    }
}

#[macro_export]
macro_rules! make_arenas {
    ($(#[$arena_attr:meta])* pub struct $arena_name:ident { $($name:ident: $type:ty,)* }) => (
        $(#[$arena_attr])*
        pub struct $arena_name {
            $(pub $name: Arena<$type>,)*
        }

        impl Default for $arena_name {
            fn default() -> $arena_name {
                $arena_name {
                    $($name: Arena::new(),)*
                }
            }
        }

        $(
            impl $crate::arenas::Alloc<$type> for $arena_name {
                fn alloc(&self, value: $type) -> &mut $type {
                    self.$name.alloc(value)
                }
            }
        )*
    );
}
<|endoftext|>


<commit_before>use chrono::DateTime;
use chrono::offset::Utc;
use diesel;
use diesel::prelude::*;
use diesel::pg::PgConnection;
use diesel::pg::upsert::*;
use futures::{Future, stream, Stream};
use futures_cpupool::CpuPool;
use hyper::{Body, Client, Uri};
use hyper::client::HttpConnector;
use hyper_tls::HttpsConnector;
use InsertError;
use kuchiki;
use kuchiki::traits::*;
use r2d2_diesel::ConnectionManager;
use r2d2::Pool;
use rusoto_core::{default_tls_client, Region};
use rusoto_credential::DefaultCredentialsProvider;
use rusoto_s3::{PutObjectRequest, S3Client, S3};
use schema::ads;
use server::AdPost;

const ENDPOINT: &'static str = "https://pp-facebook-ads.s3.amazonaws.com/";

#[derive(Queryable, Debug, Clone)]
pub struct Ad {
    id: String,
    html: String,
    political: i32,
    not_political: i32,

    fuzzy_id: Option<i32>,
    title: String,
    message: String,
    thumbnail: String,

    created_at: DateTime<Utc>,
    updated_at: DateTime<Utc>,

    browser_lang: String,

    images: Vec<String>,
}

#[derive(AsChangeset)]
#[table_name = "ads"]
pub struct Images {
    thumbnail: Option<String>,
    images: Vec<String>,
}

impl Ad {
    // This will asynchronously save the images to s3 we may very well end up
    // with broken images. but I can't see any way around it right now.  Also we
    // should think about splitting this up, but I'm fine -- if a little
    // embarassed about it right now. This function swallows errors, and there's
    // a chance we'll end up with no images at the end, but I think we can
    // handle that in the extension's UI.
    pub fn grab_and_store(
        &self,
        client: Client<HttpsConnector<HttpConnector>, Body>,
        db: &Pool<ConnectionManager<PgConnection>>,
        pool: CpuPool,
    ) -> Box<Future<Item = (), Error = ()>> {
        let ad = self.clone();
        let pool_s3 = pool.clone();
        let pool_db = pool.clone();
        let db = db.clone();

        let images = [vec![self.thumbnail.clone()], self.images.clone()];
        let urls = images
            .concat()
            .iter()
            .map(|a| {
                let a: Uri = a.parse().map_err(InsertError::Uri)?;
                Ok(a)
            })
            .collect::<Vec<Result<Uri, InsertError>>>();
        let future = stream::iter(urls)
            // filter ones we already have in the db and ones we can verify as
            // coming from fb, we don't want to become a malware vector :)
            .filter(|u| match u.host() {
                Some(h) => h != "pp-facebook-ads.s3.amazonaws.com" || !h.ends_with("fbcdn.net"),
                None => false
            })
            // grab image
            .and_then(move |img| {
                info!("getting {:?}", img);
                let cloned = img.clone();
                client
                    .get(img)
                    .and_then(|res| {
                        res.body().concat2().and_then(|chunk| Ok((chunk, cloned)))
                    })
                    .map_err(InsertError::Hyper)
            })
            // upload them to s3
            .and_then(move |tuple| {
                let pool = pool_s3.clone();
                // we do this in a worker thread because rusoto isn't on
                // Hyper async yet.
                pool.spawn_fn(move || {
                    let client =
                        S3Client::new(default_tls_client().map_err(InsertError::TLS)?,
                                      DefaultCredentialsProvider::new().map_err(InsertError::AWS)?,
                                      Region::UsEast1);
                    let req = PutObjectRequest {
                        bucket: "pp-facebook-ads".to_string(),
                        key: tuple.1.path().trim_left_matches("/").to_string(),
                        acl: Some("public-read".to_string()),
                        body: Some(tuple.0.to_vec()),
                        ..PutObjectRequest::default()
                    };

                    client.put_object(&req).map_err(InsertError::S3)?;
                    Ok(tuple.1)
                })
            })
            .map_err(|e| {
                warn!("{:?}", e);
                ()
            })
            .collect()
            // save the new urls to the database. the images variable will
            // include only those that we've successfully saved to s3, so we
            // have to do a funky merge here.
            .and_then(move |images| {
                let imgs = images.clone();
                pool_db.spawn_fn(move || {
                    use schema::ads::dsl::*;
                    let thumb = imgs
                        .iter()
                        .filter(|i| ad.thumbnail.contains(i.path()))
                        .map(|i| ENDPOINT.to_string() + i.path().trim_left_matches("/"))
                        .nth(0);

                    let mut rest = imgs.clone();
                    if let Some(thumb) = thumb.clone() {
                      rest.retain(|x| !thumb.contains(x.path()))
                    };

                    let collection = rest
                        .iter()
                        .filter(|i| ad.images.iter().any(|a| a.contains(i.path())))
                        .map(|i| ENDPOINT.to_string() + i.path().trim_left_matches("/"))
                        .collect::<Vec<String>>();

                    let update = Images {
                        thumbnail: thumb,
                        images: collection
                    };
                    // I don't understand why we're zeroing out the errors here
                    // but ok.
                    let connection = db.get().map_err(|e| {warn!("{:?}", e); ()})?;
                    let adid = ad.id.clone();
                    diesel::update(ads.filter(id.eq(ad.id)))
                        .set(&update)
                        .execute(&*connection)
                        .map_err(|e| {warn!("{:?}", e); ()})?;
                    info!("saved {:?}", adid);
                    Ok(())
                })
            });
        Box::new(future)
    }
}

#[derive(Insertable)]
#[table_name = "ads"]
pub struct NewAd<'a> {
    id: &'a str,
    html: String,
    political: i32,
    not_political: i32,

    title: String,
    message: String,
    thumbnail: String,

    browser_lang: &'a str,
    images: Vec<String>,
}

impl<'a> NewAd<'a> {
    fn get_title(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
        document
            .select("h5 a, h6 a, strong")
            .map_err(InsertError::HTML)?
            .nth(0)
            .and_then(|a| Some(a.text_contents()))
            .ok_or(InsertError::HTML(()))
    }

    fn get_image(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
        document
            .select("img")
            .map_err(InsertError::HTML)?
            .nth(0)
            .and_then(|a| {
                a.attributes.borrow().get("src").and_then(
                    |src| Some(src.to_string()),
                )
            })
            .ok_or(InsertError::HTML(()))
    }
    // Video ads make this a bit messy
    fn get_message(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
        let selectors = vec![".userContent p", "span"];
        let iters = selectors.iter().map(|s| document.select(s)).flat_map(|a| a);

        iters
            .map(|i| {
                i.fold(String::new(), |m, a| m + &a.as_node().to_string())
            })
            .filter(|i| i.len() > 0)
            .nth(0)
            .ok_or(InsertError::HTML(()))
    }

    fn get_images(document: &kuchiki::NodeRef) -> Result<Vec<String>, InsertError> {
        Ok(
            document
                .select("img")
                .map_err(InsertError::HTML)?
                .skip(1)
                .map(|a| {
                    a.attributes.borrow().get("src").and_then(
                        |s| Some(s.to_string()),
                    )
                })
                .filter(|s| s.is_some())
                .map(|s| s.unwrap())
                .collect::<Vec<String>>(),
        )
    }

    fn rewrite_images(document: &kuchiki::NodeRef) -> Result<&kuchiki::NodeRef, InsertError> {
        // rewrite
        for a in document.select("img").map_err(InsertError::HTML)? {
            if let Some(x) = a.attributes.borrow_mut().get_mut("src") {
                if let Ok(u) = x.parse::<Uri>() {
                    *x = ENDPOINT.to_string() + u.path().trim_left_matches("/");
                }
            };
        }

        Ok(&document)
    }

    pub fn new(ad: &'a AdPost) -> Result<NewAd<'a>, InsertError> {
        let document = kuchiki::parse_html().one(ad.html.clone());

        let message = NewAd::get_message(&document)?;
        let title = NewAd::get_title(&document)?;
        let thumb = NewAd::get_image(&document)?;
        let images = NewAd::get_images(&document)?;
        let doc = NewAd::rewrite_images(&document)?;
        let html = doc.to_string();

        Ok(NewAd {
            id: &ad.id,
            html: html,
            political: if ad.political { 1 } else { 0 },
            not_political: if !ad.political { 1 } else { 0 },
            title: title,
            message: message,
            thumbnail: thumb,
            browser_lang: &ad.browser_lang,
            images: images,
        })
    }

    pub fn save(&self, pool: &Pool<ConnectionManager<PgConnection>>) -> Result<Ad, InsertError> {
        use schema::ads;
        use schema::ads::dsl::*;
        let connection = pool.get().map_err(InsertError::Timeout)?;
        let ad: Ad = diesel::insert(&self.on_conflict(
            id,
            do_update().set((
                political.eq(political + self.political),
                not_political.eq(
                    not_political +
                        self.not_political,
                ),
                updated_at.eq(Utc::now()),
            )),
        )).into(ads::table)
            .get_result(&*connection)
            .map_err(InsertError::DataBase)?;
        Ok(ad)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn ad_parsing() {
        let ad = include_str!("./html-test.txt");
        let post = AdPost {
            id: "test".to_string(),
            html: ad.to_string(),
            political: true,
            browser_lang: "en-US".to_string(),
        };
        let new_ad = NewAd::new(&post).unwrap();
        assert!(new_ad.thumbnail.len() > 0);
        assert_eq!(new_ad.images.len(), 2);
        assert!(new_ad.title.len() > 0);
        assert!(new_ad.html != post.html);
        assert!(!new_ad.html.contains("fbcdn"));
        assert_eq!(
            new_ad.message,
            "
#Valerian is “the best experience since ‘Avatar.’” See it in 3D and RealD3D theaters this Friday. Get tickets now: ValerianTickets.com
"
        );
    }
}
<commit_msg>munge the html after we grab images<commit_after>use chrono::DateTime;
use chrono::offset::Utc;
use diesel;
use diesel::prelude::*;
use diesel::pg::PgConnection;
use diesel::pg::upsert::*;
use futures::{Future, stream, Stream};
use futures_cpupool::CpuPool;
use hyper::{Body, Client, Uri};
use hyper::client::HttpConnector;
use hyper_tls::HttpsConnector;
use InsertError;
use kuchiki;
use kuchiki::traits::*;
use r2d2_diesel::ConnectionManager;
use r2d2::Pool;
use rusoto_core::{default_tls_client, Region};
use rusoto_credential::DefaultCredentialsProvider;
use rusoto_s3::{PutObjectRequest, S3Client, S3};
use schema::ads;
use server::AdPost;

const ENDPOINT: &'static str = "https://pp-facebook-ads.s3.amazonaws.com/";

fn get_title(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    document
        .select("h5 a, h6 a, strong")
        .map_err(InsertError::HTML)?
        .nth(0)
        .and_then(|a| Some(a.text_contents()))
        .ok_or(InsertError::HTML(()))
}

fn get_image(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    document
        .select("img")
        .map_err(InsertError::HTML)?
        .nth(0)
        .and_then(|a| {
            a.attributes.borrow().get("src").and_then(
                |src| Some(src.to_string()),
            )
        })
        .ok_or(InsertError::HTML(()))
}
// Video ads make this a bit messy
fn get_message(document: &kuchiki::NodeRef) -> Result<String, InsertError> {
    let selectors = vec![".userContent p", "span"];
    let iters = selectors.iter().map(|s| document.select(s)).flat_map(|a| a);

    iters
        .map(|i| {
            i.fold(String::new(), |m, a| m + &a.as_node().to_string())
        })
        .filter(|i| i.len() > 0)
        .nth(0)
        .ok_or(InsertError::HTML(()))
}

fn get_images(document: &kuchiki::NodeRef) -> Result<Vec<String>, InsertError> {
    Ok(
        document
            .select("img")
            .map_err(InsertError::HTML)?
            .skip(1)
            .map(|a| {
                a.attributes.borrow().get("src").and_then(
                    |s| Some(s.to_string()),
                )
            })
            .filter(|s| s.is_some())
            .map(|s| s.unwrap())
            .collect::<Vec<String>>(),
    )
}

#[derive(AsChangeset, Debug)]
#[table_name = "ads"]
pub struct Images {
    thumbnail: Option<String>,
    images: Vec<String>,
    title: String,
    message: String,
    html: String,
}

impl Images {
    fn from_ad(ad: &Ad, images: Vec<Uri>) -> Result<Images, InsertError> {
        let thumb = images
            .iter()
            .filter(|i| ad.thumbnail.contains(i.path()))
            .map(|i| ENDPOINT.to_string() + i.path().trim_left_matches("/"))
            .nth(0);

        let mut rest = images.clone();
        if let Some(thumb) = thumb.clone() {
            rest.retain(|x| !thumb.contains(x.path()))
        };

        let collection = rest.iter()
            .filter(|i| ad.images.iter().any(|a| a.contains(i.path())))
            .map(|i| ENDPOINT.to_string() + i.path().trim_left_matches("/"))
            .collect::<Vec<String>>();

        let document = kuchiki::parse_html().one(ad.html.clone());
        for a in document.select("img").map_err(InsertError::HTML)? {
            if let Some(x) = a.attributes.borrow_mut().get_mut("src") {
                if let Ok(u) = x.parse::<Uri>() {
                    if let Some(i) = images.iter().find(|i| i.path() == u.path()) {
                        *x = ENDPOINT.to_string() + i.path().trim_left_matches("/");
                    } else {
                        *x = "".to_string();
                    }
                }
            };
        }

        let title = get_title(&document)?;
        let message = get_title(&document)?;
        Ok(Images {
            thumbnail: thumb,
            images: collection,
            title: title,
            html: document.to_string(),
            message: message,
        })
    }
}

#[derive(Queryable, Debug, Clone)]
pub struct Ad {
    id: String,
    html: String,
    political: i32,
    not_political: i32,

    fuzzy_id: Option<i32>,
    title: String,
    message: String,
    thumbnail: String,

    created_at: DateTime<Utc>,
    updated_at: DateTime<Utc>,

    browser_lang: String,

    images: Vec<String>,
}

impl Ad {
    // This will asynchronously save the images to s3 we may very well end up
    // dropping images. but I can't see any way around it right now. Also we
    // should think about splitting this up, but I'm fine -- if a little
    // embarassed about it right now. This function swallows errors, and there's
    // a chance we'll end up with no images at the end, but I think we can
    // handle that in the extension's UI.
    pub fn grab_and_store(
        &self,
        client: Client<HttpsConnector<HttpConnector>, Body>,
        db: &Pool<ConnectionManager<PgConnection>>,
        pool: CpuPool,
    ) -> Box<Future<Item = (), Error = ()>> {
        let ad = self.clone();
        let pool_s3 = pool.clone();
        let pool_db = pool.clone();
        let db = db.clone();
        let future = stream::iter(self.image_urls())
            // filter ones we already have in the db and ones we can verify as
            // coming from fb, we don't want to become a malware vector :)
            .filter(|u| match u.host() {
                Some(h) => h != "pp-facebook-ads.s3.amazonaws.com" || !h.ends_with("fbcdn.net"),
                None => false
            })
            // grab image
            .and_then(move |img| {
                info!("getting {:?}", img);
                let cloned = img.clone();
                client
                    .get(img)
                    .and_then(|res| {
                        res.body().concat2().and_then(|chunk| Ok((chunk, cloned)))
                    })
                    .map_err(InsertError::Hyper)
            })
            // upload them to s3
            .and_then(move |tuple| {
                let pool = pool_s3.clone();
                // we do this in a worker thread because rusoto isn't on
                // Hyper async yet.
                pool.spawn_fn(move || {
                    let client =
                        S3Client::new(default_tls_client().map_err(InsertError::TLS)?,
                                      DefaultCredentialsProvider::new().map_err(InsertError::AWS)?,
                                      Region::UsEast1);
                    let req = PutObjectRequest {
                        bucket: "pp-facebook-ads".to_string(),
                        key: tuple.1.path().trim_left_matches("/").to_string(),
                        acl: Some("public-read".to_string()),
                        body: Some(tuple.0.to_vec()),
                        ..PutObjectRequest::default()
                    };

                    client.put_object(&req).map_err(InsertError::S3)?;
                    Ok(tuple.1)
                })
            })
            .map_err(|e| {
                warn!("{:?}", e);
                ()
            })
            .collect()
            // save the new urls to the database. the images variable will
            // include only those that we've successfully saved to s3, so we
            // have to do a funky merge here.
            .and_then(move |images| {
                let imgs = images.clone();
                pool_db.spawn_fn(move || {
                    use schema::ads::dsl::*;
                    // I don't understand why we're zeroing out the errors here
                    // but ok.
                    let update = Images::from_ad(&ad, imgs).map_err(|e| {warn!("{:?}", e); ()})?;
                    let connection = db.get().map_err(|e| {warn!("{:?}", e); ()})?;
                    diesel::update(ads.filter(id.eq(&ad.id)))
                        .set(&update)
                        .execute(&*connection)
                        .map_err(|e| {warn!("{:?}", e); ()})?;
                    info!("saved {:?}", ad.id);
                    Ok(())
                })
            });
        Box::new(future)
    }

    pub fn image_urls(&self) -> Vec<Result<Uri, InsertError>> {
        let images = [vec![self.thumbnail.clone()], self.images.clone()];
        images
            .concat()
            .iter()
            .map(|a| {
                let a: Uri = a.parse().map_err(InsertError::Uri)?;
                Ok(a)
            })
            .collect()
    }
}

#[derive(Insertable)]
#[table_name = "ads"]
pub struct NewAd<'a> {
    id: &'a str,
    html: &'a str,
    political: i32,
    not_political: i32,

    title: String,
    message: String,
    thumbnail: String,

    browser_lang: &'a str,
    images: Vec<String>,
}

impl<'a> NewAd<'a> {
    pub fn new(ad: &'a AdPost) -> Result<NewAd<'a>, InsertError> {
        let document = kuchiki::parse_html().one(ad.html.clone());

        let thumb = get_image(&document)?;
        let images = get_images(&document)?;
        let message = get_message(&document)?;
        let title = get_title(&document)?;

        Ok(NewAd {
            id: &ad.id,
            html: &ad.html,
            political: if ad.political { 1 } else { 0 },
            not_political: if !ad.political { 1 } else { 0 },
            title: title,
            message: message,
            thumbnail: thumb,
            browser_lang: &ad.browser_lang,
            images: images,
        })
    }

    pub fn save(&self, pool: &Pool<ConnectionManager<PgConnection>>) -> Result<Ad, InsertError> {
        use schema::ads;
        use schema::ads::dsl::*;
        let connection = pool.get().map_err(InsertError::Timeout)?;
        let ad: Ad = diesel::insert(&self.on_conflict(
            id,
            do_update().set((
                political.eq(political + self.political),
                not_political.eq(
                    not_political +
                        self.not_political,
                ),
                updated_at.eq(Utc::now()),
            )),
        )).into(ads::table)
            .get_result(&*connection)
            .map_err(InsertError::DataBase)?;
        Ok(ad)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn ad_parsing() {
        let ad = include_str!("./html-test.txt");
        let post = AdPost {
            id: "test".to_string(),
            html: ad.to_string(),
            political: true,
            browser_lang: "en-US".to_string(),
        };
        let new_ad = NewAd::new(&post).unwrap();
        assert!(new_ad.thumbnail.len() > 0);
        assert_eq!(new_ad.images.len(), 2);
        assert!(new_ad.title.len() > 0);

        assert_eq!(
            new_ad.message,
            "
#Valerian is “the best experience since ‘Avatar.’” See it in 3D and RealD3D theaters this Friday. Get tickets now: ValerianTickets.com
"
        );
    }

    #[test]
    fn image_parsing() {
        use chrono::prelude::*;
        let ad = include_str!("./html-test.txt");
        let document = kuchiki::parse_html().one(ad);

        let saved_ad = Ad {
            id: "test".to_string(),
            html: ad.to_string(),
            political: 1,
            not_political: 2,
            fuzzy_id: None,
            title: get_title(&document).unwrap(),
            message: get_message(&document).unwrap(),
            thumbnail: get_image(&document).unwrap(),
            created_at: Utc::now(),
            updated_at: Utc::now(),
            browser_lang: "en-US".to_string(),
            images: get_images(&document).unwrap(),
        };
        let urls = saved_ad
            .image_urls()
            .into_iter()
            .map(|x| x.unwrap())
            .collect();
        let images = Images::from_ad(&saved_ad, urls).unwrap();
        assert!(images.html != saved_ad.html);
        assert!(!images.html.contains("fbcdn"));
        assert!(images.images.len() == saved_ad.images.len());
        println!("{:?}", images);
    }
}
<|endoftext|>


<commit_before>//! Package up unwind recovery. Note that if you are in some sensitive
//! place, you can use the `AbortIfPanic` helper to protect against
//! accidental panics in the rayon code itself.

use libc;
use std::any::Any;
use std::panic::{self, AssertUnwindSafe};
use std::io::stderr;
use std::io::prelude::*;
use std::thread;

/// Executes `f` and captures any panic, translating that panic into a
/// `Err` result. The assumption is that any panic will be propagated
/// later with `resume_unwinding`, and hence `f` can be treated as
/// exception safe.
pub fn halt_unwinding<F, R>(func: F) -> thread::Result<R>
    where F: FnOnce() -> R
{
    panic::catch_unwind(AssertUnwindSafe(func))
}

pub fn resume_unwinding(payload: Box<Any + Send>) -> ! {
    panic::resume_unwind(payload)
}

pub struct AbortIfPanic;

impl Drop for AbortIfPanic {
    fn drop(&mut self) {
        unsafe {
            let _ = writeln!(&mut stderr(), "Rayon: detected unexpected panic; aborting");
            libc::abort();
        }
    }
}
<commit_msg>Use process::abort() instead of libc::abort() for wasm32<commit_after>//! Package up unwind recovery. Note that if you are in some sensitive
//! place, you can use the `AbortIfPanic` helper to protect against
//! accidental panics in the rayon code itself.

#[cfg(any(not(target_arch = "wasm32"), target_os = "emscripten"))]
use libc;
#[cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))]
use std::process;
use std::any::Any;
use std::panic::{self, AssertUnwindSafe};
use std::io::stderr;
use std::io::prelude::*;
use std::thread;

/// Executes `f` and captures any panic, translating that panic into a
/// `Err` result. The assumption is that any panic will be propagated
/// later with `resume_unwinding`, and hence `f` can be treated as
/// exception safe.
pub fn halt_unwinding<F, R>(func: F) -> thread::Result<R>
    where F: FnOnce() -> R
{
    panic::catch_unwind(AssertUnwindSafe(func))
}

pub fn resume_unwinding(payload: Box<Any + Send>) -> ! {
    panic::resume_unwind(payload)
}

pub struct AbortIfPanic;

impl Drop for AbortIfPanic {
    fn drop(&mut self) {
        let _ = writeln!(&mut stderr(), "Rayon: detected unexpected panic; aborting");
        #[cfg(any(not(target_arch = "wasm32"), target_os = "emscripten"))]
        unsafe {
            libc::abort();
        }
        #[cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))]
        process::abort();
    }
}
<|endoftext|>


<commit_before>use types::*;

use nom::{be_i8, be_u8, be_i16, be_u16, be_i32, be_u32, be_i64, be_u64, float, double};

named!(pub parse_value<AMQPValue>,                 switch!(parse_type,
    AMQPType::Boolean        => map!(call!(parse_boolean),          |b| AMQPValue::Boolean(b))        |
    AMQPType::ShortShortInt  => map!(call!(parse_short_short_int),  |i| AMQPValue::ShortShortInt(i))  |
    AMQPType::ShortShortUInt => map!(call!(parse_short_short_uint), |u| AMQPValue::ShortShortUInt(u)) |
    AMQPType::ShortInt       => map!(call!(parse_short_int),        |i| AMQPValue::ShortInt(i))       |
    AMQPType::ShortUInt      => map!(call!(parse_short_uint),       |u| AMQPValue::ShortUInt(u))      |
    AMQPType::LongInt        => map!(call!(parse_long_int),         |i| AMQPValue::LongInt(i))        |
    AMQPType::LongUInt       => map!(call!(parse_long_uint),        |u| AMQPValue::LongUInt(u))       |
    AMQPType::LongLongInt    => map!(call!(parse_long_long_int),    |i| AMQPValue::LongLongInt(i))    |
    AMQPType::LongLongUInt   => map!(call!(parse_long_long_uint),   |u| AMQPValue::LongLongUInt(u))   |
    AMQPType::Float          => map!(call!(parse_float),            |f| AMQPValue::Float(f))          |
    AMQPType::Double         => map!(call!(parse_double),           |d| AMQPValue::Double(d))         |
    AMQPType::DecimalValue   => map!(call!(parse_decimal_value),    |d| AMQPValue::DecimalValue(d))   |
    AMQPType::ShortString    => map!(call!(parse_short_string),     |s| AMQPValue::ShortString(s))    |
    AMQPType::LongString     => map!(call!(parse_long_string),      |s| AMQPValue::LongString(s))     |
//    AMQPType::FieldArray     => map!(call!(parse_field_array),      |a| AMQPValue::FieldArray(a))     |
    AMQPType::Timestamp      => map!(call!(parse_timestamp),        |t| AMQPValue::Timestamp(t))      |
//    AMQPType::FieldTable     => map!(call!(parse_field_table),      |t| AMQPValue::FieldTable(t))     |
    AMQPType::Void           => value!(AMQPValue::Void)
));

named!(pub parse_type<AMQPType>,                   map_opt!(be_u8, |t| AMQPType::from_id(t as char)));

named!(pub parse_boolean<Boolean>,                 map!(be_u8, |b| b != 0));
named!(pub parse_short_short_int<ShortShortInt>,   call!(be_i8));
named!(pub parse_short_short_uint<ShortShortUInt>, call!(be_u8));
named!(pub parse_short_int<ShortInt>,              call!(be_i16));
named!(pub parse_short_uint<ShortUInt>,            call!(be_u16));
named!(pub parse_long_int<LongInt>,                call!(be_i32));
named!(pub parse_long_uint<LongUInt>,              call!(be_u32));
named!(pub parse_long_long_int<LongLongInt>,       call!(be_i64));
named!(pub parse_long_long_uint<LongLongUInt>,     call!(be_u64));
named!(pub parse_float<Float>,                     call!(float));
named!(pub parse_double<Double>,                   call!(double));
named!(pub parse_decimal_value<DecimalValue>,      do_parse!(scale: parse_short_short_uint >> value: parse_long_uint >> (DecimalValue { scale: scale, value: value, })));
named!(pub parse_short_string<ShortString>,        do_parse!(length: parse_short_short_uint >> s: take_str!(length) >> (s.to_string())));
named!(pub parse_long_string<LongString>,          do_parse!(length: parse_long_uint >> s: take_str!(length) >> (s.to_string())));
named!(pub parse_timestamp<Timestamp>,             call!(parse_long_long_uint));
<commit_msg>types: parse field array<commit_after>use types::*;

use nom::{be_i8, be_u8, be_i16, be_u16, be_i32, be_u32, be_i64, be_u64, float, double};

named!(pub parse_value<AMQPValue>,                 switch!(parse_type,
    AMQPType::Boolean        => map!(call!(parse_boolean),          |b| AMQPValue::Boolean(b))        |
    AMQPType::ShortShortInt  => map!(call!(parse_short_short_int),  |i| AMQPValue::ShortShortInt(i))  |
    AMQPType::ShortShortUInt => map!(call!(parse_short_short_uint), |u| AMQPValue::ShortShortUInt(u)) |
    AMQPType::ShortInt       => map!(call!(parse_short_int),        |i| AMQPValue::ShortInt(i))       |
    AMQPType::ShortUInt      => map!(call!(parse_short_uint),       |u| AMQPValue::ShortUInt(u))      |
    AMQPType::LongInt        => map!(call!(parse_long_int),         |i| AMQPValue::LongInt(i))        |
    AMQPType::LongUInt       => map!(call!(parse_long_uint),        |u| AMQPValue::LongUInt(u))       |
    AMQPType::LongLongInt    => map!(call!(parse_long_long_int),    |i| AMQPValue::LongLongInt(i))    |
    AMQPType::LongLongUInt   => map!(call!(parse_long_long_uint),   |u| AMQPValue::LongLongUInt(u))   |
    AMQPType::Float          => map!(call!(parse_float),            |f| AMQPValue::Float(f))          |
    AMQPType::Double         => map!(call!(parse_double),           |d| AMQPValue::Double(d))         |
    AMQPType::DecimalValue   => map!(call!(parse_decimal_value),    |d| AMQPValue::DecimalValue(d))   |
    AMQPType::ShortString    => map!(call!(parse_short_string),     |s| AMQPValue::ShortString(s))    |
    AMQPType::LongString     => map!(call!(parse_long_string),      |s| AMQPValue::LongString(s))     |
    AMQPType::FieldArray     => map!(call!(parse_field_array),      |a| AMQPValue::FieldArray(a))     |
    AMQPType::Timestamp      => map!(call!(parse_timestamp),        |t| AMQPValue::Timestamp(t))      |
//    AMQPType::FieldTable     => map!(call!(parse_field_table),      |t| AMQPValue::FieldTable(t))     |
    AMQPType::Void           => value!(AMQPValue::Void)
));

named!(pub parse_type<AMQPType>,                   map_opt!(be_u8, |t| AMQPType::from_id(t as char)));

named!(pub parse_boolean<Boolean>,                 map!(be_u8, |b| b != 0));
named!(pub parse_short_short_int<ShortShortInt>,   call!(be_i8));
named!(pub parse_short_short_uint<ShortShortUInt>, call!(be_u8));
named!(pub parse_short_int<ShortInt>,              call!(be_i16));
named!(pub parse_short_uint<ShortUInt>,            call!(be_u16));
named!(pub parse_long_int<LongInt>,                call!(be_i32));
named!(pub parse_long_uint<LongUInt>,              call!(be_u32));
named!(pub parse_long_long_int<LongLongInt>,       call!(be_i64));
named!(pub parse_long_long_uint<LongLongUInt>,     call!(be_u64));
named!(pub parse_float<Float>,                     call!(float));
named!(pub parse_double<Double>,                   call!(double));
named!(pub parse_decimal_value<DecimalValue>,      do_parse!(scale: parse_short_short_uint >> value: parse_long_uint >> (DecimalValue { scale: scale, value: value, })));
named!(pub parse_short_string<ShortString>,        do_parse!(length: parse_short_short_uint >> s: take_str!(length) >> (s.to_string())));
named!(pub parse_long_string<LongString>,          do_parse!(length: parse_long_uint >> s: take_str!(length) >> (s.to_string())));
named!(pub parse_field_array<FieldArray>,          do_parse!(length: parse_long_int >> array: count!(parse_value, length as usize) >> (array)));
named!(pub parse_timestamp<Timestamp>,             call!(parse_long_long_uint));
<|endoftext|>