<commit_before>// Copyright (C) 2018 Sebastian Dröge <sebastian@centricular.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the
// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
// Boston, MA 02110-1335, USA.

use std::collections::HashMap;
use std::io;
use std::mem;
use std::sync::atomic;
use std::sync::{Arc, Mutex, Weak};
use std::thread;

use futures::future;
use futures::stream::futures_unordered::FuturesUnordered;
use futures::sync::oneshot;
use futures::{Future, Stream};
use tokio::executor::thread_pool;
use tokio::reactor;
use tokio_timer::timer;

use gst;

use either::Either;

lazy_static! {
    static ref CONTEXTS: Mutex<HashMap<String, Weak<IOContextInner>>> = Mutex::new(HashMap::new());
    static ref CONTEXT_CAT: gst::DebugCategory = gst::DebugCategory::new(
        "ts-context",
        gst::DebugColorFlags::empty(),
        "Thread-sharing Context",
    );
}

// Our own simplified implementation of reactor::Background to allow hooking into its internals
const RUNNING: usize = 0;
const SHUTDOWN_NOW: usize = 1;

struct IOContextRunner {
    name: String,
    shutdown: Arc<atomic::AtomicUsize>,
    pending_futures: Option<Arc<Mutex<Vec<Box<Future<Item = (), Error = ()> + Send + 'static>>>>>,
}

impl IOContextRunner {
    fn start_single_threaded(
        name: &str,
        wait: u32,
        reactor: reactor::Reactor,
    ) -> (IOContextExecutor, IOContextShutdown) {
        let handle = reactor.handle().clone();
        let handle2 = reactor.handle().clone();
        let shutdown = Arc::new(atomic::AtomicUsize::new(RUNNING));
        let shutdown_clone = shutdown.clone();
        let name_clone = name.into();

        let pending_futures = Arc::new(Mutex::new(Vec::new()));
        let pending_futures_clone = pending_futures.clone();

        let mut runner = IOContextRunner {
            shutdown: shutdown_clone,
            name: name_clone,
            pending_futures: Some(pending_futures),
        };
        let join = thread::spawn(move || {
            runner.run(wait, reactor);
        });

        let executor = IOContextExecutor {
            handle: handle,
            pending_futures: pending_futures_clone,
        };

        let shutdown = IOContextShutdown {
            name: name.into(),
            shutdown: shutdown,
            handle: handle2,
            join: Some(join),
        };

        (executor, shutdown)
    }

    fn start(name: &str, wait: u32, reactor: reactor::Reactor) -> IOContextShutdown {
        let handle = reactor.handle().clone();
        let shutdown = Arc::new(atomic::AtomicUsize::new(RUNNING));
        let shutdown_clone = shutdown.clone();
        let name_clone = name.into();

        let mut runner = IOContextRunner {
            shutdown: shutdown_clone,
            name: name_clone,
            pending_futures: None,
        };
        let join = thread::spawn(move || {
            runner.run(wait, reactor);
        });

        let shutdown = IOContextShutdown {
            name: name.into(),
            shutdown: shutdown,
            handle: handle,
            join: Some(join),
        };

        shutdown
    }

    fn run(&mut self, wait: u32, reactor: reactor::Reactor) {
        use std::time;
        let wait = time::Duration::from_millis(wait as u64);

        gst_debug!(CONTEXT_CAT, "Started reactor thread '{}'", self.name);

        if let Some(ref pending_futures) = self.pending_futures {
            use tokio::executor::current_thread;

            let handle = reactor.handle();
            let mut enter = ::tokio_executor::enter().unwrap();
            let timer = timer::Timer::new(reactor);
            let timer_handle = timer.handle();
            let mut current_thread = current_thread::CurrentThread::new_with_park(timer);

            ::tokio_reactor::with_default(&handle, &mut enter, |mut enter| {
                ::tokio_timer::with_default(&timer_handle, &mut enter, |enter| loop {
                    use tokio::executor::current_thread::Turn;

                    let now = time::Instant::now();

                    if self.shutdown.load(atomic::Ordering::SeqCst) > RUNNING {
                        break;
                    }

                    {
                        let mut pending_futures = pending_futures.lock().unwrap();
                        while let Some(future) = pending_futures.pop() {
                            current_thread.spawn(future);
                        }
                    }

                    gst_trace!(CONTEXT_CAT, "Turning current thread '{}'", self.name);
                    while let Turn(true) = current_thread
                        .enter(enter)
                        .turn(Some(time::Duration::from_millis(0)))
                        .unwrap()
                    {}
                    gst_trace!(CONTEXT_CAT, "Turned current thread '{}'", self.name);

                    let elapsed = now.elapsed();
                    if elapsed < wait {
                        gst_trace!(
                            CONTEXT_CAT,
                            "Waiting for {:?} before polling again",
                            wait - elapsed
                        );
                        thread::sleep(wait - elapsed);
                    }
                })
            });
        } else {
            let mut reactor = reactor;

            loop {
                let now = time::Instant::now();

                if self.shutdown.load(atomic::Ordering::SeqCst) > RUNNING {
                    break;
                }

                gst_trace!(CONTEXT_CAT, "Turning reactor '{}'", self.name);
                reactor.turn(None).unwrap();
                gst_trace!(CONTEXT_CAT, "Turned reactor '{}'", self.name);

                let elapsed = now.elapsed();
                if elapsed < wait {
                    gst_trace!(
                        CONTEXT_CAT,
                        "Waiting for {:?} before polling again",
                        wait - elapsed
                    );
                    thread::sleep(wait - elapsed);
                }
            }
        }
    }
}

impl Drop for IOContextRunner {
    fn drop(&mut self) {
        gst_debug!(CONTEXT_CAT, "Shut down reactor thread '{}'", self.name);
    }
}

struct IOContextShutdown {
    name: String,
    shutdown: Arc<atomic::AtomicUsize>,
    handle: reactor::Handle,
    join: Option<thread::JoinHandle<()>>,
}

impl Drop for IOContextShutdown {
    fn drop(&mut self) {
        use tokio_executor::park::Unpark;

        gst_debug!(CONTEXT_CAT, "Shutting down reactor thread '{}'", self.name);
        self.shutdown.store(SHUTDOWN_NOW, atomic::Ordering::SeqCst);
        gst_trace!(CONTEXT_CAT, "Waiting for reactor '{}' shutdown", self.name);
        // After being unparked, the next turn() is guaranteed to finish immediately,
        // as such there is no race condition between checking for shutdown and setting
        // shutdown.
        self.handle.unpark();
        let _ = self.join.take().unwrap().join();
    }
}

struct IOContextExecutor {
    handle: reactor::Handle,
    pending_futures: Arc<Mutex<Vec<Box<Future<Item = (), Error = ()> + Send + 'static>>>>,
}

impl IOContextExecutor {
    fn spawn<F>(&self, future: F)
    where
        F: Future<Item = (), Error = ()> + Send + 'static,
    {
        use tokio_executor::park::Unpark;

        self.pending_futures.lock().unwrap().push(Box::new(future));
        self.handle.unpark();
    }
}

#[derive(Clone)]
pub struct IOContext(Arc<IOContextInner>);

struct IOContextInner {
    name: String,
    pool: Either<thread_pool::ThreadPool, IOContextExecutor>,
    // Only used for dropping
    _shutdown: IOContextShutdown,
    pending_futures: Mutex<(
        u64,
        HashMap<u64, FuturesUnordered<Box<Future<Item = (), Error = ()> + Send + 'static>>>,
    )>,
}

impl Drop for IOContextInner {
    fn drop(&mut self) {
        let mut contexts = CONTEXTS.lock().unwrap();
        gst_debug!(CONTEXT_CAT, "Finalizing context '{}'", self.name);
        contexts.remove(&self.name);
    }
}

impl IOContext {
    pub fn new(name: &str, n_threads: isize, wait: u32) -> Result<Self, io::Error> {
        let mut contexts = CONTEXTS.lock().unwrap();
        if let Some(context) = contexts.get(name) {
            if let Some(context) = context.upgrade() {
                gst_debug!(CONTEXT_CAT, "Reusing existing context '{}'", name);
                return Ok(IOContext(context));
            }
        }

        let reactor = reactor::Reactor::new()?;

        let (pool, shutdown) = if n_threads >= 0 {
            let handle = reactor.handle().clone();

            let timers = Arc::new(Mutex::new(HashMap::<_, timer::Handle>::new()));
            let t1 = timers.clone();

            let shutdown = IOContextRunner::start(name, wait, reactor);

            // FIXME: The executor threads are not throttled at all, only the reactor
            let mut pool_builder = thread_pool::Builder::new();
            pool_builder
                .around_worker(move |w, enter| {
                    let timer_handle = t1.lock().unwrap().get(w.id()).unwrap().clone();

                    ::tokio_reactor::with_default(&handle, enter, |enter| {
                        timer::with_default(&timer_handle, enter, |_| {
                            w.run();
                        });
                    });
                })
                .custom_park(move |worker_id| {
                    // Create a new timer
                    let timer = timer::Timer::new(::tokio_threadpool::park::DefaultPark::new());

                    timers
                        .lock()
                        .unwrap()
                        .insert(worker_id.clone(), timer.handle());

                    timer
                });

            if n_threads > 0 {
                pool_builder.pool_size(n_threads as usize);
            }
            (Either::Left(pool_builder.build()), shutdown)
        } else {
            let (executor, shutdown) = IOContextRunner::start_single_threaded(name, wait, reactor);

            (Either::Right(executor), shutdown)
        };

        let context = Arc::new(IOContextInner {
            name: name.into(),
            pool,
            _shutdown: shutdown,
            pending_futures: Mutex::new((0, HashMap::new())),
        });
        contexts.insert(name.into(), Arc::downgrade(&context));

        gst_debug!(CONTEXT_CAT, "Created new context '{}'", name);
        Ok(IOContext(context))
    }

    pub fn spawn<F>(&self, future: F)
    where
        F: Future<Item = (), Error = ()> + Send + 'static,
    {
        match self.0.pool {
            Either::Left(ref pool) => pool.spawn(future),
            Either::Right(ref pool) => pool.spawn(future),
        }
    }

    pub fn acquire_pending_future_id(&self) -> PendingFutureId {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        let id = pending_futures.0;
        pending_futures.0 += 1;
        pending_futures.1.insert(id, FuturesUnordered::new());

        PendingFutureId(id)
    }

    pub fn release_pending_future_id(&self, id: PendingFutureId) {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        if let Some(fs) = pending_futures.1.remove(&id.0) {
            self.spawn(fs.for_each(|_| Ok(())));
        }
    }

    pub fn add_pending_future<F>(&self, id: PendingFutureId, future: F)
    where
        F: Future<Item = (), Error = ()> + Send + 'static,
    {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        let fs = pending_futures.1.get_mut(&id.0).unwrap();
        fs.push(Box::new(future))
    }

    pub fn drain_pending_futures<E: Send + 'static>(
        &self,
        id: PendingFutureId,
    ) -> (Option<oneshot::Sender<()>>, PendingFuturesFuture<E>) {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        let fs = pending_futures.1.get_mut(&id.0).unwrap();

        let pending_futures = mem::replace(fs, FuturesUnordered::new());

        if !pending_futures.is_empty() {
            gst_log!(
                CONTEXT_CAT,
                "Scheduling {} pending futures for context '{}' with pending future id {:?}",
                pending_futures.len(),
                self.0.name,
                id,
            );

            let (sender, receiver) = oneshot::channel();

            let future = pending_futures
                .for_each(|_| Ok(()))
                .select(receiver.then(|_| Ok(())))
                .then(|_| Ok(()));

            (Some(sender), future::Either::A(Box::new(future)))
        } else {
            (None, future::Either::B(future::ok(())))
        }
    }
}

pub type PendingFuturesFuture<E> =
    future::Either<Box<Future<Item = (), Error = E> + Send + 'static>, future::FutureResult<(), E>>;

#[derive(Clone, Copy, Eq, PartialEq, Hash, Debug)]
pub struct PendingFutureId(u64);
<commit_msg>threadshare: Update for tokio API changes<commit_after>// Copyright (C) 2018 Sebastian Dröge <sebastian@centricular.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the
// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
// Boston, MA 02110-1335, USA.

use std::collections::HashMap;
use std::io;
use std::mem;
use std::sync::atomic;
use std::sync::{Arc, Mutex, Weak};
use std::thread;

use futures::future;
use futures::stream::futures_unordered::FuturesUnordered;
use futures::sync::oneshot;
use futures::{Future, Stream};
use tokio::executor::thread_pool;
use tokio::reactor;
use tokio_timer::timer;

use gst;

use either::Either;

lazy_static! {
    static ref CONTEXTS: Mutex<HashMap<String, Weak<IOContextInner>>> = Mutex::new(HashMap::new());
    static ref CONTEXT_CAT: gst::DebugCategory = gst::DebugCategory::new(
        "ts-context",
        gst::DebugColorFlags::empty(),
        "Thread-sharing Context",
    );
}

// Our own simplified implementation of reactor::Background to allow hooking into its internals
const RUNNING: usize = 0;
const SHUTDOWN_NOW: usize = 1;

struct IOContextRunner {
    name: String,
    shutdown: Arc<atomic::AtomicUsize>,
    pending_futures: Option<Arc<Mutex<Vec<Box<Future<Item = (), Error = ()> + Send + 'static>>>>>,
}

impl IOContextRunner {
    fn start_single_threaded(
        name: &str,
        wait: u32,
        reactor: reactor::Reactor,
    ) -> (IOContextExecutor, IOContextShutdown) {
        let handle = reactor.handle().clone();
        let handle2 = reactor.handle().clone();
        let shutdown = Arc::new(atomic::AtomicUsize::new(RUNNING));
        let shutdown_clone = shutdown.clone();
        let name_clone = name.into();

        let pending_futures = Arc::new(Mutex::new(Vec::new()));
        let pending_futures_clone = pending_futures.clone();

        let mut runner = IOContextRunner {
            shutdown: shutdown_clone,
            name: name_clone,
            pending_futures: Some(pending_futures),
        };
        let join = thread::spawn(move || {
            runner.run(wait, reactor);
        });

        let executor = IOContextExecutor {
            handle: handle,
            pending_futures: pending_futures_clone,
        };

        let shutdown = IOContextShutdown {
            name: name.into(),
            shutdown: shutdown,
            handle: handle2,
            join: Some(join),
        };

        (executor, shutdown)
    }

    fn start(name: &str, wait: u32, reactor: reactor::Reactor) -> IOContextShutdown {
        let handle = reactor.handle().clone();
        let shutdown = Arc::new(atomic::AtomicUsize::new(RUNNING));
        let shutdown_clone = shutdown.clone();
        let name_clone = name.into();

        let mut runner = IOContextRunner {
            shutdown: shutdown_clone,
            name: name_clone,
            pending_futures: None,
        };
        let join = thread::spawn(move || {
            runner.run(wait, reactor);
        });

        let shutdown = IOContextShutdown {
            name: name.into(),
            shutdown: shutdown,
            handle: handle,
            join: Some(join),
        };

        shutdown
    }

    fn run(&mut self, wait: u32, reactor: reactor::Reactor) {
        use std::time;
        let wait = time::Duration::from_millis(wait as u64);

        gst_debug!(CONTEXT_CAT, "Started reactor thread '{}'", self.name);

        if let Some(ref pending_futures) = self.pending_futures {
            use tokio::executor::current_thread;

            let handle = reactor.handle();
            let mut enter = ::tokio_executor::enter().unwrap();
            let timer = timer::Timer::new(reactor);
            let timer_handle = timer.handle();
            let mut current_thread = current_thread::CurrentThread::new_with_park(timer);

            ::tokio_reactor::with_default(&handle, &mut enter, |mut enter| {
                ::tokio_timer::with_default(&timer_handle, &mut enter, |enter| loop {
                    let now = time::Instant::now();

                    if self.shutdown.load(atomic::Ordering::SeqCst) > RUNNING {
                        break;
                    }

                    {
                        let mut pending_futures = pending_futures.lock().unwrap();
                        while let Some(future) = pending_futures.pop() {
                            current_thread.spawn(future);
                        }
                    }

                    gst_trace!(CONTEXT_CAT, "Turning current thread '{}'", self.name);
                    while current_thread
                        .enter(enter)
                        .turn(Some(time::Duration::from_millis(0)))
                        .unwrap()
                        .has_polled()
                    {}
                    gst_trace!(CONTEXT_CAT, "Turned current thread '{}'", self.name);

                    let elapsed = now.elapsed();
                    if elapsed < wait {
                        gst_trace!(
                            CONTEXT_CAT,
                            "Waiting for {:?} before polling again",
                            wait - elapsed
                        );
                        thread::sleep(wait - elapsed);
                    }
                })
            });
        } else {
            let mut reactor = reactor;

            loop {
                let now = time::Instant::now();

                if self.shutdown.load(atomic::Ordering::SeqCst) > RUNNING {
                    break;
                }

                gst_trace!(CONTEXT_CAT, "Turning reactor '{}'", self.name);
                reactor.turn(None).unwrap();
                gst_trace!(CONTEXT_CAT, "Turned reactor '{}'", self.name);

                let elapsed = now.elapsed();
                if elapsed < wait {
                    gst_trace!(
                        CONTEXT_CAT,
                        "Waiting for {:?} before polling again",
                        wait - elapsed
                    );
                    thread::sleep(wait - elapsed);
                }
            }
        }
    }
}

impl Drop for IOContextRunner {
    fn drop(&mut self) {
        gst_debug!(CONTEXT_CAT, "Shut down reactor thread '{}'", self.name);
    }
}

struct IOContextShutdown {
    name: String,
    shutdown: Arc<atomic::AtomicUsize>,
    handle: reactor::Handle,
    join: Option<thread::JoinHandle<()>>,
}

impl Drop for IOContextShutdown {
    fn drop(&mut self) {
        use tokio_executor::park::Unpark;

        gst_debug!(CONTEXT_CAT, "Shutting down reactor thread '{}'", self.name);
        self.shutdown.store(SHUTDOWN_NOW, atomic::Ordering::SeqCst);
        gst_trace!(CONTEXT_CAT, "Waiting for reactor '{}' shutdown", self.name);
        // After being unparked, the next turn() is guaranteed to finish immediately,
        // as such there is no race condition between checking for shutdown and setting
        // shutdown.
        self.handle.unpark();
        let _ = self.join.take().unwrap().join();
    }
}

struct IOContextExecutor {
    handle: reactor::Handle,
    pending_futures: Arc<Mutex<Vec<Box<Future<Item = (), Error = ()> + Send + 'static>>>>,
}

impl IOContextExecutor {
    fn spawn<F>(&self, future: F)
    where
        F: Future<Item = (), Error = ()> + Send + 'static,
    {
        use tokio_executor::park::Unpark;

        self.pending_futures.lock().unwrap().push(Box::new(future));
        self.handle.unpark();
    }
}

#[derive(Clone)]
pub struct IOContext(Arc<IOContextInner>);

struct IOContextInner {
    name: String,
    pool: Either<thread_pool::ThreadPool, IOContextExecutor>,
    // Only used for dropping
    _shutdown: IOContextShutdown,
    pending_futures: Mutex<(
        u64,
        HashMap<u64, FuturesUnordered<Box<Future<Item = (), Error = ()> + Send + 'static>>>,
    )>,
}

impl Drop for IOContextInner {
    fn drop(&mut self) {
        let mut contexts = CONTEXTS.lock().unwrap();
        gst_debug!(CONTEXT_CAT, "Finalizing context '{}'", self.name);
        contexts.remove(&self.name);
    }
}

impl IOContext {
    pub fn new(name: &str, n_threads: isize, wait: u32) -> Result<Self, io::Error> {
        let mut contexts = CONTEXTS.lock().unwrap();
        if let Some(context) = contexts.get(name) {
            if let Some(context) = context.upgrade() {
                gst_debug!(CONTEXT_CAT, "Reusing existing context '{}'", name);
                return Ok(IOContext(context));
            }
        }

        let reactor = reactor::Reactor::new()?;

        let (pool, shutdown) = if n_threads >= 0 {
            let handle = reactor.handle().clone();

            let timers = Arc::new(Mutex::new(HashMap::<_, timer::Handle>::new()));
            let t1 = timers.clone();

            let shutdown = IOContextRunner::start(name, wait, reactor);

            // FIXME: The executor threads are not throttled at all, only the reactor
            let mut pool_builder = thread_pool::Builder::new();
            pool_builder
                .around_worker(move |w, enter| {
                    let timer_handle = t1.lock().unwrap().get(w.id()).unwrap().clone();

                    ::tokio_reactor::with_default(&handle, enter, |enter| {
                        timer::with_default(&timer_handle, enter, |_| {
                            w.run();
                        });
                    });
                })
                .custom_park(move |worker_id| {
                    // Create a new timer
                    let timer = timer::Timer::new(::tokio_threadpool::park::DefaultPark::new());

                    timers
                        .lock()
                        .unwrap()
                        .insert(worker_id.clone(), timer.handle());

                    timer
                });

            if n_threads > 0 {
                pool_builder.pool_size(n_threads as usize);
            }
            (Either::Left(pool_builder.build()), shutdown)
        } else {
            let (executor, shutdown) = IOContextRunner::start_single_threaded(name, wait, reactor);

            (Either::Right(executor), shutdown)
        };

        let context = Arc::new(IOContextInner {
            name: name.into(),
            pool,
            _shutdown: shutdown,
            pending_futures: Mutex::new((0, HashMap::new())),
        });
        contexts.insert(name.into(), Arc::downgrade(&context));

        gst_debug!(CONTEXT_CAT, "Created new context '{}'", name);
        Ok(IOContext(context))
    }

    pub fn spawn<F>(&self, future: F)
    where
        F: Future<Item = (), Error = ()> + Send + 'static,
    {
        match self.0.pool {
            Either::Left(ref pool) => pool.spawn(future),
            Either::Right(ref pool) => pool.spawn(future),
        }
    }

    pub fn acquire_pending_future_id(&self) -> PendingFutureId {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        let id = pending_futures.0;
        pending_futures.0 += 1;
        pending_futures.1.insert(id, FuturesUnordered::new());

        PendingFutureId(id)
    }

    pub fn release_pending_future_id(&self, id: PendingFutureId) {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        if let Some(fs) = pending_futures.1.remove(&id.0) {
            self.spawn(fs.for_each(|_| Ok(())));
        }
    }

    pub fn add_pending_future<F>(&self, id: PendingFutureId, future: F)
    where
        F: Future<Item = (), Error = ()> + Send + 'static,
    {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        let fs = pending_futures.1.get_mut(&id.0).unwrap();
        fs.push(Box::new(future))
    }

    pub fn drain_pending_futures<E: Send + 'static>(
        &self,
        id: PendingFutureId,
    ) -> (Option<oneshot::Sender<()>>, PendingFuturesFuture<E>) {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        let fs = pending_futures.1.get_mut(&id.0).unwrap();

        let pending_futures = mem::replace(fs, FuturesUnordered::new());

        if !pending_futures.is_empty() {
            gst_log!(
                CONTEXT_CAT,
                "Scheduling {} pending futures for context '{}' with pending future id {:?}",
                pending_futures.len(),
                self.0.name,
                id,
            );

            let (sender, receiver) = oneshot::channel();

            let future = pending_futures
                .for_each(|_| Ok(()))
                .select(receiver.then(|_| Ok(())))
                .then(|_| Ok(()));

            (Some(sender), future::Either::A(Box::new(future)))
        } else {
            (None, future::Either::B(future::ok(())))
        }
    }
}

pub type PendingFuturesFuture<E> =
    future::Either<Box<Future<Item = (), Error = E> + Send + 'static>, future::FutureResult<(), E>>;

#[derive(Clone, Copy, Eq, PartialEq, Hash, Debug)]
pub struct PendingFutureId(u64);
<|endoftext|>


<commit_before>// Copyright (C) 2018 Sebastian Dröge <sebastian@centricular.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the
// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
// Boston, MA 02110-1335, USA.

use std::cmp;
use std::collections::{BinaryHeap, HashMap};
use std::io;
use std::mem;
use std::sync::{atomic, mpsc};
use std::sync::{Arc, Mutex, Weak};
use std::thread;
use std::time;

use futures::future;
use futures::stream::futures_unordered::FuturesUnordered;
use futures::sync::mpsc as futures_mpsc;
use futures::sync::oneshot;
use futures::{Async, Future, Stream};
use tokio::reactor;
use tokio_current_thread;
use tokio_timer::timer;

use glib;
use gst;

lazy_static! {
    static ref CONTEXTS: Mutex<HashMap<String, Weak<IOContextInner>>> = Mutex::new(HashMap::new());
    static ref CONTEXT_CAT: gst::DebugCategory = gst::DebugCategory::new(
        "ts-context",
        gst::DebugColorFlags::empty(),
        Some("Thread-sharing Context"),
    );
}

// Our own simplified implementation of reactor::Background to allow hooking into its internals
const RUNNING: usize = 0;
const SHUTDOWN_NOW: usize = 1;

struct IOContextRunner {
    name: String,
    shutdown: Arc<atomic::AtomicUsize>,
}

impl IOContextRunner {
    fn start(
        name: &str,
        wait: u32,
        reactor: reactor::Reactor,
        timers: Arc<Mutex<BinaryHeap<TimerEntry>>>,
    ) -> (tokio_current_thread::Handle, IOContextShutdown) {
        let handle = reactor.handle().clone();
        let shutdown = Arc::new(atomic::AtomicUsize::new(RUNNING));
        let shutdown_clone = shutdown.clone();
        let name_clone = name.into();

        let mut runner = IOContextRunner {
            shutdown: shutdown_clone,
            name: name_clone,
        };

        let (sender, receiver) = mpsc::channel();

        let join = thread::spawn(move || {
            runner.run(wait, reactor, sender, timers);
        });

        let shutdown = IOContextShutdown {
            name: name.into(),
            shutdown,
            handle,
            join: Some(join),
        };

        let runtime_handle = receiver.recv().unwrap();

        (runtime_handle, shutdown)
    }

    fn run(
        &mut self,
        wait: u32,
        reactor: reactor::Reactor,
        sender: mpsc::Sender<tokio_current_thread::Handle>,
        timers: Arc<Mutex<BinaryHeap<TimerEntry>>>,
    ) {
        let wait = time::Duration::from_millis(wait as u64);

        gst_debug!(CONTEXT_CAT, "Started reactor thread '{}'", self.name);

        let handle = reactor.handle();
        let mut enter = ::tokio_executor::enter().unwrap();
        let timer = timer::Timer::new(reactor);
        let timer_handle = timer.handle();
        let mut current_thread = tokio_current_thread::CurrentThread::new_with_park(timer);

        let _ = sender.send(current_thread.handle());

        let mut now = time::Instant::now();

        ::tokio_timer::with_default(&timer_handle, &mut enter, |mut enter| {
            ::tokio_reactor::with_default(&handle, &mut enter, |enter| loop {
                if self.shutdown.load(atomic::Ordering::SeqCst) > RUNNING {
                    break;
                }

                gst_trace!(CONTEXT_CAT, "Elapsed {:?} since last loop", now.elapsed());

                // Handle timers
                {
                    // Trigger all timers that would be expired before the middle of the loop wait
                    // time
                    let timer_threshold = now + wait / 2;
                    let mut timers = timers.lock().unwrap();
                    while timers
                        .peek()
                        .and_then(|entry| {
                            if entry.time < timer_threshold {
                                Some(())
                            } else {
                                None
                            }
                        })
                        .is_some()
                    {
                        let TimerEntry {
                            time,
                            interval,
                            sender,
                            ..
                        } = timers.pop().unwrap();

                        if sender.is_closed() {
                            continue;
                        }

                        let _ = sender.unbounded_send(());
                        if let Some(interval) = interval {
                            timers.push(TimerEntry {
                                time: time + interval,
                                id: TIMER_ENTRY_ID.fetch_add(1, atomic::Ordering::Relaxed),
                                interval: Some(interval),
                                sender,
                            });
                        }
                    }
                }

                gst_trace!(CONTEXT_CAT, "Turning current thread '{}'", self.name);
                while current_thread
                    .enter(enter)
                    .turn(Some(time::Duration::from_millis(0)))
                    .unwrap()
                    .has_polled()
                {}
                gst_trace!(CONTEXT_CAT, "Turned current thread '{}'", self.name);

                let elapsed = now.elapsed();
                gst_trace!(CONTEXT_CAT, "Elapsed {:?} after handling futures", elapsed);

                if wait == time::Duration::from_millis(0) {
                    let timers = timers.lock().unwrap();
                    let wait = match timers.peek().map(|entry| entry.time) {
                        None => None,
                        Some(time) => Some({
                            let tmp = time::Instant::now();

                            if time < tmp {
                                time::Duration::from_millis(0)
                            } else {
                                time.duration_since(tmp)
                            }
                        }),
                    };
                    drop(timers);

                    gst_trace!(CONTEXT_CAT, "Sleeping for up to {:?}", wait);
                    current_thread.enter(enter).turn(wait).unwrap();
                    gst_trace!(CONTEXT_CAT, "Slept for {:?}", now.elapsed());
                    now = time::Instant::now();
                } else {
                    if elapsed < wait {
                        gst_trace!(
                            CONTEXT_CAT,
                            "Waiting for {:?} before polling again",
                            wait - elapsed
                        );
                        thread::sleep(wait - elapsed);
                        gst_trace!(CONTEXT_CAT, "Slept for {:?}", now.elapsed());
                    }

                    now += wait;
                }
            })
        });
    }
}

impl Drop for IOContextRunner {
    fn drop(&mut self) {
        gst_debug!(CONTEXT_CAT, "Shut down reactor thread '{}'", self.name);
    }
}

struct IOContextShutdown {
    name: String,
    shutdown: Arc<atomic::AtomicUsize>,
    handle: reactor::Handle,
    join: Option<thread::JoinHandle<()>>,
}

impl Drop for IOContextShutdown {
    fn drop(&mut self) {
        use tokio_executor::park::Unpark;

        gst_debug!(CONTEXT_CAT, "Shutting down reactor thread '{}'", self.name);
        self.shutdown.store(SHUTDOWN_NOW, atomic::Ordering::SeqCst);
        gst_trace!(CONTEXT_CAT, "Waiting for reactor '{}' shutdown", self.name);
        // After being unparked, the next turn() is guaranteed to finish immediately,
        // as such there is no race condition between checking for shutdown and setting
        // shutdown.
        self.handle.unpark();
        let _ = self.join.take().unwrap().join();
    }
}

#[derive(Clone)]
pub struct IOContext(Arc<IOContextInner>);

impl glib::subclass::boxed::BoxedType for IOContext {
    const NAME: &'static str = "TsIOContext";

    glib_boxed_type!();
}

glib_boxed_derive_traits!(IOContext);

type PendingFutures = Mutex<(
    u64,
    HashMap<u64, FuturesUnordered<Box<dyn Future<Item = (), Error = ()> + Send + 'static>>>,
)>;

struct IOContextInner {
    name: String,
    runtime_handle: Mutex<tokio_current_thread::Handle>,
    reactor_handle: reactor::Handle,
    timers: Arc<Mutex<BinaryHeap<TimerEntry>>>,
    // Only used for dropping
    _shutdown: IOContextShutdown,
    pending_futures: PendingFutures,
}

impl Drop for IOContextInner {
    fn drop(&mut self) {
        let mut contexts = CONTEXTS.lock().unwrap();
        gst_debug!(CONTEXT_CAT, "Finalizing context '{}'", self.name);
        contexts.remove(&self.name);
    }
}

impl IOContext {
    pub fn new(name: &str, wait: u32) -> Result<Self, io::Error> {
        let mut contexts = CONTEXTS.lock().unwrap();
        if let Some(context) = contexts.get(name) {
            if let Some(context) = context.upgrade() {
                gst_debug!(CONTEXT_CAT, "Reusing existing context '{}'", name);
                return Ok(IOContext(context));
            }
        }

        let reactor = reactor::Reactor::new()?;
        let reactor_handle = reactor.handle().clone();

        let timers = Arc::new(Mutex::new(BinaryHeap::new()));

        let (runtime_handle, shutdown) =
            IOContextRunner::start(name, wait, reactor, timers.clone());

        let context = Arc::new(IOContextInner {
            name: name.into(),
            runtime_handle: Mutex::new(runtime_handle),
            reactor_handle,
            timers,
            _shutdown: shutdown,
            pending_futures: Mutex::new((0, HashMap::new())),
        });
        contexts.insert(name.into(), Arc::downgrade(&context));

        gst_debug!(CONTEXT_CAT, "Created new context '{}'", name);
        Ok(IOContext(context))
    }

    pub fn spawn<F>(&self, future: F)
    where
        F: Future<Item = (), Error = ()> + Send + 'static,
    {
        self.0.runtime_handle.lock().unwrap().spawn(future).unwrap();
    }

    pub fn reactor_handle(&self) -> &reactor::Handle {
        &self.0.reactor_handle
    }

    pub fn acquire_pending_future_id(&self) -> PendingFutureId {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        let id = pending_futures.0;
        pending_futures.0 += 1;
        pending_futures.1.insert(id, FuturesUnordered::new());

        PendingFutureId(id)
    }

    pub fn release_pending_future_id(&self, id: PendingFutureId) {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        if let Some(fs) = pending_futures.1.remove(&id.0) {
            self.spawn(fs.for_each(|_| Ok(())));
        }
    }

    pub fn add_pending_future<F>(&self, id: PendingFutureId, future: F)
    where
        F: Future<Item = (), Error = ()> + Send + 'static,
    {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        let fs = pending_futures.1.get_mut(&id.0).unwrap();
        fs.push(Box::new(future))
    }

    pub fn drain_pending_futures<E: Send + 'static>(
        &self,
        id: PendingFutureId,
    ) -> (Option<oneshot::Sender<()>>, PendingFuturesFuture<E>) {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        let fs = pending_futures.1.get_mut(&id.0).unwrap();

        let pending_futures = mem::replace(fs, FuturesUnordered::new());

        if !pending_futures.is_empty() {
            gst_log!(
                CONTEXT_CAT,
                "Scheduling {} pending futures for context '{}' with pending future id {:?}",
                pending_futures.len(),
                self.0.name,
                id,
            );

            let (sender, receiver) = oneshot::channel();

            let future = pending_futures
                .for_each(|_| Ok(()))
                .select(receiver.then(|_| Ok(())))
                .then(|_| Ok(()));

            (Some(sender), future::Either::A(Box::new(future)))
        } else {
            (None, future::Either::B(future::ok(())))
        }
    }
}

pub type PendingFuturesFuture<E> = future::Either<
    Box<dyn Future<Item = (), Error = E> + Send + 'static>,
    future::FutureResult<(), E>,
>;

#[derive(Clone, Copy, Eq, PartialEq, Hash, Debug)]
pub struct PendingFutureId(u64);

impl glib::subclass::boxed::BoxedType for PendingFutureId {
    const NAME: &'static str = "TsPendingFutureId";

    glib_boxed_type!();
}

glib_boxed_derive_traits!(PendingFutureId);

static TIMER_ENTRY_ID: atomic::AtomicUsize = atomic::AtomicUsize::new(0);

// Ad-hoc interval timer implementation for our throttled event loop above
pub struct TimerEntry {
    time: time::Instant,
    id: usize, // for producing a total order
    interval: Option<time::Duration>,
    sender: futures_mpsc::UnboundedSender<()>,
}

impl PartialEq for TimerEntry {
    fn eq(&self, other: &Self) -> bool {
        self.time.eq(&other.time) && self.id.eq(&other.id)
    }
}

impl Eq for TimerEntry {}

impl PartialOrd for TimerEntry {
    fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
        Some(self.cmp(&other))
    }
}

impl Ord for TimerEntry {
    fn cmp(&self, other: &Self) -> cmp::Ordering {
        other
            .time
            .cmp(&self.time)
            .then_with(|| other.id.cmp(&self.id))
    }
}

#[allow(unused)]
pub struct Interval {
    receiver: futures_mpsc::UnboundedReceiver<()>,
}

impl Interval {
    #[allow(unused)]
    pub fn new(context: &IOContext, interval: time::Duration) -> Self {
        use tokio_executor::park::Unpark;

        let (sender, receiver) = futures_mpsc::unbounded();

        let mut timers = context.0.timers.lock().unwrap();
        let entry = TimerEntry {
            time: time::Instant::now(),
            id: TIMER_ENTRY_ID.fetch_add(1, atomic::Ordering::Relaxed),
            interval: Some(interval),
            sender,
        };
        timers.push(entry);
        context.reactor_handle().unpark();

        Self { receiver }
    }
}

impl Stream for Interval {
    type Item = ();
    type Error = ();

    fn poll(&mut self) -> futures::Poll<Option<Self::Item>, Self::Error> {
        self.receiver.poll()
    }
}

pub struct Timeout {
    receiver: futures_mpsc::UnboundedReceiver<()>,
}

impl Timeout {
    pub fn new(context: &IOContext, timeout: time::Duration) -> Self {
        let (sender, receiver) = futures_mpsc::unbounded();

        let mut timers = context.0.timers.lock().unwrap();
        let entry = TimerEntry {
            time: time::Instant::now() + timeout,
            id: TIMER_ENTRY_ID.fetch_add(1, atomic::Ordering::Relaxed),
            interval: None,
            sender,
        };
        timers.push(entry);

        Self { receiver }
    }
}

impl Future for Timeout {
    type Item = ();
    type Error = ();

    fn poll(&mut self) -> futures::Poll<Self::Item, Self::Error> {
        let res = self.receiver.poll()?;

        match res {
            Async::NotReady => Ok(Async::NotReady),
            Async::Ready(None) => unreachable!(),
            Async::Ready(Some(_)) => Ok(Async::Ready(())),
        }
    }
}
<commit_msg>threadshare: Fix shutdown race condition<commit_after>// Copyright (C) 2018 Sebastian Dröge <sebastian@centricular.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the
// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
// Boston, MA 02110-1335, USA.

use std::cmp;
use std::collections::{BinaryHeap, HashMap};
use std::io;
use std::mem;
use std::sync::{atomic, mpsc};
use std::sync::{Arc, Mutex, Weak};
use std::thread;
use std::time;

use futures::future;
use futures::stream::futures_unordered::FuturesUnordered;
use futures::sync::mpsc as futures_mpsc;
use futures::sync::oneshot;
use futures::{Async, Future, Stream};
use tokio::reactor;
use tokio_current_thread;
use tokio_timer::timer;

use glib;
use gst;

lazy_static! {
    static ref CONTEXTS: Mutex<HashMap<String, Weak<IOContextInner>>> = Mutex::new(HashMap::new());
    static ref CONTEXT_CAT: gst::DebugCategory = gst::DebugCategory::new(
        "ts-context",
        gst::DebugColorFlags::empty(),
        Some("Thread-sharing Context"),
    );
}

// Our own simplified implementation of reactor::Background to allow hooking into its internals
const RUNNING: usize = 0;
const SHUTDOWN_NOW: usize = 1;

struct IOContextRunner {
    name: String,
    shutdown: Arc<atomic::AtomicUsize>,
}

impl IOContextRunner {
    fn start(
        name: &str,
        wait: u32,
        reactor: reactor::Reactor,
        timers: Arc<Mutex<BinaryHeap<TimerEntry>>>,
    ) -> (tokio_current_thread::Handle, IOContextShutdown) {
        let handle = reactor.handle().clone();
        let shutdown = Arc::new(atomic::AtomicUsize::new(RUNNING));
        let shutdown_clone = shutdown.clone();
        let name_clone = name.into();

        let mut runner = IOContextRunner {
            shutdown: shutdown_clone,
            name: name_clone,
        };

        let (sender, receiver) = mpsc::channel();

        let join = thread::spawn(move || {
            runner.run(wait, reactor, sender, timers);
        });

        let shutdown = IOContextShutdown {
            name: name.into(),
            shutdown,
            handle,
            join: Some(join),
        };

        let runtime_handle = receiver.recv().unwrap();

        (runtime_handle, shutdown)
    }

    fn run(
        &mut self,
        wait: u32,
        reactor: reactor::Reactor,
        sender: mpsc::Sender<tokio_current_thread::Handle>,
        timers: Arc<Mutex<BinaryHeap<TimerEntry>>>,
    ) {
        let wait = time::Duration::from_millis(wait as u64);

        gst_debug!(CONTEXT_CAT, "Started reactor thread '{}'", self.name);

        let handle = reactor.handle();
        let mut enter = ::tokio_executor::enter().unwrap();
        let timer = timer::Timer::new(reactor);
        let timer_handle = timer.handle();
        let mut current_thread = tokio_current_thread::CurrentThread::new_with_park(timer);

        let _ = sender.send(current_thread.handle());

        let mut now = time::Instant::now();

        ::tokio_timer::with_default(&timer_handle, &mut enter, |mut enter| {
            ::tokio_reactor::with_default(&handle, &mut enter, |enter| loop {
                if self.shutdown.load(atomic::Ordering::SeqCst) > RUNNING {
                    gst_debug!(CONTEXT_CAT, "Shutting down loop");
                    break;
                }

                gst_trace!(CONTEXT_CAT, "Elapsed {:?} since last loop", now.elapsed());

                // Handle timers
                {
                    // Trigger all timers that would be expired before the middle of the loop wait
                    // time
                    let timer_threshold = now + wait / 2;
                    let mut timers = timers.lock().unwrap();
                    while timers
                        .peek()
                        .and_then(|entry| {
                            if entry.time < timer_threshold {
                                Some(())
                            } else {
                                None
                            }
                        })
                        .is_some()
                    {
                        let TimerEntry {
                            time,
                            interval,
                            sender,
                            ..
                        } = timers.pop().unwrap();

                        if sender.is_closed() {
                            continue;
                        }

                        let _ = sender.unbounded_send(());
                        if let Some(interval) = interval {
                            timers.push(TimerEntry {
                                time: time + interval,
                                id: TIMER_ENTRY_ID.fetch_add(1, atomic::Ordering::Relaxed),
                                interval: Some(interval),
                                sender,
                            });
                        }
                    }
                }

                gst_trace!(CONTEXT_CAT, "Turning current thread '{}'", self.name);
                while current_thread
                    .enter(enter)
                    .turn(Some(time::Duration::from_millis(0)))
                    .unwrap()
                    .has_polled()
                {}
                gst_trace!(CONTEXT_CAT, "Turned current thread '{}'", self.name);

                // We have to check again after turning in case we're supposed to shut down now
                // and already handled the unpark above
                if self.shutdown.load(atomic::Ordering::SeqCst) > RUNNING {
                    gst_debug!(CONTEXT_CAT, "Shutting down loop");
                    break;
                }

                let elapsed = now.elapsed();
                gst_trace!(CONTEXT_CAT, "Elapsed {:?} after handling futures", elapsed);

                if wait == time::Duration::from_millis(0) {
                    let timers = timers.lock().unwrap();
                    let wait = match timers.peek().map(|entry| entry.time) {
                        None => None,
                        Some(time) => Some({
                            let tmp = time::Instant::now();

                            if time < tmp {
                                time::Duration::from_millis(0)
                            } else {
                                time.duration_since(tmp)
                            }
                        }),
                    };
                    drop(timers);

                    gst_trace!(CONTEXT_CAT, "Sleeping for up to {:?}", wait);
                    current_thread.enter(enter).turn(wait).unwrap();
                    gst_trace!(CONTEXT_CAT, "Slept for {:?}", now.elapsed());
                    now = time::Instant::now();
                } else {
                    if elapsed < wait {
                        gst_trace!(
                            CONTEXT_CAT,
                            "Waiting for {:?} before polling again",
                            wait - elapsed
                        );
                        thread::sleep(wait - elapsed);
                        gst_trace!(CONTEXT_CAT, "Slept for {:?}", now.elapsed());
                    }

                    now += wait;
                }
            })
        });
    }
}

impl Drop for IOContextRunner {
    fn drop(&mut self) {
        gst_debug!(CONTEXT_CAT, "Shut down reactor thread '{}'", self.name);
    }
}

struct IOContextShutdown {
    name: String,
    shutdown: Arc<atomic::AtomicUsize>,
    handle: reactor::Handle,
    join: Option<thread::JoinHandle<()>>,
}

impl Drop for IOContextShutdown {
    fn drop(&mut self) {
        use tokio_executor::park::Unpark;

        gst_debug!(CONTEXT_CAT, "Shutting down reactor thread '{}'", self.name);
        self.shutdown.store(SHUTDOWN_NOW, atomic::Ordering::SeqCst);
        gst_trace!(CONTEXT_CAT, "Waiting for reactor '{}' shutdown", self.name);
        // After being unparked, the next turn() is guaranteed to finish immediately,
        // as such there is no race condition between checking for shutdown and setting
        // shutdown.
        self.handle.unpark();
        let _ = self.join.take().unwrap().join();
    }
}

#[derive(Clone)]
pub struct IOContext(Arc<IOContextInner>);

impl glib::subclass::boxed::BoxedType for IOContext {
    const NAME: &'static str = "TsIOContext";

    glib_boxed_type!();
}

glib_boxed_derive_traits!(IOContext);

type PendingFutures = Mutex<(
    u64,
    HashMap<u64, FuturesUnordered<Box<dyn Future<Item = (), Error = ()> + Send + 'static>>>,
)>;

struct IOContextInner {
    name: String,
    runtime_handle: Mutex<tokio_current_thread::Handle>,
    reactor_handle: reactor::Handle,
    timers: Arc<Mutex<BinaryHeap<TimerEntry>>>,
    // Only used for dropping
    _shutdown: IOContextShutdown,
    pending_futures: PendingFutures,
}

impl Drop for IOContextInner {
    fn drop(&mut self) {
        let mut contexts = CONTEXTS.lock().unwrap();
        gst_debug!(CONTEXT_CAT, "Finalizing context '{}'", self.name);
        contexts.remove(&self.name);
    }
}

impl IOContext {
    pub fn new(name: &str, wait: u32) -> Result<Self, io::Error> {
        let mut contexts = CONTEXTS.lock().unwrap();
        if let Some(context) = contexts.get(name) {
            if let Some(context) = context.upgrade() {
                gst_debug!(CONTEXT_CAT, "Reusing existing context '{}'", name);
                return Ok(IOContext(context));
            }
        }

        let reactor = reactor::Reactor::new()?;
        let reactor_handle = reactor.handle().clone();

        let timers = Arc::new(Mutex::new(BinaryHeap::new()));

        let (runtime_handle, shutdown) =
            IOContextRunner::start(name, wait, reactor, timers.clone());

        let context = Arc::new(IOContextInner {
            name: name.into(),
            runtime_handle: Mutex::new(runtime_handle),
            reactor_handle,
            timers,
            _shutdown: shutdown,
            pending_futures: Mutex::new((0, HashMap::new())),
        });
        contexts.insert(name.into(), Arc::downgrade(&context));

        gst_debug!(CONTEXT_CAT, "Created new context '{}'", name);
        Ok(IOContext(context))
    }

    pub fn spawn<F>(&self, future: F)
    where
        F: Future<Item = (), Error = ()> + Send + 'static,
    {
        self.0.runtime_handle.lock().unwrap().spawn(future).unwrap();
    }

    pub fn reactor_handle(&self) -> &reactor::Handle {
        &self.0.reactor_handle
    }

    pub fn acquire_pending_future_id(&self) -> PendingFutureId {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        let id = pending_futures.0;
        pending_futures.0 += 1;
        pending_futures.1.insert(id, FuturesUnordered::new());

        PendingFutureId(id)
    }

    pub fn release_pending_future_id(&self, id: PendingFutureId) {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        if let Some(fs) = pending_futures.1.remove(&id.0) {
            self.spawn(fs.for_each(|_| Ok(())));
        }
    }

    pub fn add_pending_future<F>(&self, id: PendingFutureId, future: F)
    where
        F: Future<Item = (), Error = ()> + Send + 'static,
    {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        let fs = pending_futures.1.get_mut(&id.0).unwrap();
        fs.push(Box::new(future))
    }

    pub fn drain_pending_futures<E: Send + 'static>(
        &self,
        id: PendingFutureId,
    ) -> (Option<oneshot::Sender<()>>, PendingFuturesFuture<E>) {
        let mut pending_futures = self.0.pending_futures.lock().unwrap();
        let fs = pending_futures.1.get_mut(&id.0).unwrap();

        let pending_futures = mem::replace(fs, FuturesUnordered::new());

        if !pending_futures.is_empty() {
            gst_log!(
                CONTEXT_CAT,
                "Scheduling {} pending futures for context '{}' with pending future id {:?}",
                pending_futures.len(),
                self.0.name,
                id,
            );

            let (sender, receiver) = oneshot::channel();

            let future = pending_futures
                .for_each(|_| Ok(()))
                .select(receiver.then(|_| Ok(())))
                .then(|_| Ok(()));

            (Some(sender), future::Either::A(Box::new(future)))
        } else {
            (None, future::Either::B(future::ok(())))
        }
    }
}

pub type PendingFuturesFuture<E> = future::Either<
    Box<dyn Future<Item = (), Error = E> + Send + 'static>,
    future::FutureResult<(), E>,
>;

#[derive(Clone, Copy, Eq, PartialEq, Hash, Debug)]
pub struct PendingFutureId(u64);

impl glib::subclass::boxed::BoxedType for PendingFutureId {
    const NAME: &'static str = "TsPendingFutureId";

    glib_boxed_type!();
}

glib_boxed_derive_traits!(PendingFutureId);

static TIMER_ENTRY_ID: atomic::AtomicUsize = atomic::AtomicUsize::new(0);

// Ad-hoc interval timer implementation for our throttled event loop above
pub struct TimerEntry {
    time: time::Instant,
    id: usize, // for producing a total order
    interval: Option<time::Duration>,
    sender: futures_mpsc::UnboundedSender<()>,
}

impl PartialEq for TimerEntry {
    fn eq(&self, other: &Self) -> bool {
        self.time.eq(&other.time) && self.id.eq(&other.id)
    }
}

impl Eq for TimerEntry {}

impl PartialOrd for TimerEntry {
    fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
        Some(self.cmp(&other))
    }
}

impl Ord for TimerEntry {
    fn cmp(&self, other: &Self) -> cmp::Ordering {
        other
            .time
            .cmp(&self.time)
            .then_with(|| other.id.cmp(&self.id))
    }
}

#[allow(unused)]
pub struct Interval {
    receiver: futures_mpsc::UnboundedReceiver<()>,
}

impl Interval {
    #[allow(unused)]
    pub fn new(context: &IOContext, interval: time::Duration) -> Self {
        use tokio_executor::park::Unpark;

        let (sender, receiver) = futures_mpsc::unbounded();

        let mut timers = context.0.timers.lock().unwrap();
        let entry = TimerEntry {
            time: time::Instant::now(),
            id: TIMER_ENTRY_ID.fetch_add(1, atomic::Ordering::Relaxed),
            interval: Some(interval),
            sender,
        };
        timers.push(entry);
        context.reactor_handle().unpark();

        Self { receiver }
    }
}

impl Stream for Interval {
    type Item = ();
    type Error = ();

    fn poll(&mut self) -> futures::Poll<Option<Self::Item>, Self::Error> {
        self.receiver.poll()
    }
}

pub struct Timeout {
    receiver: futures_mpsc::UnboundedReceiver<()>,
}

impl Timeout {
    pub fn new(context: &IOContext, timeout: time::Duration) -> Self {
        let (sender, receiver) = futures_mpsc::unbounded();

        let mut timers = context.0.timers.lock().unwrap();
        let entry = TimerEntry {
            time: time::Instant::now() + timeout,
            id: TIMER_ENTRY_ID.fetch_add(1, atomic::Ordering::Relaxed),
            interval: None,
            sender,
        };
        timers.push(entry);

        Self { receiver }
    }
}

impl Future for Timeout {
    type Item = ();
    type Error = ();

    fn poll(&mut self) -> futures::Poll<Self::Item, Self::Error> {
        let res = self.receiver.poll()?;

        match res {
            Async::NotReady => Ok(Async::NotReady),
            Async::Ready(None) => unreachable!(),
            Async::Ready(Some(_)) => Ok(Async::Ready(())),
        }
    }
}
<|endoftext|>


<commit_before>//! Tools to work with format string literals for the `format_args!` family of macros.
use syntax::{
    ast::{self, IsString},
    AstNode, AstToken, TextRange, TextSize,
};

pub fn is_format_string(string: &ast::String) -> bool {
    // Check if `string` is a format string argument of a macro invocation.
    // `string` is a string literal, mapped down into the innermost macro expansion.
    // Since `format_args!` etc. remove the format string when expanding, but place all arguments
    // in the expanded output, we know that the string token is (part of) the format string if it
    // appears in `format_args!` (otherwise it would have been mapped down further).
    //
    // This setup lets us correctly highlight the components of `concat!("{}", "bla")` format
    // strings. It still fails for `concat!("{", "}")`, but that is rare.
    (|| {
        let macro_call = string.syntax().parent_ancestors().find_map(ast::MacroCall::cast)?;
        let name = macro_call.path()?.segment()?.name_ref()?;

        if !matches!(
            name.text().as_str(),
            "format_args" | "format_args_nl" | "const_format_args" | "panic_2015" | "panic_2021"
        ) {
            return None;
        }

        // NB: we match against `panic_2015`/`panic_2021` here because they have a special-cased arm for
        // `"{}"`, which otherwise wouldn't get highlighted.

        Some(())
    })()
    .is_some()
}

#[derive(Debug)]
pub enum FormatSpecifier {
    Open,
    Close,
    Integer,
    Identifier,
    Colon,
    Fill,
    Align,
    Sign,
    NumberSign,
    Zero,
    DollarSign,
    Dot,
    Asterisk,
    QuestionMark,
    Escape,
}

pub fn lex_format_specifiers(
    string: &ast::String,
    mut callback: &mut dyn FnMut(TextRange, FormatSpecifier),
) {
    let mut char_ranges = Vec::new();
    string.escaped_char_ranges(&mut |range, res| char_ranges.push((range, res)));
    let mut chars = char_ranges
        .iter()
        .filter_map(|(range, res)| Some((*range, *res.as_ref().ok()?)))
        .peekable();

    while let Some((range, first_char)) = chars.next() {
        if let '{' = first_char {
            // Format specifier, see syntax at https://doc.rust-lang.org/std/fmt/index.html#syntax
            if let Some((_, '{')) = chars.peek() {
                // Escaped format specifier, `{{`
                read_escaped_format_specifier(&mut chars, &mut callback);
                continue;
            }

            callback(range, FormatSpecifier::Open);

            // check for integer/identifier
            let (_, int_char) = chars.peek().copied().unwrap_or_default();
            match int_char {
                // integer
                '0'..='9' => read_integer(&mut chars, &mut callback),
                // identifier
                c if c == '_' || c.is_alphabetic() => read_identifier(&mut chars, &mut callback),
                _ => {}
            }

            if let Some((_, ':')) = chars.peek() {
                skip_char_and_emit(&mut chars, FormatSpecifier::Colon, &mut callback);

                // check for fill/align
                let mut cloned = chars.clone().take(2);
                let (_, first) = cloned.next().unwrap_or_default();
                let (_, second) = cloned.next().unwrap_or_default();
                match second {
                    '<' | '^' | '>' => {
                        // alignment specifier, first char specifies fillment
                        skip_char_and_emit(&mut chars, FormatSpecifier::Fill, &mut callback);
                        skip_char_and_emit(&mut chars, FormatSpecifier::Align, &mut callback);
                    }
                    _ => {
                        if let '<' | '^' | '>' = first {
                            skip_char_and_emit(&mut chars, FormatSpecifier::Align, &mut callback);
                        }
                    }
                }

                // check for sign
                match chars.peek().copied().unwrap_or_default().1 {
                    '+' | '-' => {
                        skip_char_and_emit(&mut chars, FormatSpecifier::Sign, &mut callback);
                    }
                    _ => {}
                }

                // check for `#`
                if let Some((_, '#')) = chars.peek() {
                    skip_char_and_emit(&mut chars, FormatSpecifier::NumberSign, &mut callback);
                }

                // check for `0`
                let mut cloned = chars.clone().take(2);
                let first = cloned.next().map(|next| next.1);
                let second = cloned.next().map(|next| next.1);

                if first == Some('0') && second != Some('$') {
                    skip_char_and_emit(&mut chars, FormatSpecifier::Zero, &mut callback);
                }

                // width
                match chars.peek().copied().unwrap_or_default().1 {
                    '0'..='9' => {
                        read_integer(&mut chars, &mut callback);
                        if let Some((_, '$')) = chars.peek() {
                            skip_char_and_emit(
                                &mut chars,
                                FormatSpecifier::DollarSign,
                                &mut callback,
                            );
                        }
                    }
                    c if c == '_' || c.is_alphabetic() => {
                        read_identifier(&mut chars, &mut callback);

                        if chars.peek().map(|&(_, c)| c) == Some('?') {
                            skip_char_and_emit(
                                &mut chars,
                                FormatSpecifier::QuestionMark,
                                &mut callback,
                            );
                        }

                        // can be either width (indicated by dollar sign, or type in which case
                        // the next sign has to be `}`)
                        let next = chars.peek().map(|&(_, c)| c);

                        match next {
                            Some('$') => skip_char_and_emit(
                                &mut chars,
                                FormatSpecifier::DollarSign,
                                &mut callback,
                            ),
                            Some('}') => {
                                skip_char_and_emit(
                                    &mut chars,
                                    FormatSpecifier::Close,
                                    &mut callback,
                                );
                                continue;
                            }
                            _ => continue,
                        };
                    }
                    _ => {}
                }

                // precision
                if let Some((_, '.')) = chars.peek() {
                    skip_char_and_emit(&mut chars, FormatSpecifier::Dot, &mut callback);

                    match chars.peek().copied().unwrap_or_default().1 {
                        '*' => {
                            skip_char_and_emit(
                                &mut chars,
                                FormatSpecifier::Asterisk,
                                &mut callback,
                            );
                        }
                        '0'..='9' => {
                            read_integer(&mut chars, &mut callback);
                            if let Some((_, '$')) = chars.peek() {
                                skip_char_and_emit(
                                    &mut chars,
                                    FormatSpecifier::DollarSign,
                                    &mut callback,
                                );
                            }
                        }
                        c if c == '_' || c.is_alphabetic() => {
                            read_identifier(&mut chars, &mut callback);
                            if chars.peek().map(|&(_, c)| c) != Some('$') {
                                continue;
                            }
                            skip_char_and_emit(
                                &mut chars,
                                FormatSpecifier::DollarSign,
                                &mut callback,
                            );
                        }
                        _ => {
                            continue;
                        }
                    }
                }

                // type
                match chars.peek().copied().unwrap_or_default().1 {
                    '?' => {
                        skip_char_and_emit(
                            &mut chars,
                            FormatSpecifier::QuestionMark,
                            &mut callback,
                        );
                    }
                    c if c == '_' || c.is_alphabetic() => {
                        read_identifier(&mut chars, &mut callback);

                        if chars.peek().map(|&(_, c)| c) == Some('?') {
                            skip_char_and_emit(
                                &mut chars,
                                FormatSpecifier::QuestionMark,
                                &mut callback,
                            );
                        }
                    }
                    _ => {}
                }
            }

            if let Some((_, '}')) = chars.peek() {
                skip_char_and_emit(&mut chars, FormatSpecifier::Close, &mut callback);
            }
            continue;
        } else if let '}' = first_char {
            if let Some((_, '}')) = chars.peek() {
                // Escaped format specifier, `}}`
                read_escaped_format_specifier(&mut chars, &mut callback);
            }
        }
    }

    fn skip_char_and_emit<I, F>(
        chars: &mut std::iter::Peekable<I>,
        emit: FormatSpecifier,
        callback: &mut F,
    ) where
        I: Iterator<Item = (TextRange, char)>,
        F: FnMut(TextRange, FormatSpecifier),
    {
        let (range, _) = chars.next().unwrap();
        callback(range, emit);
    }

    fn read_integer<I, F>(chars: &mut std::iter::Peekable<I>, callback: &mut F)
    where
        I: Iterator<Item = (TextRange, char)>,
        F: FnMut(TextRange, FormatSpecifier),
    {
        let (mut range, c) = chars.next().unwrap();
        assert!(c.is_ascii_digit());
        while let Some(&(r, next_char)) = chars.peek() {
            if next_char.is_ascii_digit() {
                chars.next();
                range = range.cover(r);
            } else {
                break;
            }
        }
        callback(range, FormatSpecifier::Integer);
    }

    fn read_identifier<I, F>(chars: &mut std::iter::Peekable<I>, callback: &mut F)
    where
        I: Iterator<Item = (TextRange, char)>,
        F: FnMut(TextRange, FormatSpecifier),
    {
        let (mut range, c) = chars.next().unwrap();
        assert!(c.is_alphabetic() || c == '_');
        while let Some(&(r, next_char)) = chars.peek() {
            if next_char == '_' || next_char.is_ascii_digit() || next_char.is_alphabetic() {
                chars.next();
                range = range.cover(r);
            } else {
                break;
            }
        }
        callback(range, FormatSpecifier::Identifier);
    }

    fn read_escaped_format_specifier<I, F>(chars: &mut std::iter::Peekable<I>, callback: &mut F)
    where
        I: Iterator<Item = (TextRange, char)>,
        F: FnMut(TextRange, FormatSpecifier),
    {
        let (range, _) = chars.peek().unwrap();
        let offset = TextSize::from(1);
        callback(TextRange::new(range.start() - offset, range.end()), FormatSpecifier::Escape);
        chars.next();
    }
}
<commit_msg>Use helper method in is_format_string<commit_after>//! Tools to work with format string literals for the `format_args!` family of macros.
use syntax::{
    ast::{self, IsString}, TextRange, TextSize,
};

use super::node_ext::get_outer_macro_name;

pub fn is_format_string(string: &ast::String) -> bool {
    // Check if `string` is a format string argument of a macro invocation.
    // `string` is a string literal, mapped down into the innermost macro expansion.
    // Since `format_args!` etc. remove the format string when expanding, but place all arguments
    // in the expanded output, we know that the string token is (part of) the format string if it
    // appears in `format_args!` (otherwise it would have been mapped down further).
    //
    // This setup lets us correctly highlight the components of `concat!("{}", "bla")` format
    // strings. It still fails for `concat!("{", "}")`, but that is rare.
    (|| {
        let name = get_outer_macro_name(string)?;

        if !matches!(
            name.text().as_str(),
            "format_args" | "format_args_nl" | "const_format_args" | "panic_2015" | "panic_2021"
        ) {
            return None;
        }

        // NB: we match against `panic_2015`/`panic_2021` here because they have a special-cased arm for
        // `"{}"`, which otherwise wouldn't get highlighted.

        Some(())
    })()
    .is_some()
}

#[derive(Debug)]
pub enum FormatSpecifier {
    Open,
    Close,
    Integer,
    Identifier,
    Colon,
    Fill,
    Align,
    Sign,
    NumberSign,
    Zero,
    DollarSign,
    Dot,
    Asterisk,
    QuestionMark,
    Escape,
}

pub fn lex_format_specifiers(
    string: &ast::String,
    mut callback: &mut dyn FnMut(TextRange, FormatSpecifier),
) {
    let mut char_ranges = Vec::new();
    string.escaped_char_ranges(&mut |range, res| char_ranges.push((range, res)));
    let mut chars = char_ranges
        .iter()
        .filter_map(|(range, res)| Some((*range, *res.as_ref().ok()?)))
        .peekable();

    while let Some((range, first_char)) = chars.next() {
        if let '{' = first_char {
            // Format specifier, see syntax at https://doc.rust-lang.org/std/fmt/index.html#syntax
            if let Some((_, '{')) = chars.peek() {
                // Escaped format specifier, `{{`
                read_escaped_format_specifier(&mut chars, &mut callback);
                continue;
            }

            callback(range, FormatSpecifier::Open);

            // check for integer/identifier
            let (_, int_char) = chars.peek().copied().unwrap_or_default();
            match int_char {
                // integer
                '0'..='9' => read_integer(&mut chars, &mut callback),
                // identifier
                c if c == '_' || c.is_alphabetic() => read_identifier(&mut chars, &mut callback),
                _ => {}
            }

            if let Some((_, ':')) = chars.peek() {
                skip_char_and_emit(&mut chars, FormatSpecifier::Colon, &mut callback);

                // check for fill/align
                let mut cloned = chars.clone().take(2);
                let (_, first) = cloned.next().unwrap_or_default();
                let (_, second) = cloned.next().unwrap_or_default();
                match second {
                    '<' | '^' | '>' => {
                        // alignment specifier, first char specifies fillment
                        skip_char_and_emit(&mut chars, FormatSpecifier::Fill, &mut callback);
                        skip_char_and_emit(&mut chars, FormatSpecifier::Align, &mut callback);
                    }
                    _ => {
                        if let '<' | '^' | '>' = first {
                            skip_char_and_emit(&mut chars, FormatSpecifier::Align, &mut callback);
                        }
                    }
                }

                // check for sign
                match chars.peek().copied().unwrap_or_default().1 {
                    '+' | '-' => {
                        skip_char_and_emit(&mut chars, FormatSpecifier::Sign, &mut callback);
                    }
                    _ => {}
                }

                // check for `#`
                if let Some((_, '#')) = chars.peek() {
                    skip_char_and_emit(&mut chars, FormatSpecifier::NumberSign, &mut callback);
                }

                // check for `0`
                let mut cloned = chars.clone().take(2);
                let first = cloned.next().map(|next| next.1);
                let second = cloned.next().map(|next| next.1);

                if first == Some('0') && second != Some('$') {
                    skip_char_and_emit(&mut chars, FormatSpecifier::Zero, &mut callback);
                }

                // width
                match chars.peek().copied().unwrap_or_default().1 {
                    '0'..='9' => {
                        read_integer(&mut chars, &mut callback);
                        if let Some((_, '$')) = chars.peek() {
                            skip_char_and_emit(
                                &mut chars,
                                FormatSpecifier::DollarSign,
                                &mut callback,
                            );
                        }
                    }
                    c if c == '_' || c.is_alphabetic() => {
                        read_identifier(&mut chars, &mut callback);

                        if chars.peek().map(|&(_, c)| c) == Some('?') {
                            skip_char_and_emit(
                                &mut chars,
                                FormatSpecifier::QuestionMark,
                                &mut callback,
                            );
                        }

                        // can be either width (indicated by dollar sign, or type in which case
                        // the next sign has to be `}`)
                        let next = chars.peek().map(|&(_, c)| c);

                        match next {
                            Some('$') => skip_char_and_emit(
                                &mut chars,
                                FormatSpecifier::DollarSign,
                                &mut callback,
                            ),
                            Some('}') => {
                                skip_char_and_emit(
                                    &mut chars,
                                    FormatSpecifier::Close,
                                    &mut callback,
                                );
                                continue;
                            }
                            _ => continue,
                        };
                    }
                    _ => {}
                }

                // precision
                if let Some((_, '.')) = chars.peek() {
                    skip_char_and_emit(&mut chars, FormatSpecifier::Dot, &mut callback);

                    match chars.peek().copied().unwrap_or_default().1 {
                        '*' => {
                            skip_char_and_emit(
                                &mut chars,
                                FormatSpecifier::Asterisk,
                                &mut callback,
                            );
                        }
                        '0'..='9' => {
                            read_integer(&mut chars, &mut callback);
                            if let Some((_, '$')) = chars.peek() {
                                skip_char_and_emit(
                                    &mut chars,
                                    FormatSpecifier::DollarSign,
                                    &mut callback,
                                );
                            }
                        }
                        c if c == '_' || c.is_alphabetic() => {
                            read_identifier(&mut chars, &mut callback);
                            if chars.peek().map(|&(_, c)| c) != Some('$') {
                                continue;
                            }
                            skip_char_and_emit(
                                &mut chars,
                                FormatSpecifier::DollarSign,
                                &mut callback,
                            );
                        }
                        _ => {
                            continue;
                        }
                    }
                }

                // type
                match chars.peek().copied().unwrap_or_default().1 {
                    '?' => {
                        skip_char_and_emit(
                            &mut chars,
                            FormatSpecifier::QuestionMark,
                            &mut callback,
                        );
                    }
                    c if c == '_' || c.is_alphabetic() => {
                        read_identifier(&mut chars, &mut callback);

                        if chars.peek().map(|&(_, c)| c) == Some('?') {
                            skip_char_and_emit(
                                &mut chars,
                                FormatSpecifier::QuestionMark,
                                &mut callback,
                            );
                        }
                    }
                    _ => {}
                }
            }

            if let Some((_, '}')) = chars.peek() {
                skip_char_and_emit(&mut chars, FormatSpecifier::Close, &mut callback);
            }
            continue;
        } else if let '}' = first_char {
            if let Some((_, '}')) = chars.peek() {
                // Escaped format specifier, `}}`
                read_escaped_format_specifier(&mut chars, &mut callback);
            }
        }
    }

    fn skip_char_and_emit<I, F>(
        chars: &mut std::iter::Peekable<I>,
        emit: FormatSpecifier,
        callback: &mut F,
    ) where
        I: Iterator<Item = (TextRange, char)>,
        F: FnMut(TextRange, FormatSpecifier),
    {
        let (range, _) = chars.next().unwrap();
        callback(range, emit);
    }

    fn read_integer<I, F>(chars: &mut std::iter::Peekable<I>, callback: &mut F)
    where
        I: Iterator<Item = (TextRange, char)>,
        F: FnMut(TextRange, FormatSpecifier),
    {
        let (mut range, c) = chars.next().unwrap();
        assert!(c.is_ascii_digit());
        while let Some(&(r, next_char)) = chars.peek() {
            if next_char.is_ascii_digit() {
                chars.next();
                range = range.cover(r);
            } else {
                break;
            }
        }
        callback(range, FormatSpecifier::Integer);
    }

    fn read_identifier<I, F>(chars: &mut std::iter::Peekable<I>, callback: &mut F)
    where
        I: Iterator<Item = (TextRange, char)>,
        F: FnMut(TextRange, FormatSpecifier),
    {
        let (mut range, c) = chars.next().unwrap();
        assert!(c.is_alphabetic() || c == '_');
        while let Some(&(r, next_char)) = chars.peek() {
            if next_char == '_' || next_char.is_ascii_digit() || next_char.is_alphabetic() {
                chars.next();
                range = range.cover(r);
            } else {
                break;
            }
        }
        callback(range, FormatSpecifier::Identifier);
    }

    fn read_escaped_format_specifier<I, F>(chars: &mut std::iter::Peekable<I>, callback: &mut F)
    where
        I: Iterator<Item = (TextRange, char)>,
        F: FnMut(TextRange, FormatSpecifier),
    {
        let (range, _) = chars.peek().unwrap();
        let offset = TextSize::from(1);
        callback(TextRange::new(range.start() - offset, range.end()), FormatSpecifier::Escape);
        chars.next();
    }
}
<|endoftext|>


<commit_before>use crate::stable_hasher::{HashStable, StableHasher};

use std::iter::FromIterator;

/// A vector type optimized for cases where this size is usually 0 (cf. `SmallVector`).
/// The `Option<Box<..>>` wrapping allows us to represent a zero sized vector with `None`,
/// which uses only a single (null) pointer.
#[derive(Clone, Encodable, Decodable, Debug)]
pub struct ThinVec<T>(Option<Box<Vec<T>>>);

impl<T> ThinVec<T> {
    pub fn new() -> Self {
        ThinVec(None)
    }

    pub fn iter(&self) -> std::slice::Iter<'_, T> {
        self.into_iter()
    }

    pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, T> {
        self.into_iter()
    }
}

impl<T> From<Vec<T>> for ThinVec<T> {
    fn from(vec: Vec<T>) -> Self {
        if vec.is_empty() { ThinVec(None) } else { ThinVec(Some(Box::new(vec))) }
    }
}

impl<T> Into<Vec<T>> for ThinVec<T> {
    fn into(self) -> Vec<T> {
        match self {
            ThinVec(None) => Vec::new(),
            ThinVec(Some(vec)) => *vec,
        }
    }
}

impl<T> ::std::ops::Deref for ThinVec<T> {
    type Target = [T];
    fn deref(&self) -> &[T] {
        match *self {
            ThinVec(None) => &[],
            ThinVec(Some(ref vec)) => vec,
        }
    }
}

impl<T> ::std::ops::DerefMut for ThinVec<T> {
    fn deref_mut(&mut self) -> &mut [T] {
        match *self {
            ThinVec(None) => &mut [],
            ThinVec(Some(ref mut vec)) => vec,
        }
    }
}

impl<T> FromIterator<T> for ThinVec<T> {
    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
        // `Vec::from_iter()` should not allocate if the iterator is empty.
        let vec: Vec<_> = iter.into_iter().collect();
        if vec.is_empty() { ThinVec(None) } else { ThinVec(Some(Box::new(vec))) }
    }
}

impl<T> IntoIterator for ThinVec<T> {
    type Item = T;
    type IntoIter = std::vec::IntoIter<T>;

    fn into_iter(self) -> Self::IntoIter {
        // This is still performant because `Vec::new()` does not allocate.
        self.0.map_or_else(Vec::new, |ptr| *ptr).into_iter()
    }
}

impl<'a, T> IntoIterator for &'a ThinVec<T> {
    type Item = &'a T;
    type IntoIter = std::slice::Iter<'a, T>;

    fn into_iter(self) -> Self::IntoIter {
        self.as_ref().iter()
    }
}

impl<'a, T> IntoIterator for &'a mut ThinVec<T> {
    type Item = &'a mut T;
    type IntoIter = std::slice::IterMut<'a, T>;

    fn into_iter(self) -> Self::IntoIter {
        self.as_mut().iter_mut()
    }
}

impl<T> Extend<T> for ThinVec<T> {
    fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
        match *self {
            ThinVec(Some(ref mut vec)) => vec.extend(iter),
            ThinVec(None) => *self = iter.into_iter().collect::<Vec<_>>().into(),
        }
    }

    fn extend_one(&mut self, item: T) {
        match *self {
            ThinVec(Some(ref mut vec)) => vec.push(item),
            ThinVec(None) => *self = vec![item].into(),
        }
    }

    fn extend_reserve(&mut self, additional: usize) {
        match *self {
            ThinVec(Some(ref mut vec)) => vec.reserve(additional),
            ThinVec(None) => *self = Vec::with_capacity(additional).into(),
        }
    }
}

impl<T: HashStable<CTX>, CTX> HashStable<CTX> for ThinVec<T> {
    fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
        (**self).hash_stable(hcx, hasher)
    }
}

impl<T> Default for ThinVec<T> {
    fn default() -> Self {
        Self(None)
    }
}

#[cfg(test)]
mod tests;
<commit_msg>Rollup merge of #88659 - est31:update_smallvec_name, r=matthewjasper<commit_after>use crate::stable_hasher::{HashStable, StableHasher};

use std::iter::FromIterator;

/// A vector type optimized for cases where this size is usually 0 (cf. `SmallVec`).
/// The `Option<Box<..>>` wrapping allows us to represent a zero sized vector with `None`,
/// which uses only a single (null) pointer.
#[derive(Clone, Encodable, Decodable, Debug)]
pub struct ThinVec<T>(Option<Box<Vec<T>>>);

impl<T> ThinVec<T> {
    pub fn new() -> Self {
        ThinVec(None)
    }

    pub fn iter(&self) -> std::slice::Iter<'_, T> {
        self.into_iter()
    }

    pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, T> {
        self.into_iter()
    }
}

impl<T> From<Vec<T>> for ThinVec<T> {
    fn from(vec: Vec<T>) -> Self {
        if vec.is_empty() { ThinVec(None) } else { ThinVec(Some(Box::new(vec))) }
    }
}

impl<T> Into<Vec<T>> for ThinVec<T> {
    fn into(self) -> Vec<T> {
        match self {
            ThinVec(None) => Vec::new(),
            ThinVec(Some(vec)) => *vec,
        }
    }
}

impl<T> ::std::ops::Deref for ThinVec<T> {
    type Target = [T];
    fn deref(&self) -> &[T] {
        match *self {
            ThinVec(None) => &[],
            ThinVec(Some(ref vec)) => vec,
        }
    }
}

impl<T> ::std::ops::DerefMut for ThinVec<T> {
    fn deref_mut(&mut self) -> &mut [T] {
        match *self {
            ThinVec(None) => &mut [],
            ThinVec(Some(ref mut vec)) => vec,
        }
    }
}

impl<T> FromIterator<T> for ThinVec<T> {
    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
        // `Vec::from_iter()` should not allocate if the iterator is empty.
        let vec: Vec<_> = iter.into_iter().collect();
        if vec.is_empty() { ThinVec(None) } else { ThinVec(Some(Box::new(vec))) }
    }
}

impl<T> IntoIterator for ThinVec<T> {
    type Item = T;
    type IntoIter = std::vec::IntoIter<T>;

    fn into_iter(self) -> Self::IntoIter {
        // This is still performant because `Vec::new()` does not allocate.
        self.0.map_or_else(Vec::new, |ptr| *ptr).into_iter()
    }
}

impl<'a, T> IntoIterator for &'a ThinVec<T> {
    type Item = &'a T;
    type IntoIter = std::slice::Iter<'a, T>;

    fn into_iter(self) -> Self::IntoIter {
        self.as_ref().iter()
    }
}

impl<'a, T> IntoIterator for &'a mut ThinVec<T> {
    type Item = &'a mut T;
    type IntoIter = std::slice::IterMut<'a, T>;

    fn into_iter(self) -> Self::IntoIter {
        self.as_mut().iter_mut()
    }
}

impl<T> Extend<T> for ThinVec<T> {
    fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
        match *self {
            ThinVec(Some(ref mut vec)) => vec.extend(iter),
            ThinVec(None) => *self = iter.into_iter().collect::<Vec<_>>().into(),
        }
    }

    fn extend_one(&mut self, item: T) {
        match *self {
            ThinVec(Some(ref mut vec)) => vec.push(item),
            ThinVec(None) => *self = vec![item].into(),
        }
    }

    fn extend_reserve(&mut self, additional: usize) {
        match *self {
            ThinVec(Some(ref mut vec)) => vec.reserve(additional),
            ThinVec(None) => *self = Vec::with_capacity(additional).into(),
        }
    }
}

impl<T: HashStable<CTX>, CTX> HashStable<CTX> for ThinVec<T> {
    fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
        (**self).hash_stable(hcx, hasher)
    }
}

impl<T> Default for ThinVec<T> {
    fn default() -> Self {
        Self(None)
    }
}

#[cfg(test)]
mod tests;
<|endoftext|>


<commit_before>use std::{
    fs::File,
    io::{
        BufRead,
        BufReader,
        Read,
    },
    path::Path,
};

use std::marker::PhantomData;

use anyhow::{
    Context,
    Result,
};

use serde::de::DeserializeOwned;

pub struct JsonlIterator<R: Read, T: DeserializeOwned> {
    reader: BufReader<R>,
    buf: String,
    phantom: PhantomData<T>,
}

impl<R: Read, T: DeserializeOwned> JsonlIterator<R, T> {
    pub fn from_read(reader: R) -> Self {
        Self {
            reader: BufReader::new(reader),
            buf: String::new(),
            phantom: PhantomData,
        }
    }
}

impl<T: DeserializeOwned> JsonlIterator<File, T> {
    pub fn from_path(path: &Path) -> Result<Self> {
        let file = File::open(path).with_context(|| format!("Could not open file {:?}", &path))?;
        Ok(JsonlIterator::from_read(file))
    }
}

impl<R: Read, T: DeserializeOwned> Iterator for JsonlIterator<R, T> {
    type Item = T;

    fn next(&mut self) -> Option<T> {
        match self.reader.read_line(&mut self.buf).unwrap() {
            0 => None,
            _ => {
                let value: T = serde_json::from_str(&self.buf)
                    .or(serde_json::from_str(&self.buf.replace("////", "//")))
                    .with_context(|| format!("Could not parse line {}", &self.buf))
                    .unwrap();
                self.buf.clear();
                Some(value)
            },
        }
    }
}
<commit_msg>Use escaped logic first before unescaped<commit_after>use std::{
    fs::File,
    io::{
        BufRead,
        BufReader,
        Read,
    },
    path::Path,
};

use std::marker::PhantomData;

use anyhow::{
    Context,
    Result,
};

use serde::de::DeserializeOwned;

pub struct JsonlIterator<R: Read, T: DeserializeOwned> {
    reader: BufReader<R>,
    buf: String,
    phantom: PhantomData<T>,
}

impl<R: Read, T: DeserializeOwned> JsonlIterator<R, T> {
    pub fn from_read(reader: R) -> Self {
        Self {
            reader: BufReader::new(reader),
            buf: String::new(),
            phantom: PhantomData,
        }
    }
}

impl<T: DeserializeOwned> JsonlIterator<File, T> {
    pub fn from_path(path: &Path) -> Result<Self> {
        let file = File::open(path).with_context(|| format!("Could not open file {:?}", &path))?;
        Ok(JsonlIterator::from_read(file))
    }
}

impl<R: Read, T: DeserializeOwned> Iterator for JsonlIterator<R, T> {
    type Item = T;

    fn next(&mut self) -> Option<T> {
        match self.reader.read_line(&mut self.buf).unwrap() {
            0 => None,
            _ => {
                let value: T = serde_json::from_str(&self.buf.replace("\\\\", "\\"))
                    .or(serde_json::from_str(&self.buf))
                    .with_context(|| format!("Could not parse line {}", &self.buf))
                    .unwrap();
                self.buf.clear();
                Some(value)
            },
        }
    }
}
<|endoftext|>