runtime: replace epoll by io-uring
This commit is contained in:
Julian Orth
parent
98cc85e2d3
commit
9416efeabe
21 changed files with 173 additions and 742 deletions
|
|
@ -1,196 +0,0 @@
|
|||
use {
|
||||
crate::{
|
||||
async_engine::{AsyncEngine, AsyncError},
|
||||
event_loop::{EventLoop, EventLoopDispatcher, EventLoopError, EventLoopId},
|
||||
utils::numcell::NumCell,
|
||||
},
|
||||
std::{
|
||||
cell::{Cell, RefCell},
|
||||
error::Error,
|
||||
fmt::{Debug, Formatter},
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
rc::Rc,
|
||||
task::{Context, Poll, Waker},
|
||||
},
|
||||
uapi::{c, OwnedFd},
|
||||
};
|
||||
|
||||
type Queue = RefCell<Vec<(Waker, Rc<Cell<Option<FdStatus>>>)>>;
|
||||
|
||||
pub(super) struct AsyncFdData {
|
||||
pub(super) ref_count: NumCell<u64>,
|
||||
pub(super) fd: Rc<OwnedFd>,
|
||||
pub(super) id: EventLoopId,
|
||||
pub(super) el: Rc<EventLoop>,
|
||||
pub(super) write_registered: Cell<bool>,
|
||||
pub(super) read_registered: Cell<bool>,
|
||||
pub(super) readers: Queue,
|
||||
pub(super) writers: Queue,
|
||||
}
|
||||
|
||||
impl AsyncFdData {
|
||||
fn update_interests(&self) -> Result<(), EventLoopError> {
|
||||
let mut events = 0;
|
||||
if self.write_registered.get() {
|
||||
events |= c::EPOLLOUT;
|
||||
}
|
||||
if self.read_registered.get() {
|
||||
events |= c::EPOLLIN;
|
||||
}
|
||||
let res = self.el.modify(self.id, events);
|
||||
if res.is_err() {
|
||||
if let Err(e) = self.el.remove(self.id) {
|
||||
log::error!(
|
||||
"Fatal error: Cannot remove file descriptor from event loop: {:?}",
|
||||
e
|
||||
);
|
||||
self.el.stop();
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
fn poll(
|
||||
&self,
|
||||
woken: &Rc<Cell<Option<FdStatus>>>,
|
||||
cx: &mut Context<'_>,
|
||||
registered: impl Fn(&AsyncFdData) -> &Cell<bool>,
|
||||
queue: impl Fn(&AsyncFdData) -> &Queue,
|
||||
) -> Poll<Result<FdStatus, AsyncError>> {
|
||||
if let Some(status) = woken.get() {
|
||||
return Poll::Ready(Ok(status));
|
||||
}
|
||||
if !registered(self).get() {
|
||||
registered(self).set(true);
|
||||
if let Err(e) = self.update_interests() {
|
||||
return Poll::Ready(Err(AsyncError::EventLoopError(e)));
|
||||
}
|
||||
}
|
||||
queue(self)
|
||||
.borrow_mut()
|
||||
.push((cx.waker().clone(), woken.clone()));
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
impl EventLoopDispatcher for AsyncFdData {
|
||||
fn dispatch(self: Rc<Self>, _fd: Option<i32>, events: i32) -> Result<(), Box<dyn Error>> {
|
||||
let mut status = FdStatus::Ok;
|
||||
if events & (c::EPOLLERR | c::EPOLLHUP) != 0 {
|
||||
status = FdStatus::Err;
|
||||
if let Err(e) = self.el.remove(self.id) {
|
||||
return Err(Box::new(e));
|
||||
}
|
||||
}
|
||||
let mut woke_any = false;
|
||||
if events & c::EPOLLIN != 0 || status == FdStatus::Err {
|
||||
let mut readers = self.readers.borrow_mut();
|
||||
woke_any |= !readers.is_empty();
|
||||
for (waker, woken) in readers.drain(..) {
|
||||
woken.set(Some(status));
|
||||
waker.wake();
|
||||
}
|
||||
}
|
||||
if events & c::EPOLLOUT != 0 || status == FdStatus::Err {
|
||||
let mut writers = self.writers.borrow_mut();
|
||||
woke_any |= !writers.is_empty();
|
||||
for (waker, woken) in writers.drain(..) {
|
||||
woken.set(Some(status));
|
||||
waker.wake();
|
||||
}
|
||||
}
|
||||
if !woke_any && status == FdStatus::Ok {
|
||||
self.read_registered.set(false);
|
||||
self.write_registered.set(false);
|
||||
if let Err(e) = self.update_interests() {
|
||||
return Err(Box::new(e));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AsyncFd {
|
||||
pub(super) engine: Rc<AsyncEngine>,
|
||||
pub(super) data: Rc<AsyncFdData>,
|
||||
}
|
||||
|
||||
impl Debug for AsyncFd {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("AsyncFd").finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl Clone for AsyncFd {
|
||||
fn clone(&self) -> Self {
|
||||
self.data.ref_count.fetch_add(1);
|
||||
Self {
|
||||
engine: self.engine.clone(),
|
||||
data: self.data.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AsyncFd {
|
||||
fn drop(&mut self) {
|
||||
if self.data.ref_count.fetch_sub(1) == 1 {
|
||||
self.engine.fds.remove(&self.data.fd.raw());
|
||||
let _ = self.data.el.remove(self.data.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncFd {
|
||||
pub fn raw(&self) -> i32 {
|
||||
self.data.fd.raw()
|
||||
}
|
||||
|
||||
pub fn readable(&self) -> AsyncFdReadable {
|
||||
AsyncFdReadable {
|
||||
fd: self,
|
||||
woken: Rc::new(Cell::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn writable(&self) -> AsyncFdWritable {
|
||||
AsyncFdWritable {
|
||||
fd: self,
|
||||
woken: Rc::new(Cell::new(None)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
|
||||
pub enum FdStatus {
|
||||
Ok,
|
||||
Err,
|
||||
}
|
||||
|
||||
pub struct AsyncFdReadable<'a> {
|
||||
fd: &'a AsyncFd,
|
||||
woken: Rc<Cell<Option<FdStatus>>>,
|
||||
}
|
||||
|
||||
impl<'a> Future for AsyncFdReadable<'a> {
|
||||
type Output = Result<FdStatus, AsyncError>;
|
||||
|
||||
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
|
||||
let data = &self.fd.data;
|
||||
data.poll(&self.woken, cx, |d| &d.read_registered, |d| &d.readers)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AsyncFdWritable<'a> {
|
||||
fd: &'a AsyncFd,
|
||||
woken: Rc<Cell<Option<FdStatus>>>,
|
||||
}
|
||||
|
||||
impl<'a> Future for AsyncFdWritable<'a> {
|
||||
type Output = Result<FdStatus, AsyncError>;
|
||||
|
||||
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
|
||||
let data = &self.fd.data;
|
||||
data.poll(&self.woken, cx, |d| &d.write_registered, |d| &d.writers)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,115 +0,0 @@
|
|||
use {
|
||||
crate::{
|
||||
async_engine::{ae_task::Runnable, AsyncError, Phase, NUM_PHASES},
|
||||
event_loop::{EventLoop, EventLoopDispatcher, EventLoopId},
|
||||
utils::{array, numcell::NumCell, syncqueue::SyncQueue},
|
||||
},
|
||||
std::{
|
||||
cell::{Cell, RefCell},
|
||||
collections::VecDeque,
|
||||
error::Error,
|
||||
rc::Rc,
|
||||
task::Waker,
|
||||
},
|
||||
};
|
||||
|
||||
pub(super) struct Dispatcher {
|
||||
queue: Rc<DispatchQueue>,
|
||||
stash: RefCell<VecDeque<Runnable>>,
|
||||
yield_stash: RefCell<VecDeque<Waker>>,
|
||||
}
|
||||
|
||||
impl Dispatcher {
|
||||
pub fn install(el: &Rc<EventLoop>) -> Result<Rc<DispatchQueue>, AsyncError> {
|
||||
let id = el.id();
|
||||
let queue = Rc::new(DispatchQueue {
|
||||
id,
|
||||
el: el.clone(),
|
||||
dispatch_scheduled: Cell::new(false),
|
||||
num_queued: Default::default(),
|
||||
queues: array::from_fn(|_| Default::default()),
|
||||
iteration: Default::default(),
|
||||
yields: Default::default(),
|
||||
});
|
||||
let slf = Rc::new(Dispatcher {
|
||||
queue: queue.clone(),
|
||||
stash: Default::default(),
|
||||
yield_stash: Default::default(),
|
||||
});
|
||||
el.insert(id, None, 0, slf)?;
|
||||
Ok(queue)
|
||||
}
|
||||
}
|
||||
|
||||
impl EventLoopDispatcher for Dispatcher {
|
||||
fn dispatch(self: Rc<Self>, _fd: Option<i32>, _events: i32) -> Result<(), Box<dyn Error>> {
|
||||
let mut stash = self.stash.borrow_mut();
|
||||
let mut yield_stash = self.yield_stash.borrow_mut();
|
||||
while self.queue.num_queued.get() > 0 {
|
||||
self.queue.iteration.fetch_add(1);
|
||||
let mut phase = 0;
|
||||
while phase < NUM_PHASES as usize {
|
||||
self.queue.queues[phase].swap(&mut *stash);
|
||||
if stash.is_empty() {
|
||||
phase += 1;
|
||||
continue;
|
||||
}
|
||||
self.queue.num_queued.fetch_sub(stash.len());
|
||||
for runnable in stash.drain(..) {
|
||||
runnable.run();
|
||||
}
|
||||
}
|
||||
self.queue.yields.swap(&mut *yield_stash);
|
||||
for waker in yield_stash.drain(..) {
|
||||
waker.wake();
|
||||
}
|
||||
}
|
||||
self.queue.dispatch_scheduled.set(false);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Dispatcher {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.queue.el.remove(self.queue.id);
|
||||
for queue in &self.queue.queues {
|
||||
queue.swap(&mut VecDeque::new());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct DispatchQueue {
|
||||
dispatch_scheduled: Cell<bool>,
|
||||
id: EventLoopId,
|
||||
el: Rc<EventLoop>,
|
||||
num_queued: NumCell<usize>,
|
||||
queues: [SyncQueue<Runnable>; NUM_PHASES],
|
||||
iteration: NumCell<u64>,
|
||||
yields: SyncQueue<Waker>,
|
||||
}
|
||||
|
||||
impl DispatchQueue {
|
||||
pub fn clear(&self) {
|
||||
self.yields.take();
|
||||
for queue in &self.queues {
|
||||
queue.take();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn push(&self, runnable: Runnable, phase: Phase) {
|
||||
self.queues[phase as usize].push(runnable);
|
||||
self.num_queued.fetch_add(1);
|
||||
if !self.dispatch_scheduled.get() {
|
||||
let _ = self.el.schedule(self.id);
|
||||
self.dispatch_scheduled.set(true);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn push_yield(&self, waker: Waker) {
|
||||
self.yields.push(waker);
|
||||
}
|
||||
|
||||
pub fn iteration(&self) -> u64 {
|
||||
self.iteration.get()
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
use {
|
||||
crate::{
|
||||
async_engine::{ae_queue::DispatchQueue, Phase},
|
||||
async_engine::{AsyncEngine, Phase},
|
||||
utils::{
|
||||
numcell::NumCell,
|
||||
ptr_ext::{MutPtrExt, PtrExt},
|
||||
|
|
@ -94,7 +94,7 @@ struct Task<T, F: Future<Output = T>> {
|
|||
state: NumCell<u32>,
|
||||
data: UnsafeCell<TaskData<T, F>>,
|
||||
waker: Cell<Option<Waker>>,
|
||||
queue: Rc<DispatchQueue>,
|
||||
queue: Rc<AsyncEngine>,
|
||||
}
|
||||
|
||||
pub(super) struct Runnable {
|
||||
|
|
@ -119,8 +119,8 @@ impl Drop for Runnable {
|
|||
}
|
||||
}
|
||||
|
||||
impl DispatchQueue {
|
||||
pub(super) fn spawn<T, F: Future<Output = T>>(
|
||||
impl AsyncEngine {
|
||||
pub(super) fn spawn_<T, F: Future<Output = T>>(
|
||||
self: &Rc<Self>,
|
||||
phase: Phase,
|
||||
f: F,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use {
|
||||
crate::async_engine::ae_queue::DispatchQueue,
|
||||
crate::async_engine::AsyncEngine,
|
||||
std::{
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
|
|
@ -10,7 +10,7 @@ use {
|
|||
|
||||
pub struct Yield {
|
||||
pub(super) iteration: u64,
|
||||
pub(super) queue: Rc<DispatchQueue>,
|
||||
pub(super) queue: Rc<AsyncEngine>,
|
||||
}
|
||||
|
||||
impl Future for Yield {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue