From 3619a51fbd630df7157c685bc50b029a224e2948 Mon Sep 17 00:00:00 2001 From: Julian Orth Date: Sat, 5 Oct 2024 16:49:22 +0200 Subject: [PATCH 1/7] gfx: add GfxStagingBuffer --- src/gfx_api.rs | 35 +++++++++++ src/gfx_apis/gl/renderer/texture.rs | 5 +- src/gfx_apis/vulkan.rs | 19 +++++- src/gfx_apis/vulkan/image.rs | 14 ++++- src/gfx_apis/vulkan/shm_image.rs | 39 +++++++----- src/gfx_apis/vulkan/staging.rs | 89 +++++++++++++++++++++++---- src/ifs/wl_buffer.rs | 1 + src/ifs/wl_surface.rs | 7 ++- src/ifs/wl_surface/commit_timeline.rs | 24 +++++++- src/it/test_gfx_api.rs | 5 +- src/text.rs | 20 +++++- 11 files changed, 215 insertions(+), 43 deletions(-) diff --git a/src/gfx_api.rs b/src/gfx_api.rs index a8214101..e98cd032 100644 --- a/src/gfx_api.rs +++ b/src/gfx_api.rs @@ -503,6 +503,17 @@ pub trait AsyncShmGfxTextureCallback { fn completed(self: Rc, res: Result<(), GfxError>); } +bitflags! { + StagingBufferUsecase: u32; + STAGING_UPLOAD = 1 << 0, + STAGING_DOWNLOAD = 1 << 1, +} + +pub trait GfxStagingBuffer { + fn size(&self) -> usize; + fn into_any(self: Rc) -> Rc; +} + pub trait AsyncShmGfxTextureUploadCancellable { fn cancel(&self, id: u64); } @@ -539,8 +550,13 @@ impl ShmMemory for Vec> { } pub trait AsyncShmGfxTexture: GfxTexture { + fn staging_size(&self) -> usize { + 0 + } + fn async_upload( self: Rc, + staging: &Rc, callback: Rc, mem: Rc, damage: Region, @@ -605,6 +621,25 @@ pub trait GfxContext: Debug { ) -> Result, GfxError>; fn sync_obj_ctx(&self) -> Option<&Rc>; + + fn create_staging_buffer( + &self, + size: usize, + usecase: StagingBufferUsecase, + ) -> Rc { + let _ = usecase; + struct Dummy(usize); + impl GfxStagingBuffer for Dummy { + fn size(&self) -> usize { + self.0 + } + + fn into_any(self: Rc) -> Rc { + self + } + } + Rc::new(Dummy(size)) + } } #[derive(Clone, Debug)] diff --git a/src/gfx_apis/gl/renderer/texture.rs b/src/gfx_apis/gl/renderer/texture.rs index 7add2b78..7a4e1103 100644 --- a/src/gfx_apis/gl/renderer/texture.rs +++ b/src/gfx_apis/gl/renderer/texture.rs @@ -2,8 +2,8 @@ use { crate::{ format::Format, gfx_api::{ - AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxError, GfxTexture, PendingShmUpload, - ShmGfxTexture, ShmMemory, + AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxError, GfxStagingBuffer, GfxTexture, + PendingShmUpload, ShmGfxTexture, ShmMemory, }, gfx_apis::gl::{ gl::texture::GlTexture, @@ -100,6 +100,7 @@ impl ShmGfxTexture for Texture { impl AsyncShmGfxTexture for Texture { fn async_upload( self: Rc, + _staging: &Rc, _callback: Rc, mem: Rc, _damage: Region, diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs index 4562377b..0924f9ab 100644 --- a/src/gfx_apis/vulkan.rs +++ b/src/gfx_apis/vulkan.rs @@ -23,7 +23,8 @@ use { format::Format, gfx_api::{ AsyncShmGfxTexture, GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage, - ResetStatus, ShmGfxTexture, + GfxStagingBuffer, ResetStatus, ShmGfxTexture, StagingBufferUsecase, STAGING_DOWNLOAD, + STAGING_UPLOAD, }, gfx_apis::vulkan::{ image::VulkanImageMemory, instance::VulkanInstance, renderer::VulkanRenderer, @@ -203,6 +204,10 @@ pub enum VulkanError { AsyncCopyToStaging(#[source] ReadWriteJobError), #[error("The async shm texture is busy")] AsyncCopyBusy, + #[error("The staging buffer is busy")] + StagingBufferBusy, + #[error("The staging buffer does not support uploads")] + StagingBufferNoUpload, } impl From for GfxError { @@ -330,6 +335,18 @@ impl GfxContext for Context { fn sync_obj_ctx(&self) -> Option<&Rc> { Some(&self.0.device.sync_ctx) } + + fn create_staging_buffer( + &self, + size: usize, + usecase: StagingBufferUsecase, + ) -> Rc { + let upload = usecase.contains(STAGING_UPLOAD); + let download = usecase.contains(STAGING_DOWNLOAD); + self.0 + .device + .create_staging_shell(size as u64, upload, download) + } } impl Drop for Context { diff --git a/src/gfx_apis/vulkan/image.rs b/src/gfx_apis/vulkan/image.rs index 6826c2f6..6df5788a 100644 --- a/src/gfx_apis/vulkan/image.rs +++ b/src/gfx_apis/vulkan/image.rs @@ -4,7 +4,8 @@ use { gfx_api::{ AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback, AsyncShmGfxTextureUploadCancellable, GfxApiOpt, GfxError, GfxFramebuffer, GfxImage, - GfxTexture, PendingShmUpload, ReleaseSync, ShmGfxTexture, ShmMemory, SyncFile, + GfxStagingBuffer, GfxTexture, PendingShmUpload, ReleaseSync, ShmGfxTexture, ShmMemory, + SyncFile, }, gfx_apis::vulkan::{ allocator::VulkanAllocation, device::VulkanDevice, format::VulkanModifierLimits, @@ -575,8 +576,16 @@ impl ShmGfxTexture for VulkanImage { } impl AsyncShmGfxTexture for VulkanImage { + fn staging_size(&self) -> usize { + let VulkanImageMemory::Internal(shm) = &self.ty else { + unreachable!(); + }; + shm.size as _ + } + fn async_upload( self: Rc, + staging: &Rc, callback: Rc, mem: Rc, damage: Region, @@ -584,7 +593,8 @@ impl AsyncShmGfxTexture for VulkanImage { let VulkanImageMemory::Internal(shm) = &self.ty else { unreachable!(); }; - let pending = shm.async_upload(&self, &mem, damage, callback)?; + let staging = staging.clone().into_vk(&self.renderer.device.device); + let pending = shm.async_upload(&self, staging, &mem, damage, callback)?; Ok(pending) } diff --git a/src/gfx_apis/vulkan/shm_image.rs b/src/gfx_apis/vulkan/shm_image.rs index ced7ea0a..2b11f6ad 100644 --- a/src/gfx_apis/vulkan/shm_image.rs +++ b/src/gfx_apis/vulkan/shm_image.rs @@ -17,7 +17,7 @@ use { fence::VulkanFence, image::{QueueFamily, QueueState, QueueTransfer, VulkanImage, VulkanImageMemory}, renderer::{image_barrier, VulkanRenderer}, - staging::VulkanStagingBuffer, + staging::{VulkanStagingBuffer, VulkanStagingShell}, VulkanError, }, rect::{Rect, Region}, @@ -55,7 +55,7 @@ pub struct VulkanShmImageAsyncData { pub(super) busy: Cell, pub(super) io_job: Cell>>, pub(super) copy_job: Cell>>, - pub(super) staging: CloneCell>>, + pub(super) staging: CloneCell>>, pub(super) callback: Cell>>, pub(super) callback_id: Cell, pub(super) regions: RefCell>>, @@ -320,6 +320,7 @@ async fn await_upload( impl VulkanShmImageAsyncData { fn complete(&self, result: Result<(), VulkanError>) { self.busy.set(false); + self.staging.take().unwrap().busy.set(false); if let Some(cb) = self.callback.take() { cb.completed(result.map_err(|e| e.into())); } @@ -330,12 +331,13 @@ impl VulkanShmImage { pub fn async_upload( &self, img: &Rc, + staging: Rc, client_mem: &Rc, damage: Region, callback: Rc, ) -> Result, VulkanError> { let data = self.async_data.as_ref().unwrap(); - let res = self.try_async_upload(img, data, client_mem, damage); + let res = self.try_async_upload(img, staging, data, client_mem, damage); match res { Ok(()) => { let id = img.renderer.allocate_point(); @@ -350,6 +352,7 @@ impl VulkanShmImage { fn try_async_upload( &self, img: &Rc, + staging: Rc, data: &VulkanShmImageAsyncData, client_mem: &Rc, mut damage: Region, @@ -357,11 +360,19 @@ impl VulkanShmImage { if data.busy.get() { return Err(VulkanError::AsyncCopyBusy); } + if staging.busy.get() { + return Err(VulkanError::StagingBufferBusy); + } + if !staging.upload { + return Err(VulkanError::StagingBufferNoUpload); + } if self.size > client_mem.len() as u64 { return Err(VulkanError::InvalidBufferSize); } data.busy.set(true); data.data_copied.set(false); + staging.busy.set(true); + data.staging.set(Some(staging.clone())); if img.contents_are_undefined.get() { damage = Region::new2(Rect::new_sized(0, 0, img.width as _, img.height as _).unwrap()); } @@ -416,27 +427,22 @@ impl VulkanShmImage { self.async_release_from_gfx_queue(img, data)?; - if let Some(staging) = data.staging.get() { + if let Some(staging) = staging.staging.get() { return self.async_upload_initiate_copy(img, data, &staging, copies, client_mem); } let img2 = img.clone(); let client_mem = client_mem.clone(); - img.renderer.device.create_shm_staging( - &img.renderer, - &data.cpu, - self.size, - true, - false, - move |res| { + img.renderer + .device + .fill_staging_shell(&img.renderer, &data.cpu, staging, move |res| { let VulkanImageMemory::Internal(shm) = &img2.ty else { unreachable!(); }; if let Err(e) = shm.async_upload_after_allocation(&img2, &client_mem, res) { shm.async_data.as_ref().unwrap().complete(Err(e)); } - }, - ) + }) } fn async_release_from_gfx_queue( @@ -532,11 +538,10 @@ impl VulkanShmImage { &self, img: &Rc, client_mem: &Rc, - res: Result, + res: Result, VulkanError>, ) -> Result<(), VulkanError> { - let staging = Rc::new(res?); + let staging = res?; let data = self.async_data.as_ref().unwrap(); - data.staging.set(Some(staging.clone())); let copies = &*data.regions.borrow(); self.async_upload_initiate_copy(img, data, &staging, copies, client_mem) } @@ -637,7 +642,7 @@ impl VulkanShmImage { } img.renderer.check_defunct()?; let regions = &*data.regions.borrow(); - let staging = data.staging.get().unwrap(); + let staging = data.staging.get().unwrap().staging.get().unwrap(); staging.upload(|_, _| ())?; let Some((cmd, fence, sync_file, point)) = self.submit_buffer_to_image_copy(img, &staging, regions, true)? diff --git a/src/gfx_apis/vulkan/staging.rs b/src/gfx_apis/vulkan/staging.rs index 5a3faeac..d8fa685b 100644 --- a/src/gfx_apis/vulkan/staging.rs +++ b/src/gfx_apis/vulkan/staging.rs @@ -1,17 +1,24 @@ use { crate::{ cpu_worker::CpuWorker, + gfx_api::GfxStagingBuffer, gfx_apis::vulkan::{ allocator::{VulkanAllocation, VulkanAllocator}, device::VulkanDevice, renderer::VulkanRenderer, VulkanError, }, - utils::on_drop::{OnDrop, OnDrop2}, + utils::{ + clonecell::CloneCell, + on_drop::{OnDrop, OnDrop2}, + }, + }, + ash::{ + vk::{Buffer, BufferCreateInfo, BufferUsageFlags, MappedMemoryRange}, + Device, }, - ash::vk::{Buffer, BufferCreateInfo, BufferUsageFlags, MappedMemoryRange}, gpu_alloc::UsageFlags, - std::rc::Rc, + std::{any::Any, cell::Cell, rc::Rc}, }; pub struct VulkanStagingBuffer { @@ -22,6 +29,22 @@ pub struct VulkanStagingBuffer { } impl VulkanDevice { + pub(super) fn create_staging_shell( + self: &Rc, + size: u64, + upload: bool, + download: bool, + ) -> Rc { + Rc::new(VulkanStagingShell { + device: self.clone(), + staging: Default::default(), + size, + download, + upload, + busy: Cell::new(false), + }) + } + pub(super) fn create_staging_buffer( self: &Rc, allocator: &Rc, @@ -51,17 +74,15 @@ impl VulkanDevice { }) } - pub(super) fn create_shm_staging( + pub(super) fn fill_staging_shell( self: &Rc, renderer: &Rc, cpu: &Rc, - size: u64, - upload: bool, - download: bool, - cb: impl FnOnce(Result) + 'static, + shell: Rc, + cb: impl FnOnce(Result, VulkanError>) + 'static, ) -> Result<(), VulkanError> { - let (vk_usage, usage) = get_usage(upload, download, false); - let buffer = self.create_buffer(size, vk_usage)?; + let (vk_usage, usage) = get_usage(shell.upload, shell.download, false); + let buffer = self.create_buffer(shell.size, vk_usage)?; let memory_requirements = unsafe { self.device.get_buffer_memory_requirements(buffer) }; let slf = self.clone(); let destroy_buffer = @@ -77,12 +98,14 @@ impl VulkanDevice { res.map_err(VulkanError::BindBufferMemory)?; } destroy_buffer.forget(); - Ok(VulkanStagingBuffer { + let buffer = Rc::new(VulkanStagingBuffer { device: slf.clone(), allocation, buffer, - size, - }) + size: shell.size, + }); + shell.staging.set(Some(buffer.clone())); + Ok(buffer) }; renderer.shm_allocator.async_alloc( renderer, @@ -159,3 +182,43 @@ impl Drop for VulkanStagingBuffer { } } } + +pub(super) struct VulkanStagingShell { + pub(super) device: Rc, + pub(super) staging: CloneCell>>, + pub(super) size: u64, + pub(super) download: bool, + pub(super) upload: bool, + pub(super) busy: Cell, +} + +impl GfxStagingBuffer for VulkanStagingShell { + fn size(&self) -> usize { + self.size as _ + } + + fn into_any(self: Rc) -> Rc { + self + } +} + +impl VulkanStagingShell { + fn assert_device(&self, device: &Device) { + assert_eq!( + self.device.device.handle(), + device.handle(), + "Mixed vulkan device use" + ); + } +} + +impl dyn GfxStagingBuffer { + pub(super) fn into_vk(self: Rc, device: &Device) -> Rc { + let shell: Rc = self + .into_any() + .downcast() + .expect("Non-vulkan staging buffer passed into vulkan"); + shell.assert_device(device); + shell + } +} diff --git a/src/ifs/wl_buffer.rs b/src/ifs/wl_buffer.rs index 3068d572..d60de1c9 100644 --- a/src/ifs/wl_buffer.rs +++ b/src/ifs/wl_buffer.rs @@ -175,6 +175,7 @@ impl WlBuffer { WlBufferStorage::Shm { .. } => { return match surface { Some(s) => { + s.shm_staging.take(); s.shm_textures.back().tex.take(); s.shm_textures.front().tex.take().is_some() } diff --git a/src/ifs/wl_surface.rs b/src/ifs/wl_surface.rs index 61df4dd0..f8559cb2 100644 --- a/src/ifs/wl_surface.rs +++ b/src/ifs/wl_surface.rs @@ -23,8 +23,8 @@ use { drm_feedback::DrmFeedback, fixed::Fixed, gfx_api::{ - AsyncShmGfxTexture, BufferResv, BufferResvUser, GfxError, ReleaseSync, SampleRect, - SyncFile, + AsyncShmGfxTexture, BufferResv, BufferResvUser, GfxError, GfxStagingBuffer, + ReleaseSync, SampleRect, SyncFile, }, ifs::{ wl_buffer::WlBuffer, @@ -276,6 +276,7 @@ pub struct WlSurface { pub buffer_abs_pos: Cell, pub need_extents_update: Cell, pub buffer: CloneCell>>, + pub shm_staging: CloneCell>>, pub shm_textures: DoubleBuffered, pub buf_x: NumCell, pub buf_y: NumCell, @@ -593,6 +594,7 @@ impl WlSurface { buffer_abs_pos: Cell::new(Default::default()), need_extents_update: Default::default(), buffer: Default::default(), + shm_staging: Default::default(), shm_textures: DoubleBuffered::new(DamageQueue::new().map(|damage| SurfaceShmTexture { tex: Default::default(), damage, @@ -1327,6 +1329,7 @@ impl WlSurface { } pub fn reset_shm_textures(&self) { + self.shm_staging.take(); for tex in &*self.shm_textures { tex.tex.take(); tex.damage.clear(); diff --git a/src/ifs/wl_surface/commit_timeline.rs b/src/ifs/wl_surface/commit_timeline.rs index 544d98a6..a973ddbb 100644 --- a/src/ifs/wl_surface/commit_timeline.rs +++ b/src/ifs/wl_surface/commit_timeline.rs @@ -1,6 +1,6 @@ use { crate::{ - gfx_api::{AsyncShmGfxTextureCallback, GfxError, PendingShmUpload}, + gfx_api::{AsyncShmGfxTextureCallback, GfxError, PendingShmUpload, STAGING_UPLOAD}, ifs::{ wl_buffer::WlBufferStorage, wl_surface::{PendingState, WlSurface, WlSurfaceError}, @@ -459,8 +459,28 @@ fn schedule_async_upload( back_tex } }; + let mut staging_opt = surface.shm_staging.get(); + if let Some(staging) = &staging_opt { + if staging.size() != back_tex.staging_size() { + staging_opt = None; + } + } + let staging = match staging_opt { + Some(s) => s, + None => { + let s = surface + .client + .state + .render_ctx + .get() + .ok_or(WlSurfaceError::NoRenderContext)? + .create_staging_buffer(back_tex.staging_size(), STAGING_UPLOAD); + surface.shm_staging.set(Some(s.clone())); + s + } + }; back_tex - .async_upload(node_ref.clone(), mem.clone(), back.damage.get()) + .async_upload(&staging, node_ref.clone(), mem.clone(), back.damage.get()) .map_err(WlSurfaceError::PrepareAsyncUpload) } diff --git a/src/it/test_gfx_api.rs b/src/it/test_gfx_api.rs index 95c52864..6ba86de9 100644 --- a/src/it/test_gfx_api.rs +++ b/src/it/test_gfx_api.rs @@ -6,8 +6,8 @@ use { gfx_api::{ AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback, CopyTexture, FillRect, FramebufferRect, GfxApiOpt, GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage, - GfxTexture, GfxWriteModifier, PendingShmUpload, ReleaseSync, ResetStatus, - ShmGfxTexture, ShmMemory, SyncFile, + GfxStagingBuffer, GfxTexture, GfxWriteModifier, PendingShmUpload, ReleaseSync, + ResetStatus, ShmGfxTexture, ShmMemory, SyncFile, }, rect::{Rect, Region}, theme::Color, @@ -335,6 +335,7 @@ impl ShmGfxTexture for TestGfxImage { impl AsyncShmGfxTexture for TestGfxImage { fn async_upload( self: Rc, + _staging: &Rc, _callback: Rc, mem: Rc, _damage: Region, diff --git a/src/text.rs b/src/text.rs index 7eed4e21..a097a511 100644 --- a/src/text.rs +++ b/src/text.rs @@ -3,8 +3,8 @@ use { cpu_worker::{AsyncCpuWork, CpuJob, CpuWork, CpuWorker, PendingJob}, format::ARGB8888, gfx_api::{ - AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxContext, GfxError, GfxTexture, - PendingShmUpload, + AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxContext, GfxError, GfxStagingBuffer, + GfxTexture, PendingShmUpload, STAGING_UPLOAD, }, pango::{ consts::{ @@ -302,6 +302,7 @@ impl Drop for TextTexture { struct Shared { cpu_worker: Rc, ctx: Rc, + staging: CloneCell>>, textures: DoubleBuffered, pending_render: Cell>, pending_upload: Cell>, @@ -367,6 +368,7 @@ impl TextTexture { let data = Rc::new(Shared { cpu_worker: cpu_worker.clone(), ctx: ctx.clone(), + staging: Default::default(), textures: Default::default(), pending_render: Default::default(), pending_upload: Default::default(), @@ -528,9 +530,22 @@ impl CpuJob for RenderJob { } } }; + let mut staging_opt = data.staging.take(); + if let Some(staging) = &staging_opt { + if staging.size() != tex.staging_size() { + staging_opt = None; + } + } + let staging = match staging_opt { + Some(s) => s, + None => data + .ctx + .create_staging_buffer(tex.staging_size(), STAGING_UPLOAD), + }; let pending = tex .clone() .async_upload( + &staging, data.clone(), Rc::new(rt.data), Region::new2(Rect::new_sized_unchecked(0, 0, rt.width, rt.height)), @@ -538,6 +553,7 @@ impl CpuJob for RenderJob { .map_err(TextError::Upload); if pending.is_ok() { data.textures.back().tex.set(Some(tex)); + data.staging.set(Some(staging)); } match pending { Ok(Some(p)) => data.pending_upload.set(Some(p)), From 59f06dc2081e54d5c7395af7caf2221a2585f54a Mon Sep 17 00:00:00 2001 From: Julian Orth Date: Sun, 6 Oct 2024 13:07:18 +0200 Subject: [PATCH 2/7] vulkan: move async transfer code to separate module --- src/gfx_apis/vulkan.rs | 1 + src/gfx_apis/vulkan/shm_image.rs | 512 +----------------------------- src/gfx_apis/vulkan/transfer.rs | 515 +++++++++++++++++++++++++++++++ 3 files changed, 526 insertions(+), 502 deletions(-) create mode 100644 src/gfx_apis/vulkan/transfer.rs diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs index 0924f9ab..bbeb6c5f 100644 --- a/src/gfx_apis/vulkan.rs +++ b/src/gfx_apis/vulkan.rs @@ -14,6 +14,7 @@ mod semaphore; mod shaders; mod shm_image; mod staging; +mod transfer; use { crate::{ diff --git a/src/gfx_apis/vulkan/shm_image.rs b/src/gfx_apis/vulkan/shm_image.rs index 2b11f6ad..a342a453 100644 --- a/src/gfx_apis/vulkan/shm_image.rs +++ b/src/gfx_apis/vulkan/shm_image.rs @@ -1,32 +1,24 @@ use { crate::{ - cpu_worker::{ - jobs::{ - img_copy::ImgCopyWork, - read_write::{ReadWriteJobError, ReadWriteWork}, - }, - CpuJob, CpuWork, CpuWorker, - }, + cpu_worker::CpuWorker, format::{Format, FormatShmInfo}, - gfx_api::{ - AsyncShmGfxTextureCallback, PendingShmUpload, ShmMemory, ShmMemoryBacking, SyncFile, - }, + gfx_api::SyncFile, gfx_apis::vulkan::{ allocator::VulkanAllocation, command::VulkanCommandBuffer, fence::VulkanFence, - image::{QueueFamily, QueueState, QueueTransfer, VulkanImage, VulkanImageMemory}, + image::{QueueFamily, QueueState, VulkanImage, VulkanImageMemory}, renderer::{image_barrier, VulkanRenderer}, - staging::{VulkanStagingBuffer, VulkanStagingShell}, + staging::VulkanStagingBuffer, + transfer::VulkanShmImageAsyncData, VulkanError, }, - rect::{Rect, Region}, - utils::{clonecell::CloneCell, errorfmt::ErrorFmt, on_drop::OnDrop}, + rect::Rect, + utils::{errorfmt::ErrorFmt, on_drop::OnDrop}, }, - arrayvec::ArrayVec, ash::vk::{ AccessFlags2, BufferImageCopy2, BufferMemoryBarrier2, CommandBufferBeginInfo, - CommandBufferSubmitInfo, CommandBufferUsageFlags, CopyBufferToImageInfo2, DependencyInfo, + CommandBufferSubmitInfo, CommandBufferUsageFlags, CopyBufferToImageInfo2, DependencyInfoKHR, DeviceSize, Extent3D, ImageAspectFlags, ImageCreateInfo, ImageLayout, ImageSubresourceLayers, ImageSubresourceRange, ImageTiling, ImageType, ImageUsageFlags, ImageViewCreateInfo, ImageViewType, Offset3D, PipelineStageFlags2, SampleCountFlags, @@ -34,13 +26,7 @@ use { }, gpu_alloc::UsageFlags, isnt::std_1::primitive::IsntSliceExt, - std::{ - cell::{Cell, RefCell}, - ptr, - rc::Rc, - slice, - }, - uapi::OwnedFd, + std::{cell::Cell, ptr, rc::Rc, slice}, }; pub struct VulkanShmImage { @@ -51,19 +37,6 @@ pub struct VulkanShmImage { pub(super) async_data: Option, } -pub struct VulkanShmImageAsyncData { - pub(super) busy: Cell, - pub(super) io_job: Cell>>, - pub(super) copy_job: Cell>>, - pub(super) staging: CloneCell>>, - pub(super) callback: Cell>>, - pub(super) callback_id: Cell, - pub(super) regions: RefCell>>, - pub(super) cpu: Rc, - pub(super) last_sample: Cell>, - pub(super) data_copied: Cell, -} - impl VulkanShmImage { pub fn upload( &self, @@ -176,7 +149,7 @@ impl VulkanShmImage { Ok(()) } - fn submit_buffer_to_image_copy( + pub(super) fn submit_buffer_to_image_copy( &self, img: &Rc, staging: &VulkanStagingBuffer, @@ -317,471 +290,6 @@ async fn await_upload( img.renderer.pending_submits.remove(&id); } -impl VulkanShmImageAsyncData { - fn complete(&self, result: Result<(), VulkanError>) { - self.busy.set(false); - self.staging.take().unwrap().busy.set(false); - if let Some(cb) = self.callback.take() { - cb.completed(result.map_err(|e| e.into())); - } - } -} - -impl VulkanShmImage { - pub fn async_upload( - &self, - img: &Rc, - staging: Rc, - client_mem: &Rc, - damage: Region, - callback: Rc, - ) -> Result, VulkanError> { - let data = self.async_data.as_ref().unwrap(); - let res = self.try_async_upload(img, staging, data, client_mem, damage); - match res { - Ok(()) => { - let id = img.renderer.allocate_point(); - data.callback_id.set(id); - data.callback.set(Some(callback)); - Ok(Some(PendingShmUpload::new(img.clone(), id))) - } - Err(e) => Err(e), - } - } - - fn try_async_upload( - &self, - img: &Rc, - staging: Rc, - data: &VulkanShmImageAsyncData, - client_mem: &Rc, - mut damage: Region, - ) -> Result<(), VulkanError> { - if data.busy.get() { - return Err(VulkanError::AsyncCopyBusy); - } - if staging.busy.get() { - return Err(VulkanError::StagingBufferBusy); - } - if !staging.upload { - return Err(VulkanError::StagingBufferNoUpload); - } - if self.size > client_mem.len() as u64 { - return Err(VulkanError::InvalidBufferSize); - } - data.busy.set(true); - data.data_copied.set(false); - staging.busy.set(true); - data.staging.set(Some(staging.clone())); - if img.contents_are_undefined.get() { - damage = Region::new2(Rect::new_sized(0, 0, img.width as _, img.height as _).unwrap()); - } - - let copies = &mut *data.regions.borrow_mut(); - copies.clear(); - - let mut copy = |x, y, width, height| { - let buffer_offset = (y as u32 * img.stride + x as u32 * self.shm_info.bpp) as u64; - let copy = BufferImageCopy2::default() - .buffer_offset(buffer_offset) - .image_offset(Offset3D { x, y, z: 0 }) - .image_extent(Extent3D { - width, - height, - depth: 1, - }) - .image_subresource(ImageSubresourceLayers { - aspect_mask: ImageAspectFlags::COLOR, - mip_level: 0, - base_array_layer: 0, - layer_count: 1, - }) - .buffer_image_height(img.height) - .buffer_row_length(img.stride / self.shm_info.bpp); - copies.push(copy); - }; - let (width_mask, height_mask) = img.renderer.device.transfer_granularity_mask; - let width_mask = width_mask as i32; - let height_mask = height_mask as i32; - for damage in damage.rects() { - if damage.x2() < 0 || damage.y2() < 0 { - continue; - } - let x1 = damage.x1().max(0) & !width_mask; - let y1 = damage.y1().max(0) & !height_mask; - let x2 = ((damage.x2() + width_mask) & !width_mask).min(img.width as i32); - let y2 = ((damage.y2() + height_mask) & !height_mask).min(img.height as i32); - let Some(damage) = Rect::new(x1, y1, x2, y2) else { - continue; - }; - if damage.is_empty() { - continue; - } - copy( - damage.x1(), - damage.y1(), - damage.width() as u32, - damage.height() as u32, - ); - } - - self.async_release_from_gfx_queue(img, data)?; - - if let Some(staging) = staging.staging.get() { - return self.async_upload_initiate_copy(img, data, &staging, copies, client_mem); - } - - let img2 = img.clone(); - let client_mem = client_mem.clone(); - img.renderer - .device - .fill_staging_shell(&img.renderer, &data.cpu, staging, move |res| { - let VulkanImageMemory::Internal(shm) = &img2.ty else { - unreachable!(); - }; - if let Err(e) = shm.async_upload_after_allocation(&img2, &client_mem, res) { - shm.async_data.as_ref().unwrap().complete(Err(e)); - } - }) - } - - fn async_release_from_gfx_queue( - &self, - img: &Rc, - data: &VulkanShmImageAsyncData, - ) -> Result<(), VulkanError> { - img.renderer.check_defunct()?; - let Some(transfer_queue_idx) = img.renderer.device.distinct_transfer_queue_family_idx - else { - let Some(sync_file) = data.last_sample.take() else { - img.queue_state.set(QueueState::Released { - to: QueueFamily::Transfer, - }); - return Ok(()); - }; - let id = img.renderer.allocate_point(); - let pending = img.renderer.eng.spawn( - "await_transfer_to_transfer", - await_gfx_queue_release(id, img.clone(), None, None, sync_file), - ); - img.renderer.pending_submits.set(id, pending); - img.queue_state.set(QueueState::Releasing); - return Ok(()); - }; - let mut barriers = ArrayVec::<_, 2>::new(); - match img.queue_state.get() { - QueueState::Acquired { family } => { - assert_eq!(family, QueueFamily::Gfx); - } - QueueState::Releasing => { - unreachable!(); - } - QueueState::Released { to } => { - assert_eq!(to, QueueFamily::Gfx); - let barrier = image_barrier() - .image(img.image) - .src_queue_family_index(transfer_queue_idx) - .dst_queue_family_index(img.renderer.device.graphics_queue_idx) - .dst_stage_mask(PipelineStageFlags2::ALL_COMMANDS) - .old_layout(ImageLayout::TRANSFER_DST_OPTIMAL) - .new_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL); - barriers.push(barrier); - } - } - let barrier = image_barrier() - .image(img.image) - .src_queue_family_index(img.renderer.device.graphics_queue_idx) - .dst_queue_family_index(transfer_queue_idx) - .src_access_mask(AccessFlags2::SHADER_SAMPLED_READ) - .src_stage_mask(PipelineStageFlags2::ALL_COMMANDS) - .old_layout(if img.is_undefined.get() { - ImageLayout::UNDEFINED - } else { - ImageLayout::SHADER_READ_ONLY_OPTIMAL - }) - .new_layout(ImageLayout::TRANSFER_DST_OPTIMAL); - barriers.push(barrier); - let dep_info = DependencyInfo::default().image_memory_barriers(&barriers); - let release_fence = img.renderer.device.create_fence()?; - let dev = &img.renderer.device.device; - let begin_info = - CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT); - let cmd = img.renderer.gfx_command_buffers.allocate()?; - let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd.buffer); - let submit_info = - SubmitInfo2::default().command_buffer_infos(slice::from_ref(&command_buffer_info)); - unsafe { - dev.begin_command_buffer(cmd.buffer, &begin_info) - .map_err(VulkanError::BeginCommandBuffer)?; - dev.cmd_pipeline_barrier2(cmd.buffer, &dep_info); - dev.end_command_buffer(cmd.buffer) - .map_err(VulkanError::EndCommandBuffer)?; - dev.queue_submit2( - img.renderer.device.graphics_queue, - slice::from_ref(&submit_info), - release_fence.fence, - ) - .map_err(VulkanError::Submit)?; - } - let sync_file = release_fence.export_sync_file()?; - let id = img.renderer.allocate_point(); - let pending = img.renderer.eng.spawn( - "await_transfer_to_transfer", - await_gfx_queue_release(id, img.clone(), Some(cmd), Some(release_fence), sync_file), - ); - img.renderer.pending_submits.set(id, pending); - img.queue_state.set(QueueState::Releasing); - Ok(()) - } - - fn async_upload_after_allocation( - &self, - img: &Rc, - client_mem: &Rc, - res: Result, VulkanError>, - ) -> Result<(), VulkanError> { - let staging = res?; - let data = self.async_data.as_ref().unwrap(); - let copies = &*data.regions.borrow(); - self.async_upload_initiate_copy(img, data, &staging, copies, client_mem) - } - - fn async_upload_initiate_copy( - &self, - img: &Rc, - data: &VulkanShmImageAsyncData, - staging: &VulkanStagingBuffer, - copies: &[BufferImageCopy2], - client_mem: &Rc, - ) -> Result<(), VulkanError> { - img.renderer.check_defunct()?; - - let id = img.renderer.allocate_point(); - let pending; - match client_mem.safe_access() { - ShmMemoryBacking::Ptr(ptr) => { - let mut job = data.copy_job.take().unwrap_or_else(|| { - Box::new(CopyUploadJob { - img: None, - id, - _mem: None, - work: unsafe { ImgCopyWork::new() }, - }) - }); - job.id = id; - job.img = Some(img.clone()); - job._mem = Some(client_mem.clone()); - job.work.src = ptr as _; - job.work.dst = staging.allocation.mem.unwrap(); - job.work.width = img.width as _; - job.work.stride = img.stride as _; - job.work.bpp = self.shm_info.bpp as _; - job.work.rects.clear(); - for copy in copies { - job.work.rects.push( - Rect::new_sized( - copy.image_offset.x as _, - copy.image_offset.y as _, - copy.image_extent.width as _, - copy.image_extent.height as _, - ) - .unwrap(), - ); - } - pending = data.cpu.submit(job); - } - ShmMemoryBacking::Fd(fd, offset) => { - let mut min_offset = client_mem.len() as u64; - let mut max_offset = 0; - for copy in copies { - min_offset = min_offset.min(copy.buffer_offset); - let len = img.stride * (copy.image_extent.height - 1) - + copy.image_extent.width * self.shm_info.bpp; - max_offset = max_offset.max(copy.buffer_offset + len as u64); - } - let mut job = data.io_job.take().unwrap_or_else(|| { - Box::new(IoUploadJob { - img: None, - id, - _mem: None, - work: unsafe { ReadWriteWork::new() }, - fd: None, - }) - }); - job.id = id; - job.img = Some(img.clone()); - job._mem = Some(client_mem.clone()); - job.fd = Some(fd.clone()); - unsafe { - let config = job.work.config(); - config.fd = fd.raw(); - config.offset = offset + min_offset as usize; - config.ptr = staging.allocation.mem.unwrap().add(min_offset as _); - config.len = max_offset.saturating_sub(min_offset) as usize; - config.write = false; - } - pending = data.cpu.submit(job); - } - } - - img.renderer.pending_cpu_jobs.set(id, pending); - - Ok(()) - } - - fn async_upload_copy_buffer_to_image( - &self, - img: &Rc, - data: &VulkanShmImageAsyncData, - ) -> Result<(), VulkanError> { - if !data.data_copied.get() { - return Ok(()); - } - if img.queue_state.get().acquire(QueueFamily::Transfer) == QueueTransfer::Impossible { - return Ok(()); - } - img.renderer.check_defunct()?; - let regions = &*data.regions.borrow(); - let staging = data.staging.get().unwrap().staging.get().unwrap(); - staging.upload(|_, _| ())?; - let Some((cmd, fence, sync_file, point)) = - self.submit_buffer_to_image_copy(img, &staging, regions, true)? - else { - return Ok(()); - }; - img.queue_state.set(QueueState::Releasing); - let future = img.renderer.eng.spawn( - "await async upload", - await_async_upload(point, img.clone(), cmd, fence, sync_file), - ); - img.renderer.pending_submits.set(point, future); - Ok(()) - } -} - -pub(super) struct IoUploadJob { - img: Option>, - id: u64, - _mem: Option>, - fd: Option>, - work: ReadWriteWork, -} - -pub(super) struct CopyUploadJob { - img: Option>, - id: u64, - _mem: Option>, - work: ImgCopyWork, -} - -impl CpuJob for IoUploadJob { - fn work(&mut self) -> &mut dyn CpuWork { - &mut self.work - } - - fn completed(mut self: Box) { - self._mem = None; - self.fd = None; - let img = self.img.take().unwrap(); - let res = self.work.config().result.take().unwrap(); - complete_async_upload(&img, self.id, res, |data| data.io_job.set(Some(self))); - } -} - -impl CpuJob for CopyUploadJob { - fn work(&mut self) -> &mut dyn CpuWork { - &mut self.work - } - - fn completed(mut self: Box) { - self._mem = None; - let img = self.img.take().unwrap(); - complete_async_upload(&img, self.id, Ok(()), |data| data.copy_job.set(Some(self))); - } -} - -fn complete_async_upload( - img: &Rc, - id: u64, - res: Result<(), ReadWriteJobError>, - store: impl FnOnce(&VulkanShmImageAsyncData), -) { - img.renderer.pending_cpu_jobs.remove(&id); - let VulkanImageMemory::Internal(shm) = &img.ty else { - unreachable!(); - }; - let data = shm.async_data.as_ref().unwrap(); - store(data); - if let Err(e) = res { - data.complete(Err(VulkanError::AsyncCopyToStaging(e))); - } - data.data_copied.set(true); - if let Err(e) = shm.async_upload_copy_buffer_to_image(img, data) { - data.complete(Err(e)); - } -} - -async fn await_gfx_queue_release( - id: u64, - img: Rc, - buf: Option>, - _fence: Option>, - sync_file: SyncFile, -) { - let res = img.renderer.ring.readable(&sync_file.0).await; - if let Err(e) = res { - log::error!( - "Could not wait for sync file to become readable: {}", - ErrorFmt(e) - ); - img.renderer.block(); - } - if let Some(buf) = buf { - img.renderer.gfx_command_buffers.buffers.push(buf); - } - img.renderer.pending_submits.remove(&id); - img.queue_state.set(QueueState::Released { - to: QueueFamily::Transfer, - }); - let VulkanImageMemory::Internal(shm) = &img.ty else { - unreachable!(); - }; - let data = shm.async_data.as_ref().unwrap(); - if let Err(e) = shm.async_upload_copy_buffer_to_image(&img, data) { - data.complete(Err(e)); - } -} - -async fn await_async_upload( - id: u64, - img: Rc, - buf: Rc, - _fence: Rc, - sync_file: SyncFile, -) { - let res = img.renderer.ring.readable(&sync_file.0).await; - if let Err(e) = res { - log::error!( - "Could not wait for sync file to become readable: {}", - ErrorFmt(e) - ); - img.renderer.block(); - } - match &img.renderer.transfer_command_buffers { - Some(b) => b.buffers.push(buf), - None => img.renderer.gfx_command_buffers.buffers.push(buf), - } - img.queue_state.set(QueueState::Released { - to: QueueFamily::Gfx, - }); - img.renderer.pending_submits.remove(&id); - let VulkanImageMemory::Internal(shm) = &img.ty else { - unreachable!(); - }; - let data = shm.async_data.as_ref().unwrap(); - data.complete(Ok(())); -} - impl VulkanRenderer { pub fn create_shm_texture( self: &Rc, diff --git a/src/gfx_apis/vulkan/transfer.rs b/src/gfx_apis/vulkan/transfer.rs new file mode 100644 index 00000000..b88decce --- /dev/null +++ b/src/gfx_apis/vulkan/transfer.rs @@ -0,0 +1,515 @@ +use { + crate::{ + cpu_worker::{ + jobs::{ + img_copy::ImgCopyWork, + read_write::{ReadWriteJobError, ReadWriteWork}, + }, + CpuJob, CpuWork, CpuWorker, + }, + gfx_api::{ + AsyncShmGfxTextureCallback, PendingShmUpload, ShmMemory, ShmMemoryBacking, SyncFile, + }, + gfx_apis::vulkan::{ + command::VulkanCommandBuffer, + fence::VulkanFence, + image::{QueueFamily, QueueState, QueueTransfer, VulkanImage, VulkanImageMemory}, + renderer::image_barrier, + shm_image::VulkanShmImage, + staging::{VulkanStagingBuffer, VulkanStagingShell}, + VulkanError, + }, + rect::{Rect, Region}, + utils::{clonecell::CloneCell, errorfmt::ErrorFmt}, + }, + arrayvec::ArrayVec, + ash::vk::{ + AccessFlags2, BufferImageCopy2, CommandBufferBeginInfo, CommandBufferSubmitInfo, + CommandBufferUsageFlags, DependencyInfo, Extent3D, ImageAspectFlags, ImageLayout, + ImageSubresourceLayers, Offset3D, PipelineStageFlags2, SubmitInfo2, + }, + std::{ + cell::{Cell, RefCell}, + rc::Rc, + slice, + }, + uapi::OwnedFd, +}; + +pub struct VulkanShmImageAsyncData { + pub(super) busy: Cell, + pub(super) io_job: Cell>>, + pub(super) copy_job: Cell>>, + pub(super) staging: CloneCell>>, + pub(super) callback: Cell>>, + pub(super) callback_id: Cell, + pub(super) regions: RefCell>>, + pub(super) cpu: Rc, + pub(super) last_sample: Cell>, + pub(super) data_copied: Cell, +} + +impl VulkanShmImageAsyncData { + fn complete(&self, result: Result<(), VulkanError>) { + self.busy.set(false); + self.staging.take().unwrap().busy.set(false); + if let Some(cb) = self.callback.take() { + cb.completed(result.map_err(|e| e.into())); + } + } +} + +impl VulkanShmImage { + pub fn async_upload( + &self, + img: &Rc, + staging: Rc, + client_mem: &Rc, + damage: Region, + callback: Rc, + ) -> Result, VulkanError> { + let data = self.async_data.as_ref().unwrap(); + let res = self.try_async_upload(img, staging, data, client_mem, damage); + match res { + Ok(()) => { + let id = img.renderer.allocate_point(); + data.callback_id.set(id); + data.callback.set(Some(callback)); + Ok(Some(PendingShmUpload::new(img.clone(), id))) + } + Err(e) => Err(e), + } + } + + fn try_async_upload( + &self, + img: &Rc, + staging: Rc, + data: &VulkanShmImageAsyncData, + client_mem: &Rc, + mut damage: Region, + ) -> Result<(), VulkanError> { + if data.busy.get() { + return Err(VulkanError::AsyncCopyBusy); + } + if staging.busy.get() { + return Err(VulkanError::StagingBufferBusy); + } + if !staging.upload { + return Err(VulkanError::StagingBufferNoUpload); + } + if self.size > client_mem.len() as u64 { + return Err(VulkanError::InvalidBufferSize); + } + data.busy.set(true); + data.data_copied.set(false); + staging.busy.set(true); + data.staging.set(Some(staging.clone())); + if img.contents_are_undefined.get() { + damage = Region::new2(Rect::new_sized(0, 0, img.width as _, img.height as _).unwrap()); + } + + let copies = &mut *data.regions.borrow_mut(); + copies.clear(); + + let mut copy = |x, y, width, height| { + let buffer_offset = (y as u32 * img.stride + x as u32 * self.shm_info.bpp) as u64; + let copy = BufferImageCopy2::default() + .buffer_offset(buffer_offset) + .image_offset(Offset3D { x, y, z: 0 }) + .image_extent(Extent3D { + width, + height, + depth: 1, + }) + .image_subresource(ImageSubresourceLayers { + aspect_mask: ImageAspectFlags::COLOR, + mip_level: 0, + base_array_layer: 0, + layer_count: 1, + }) + .buffer_image_height(img.height) + .buffer_row_length(img.stride / self.shm_info.bpp); + copies.push(copy); + }; + let (width_mask, height_mask) = img.renderer.device.transfer_granularity_mask; + let width_mask = width_mask as i32; + let height_mask = height_mask as i32; + for damage in damage.rects() { + if damage.x2() < 0 || damage.y2() < 0 { + continue; + } + let x1 = damage.x1().max(0) & !width_mask; + let y1 = damage.y1().max(0) & !height_mask; + let x2 = ((damage.x2() + width_mask) & !width_mask).min(img.width as i32); + let y2 = ((damage.y2() + height_mask) & !height_mask).min(img.height as i32); + let Some(damage) = Rect::new(x1, y1, x2, y2) else { + continue; + }; + if damage.is_empty() { + continue; + } + copy( + damage.x1(), + damage.y1(), + damage.width() as u32, + damage.height() as u32, + ); + } + + self.async_release_from_gfx_queue(img, data)?; + + if let Some(staging) = staging.staging.get() { + return self.async_upload_initiate_copy(img, data, &staging, copies, client_mem); + } + + let img2 = img.clone(); + let client_mem = client_mem.clone(); + img.renderer + .device + .fill_staging_shell(&img.renderer, &data.cpu, staging, move |res| { + let VulkanImageMemory::Internal(shm) = &img2.ty else { + unreachable!(); + }; + if let Err(e) = shm.async_upload_after_allocation(&img2, &client_mem, res) { + shm.async_data.as_ref().unwrap().complete(Err(e)); + } + }) + } + + fn async_release_from_gfx_queue( + &self, + img: &Rc, + data: &VulkanShmImageAsyncData, + ) -> Result<(), VulkanError> { + img.renderer.check_defunct()?; + let Some(transfer_queue_idx) = img.renderer.device.distinct_transfer_queue_family_idx + else { + let Some(sync_file) = data.last_sample.take() else { + img.queue_state.set(QueueState::Released { + to: QueueFamily::Transfer, + }); + return Ok(()); + }; + let id = img.renderer.allocate_point(); + let pending = img.renderer.eng.spawn( + "await_transfer_to_transfer", + await_gfx_queue_release(id, img.clone(), None, None, sync_file), + ); + img.renderer.pending_submits.set(id, pending); + img.queue_state.set(QueueState::Releasing); + return Ok(()); + }; + let mut barriers = ArrayVec::<_, 2>::new(); + match img.queue_state.get() { + QueueState::Acquired { family } => { + assert_eq!(family, QueueFamily::Gfx); + } + QueueState::Releasing => { + unreachable!(); + } + QueueState::Released { to } => { + assert_eq!(to, QueueFamily::Gfx); + let barrier = image_barrier() + .image(img.image) + .src_queue_family_index(transfer_queue_idx) + .dst_queue_family_index(img.renderer.device.graphics_queue_idx) + .dst_stage_mask(PipelineStageFlags2::ALL_COMMANDS) + .old_layout(ImageLayout::TRANSFER_DST_OPTIMAL) + .new_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL); + barriers.push(barrier); + } + } + let barrier = image_barrier() + .image(img.image) + .src_queue_family_index(img.renderer.device.graphics_queue_idx) + .dst_queue_family_index(transfer_queue_idx) + .src_access_mask(AccessFlags2::SHADER_SAMPLED_READ) + .src_stage_mask(PipelineStageFlags2::ALL_COMMANDS) + .old_layout(if img.is_undefined.get() { + ImageLayout::UNDEFINED + } else { + ImageLayout::SHADER_READ_ONLY_OPTIMAL + }) + .new_layout(ImageLayout::TRANSFER_DST_OPTIMAL); + barriers.push(barrier); + let dep_info = DependencyInfo::default().image_memory_barriers(&barriers); + let release_fence = img.renderer.device.create_fence()?; + let dev = &img.renderer.device.device; + let begin_info = + CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT); + let cmd = img.renderer.gfx_command_buffers.allocate()?; + let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd.buffer); + let submit_info = + SubmitInfo2::default().command_buffer_infos(slice::from_ref(&command_buffer_info)); + unsafe { + dev.begin_command_buffer(cmd.buffer, &begin_info) + .map_err(VulkanError::BeginCommandBuffer)?; + dev.cmd_pipeline_barrier2(cmd.buffer, &dep_info); + dev.end_command_buffer(cmd.buffer) + .map_err(VulkanError::EndCommandBuffer)?; + dev.queue_submit2( + img.renderer.device.graphics_queue, + slice::from_ref(&submit_info), + release_fence.fence, + ) + .map_err(VulkanError::Submit)?; + } + let sync_file = release_fence.export_sync_file()?; + let id = img.renderer.allocate_point(); + let pending = img.renderer.eng.spawn( + "await_transfer_to_transfer", + await_gfx_queue_release(id, img.clone(), Some(cmd), Some(release_fence), sync_file), + ); + img.renderer.pending_submits.set(id, pending); + img.queue_state.set(QueueState::Releasing); + Ok(()) + } + + fn async_upload_after_allocation( + &self, + img: &Rc, + client_mem: &Rc, + res: Result, VulkanError>, + ) -> Result<(), VulkanError> { + let staging = res?; + let data = self.async_data.as_ref().unwrap(); + let copies = &*data.regions.borrow(); + self.async_upload_initiate_copy(img, data, &staging, copies, client_mem) + } + + fn async_upload_initiate_copy( + &self, + img: &Rc, + data: &VulkanShmImageAsyncData, + staging: &VulkanStagingBuffer, + copies: &[BufferImageCopy2], + client_mem: &Rc, + ) -> Result<(), VulkanError> { + img.renderer.check_defunct()?; + + let id = img.renderer.allocate_point(); + let pending; + match client_mem.safe_access() { + ShmMemoryBacking::Ptr(ptr) => { + let mut job = data.copy_job.take().unwrap_or_else(|| { + Box::new(CopyUploadJob { + img: None, + id, + _mem: None, + work: unsafe { ImgCopyWork::new() }, + }) + }); + job.id = id; + job.img = Some(img.clone()); + job._mem = Some(client_mem.clone()); + job.work.src = ptr as _; + job.work.dst = staging.allocation.mem.unwrap(); + job.work.width = img.width as _; + job.work.stride = img.stride as _; + job.work.bpp = self.shm_info.bpp as _; + job.work.rects.clear(); + for copy in copies { + job.work.rects.push( + Rect::new_sized( + copy.image_offset.x as _, + copy.image_offset.y as _, + copy.image_extent.width as _, + copy.image_extent.height as _, + ) + .unwrap(), + ); + } + pending = data.cpu.submit(job); + } + ShmMemoryBacking::Fd(fd, offset) => { + let mut min_offset = client_mem.len() as u64; + let mut max_offset = 0; + for copy in copies { + min_offset = min_offset.min(copy.buffer_offset); + let len = img.stride * (copy.image_extent.height - 1) + + copy.image_extent.width * self.shm_info.bpp; + max_offset = max_offset.max(copy.buffer_offset + len as u64); + } + let mut job = data.io_job.take().unwrap_or_else(|| { + Box::new(IoUploadJob { + img: None, + id, + _mem: None, + work: unsafe { ReadWriteWork::new() }, + fd: None, + }) + }); + job.id = id; + job.img = Some(img.clone()); + job._mem = Some(client_mem.clone()); + job.fd = Some(fd.clone()); + unsafe { + let config = job.work.config(); + config.fd = fd.raw(); + config.offset = offset + min_offset as usize; + config.ptr = staging.allocation.mem.unwrap().add(min_offset as _); + config.len = max_offset.saturating_sub(min_offset) as usize; + config.write = false; + } + pending = data.cpu.submit(job); + } + } + + img.renderer.pending_cpu_jobs.set(id, pending); + + Ok(()) + } + + fn async_upload_copy_buffer_to_image( + &self, + img: &Rc, + data: &VulkanShmImageAsyncData, + ) -> Result<(), VulkanError> { + if !data.data_copied.get() { + return Ok(()); + } + if img.queue_state.get().acquire(QueueFamily::Transfer) == QueueTransfer::Impossible { + return Ok(()); + } + img.renderer.check_defunct()?; + let regions = &*data.regions.borrow(); + let staging = data.staging.get().unwrap().staging.get().unwrap(); + staging.upload(|_, _| ())?; + let Some((cmd, fence, sync_file, point)) = + self.submit_buffer_to_image_copy(img, &staging, regions, true)? + else { + return Ok(()); + }; + img.queue_state.set(QueueState::Releasing); + let future = img.renderer.eng.spawn( + "await async upload", + await_async_upload(point, img.clone(), cmd, fence, sync_file), + ); + img.renderer.pending_submits.set(point, future); + Ok(()) + } +} + +pub(super) struct IoUploadJob { + img: Option>, + id: u64, + _mem: Option>, + fd: Option>, + work: ReadWriteWork, +} + +pub(super) struct CopyUploadJob { + img: Option>, + id: u64, + _mem: Option>, + work: ImgCopyWork, +} + +impl CpuJob for IoUploadJob { + fn work(&mut self) -> &mut dyn CpuWork { + &mut self.work + } + + fn completed(mut self: Box) { + self._mem = None; + self.fd = None; + let img = self.img.take().unwrap(); + let res = self.work.config().result.take().unwrap(); + complete_async_upload(&img, self.id, res, |data| data.io_job.set(Some(self))); + } +} + +impl CpuJob for CopyUploadJob { + fn work(&mut self) -> &mut dyn CpuWork { + &mut self.work + } + + fn completed(mut self: Box) { + self._mem = None; + let img = self.img.take().unwrap(); + complete_async_upload(&img, self.id, Ok(()), |data| data.copy_job.set(Some(self))); + } +} + +fn complete_async_upload( + img: &Rc, + id: u64, + res: Result<(), ReadWriteJobError>, + store: impl FnOnce(&VulkanShmImageAsyncData), +) { + img.renderer.pending_cpu_jobs.remove(&id); + let VulkanImageMemory::Internal(shm) = &img.ty else { + unreachable!(); + }; + let data = shm.async_data.as_ref().unwrap(); + store(data); + if let Err(e) = res { + data.complete(Err(VulkanError::AsyncCopyToStaging(e))); + } + data.data_copied.set(true); + if let Err(e) = shm.async_upload_copy_buffer_to_image(img, data) { + data.complete(Err(e)); + } +} + +async fn await_gfx_queue_release( + id: u64, + img: Rc, + buf: Option>, + _fence: Option>, + sync_file: SyncFile, +) { + let res = img.renderer.ring.readable(&sync_file.0).await; + if let Err(e) = res { + log::error!( + "Could not wait for sync file to become readable: {}", + ErrorFmt(e) + ); + img.renderer.block(); + } + if let Some(buf) = buf { + img.renderer.gfx_command_buffers.buffers.push(buf); + } + img.renderer.pending_submits.remove(&id); + img.queue_state.set(QueueState::Released { + to: QueueFamily::Transfer, + }); + let VulkanImageMemory::Internal(shm) = &img.ty else { + unreachable!(); + }; + let data = shm.async_data.as_ref().unwrap(); + if let Err(e) = shm.async_upload_copy_buffer_to_image(&img, data) { + data.complete(Err(e)); + } +} + +async fn await_async_upload( + id: u64, + img: Rc, + buf: Rc, + _fence: Rc, + sync_file: SyncFile, +) { + let res = img.renderer.ring.readable(&sync_file.0).await; + if let Err(e) = res { + log::error!( + "Could not wait for sync file to become readable: {}", + ErrorFmt(e) + ); + img.renderer.block(); + } + match &img.renderer.transfer_command_buffers { + Some(b) => b.buffers.push(buf), + None => img.renderer.gfx_command_buffers.buffers.push(buf), + } + img.queue_state.set(QueueState::Released { + to: QueueFamily::Gfx, + }); + img.renderer.pending_submits.remove(&id); + let VulkanImageMemory::Internal(shm) = &img.ty else { + unreachable!(); + }; + let data = shm.async_data.as_ref().unwrap(); + data.complete(Ok(())); +} From 61c5ebb062e989054983b879606b5fde9aa394d0 Mon Sep 17 00:00:00 2001 From: Julian Orth Date: Sun, 6 Oct 2024 13:23:10 +0200 Subject: [PATCH 3/7] vulkan: make async transfers generic over upload/download --- src/gfx_api.rs | 14 +-- src/gfx_apis/gl/renderer/texture.rs | 4 +- src/gfx_apis/vulkan.rs | 4 + src/gfx_apis/vulkan/image.rs | 16 +-- src/gfx_apis/vulkan/transfer.rs | 157 +++++++++++++++++++------- src/ifs/wl_surface/commit_timeline.rs | 8 +- src/it/test_gfx_api.rs | 4 +- src/text.rs | 4 +- 8 files changed, 148 insertions(+), 63 deletions(-) diff --git a/src/gfx_api.rs b/src/gfx_api.rs index e98cd032..d36b2e11 100644 --- a/src/gfx_api.rs +++ b/src/gfx_api.rs @@ -514,12 +514,12 @@ pub trait GfxStagingBuffer { fn into_any(self: Rc) -> Rc; } -pub trait AsyncShmGfxTextureUploadCancellable { +pub trait AsyncShmGfxTextureTransferCancellable { fn cancel(&self, id: u64); } -pub struct PendingShmUpload { - cancel: Rc, +pub struct PendingShmTransfer { + cancel: Rc, id: u64, } @@ -560,7 +560,7 @@ pub trait AsyncShmGfxTexture: GfxTexture { callback: Rc, mem: Rc, damage: Region, - ) -> Result, GfxError>; + ) -> Result, GfxError>; fn sync_upload(self: Rc, shm: &[Cell], damage: Region) -> Result<(), GfxError>; @@ -706,13 +706,13 @@ pub fn cross_intersect_formats( res } -impl PendingShmUpload { - pub fn new(cancel: Rc, id: u64) -> Self { +impl PendingShmTransfer { + pub fn new(cancel: Rc, id: u64) -> Self { Self { cancel, id } } } -impl Drop for PendingShmUpload { +impl Drop for PendingShmTransfer { fn drop(&mut self) { self.cancel.cancel(self.id); } diff --git a/src/gfx_apis/gl/renderer/texture.rs b/src/gfx_apis/gl/renderer/texture.rs index 7a4e1103..5400273a 100644 --- a/src/gfx_apis/gl/renderer/texture.rs +++ b/src/gfx_apis/gl/renderer/texture.rs @@ -3,7 +3,7 @@ use { format::Format, gfx_api::{ AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxError, GfxStagingBuffer, GfxTexture, - PendingShmUpload, ShmGfxTexture, ShmMemory, + PendingShmTransfer, ShmGfxTexture, ShmMemory, }, gfx_apis::gl::{ gl::texture::GlTexture, @@ -104,7 +104,7 @@ impl AsyncShmGfxTexture for Texture { _callback: Rc, mem: Rc, _damage: Region, - ) -> Result, GfxError> { + ) -> Result, GfxError> { let mut res = Ok(()); mem.access(&mut |data| { res = self.clone().sync_upload(data, Region::default()); diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs index bbeb6c5f..acaa83dd 100644 --- a/src/gfx_apis/vulkan.rs +++ b/src/gfx_apis/vulkan.rs @@ -209,6 +209,10 @@ pub enum VulkanError { StagingBufferBusy, #[error("The staging buffer does not support uploads")] StagingBufferNoUpload, + #[error("The staging buffer does not support downloads")] + StagingBufferNoDownload, + #[error("Image contents are undefined")] + UndefinedContents, } impl From for GfxError { diff --git a/src/gfx_apis/vulkan/image.rs b/src/gfx_apis/vulkan/image.rs index 6df5788a..d40ae715 100644 --- a/src/gfx_apis/vulkan/image.rs +++ b/src/gfx_apis/vulkan/image.rs @@ -3,13 +3,14 @@ use { format::Format, gfx_api::{ AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback, - AsyncShmGfxTextureUploadCancellable, GfxApiOpt, GfxError, GfxFramebuffer, GfxImage, - GfxStagingBuffer, GfxTexture, PendingShmUpload, ReleaseSync, ShmGfxTexture, ShmMemory, - SyncFile, + AsyncShmGfxTextureTransferCancellable, GfxApiOpt, GfxError, GfxFramebuffer, GfxImage, + GfxStagingBuffer, GfxTexture, PendingShmTransfer, ReleaseSync, ShmGfxTexture, + ShmMemory, SyncFile, }, gfx_apis::vulkan::{ allocator::VulkanAllocation, device::VulkanDevice, format::VulkanModifierLimits, - renderer::VulkanRenderer, shm_image::VulkanShmImage, VulkanError, + renderer::VulkanRenderer, shm_image::VulkanShmImage, transfer::TransferType, + VulkanError, }, rect::Region, theme::Color, @@ -589,12 +590,13 @@ impl AsyncShmGfxTexture for VulkanImage { callback: Rc, mem: Rc, damage: Region, - ) -> Result, GfxError> { + ) -> Result, GfxError> { let VulkanImageMemory::Internal(shm) = &self.ty else { unreachable!(); }; let staging = staging.clone().into_vk(&self.renderer.device.device); - let pending = shm.async_upload(&self, staging, &mem, damage, callback)?; + let pending = + shm.async_transfer(&self, staging, &mem, damage, callback, TransferType::Upload)?; Ok(pending) } @@ -627,7 +629,7 @@ impl AsyncShmGfxTexture for VulkanImage { } } -impl AsyncShmGfxTextureUploadCancellable for VulkanImage { +impl AsyncShmGfxTextureTransferCancellable for VulkanImage { fn cancel(&self, id: u64) { let VulkanImageMemory::Internal(shm) = &self.ty else { unreachable!(); diff --git a/src/gfx_apis/vulkan/transfer.rs b/src/gfx_apis/vulkan/transfer.rs index b88decce..901e7cf0 100644 --- a/src/gfx_apis/vulkan/transfer.rs +++ b/src/gfx_apis/vulkan/transfer.rs @@ -8,7 +8,7 @@ use { CpuJob, CpuWork, CpuWorker, }, gfx_api::{ - AsyncShmGfxTextureCallback, PendingShmUpload, ShmMemory, ShmMemoryBacking, SyncFile, + AsyncShmGfxTextureCallback, PendingShmTransfer, ShmMemory, ShmMemoryBacking, SyncFile, }, gfx_apis::vulkan::{ command::VulkanCommandBuffer, @@ -38,8 +38,8 @@ use { pub struct VulkanShmImageAsyncData { pub(super) busy: Cell, - pub(super) io_job: Cell>>, - pub(super) copy_job: Cell>>, + pub(super) io_job: Cell>>, + pub(super) copy_job: Cell>>, pub(super) staging: CloneCell>>, pub(super) callback: Cell>>, pub(super) callback_id: Cell, @@ -59,35 +59,43 @@ impl VulkanShmImageAsyncData { } } +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(super) enum TransferType { + Upload, + Download, +} + impl VulkanShmImage { - pub fn async_upload( + pub fn async_transfer( &self, img: &Rc, staging: Rc, client_mem: &Rc, damage: Region, callback: Rc, - ) -> Result, VulkanError> { + tt: TransferType, + ) -> Result, VulkanError> { let data = self.async_data.as_ref().unwrap(); - let res = self.try_async_upload(img, staging, data, client_mem, damage); + let res = self.try_async_transfer(img, staging, data, client_mem, damage, tt); match res { Ok(()) => { let id = img.renderer.allocate_point(); data.callback_id.set(id); data.callback.set(Some(callback)); - Ok(Some(PendingShmUpload::new(img.clone(), id))) + Ok(Some(PendingShmTransfer::new(img.clone(), id))) } Err(e) => Err(e), } } - fn try_async_upload( + fn try_async_transfer( &self, img: &Rc, staging: Rc, data: &VulkanShmImageAsyncData, client_mem: &Rc, mut damage: Region, + tt: TransferType, ) -> Result<(), VulkanError> { if data.busy.get() { return Err(VulkanError::AsyncCopyBusy); @@ -95,8 +103,17 @@ impl VulkanShmImage { if staging.busy.get() { return Err(VulkanError::StagingBufferBusy); } - if !staging.upload { - return Err(VulkanError::StagingBufferNoUpload); + match tt { + TransferType::Upload => { + if !staging.upload { + return Err(VulkanError::StagingBufferNoUpload); + } + } + TransferType::Download => { + if !staging.download { + return Err(VulkanError::StagingBufferNoDownload); + } + } } if self.size > client_mem.len() as u64 { return Err(VulkanError::InvalidBufferSize); @@ -106,6 +123,9 @@ impl VulkanShmImage { staging.busy.set(true); data.staging.set(Some(staging.clone())); if img.contents_are_undefined.get() { + if tt == TransferType::Download { + return Err(VulkanError::UndefinedContents); + } damage = Region::new2(Rect::new_sized(0, 0, img.width as _, img.height as _).unwrap()); } @@ -157,10 +177,14 @@ impl VulkanShmImage { ); } - self.async_release_from_gfx_queue(img, data)?; + self.async_release_from_gfx_queue(img, data, tt)?; if let Some(staging) = staging.staging.get() { - return self.async_upload_initiate_copy(img, data, &staging, copies, client_mem); + return match tt { + TransferType::Upload => self + .async_transfer_initiate_host_copy(img, data, &staging, copies, client_mem, tt), + TransferType::Download => unreachable!(), + }; } let img2 = img.clone(); @@ -171,7 +195,7 @@ impl VulkanShmImage { let VulkanImageMemory::Internal(shm) = &img2.ty else { unreachable!(); }; - if let Err(e) = shm.async_upload_after_allocation(&img2, &client_mem, res) { + if let Err(e) = shm.async_transfer_after_allocation(&img2, &client_mem, res, tt) { shm.async_data.as_ref().unwrap().complete(Err(e)); } }) @@ -181,6 +205,7 @@ impl VulkanShmImage { &self, img: &Rc, data: &VulkanShmImageAsyncData, + tt: TransferType, ) -> Result<(), VulkanError> { img.renderer.check_defunct()?; let Some(transfer_queue_idx) = img.renderer.device.distinct_transfer_queue_family_idx @@ -194,12 +219,24 @@ impl VulkanShmImage { let id = img.renderer.allocate_point(); let pending = img.renderer.eng.spawn( "await_transfer_to_transfer", - await_gfx_queue_release(id, img.clone(), None, None, sync_file), + await_gfx_queue_release(id, img.clone(), None, None, sync_file, tt), ); img.renderer.pending_submits.set(id, pending); img.queue_state.set(QueueState::Releasing); return Ok(()); }; + let (gfx_access_mask, gfx_layout, transfer_layout) = match tt { + TransferType::Upload => ( + AccessFlags2::SHADER_SAMPLED_READ, + ImageLayout::SHADER_READ_ONLY_OPTIMAL, + ImageLayout::TRANSFER_DST_OPTIMAL, + ), + TransferType::Download => ( + AccessFlags2::COLOR_ATTACHMENT_WRITE, + ImageLayout::COLOR_ATTACHMENT_OPTIMAL, + ImageLayout::TRANSFER_SRC_OPTIMAL, + ), + }; let mut barriers = ArrayVec::<_, 2>::new(); match img.queue_state.get() { QueueState::Acquired { family } => { @@ -215,8 +252,8 @@ impl VulkanShmImage { .src_queue_family_index(transfer_queue_idx) .dst_queue_family_index(img.renderer.device.graphics_queue_idx) .dst_stage_mask(PipelineStageFlags2::ALL_COMMANDS) - .old_layout(ImageLayout::TRANSFER_DST_OPTIMAL) - .new_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL); + .old_layout(transfer_layout) + .new_layout(gfx_layout); barriers.push(barrier); } } @@ -224,14 +261,14 @@ impl VulkanShmImage { .image(img.image) .src_queue_family_index(img.renderer.device.graphics_queue_idx) .dst_queue_family_index(transfer_queue_idx) - .src_access_mask(AccessFlags2::SHADER_SAMPLED_READ) + .src_access_mask(gfx_access_mask) .src_stage_mask(PipelineStageFlags2::ALL_COMMANDS) .old_layout(if img.is_undefined.get() { ImageLayout::UNDEFINED } else { - ImageLayout::SHADER_READ_ONLY_OPTIMAL + gfx_layout }) - .new_layout(ImageLayout::TRANSFER_DST_OPTIMAL); + .new_layout(transfer_layout); barriers.push(barrier); let dep_info = DependencyInfo::default().image_memory_barriers(&barriers); let release_fence = img.renderer.device.create_fence()?; @@ -259,32 +296,46 @@ impl VulkanShmImage { let id = img.renderer.allocate_point(); let pending = img.renderer.eng.spawn( "await_transfer_to_transfer", - await_gfx_queue_release(id, img.clone(), Some(cmd), Some(release_fence), sync_file), + await_gfx_queue_release( + id, + img.clone(), + Some(cmd), + Some(release_fence), + sync_file, + tt, + ), ); img.renderer.pending_submits.set(id, pending); img.queue_state.set(QueueState::Releasing); Ok(()) } - fn async_upload_after_allocation( + fn async_transfer_after_allocation( &self, img: &Rc, client_mem: &Rc, res: Result, VulkanError>, + tt: TransferType, ) -> Result<(), VulkanError> { let staging = res?; let data = self.async_data.as_ref().unwrap(); let copies = &*data.regions.borrow(); - self.async_upload_initiate_copy(img, data, &staging, copies, client_mem) + match tt { + TransferType::Upload => { + self.async_transfer_initiate_host_copy(img, data, &staging, copies, client_mem, tt) + } + TransferType::Download => unreachable!(), + } } - fn async_upload_initiate_copy( + pub(super) fn async_transfer_initiate_host_copy( &self, img: &Rc, data: &VulkanShmImageAsyncData, staging: &VulkanStagingBuffer, copies: &[BufferImageCopy2], client_mem: &Rc, + tt: TransferType, ) -> Result<(), VulkanError> { img.renderer.check_defunct()?; @@ -293,18 +344,28 @@ impl VulkanShmImage { match client_mem.safe_access() { ShmMemoryBacking::Ptr(ptr) => { let mut job = data.copy_job.take().unwrap_or_else(|| { - Box::new(CopyUploadJob { + Box::new(CopyTransferJob { img: None, id, _mem: None, work: unsafe { ImgCopyWork::new() }, + tt, }) }); job.id = id; job.img = Some(img.clone()); job._mem = Some(client_mem.clone()); - job.work.src = ptr as _; - job.work.dst = staging.allocation.mem.unwrap(); + job.tt = tt; + match tt { + TransferType::Upload => { + job.work.src = ptr as _; + job.work.dst = staging.allocation.mem.unwrap(); + } + TransferType::Download => { + job.work.src = staging.allocation.mem.unwrap(); + job.work.dst = ptr as _; + } + } job.work.width = img.width as _; job.work.stride = img.stride as _; job.work.bpp = self.shm_info.bpp as _; @@ -332,25 +393,27 @@ impl VulkanShmImage { max_offset = max_offset.max(copy.buffer_offset + len as u64); } let mut job = data.io_job.take().unwrap_or_else(|| { - Box::new(IoUploadJob { + Box::new(IoTransferJob { img: None, id, _mem: None, work: unsafe { ReadWriteWork::new() }, fd: None, + tt, }) }); job.id = id; job.img = Some(img.clone()); job._mem = Some(client_mem.clone()); job.fd = Some(fd.clone()); + job.tt = tt; unsafe { let config = job.work.config(); config.fd = fd.raw(); config.offset = offset + min_offset as usize; config.ptr = staging.allocation.mem.unwrap().add(min_offset as _); config.len = max_offset.saturating_sub(min_offset) as usize; - config.write = false; + config.write = tt == TransferType::Download; } pending = data.cpu.submit(job); } @@ -384,29 +447,31 @@ impl VulkanShmImage { img.queue_state.set(QueueState::Releasing); let future = img.renderer.eng.spawn( "await async upload", - await_async_upload(point, img.clone(), cmd, fence, sync_file), + await_async_transfer_release_to_gfx(point, img.clone(), cmd, fence, sync_file), ); img.renderer.pending_submits.set(point, future); Ok(()) } } -pub(super) struct IoUploadJob { +pub(super) struct IoTransferJob { img: Option>, id: u64, _mem: Option>, fd: Option>, work: ReadWriteWork, + tt: TransferType, } -pub(super) struct CopyUploadJob { +pub(super) struct CopyTransferJob { img: Option>, id: u64, _mem: Option>, work: ImgCopyWork, + tt: TransferType, } -impl CpuJob for IoUploadJob { +impl CpuJob for IoTransferJob { fn work(&mut self) -> &mut dyn CpuWork { &mut self.work } @@ -416,11 +481,13 @@ impl CpuJob for IoUploadJob { self.fd = None; let img = self.img.take().unwrap(); let res = self.work.config().result.take().unwrap(); - complete_async_upload(&img, self.id, res, |data| data.io_job.set(Some(self))); + complete_async_host_copy(&img, self.id, res, self.tt, |data| { + data.io_job.set(Some(self)) + }); } } -impl CpuJob for CopyUploadJob { +impl CpuJob for CopyTransferJob { fn work(&mut self) -> &mut dyn CpuWork { &mut self.work } @@ -428,14 +495,17 @@ impl CpuJob for CopyUploadJob { fn completed(mut self: Box) { self._mem = None; let img = self.img.take().unwrap(); - complete_async_upload(&img, self.id, Ok(()), |data| data.copy_job.set(Some(self))); + complete_async_host_copy(&img, self.id, Ok(()), self.tt, |data| { + data.copy_job.set(Some(self)) + }); } } -fn complete_async_upload( +fn complete_async_host_copy( img: &Rc, id: u64, res: Result<(), ReadWriteJobError>, + tt: TransferType, store: impl FnOnce(&VulkanShmImageAsyncData), ) { img.renderer.pending_cpu_jobs.remove(&id); @@ -448,7 +518,11 @@ fn complete_async_upload( data.complete(Err(VulkanError::AsyncCopyToStaging(e))); } data.data_copied.set(true); - if let Err(e) = shm.async_upload_copy_buffer_to_image(img, data) { + let res = match tt { + TransferType::Upload => shm.async_upload_copy_buffer_to_image(img, data), + TransferType::Download => unreachable!(), + }; + if let Err(e) = res { data.complete(Err(e)); } } @@ -459,6 +533,7 @@ async fn await_gfx_queue_release( buf: Option>, _fence: Option>, sync_file: SyncFile, + tt: TransferType, ) { let res = img.renderer.ring.readable(&sync_file.0).await; if let Err(e) = res { @@ -479,12 +554,16 @@ async fn await_gfx_queue_release( unreachable!(); }; let data = shm.async_data.as_ref().unwrap(); - if let Err(e) = shm.async_upload_copy_buffer_to_image(&img, data) { + let res = match tt { + TransferType::Upload => shm.async_upload_copy_buffer_to_image(&img, data), + TransferType::Download => unreachable!(), + }; + if let Err(e) = res { data.complete(Err(e)); } } -async fn await_async_upload( +pub async fn await_async_transfer_release_to_gfx( id: u64, img: Rc, buf: Rc, diff --git a/src/ifs/wl_surface/commit_timeline.rs b/src/ifs/wl_surface/commit_timeline.rs index a973ddbb..04e64f7a 100644 --- a/src/ifs/wl_surface/commit_timeline.rs +++ b/src/ifs/wl_surface/commit_timeline.rs @@ -1,6 +1,6 @@ use { crate::{ - gfx_api::{AsyncShmGfxTextureCallback, GfxError, PendingShmUpload, STAGING_UPLOAD}, + gfx_api::{AsyncShmGfxTextureCallback, GfxError, PendingShmTransfer, STAGING_UPLOAD}, ifs::{ wl_buffer::WlBufferStorage, wl_surface::{PendingState, WlSurface, WlSurfaceError}, @@ -310,7 +310,7 @@ enum EntryKind { enum ShmUploadState { None, Todo(Rc>), - Scheduled(#[expect(dead_code)] SmallVec<[PendingShmUpload; 1]>), + Scheduled(#[expect(dead_code)] SmallVec<[PendingShmTransfer; 1]>), } struct Commit { @@ -395,7 +395,7 @@ fn schedule_async_uploads( node_ref: &Rc>, surface: &WlSurface, pending: &PendingState, - uploads: &mut SmallVec<[PendingShmUpload; 1]>, + uploads: &mut SmallVec<[PendingShmTransfer; 1]>, ) -> Result<(), WlSurfaceError> { if let Some(pending) = schedule_async_upload(node_ref, surface, pending)? { uploads.push(pending); @@ -412,7 +412,7 @@ fn schedule_async_upload( node_ref: &Rc>, surface: &WlSurface, pending: &PendingState, -) -> Result, WlSurfaceError> { +) -> Result, WlSurfaceError> { let Some(Some(buf)) = &pending.buffer else { return Ok(None); }; diff --git a/src/it/test_gfx_api.rs b/src/it/test_gfx_api.rs index 6ba86de9..e8b092c4 100644 --- a/src/it/test_gfx_api.rs +++ b/src/it/test_gfx_api.rs @@ -6,7 +6,7 @@ use { gfx_api::{ AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback, CopyTexture, FillRect, FramebufferRect, GfxApiOpt, GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage, - GfxStagingBuffer, GfxTexture, GfxWriteModifier, PendingShmUpload, ReleaseSync, + GfxStagingBuffer, GfxTexture, GfxWriteModifier, PendingShmTransfer, ReleaseSync, ResetStatus, ShmGfxTexture, ShmMemory, SyncFile, }, rect::{Rect, Region}, @@ -339,7 +339,7 @@ impl AsyncShmGfxTexture for TestGfxImage { _callback: Rc, mem: Rc, _damage: Region, - ) -> Result, GfxError> { + ) -> Result, GfxError> { let mut res = Ok(()); mem.access(&mut |d| { res = self.clone().sync_upload(d, Region::default()); diff --git a/src/text.rs b/src/text.rs index a097a511..0a9408d4 100644 --- a/src/text.rs +++ b/src/text.rs @@ -4,7 +4,7 @@ use { format::ARGB8888, gfx_api::{ AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxContext, GfxError, GfxStagingBuffer, - GfxTexture, PendingShmUpload, STAGING_UPLOAD, + GfxTexture, PendingShmTransfer, STAGING_UPLOAD, }, pango::{ consts::{ @@ -305,7 +305,7 @@ struct Shared { staging: CloneCell>>, textures: DoubleBuffered, pending_render: Cell>, - pending_upload: Cell>, + pending_upload: Cell>, render_job: Cell>>, result: Cell>>, waiter: Cell>>, From 156785d7c8c5aaa594d83d8c722efa8207ebc5e4 Mon Sep 17 00:00:00 2001 From: Julian Orth Date: Sun, 6 Oct 2024 14:07:16 +0200 Subject: [PATCH 4/7] vulkan: implement async shm downloads --- src/gfx_apis/vulkan.rs | 2 + src/gfx_apis/vulkan/renderer.rs | 132 ++++++++++++++++++++----------- src/gfx_apis/vulkan/shm_image.rs | 96 ++++++++++++++++------ src/gfx_apis/vulkan/transfer.rs | 114 +++++++++++++++++++++++--- 4 files changed, 260 insertions(+), 84 deletions(-) diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs index acaa83dd..fd8287c1 100644 --- a/src/gfx_apis/vulkan.rs +++ b/src/gfx_apis/vulkan.rs @@ -213,6 +213,8 @@ pub enum VulkanError { StagingBufferNoDownload, #[error("Image contents are undefined")] UndefinedContents, + #[error("The framebuffer is being used by the transfer queue")] + BusyInTransfer, } impl From for GfxError { diff --git a/src/gfx_apis/vulkan/renderer.rs b/src/gfx_apis/vulkan/renderer.rs index 10592eba..cc225064 100644 --- a/src/gfx_apis/vulkan/renderer.rs +++ b/src/gfx_apis/vulkan/renderer.rs @@ -337,38 +337,69 @@ impl VulkanRenderer { } } - fn initial_barriers(&self, buf: CommandBuffer, fb: &VulkanImage) { + fn initial_barriers(&self, buf: CommandBuffer, fb: &VulkanImage) -> Result<(), VulkanError> { zone!("initial_barriers"); let mut memory = self.memory.borrow_mut(); let memory = &mut *memory; memory.image_barriers.clear(); - let mut fb_image_memory_barrier = image_barrier() - .image(fb.image) - .new_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL) - .dst_access_mask( - AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ, - ) - .dst_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT); - if fb.bridge.is_some() { - fb_image_memory_barrier = fb_image_memory_barrier - .src_access_mask(AccessFlags2::TRANSFER_READ) - .src_stage_mask(PipelineStageFlags2::TRANSFER) - .old_layout(if fb.is_undefined.get() { - ImageLayout::UNDEFINED - } else { - ImageLayout::TRANSFER_SRC_OPTIMAL - }); - } else { - fb_image_memory_barrier = fb_image_memory_barrier - .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) - .dst_queue_family_index(self.device.graphics_queue_idx) - .old_layout(if fb.is_undefined.get() { - ImageLayout::UNDEFINED - } else { - ImageLayout::GENERAL - }); + let mut need_fb_barrier = true; + if let VulkanImageMemory::Internal(..) = &fb.ty { + need_fb_barrier = fb.is_undefined.get() + || (self.device.distinct_transfer_queue_family_idx.is_some() + && fb.queue_state.get().acquire(QueueFamily::Gfx) + != QueueTransfer::Unnecessary); + } + if need_fb_barrier { + let mut fb_image_memory_barrier = image_barrier() + .image(fb.image) + .new_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL) + .dst_access_mask( + AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ, + ) + .dst_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT); + if fb.bridge.is_some() { + fb_image_memory_barrier = fb_image_memory_barrier + .src_access_mask(AccessFlags2::TRANSFER_READ) + .src_stage_mask(PipelineStageFlags2::TRANSFER) + .old_layout(if fb.is_undefined.get() { + ImageLayout::UNDEFINED + } else { + ImageLayout::TRANSFER_SRC_OPTIMAL + }); + } else if let VulkanImageMemory::Internal(..) = &fb.ty { + let mut queue_transfer = QueueTransfer::Unnecessary; + if self.device.distinct_transfer_queue_family_idx.is_some() { + queue_transfer = fb.queue_state.get().acquire(QueueFamily::Gfx); + } + match queue_transfer { + QueueTransfer::Unnecessary => { + fb_image_memory_barrier = + fb_image_memory_barrier.old_layout(ImageLayout::UNDEFINED); + } + QueueTransfer::Possible => { + if let Some(transfer_queue_idx) = + self.device.distinct_transfer_queue_family_idx + { + fb_image_memory_barrier = fb_image_memory_barrier + .src_queue_family_index(transfer_queue_idx) + .dst_queue_family_index(self.device.graphics_queue_idx) + .old_layout(ImageLayout::TRANSFER_SRC_OPTIMAL); + } + } + QueueTransfer::Impossible => return Err(VulkanError::BusyInTransfer), + } + } else { + fb_image_memory_barrier = fb_image_memory_barrier + .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) + .dst_queue_family_index(self.device.graphics_queue_idx) + .old_layout(if fb.is_undefined.get() { + ImageLayout::UNDEFINED + } else { + ImageLayout::GENERAL + }); + } + memory.image_barriers.push(fb_image_memory_barrier); } - memory.image_barriers.push(fb_image_memory_barrier); for img in &memory.dmabuf_sample { let image_memory_barrier = image_barrier() .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) @@ -397,6 +428,7 @@ impl VulkanRenderer { unsafe { self.device.device.cmd_pipeline_barrier2(buf, &dep_info); } + Ok(()) } fn begin_rendering(&self, buf: CommandBuffer, fb: &VulkanImage, clear: Option<&Color>) { @@ -643,26 +675,28 @@ impl VulkanRenderer { let mut memory = self.memory.borrow_mut(); let memory = &mut *memory; memory.image_barriers.clear(); - let mut fb_image_memory_barrier = image_barrier() - .src_queue_family_index(self.device.graphics_queue_idx) - .dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) - .new_layout(ImageLayout::GENERAL); - if let Some(bridge) = &fb.bridge { - fb_image_memory_barrier = fb_image_memory_barrier - .image(bridge.dmabuf_image) - .old_layout(ImageLayout::TRANSFER_DST_OPTIMAL) - .src_access_mask(AccessFlags2::TRANSFER_WRITE) - .src_stage_mask(PipelineStageFlags2::TRANSFER); - } else { - fb_image_memory_barrier = fb_image_memory_barrier - .image(fb.image) - .old_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL) - .src_access_mask( - AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ, - ) - .src_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT); + if let VulkanImageMemory::DmaBuf(..) = fb.ty { + let mut fb_image_memory_barrier = image_barrier() + .src_queue_family_index(self.device.graphics_queue_idx) + .dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) + .new_layout(ImageLayout::GENERAL); + if let Some(bridge) = &fb.bridge { + fb_image_memory_barrier = fb_image_memory_barrier + .image(bridge.dmabuf_image) + .old_layout(ImageLayout::TRANSFER_DST_OPTIMAL) + .src_access_mask(AccessFlags2::TRANSFER_WRITE) + .src_stage_mask(PipelineStageFlags2::TRANSFER); + } else { + fb_image_memory_barrier = fb_image_memory_barrier + .image(fb.image) + .old_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL) + .src_access_mask( + AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ, + ) + .src_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT); + } + memory.image_barriers.push(fb_image_memory_barrier); } - memory.image_barriers.push(fb_image_memory_barrier); for img in &memory.dmabuf_sample { let image_memory_barrier = image_barrier() .src_queue_family_index(self.device.graphics_queue_idx) @@ -833,6 +867,10 @@ impl VulkanRenderer { fn store_layouts(&self, fb: &VulkanImage) { fb.is_undefined.set(false); + fb.contents_are_undefined.set(false); + fb.queue_state.set(QueueState::Acquired { + family: QueueFamily::Gfx, + }); let memory = self.memory.borrow(); for img in &*memory.queue_transfer { img.queue_state.set(QueueState::Acquired { @@ -1101,7 +1139,7 @@ impl VulkanRenderer { let buf = self.gfx_command_buffers.allocate()?; self.collect_memory(opts); self.begin_command_buffer(buf.buffer)?; - self.initial_barriers(buf.buffer, fb); + self.initial_barriers(buf.buffer, fb)?; self.begin_rendering(buf.buffer, fb, clear); self.set_viewport(buf.buffer, fb); self.record_draws(buf.buffer, fb, opts)?; diff --git a/src/gfx_apis/vulkan/shm_image.rs b/src/gfx_apis/vulkan/shm_image.rs index a342a453..86155e2b 100644 --- a/src/gfx_apis/vulkan/shm_image.rs +++ b/src/gfx_apis/vulkan/shm_image.rs @@ -10,7 +10,7 @@ use { image::{QueueFamily, QueueState, VulkanImage, VulkanImageMemory}, renderer::{image_barrier, VulkanRenderer}, staging::VulkanStagingBuffer, - transfer::VulkanShmImageAsyncData, + transfer::{TransferType, VulkanShmImageAsyncData}, VulkanError, }, rect::Rect, @@ -19,10 +19,10 @@ use { ash::vk::{ AccessFlags2, BufferImageCopy2, BufferMemoryBarrier2, CommandBufferBeginInfo, CommandBufferSubmitInfo, CommandBufferUsageFlags, CopyBufferToImageInfo2, - DependencyInfoKHR, DeviceSize, Extent3D, ImageAspectFlags, ImageCreateInfo, ImageLayout, - ImageSubresourceLayers, ImageSubresourceRange, ImageTiling, ImageType, ImageUsageFlags, - ImageViewCreateInfo, ImageViewType, Offset3D, PipelineStageFlags2, SampleCountFlags, - SharingMode, SubmitInfo2, + CopyImageToBufferInfo2, DependencyInfoKHR, DeviceSize, Extent3D, ImageAspectFlags, + ImageCreateInfo, ImageLayout, ImageSubresourceLayers, ImageSubresourceRange, ImageTiling, + ImageType, ImageUsageFlags, ImageViewCreateInfo, ImageViewType, Offset3D, + PipelineStageFlags2, SampleCountFlags, SharingMode, SubmitInfo2, }, gpu_alloc::UsageFlags, isnt::std_1::primitive::IsntSliceExt, @@ -137,7 +137,7 @@ impl VulkanShmImage { } })?; let Some((cmd, fence, sync_file, point)) = - self.submit_buffer_to_image_copy(img, &staging, cpy, false)? + self.submit_buffer_image_copy(img, &staging, cpy, false, TransferType::Upload)? else { return Ok(()); }; @@ -149,12 +149,13 @@ impl VulkanShmImage { Ok(()) } - pub(super) fn submit_buffer_to_image_copy( + pub(super) fn submit_buffer_image_copy( &self, img: &Rc, staging: &VulkanStagingBuffer, regions: &[BufferImageCopy2], use_transfer_queue: bool, + tt: TransferType, ) -> Result, Rc, SyncFile, u64)>, VulkanError> { let memory_barrier = |sam, ssm, dam, dsm| { @@ -182,18 +183,30 @@ impl VulkanShmImage { .old_layout(if img.is_undefined.get() { ImageLayout::UNDEFINED } else { - ImageLayout::SHADER_READ_ONLY_OPTIMAL + match tt { + TransferType::Upload => ImageLayout::SHADER_READ_ONLY_OPTIMAL, + TransferType::Download => ImageLayout::COLOR_ATTACHMENT_OPTIMAL, + } }) - .new_layout(ImageLayout::TRANSFER_DST_OPTIMAL); + .new_layout(match tt { + TransferType::Upload => ImageLayout::TRANSFER_DST_OPTIMAL, + TransferType::Download => ImageLayout::TRANSFER_SRC_OPTIMAL, + }); if transfer_queue_family_idx == img.renderer.device.graphics_queue_idx { initial_image_barrier = initial_image_barrier .src_access_mask(AccessFlags2::SHADER_SAMPLED_READ) .src_stage_mask(PipelineStageFlags2::FRAGMENT_SHADER) } let initial_buffer_barrier = memory_barrier( - AccessFlags2::HOST_WRITE, + match tt { + TransferType::Upload => AccessFlags2::HOST_WRITE, + TransferType::Download => AccessFlags2::HOST_READ, + }, PipelineStageFlags2::HOST, - AccessFlags2::TRANSFER_READ, + match tt { + TransferType::Upload => AccessFlags2::TRANSFER_READ, + TransferType::Download => AccessFlags2::TRANSFER_WRITE, + }, PipelineStageFlags2::TRANSFER, ); let initial_dep_info = DependencyInfoKHR::default() @@ -203,29 +216,42 @@ impl VulkanShmImage { .image(img.image) .src_queue_family_index(transfer_queue_family_idx) .dst_queue_family_index(img.renderer.device.graphics_queue_idx) - .src_access_mask(AccessFlags2::TRANSFER_WRITE) + .src_access_mask(match tt { + TransferType::Upload => AccessFlags2::TRANSFER_WRITE, + TransferType::Download => AccessFlags2::TRANSFER_READ, + }) .src_stage_mask(PipelineStageFlags2::TRANSFER) - .old_layout(ImageLayout::TRANSFER_DST_OPTIMAL) - .new_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL); + .old_layout(match tt { + TransferType::Upload => ImageLayout::TRANSFER_DST_OPTIMAL, + TransferType::Download => ImageLayout::TRANSFER_SRC_OPTIMAL, + }) + .new_layout(match tt { + TransferType::Upload => ImageLayout::SHADER_READ_ONLY_OPTIMAL, + TransferType::Download => ImageLayout::COLOR_ATTACHMENT_OPTIMAL, + }); if transfer_queue_family_idx == img.renderer.device.graphics_queue_idx { final_image_barrier = final_image_barrier - .dst_access_mask(AccessFlags2::SHADER_SAMPLED_READ) + .dst_access_mask(match tt { + TransferType::Upload => AccessFlags2::SHADER_SAMPLED_READ, + TransferType::Download => AccessFlags2::COLOR_ATTACHMENT_WRITE, + }) .dst_stage_mask(PipelineStageFlags2::FRAGMENT_SHADER); } let final_buffer_barrier = memory_barrier( - AccessFlags2::TRANSFER_READ, + match tt { + TransferType::Upload => AccessFlags2::TRANSFER_READ, + TransferType::Download => AccessFlags2::TRANSFER_WRITE, + }, PipelineStageFlags2::TRANSFER, - AccessFlags2::HOST_WRITE, + match tt { + TransferType::Upload => AccessFlags2::HOST_WRITE, + TransferType::Download => AccessFlags2::HOST_READ, + }, PipelineStageFlags2::HOST, ); let final_dep_info = DependencyInfoKHR::default() .buffer_memory_barriers(slice::from_ref(&final_buffer_barrier)) .image_memory_barriers(slice::from_ref(&final_image_barrier)); - let cpy_info = CopyBufferToImageInfo2::default() - .src_buffer(staging.buffer) - .dst_image(img.image) - .dst_image_layout(ImageLayout::TRANSFER_DST_OPTIMAL) - .regions(regions); let cmd = match &img.renderer.transfer_command_buffers { Some(b) if use_transfer_queue => b.allocate()?, _ => img.renderer.gfx_command_buffers.allocate()?, @@ -241,7 +267,24 @@ impl VulkanShmImage { dev.begin_command_buffer(cmd.buffer, &begin_info) .map_err(VulkanError::BeginCommandBuffer)?; dev.cmd_pipeline_barrier2(cmd.buffer, &initial_dep_info); - dev.cmd_copy_buffer_to_image2(cmd.buffer, &cpy_info); + match tt { + TransferType::Upload => { + let cpy_info = CopyBufferToImageInfo2::default() + .src_buffer(staging.buffer) + .dst_image(img.image) + .dst_image_layout(ImageLayout::TRANSFER_DST_OPTIMAL) + .regions(regions); + dev.cmd_copy_buffer_to_image2(cmd.buffer, &cpy_info); + } + TransferType::Download => { + let cpy_info = CopyImageToBufferInfo2::default() + .dst_buffer(staging.buffer) + .src_image(img.image) + .src_image_layout(ImageLayout::TRANSFER_SRC_OPTIMAL) + .regions(regions); + dev.cmd_copy_image_to_buffer2(cmd.buffer, &cpy_info); + } + } dev.cmd_pipeline_barrier2(cmd.buffer, &final_dep_info); dev.end_command_buffer(cmd.buffer) .map_err(VulkanError::EndCommandBuffer)?; @@ -255,8 +298,10 @@ impl VulkanShmImage { ) .map_err(VulkanError::Submit)?; } - img.is_undefined.set(false); - img.contents_are_undefined.set(false); + if tt == TransferType::Upload { + img.is_undefined.set(false); + img.contents_are_undefined.set(false); + } let release_sync_file = match release_fence.export_sync_file() { Ok(s) => s, Err(e) => { @@ -381,6 +426,7 @@ impl VulkanRenderer { io_job: Default::default(), copy_job: Default::default(), staging: Default::default(), + client_mem: Default::default(), callback: Default::default(), callback_id: Cell::new(0), regions: Default::default(), diff --git a/src/gfx_apis/vulkan/transfer.rs b/src/gfx_apis/vulkan/transfer.rs index 901e7cf0..9062df13 100644 --- a/src/gfx_apis/vulkan/transfer.rs +++ b/src/gfx_apis/vulkan/transfer.rs @@ -41,6 +41,7 @@ pub struct VulkanShmImageAsyncData { pub(super) io_job: Cell>>, pub(super) copy_job: Cell>>, pub(super) staging: CloneCell>>, + pub(super) client_mem: CloneCell>>, pub(super) callback: Cell>>, pub(super) callback_id: Cell, pub(super) regions: RefCell>>, @@ -53,6 +54,7 @@ impl VulkanShmImageAsyncData { fn complete(&self, result: Result<(), VulkanError>) { self.busy.set(false); self.staging.take().unwrap().busy.set(false); + self.client_mem.take(); if let Some(cb) = self.callback.take() { cb.completed(result.map_err(|e| e.into())); } @@ -122,6 +124,7 @@ impl VulkanShmImage { data.data_copied.set(false); staging.busy.set(true); data.staging.set(Some(staging.clone())); + data.client_mem.set(Some(client_mem.clone())); if img.contents_are_undefined.get() { if tt == TransferType::Download { return Err(VulkanError::UndefinedContents); @@ -183,7 +186,9 @@ impl VulkanShmImage { return match tt { TransferType::Upload => self .async_transfer_initiate_host_copy(img, data, &staging, copies, client_mem, tt), - TransferType::Download => unreachable!(), + TransferType::Download => { + self.async_download_copy_image_to_buffer(img, &staging, copies) + } }; } @@ -324,7 +329,9 @@ impl VulkanShmImage { TransferType::Upload => { self.async_transfer_initiate_host_copy(img, data, &staging, copies, client_mem, tt) } - TransferType::Download => unreachable!(), + TransferType::Download => { + self.async_download_copy_image_to_buffer(img, &staging, copies) + } } } @@ -339,6 +346,9 @@ impl VulkanShmImage { ) -> Result<(), VulkanError> { img.renderer.check_defunct()?; + if tt == TransferType::Download { + staging.download(|_, _| ())?; + } let id = img.renderer.allocate_point(); let pending; match client_mem.safe_access() { @@ -440,14 +450,64 @@ impl VulkanShmImage { let staging = data.staging.get().unwrap().staging.get().unwrap(); staging.upload(|_, _| ())?; let Some((cmd, fence, sync_file, point)) = - self.submit_buffer_to_image_copy(img, &staging, regions, true)? + self.submit_buffer_image_copy(img, &staging, regions, true, TransferType::Upload)? else { return Ok(()); }; img.queue_state.set(QueueState::Releasing); let future = img.renderer.eng.spawn( "await async upload", - await_async_transfer_release_to_gfx(point, img.clone(), cmd, fence, sync_file), + await_async_transfer_release_to_gfx( + point, + img.clone(), + cmd, + fence, + sync_file, + TransferType::Upload, + ), + ); + img.renderer.pending_submits.set(point, future); + Ok(()) + } + + fn async_download_copy_image_to_buffer( + &self, + img: &Rc, + staging: &VulkanStagingBuffer, + copies: &[BufferImageCopy2], + ) -> Result<(), VulkanError> { + if img.queue_state.get().acquire(QueueFamily::Transfer) == QueueTransfer::Impossible { + return Ok(()); + } + img.renderer.check_defunct()?; + let Some((cmd, fence, sync_file, point)) = + self.submit_buffer_image_copy(img, &staging, copies, true, TransferType::Download)? + else { + img.queue_state.set(QueueState::Released { + to: QueueFamily::Gfx, + }); + let data = self.async_data.as_ref().unwrap(); + let client_mem = data.client_mem.get().unwrap(); + return self.async_transfer_initiate_host_copy( + &img, + data, + &staging, + copies, + &client_mem, + TransferType::Download, + ); + }; + img.queue_state.set(QueueState::Releasing); + let future = img.renderer.eng.spawn( + "await async image to buffer copy", + await_async_transfer_release_to_gfx( + point, + img.clone(), + cmd, + fence, + sync_file, + TransferType::Download, + ), ); img.renderer.pending_submits.set(point, future); Ok(()) @@ -518,12 +578,14 @@ fn complete_async_host_copy( data.complete(Err(VulkanError::AsyncCopyToStaging(e))); } data.data_copied.set(true); - let res = match tt { - TransferType::Upload => shm.async_upload_copy_buffer_to_image(img, data), - TransferType::Download => unreachable!(), - }; - if let Err(e) = res { - data.complete(Err(e)); + match tt { + TransferType::Upload => { + let res = shm.async_upload_copy_buffer_to_image(img, data); + if let Err(e) = res { + data.complete(Err(e)); + } + } + TransferType::Download => data.complete(Ok(())), } } @@ -556,7 +618,13 @@ async fn await_gfx_queue_release( let data = shm.async_data.as_ref().unwrap(); let res = match tt { TransferType::Upload => shm.async_upload_copy_buffer_to_image(&img, data), - TransferType::Download => unreachable!(), + TransferType::Download => match data.staging.get().unwrap().staging.get() { + Some(staging) => { + let copies = &*data.regions.borrow(); + shm.async_download_copy_image_to_buffer(&img, &staging, copies) + } + None => Ok(()), + }, }; if let Err(e) = res { data.complete(Err(e)); @@ -569,6 +637,7 @@ pub async fn await_async_transfer_release_to_gfx( buf: Rc, _fence: Rc, sync_file: SyncFile, + tt: TransferType, ) { let res = img.renderer.ring.readable(&sync_file.0).await; if let Err(e) = res { @@ -590,5 +659,26 @@ pub async fn await_async_transfer_release_to_gfx( unreachable!(); }; let data = shm.async_data.as_ref().unwrap(); - data.complete(Ok(())); + match tt { + TransferType::Upload => { + data.complete(Ok(())); + } + TransferType::Download => { + let data = shm.async_data.as_ref().unwrap(); + let staging = data.staging.get().unwrap().staging.get().unwrap(); + let client_mem = data.client_mem.get().unwrap(); + let copies = &*data.regions.borrow(); + let res = shm.async_transfer_initiate_host_copy( + &img, + data, + &staging, + copies, + &client_mem, + tt, + ); + if let Err(e) = res { + data.complete(Err(e)); + } + } + } } From 17de1650a03f1071b8a41b3efd49999e26b502b1 Mon Sep 17 00:00:00 2001 From: Julian Orth Date: Sun, 6 Oct 2024 11:31:47 +0200 Subject: [PATCH 5/7] screencopy: unconditionally create copy for shm --- src/gfx_api.rs | 10 --- src/gfx_apis/gl.rs | 2 - src/gfx_apis/gl/renderer/texture.rs | 24 +------ src/gfx_apis/vulkan/image.rs | 15 ---- src/it/test_gfx_api.rs | 14 ---- src/state.rs | 104 +++++++++------------------- 6 files changed, 35 insertions(+), 134 deletions(-) diff --git a/src/gfx_api.rs b/src/gfx_api.rs index d36b2e11..abf53af2 100644 --- a/src/gfx_api.rs +++ b/src/gfx_api.rs @@ -481,16 +481,6 @@ pub trait GfxTexture: Debug { fn size(&self) -> (i32, i32); fn as_any(&self) -> &dyn Any; fn into_any(self: Rc) -> Rc; - fn read_pixels( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError>; fn dmabuf(&self) -> Option<&DmaBuf>; fn format(&self) -> &'static Format; } diff --git a/src/gfx_apis/gl.rs b/src/gfx_apis/gl.rs index dd45501a..284e8131 100644 --- a/src/gfx_apis/gl.rs +++ b/src/gfx_apis/gl.rs @@ -185,8 +185,6 @@ enum RenderError { ExternalUnsupported, #[error("OpenGL context does not support any formats")] NoSupportedFormats, - #[error("Cannot convert a shm texture into a framebuffer")] - ShmTextureToFb, #[error("Could not create EGLSyncKHR")] CreateEglSync, #[error("Could not destroy EGLSyncKHR")] diff --git a/src/gfx_apis/gl/renderer/texture.rs b/src/gfx_apis/gl/renderer/texture.rs index 5400273a..a4172059 100644 --- a/src/gfx_apis/gl/renderer/texture.rs +++ b/src/gfx_apis/gl/renderer/texture.rs @@ -7,7 +7,7 @@ use { }, gfx_apis::gl::{ gl::texture::GlTexture, - renderer::{context::GlRenderContext, framebuffer::Framebuffer}, + renderer::context::GlRenderContext, sys::{ GLint, GL_CLAMP_TO_EDGE, GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_TEXTURE_WRAP_T, GL_UNPACK_ROW_LENGTH_EXT, @@ -45,13 +45,6 @@ impl Texture { pub fn height(&self) -> i32 { self.gl.height } - - pub fn to_framebuffer(&self) -> Result, RenderError> { - match &self.gl.img { - Some(img) => self.ctx.image_to_fb(img), - _ => Err(RenderError::ShmTextureToFb), - } - } } impl GfxTexture for Texture { @@ -67,21 +60,6 @@ impl GfxTexture for Texture { self } - fn read_pixels( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - _stride: i32, - format: &Format, - shm: &[Cell], - ) -> Result<(), GfxError> { - self.to_framebuffer()? - .copy_to_shm(x, y, width, height, format, shm) - .map_err(|e| e.into()) - } - fn dmabuf(&self) -> Option<&DmaBuf> { self.gl.img.as_ref().map(|i| &i.dmabuf) } diff --git a/src/gfx_apis/vulkan/image.rs b/src/gfx_apis/vulkan/image.rs index d40ae715..082ea2a8 100644 --- a/src/gfx_apis/vulkan/image.rs +++ b/src/gfx_apis/vulkan/image.rs @@ -543,21 +543,6 @@ impl GfxTexture for VulkanImage { self } - fn read_pixels( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError> { - self.renderer - .read_pixels(&self, x, y, width, height, stride, format, shm) - .map_err(|e| e.into()) - } - fn dmabuf(&self) -> Option<&DmaBuf> { match &self.ty { VulkanImageMemory::DmaBuf(b) => Some(&b.template.dmabuf), diff --git a/src/it/test_gfx_api.rs b/src/it/test_gfx_api.rs index e8b092c4..0ec0cb4e 100644 --- a/src/it/test_gfx_api.rs +++ b/src/it/test_gfx_api.rs @@ -300,20 +300,6 @@ impl GfxTexture for TestGfxImage { self } - fn read_pixels( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError> { - self.deref() - .read_pixels(x, y, width, height, stride, format, shm) - } - fn dmabuf(&self) -> Option<&DmaBuf> { match self { TestGfxImage::Shm(_) => None, diff --git a/src/state.rs b/src/state.rs index 1871084b..fbf862f0 100644 --- a/src/state.rs +++ b/src/state.rs @@ -1006,17 +1006,6 @@ impl State { ) } - fn have_hardware_cursor(&self) -> bool { - if let Some(group) = self.cursor_user_group_hardware_cursor.get() { - if let Some(user) = group.active() { - if user.get().is_some() { - return true; - } - } - } - false - } - pub fn perform_shm_screencopy( &self, src: &Rc, @@ -1032,66 +1021,41 @@ impl State { transform: Transform, scale: Scale, ) -> Result<(), ShmScreencopyError> { - let (src_width, src_height) = src.size(); - let mut needs_copy = capture.rect.x1() < x_off - || capture.rect.x2() > x_off + src_width - || capture.rect.y1() < y_off - || capture.rect.y2() > y_off + src_height - || self.have_hardware_cursor(); - if let Some((target_width, target_height)) = size { - if (target_width, target_height) != (src_width, src_height) { - needs_copy = true; - } - } - let acc = if needs_copy { - let Some(ctx) = self.render_ctx.get() else { - return Err(ShmScreencopyError::NoRenderContext); - }; - let fb = ctx - .create_fb(capture.rect.width(), capture.rect.height(), stride, format) - .map_err(ShmScreencopyError::CreateTemporaryFb)?; - self.perform_screencopy( - src, - None, - acquire_sync, - ReleaseSync::None, - &fb, - AcquireSync::Unnecessary, - ReleaseSync::None, - transform, - position, - true, - x_off - capture.rect.x1(), - y_off - capture.rect.y1(), - size, - transform, - scale, - ) - .map_err(ShmScreencopyError::CopyToTemporary)?; - mem.access(|mem| { - fb.copy_to_shm( - 0, - 0, - capture.rect.width(), - capture.rect.height(), - stride, - format, - mem, - ) - }) - } else { - mem.access(|mem| { - src.clone().read_pixels( - capture.rect.x1() - x_off, - capture.rect.y1() - y_off, - capture.rect.width(), - capture.rect.height(), - stride, - format, - mem, - ) - }) + let Some(ctx) = self.render_ctx.get() else { + return Err(ShmScreencopyError::NoRenderContext); }; + let fb = ctx + .create_fb(capture.rect.width(), capture.rect.height(), stride, format) + .map_err(ShmScreencopyError::CreateTemporaryFb)?; + self.perform_screencopy( + src, + None, + acquire_sync, + ReleaseSync::None, + &fb, + AcquireSync::Unnecessary, + ReleaseSync::None, + transform, + position, + true, + x_off - capture.rect.x1(), + y_off - capture.rect.y1(), + size, + transform, + scale, + ) + .map_err(ShmScreencopyError::CopyToTemporary)?; + let acc = mem.access(|mem| { + fb.copy_to_shm( + 0, + 0, + capture.rect.width(), + capture.rect.height(), + stride, + format, + mem, + ) + }); match acc { Ok(res) => res.map_err(ShmScreencopyError::ReadPixels), Err(e) => { From aca14d48dd94a9ca0efabc488670bf2923053813 Mon Sep 17 00:00:00 2001 From: Julian Orth Date: Sun, 6 Oct 2024 11:50:11 +0200 Subject: [PATCH 6/7] gfx: remove incompatible shm downloads --- src/gfx_api.rs | 11 +---- src/gfx_apis/gl/renderer/framebuffer.rs | 35 +++---------- src/gfx_apis/vulkan.rs | 10 ---- src/gfx_apis/vulkan/image.rs | 13 +---- src/gfx_apis/vulkan/renderer.rs | 66 +++---------------------- src/it/test_gfx_api.rs | 56 ++++----------------- src/state.rs | 12 +---- 7 files changed, 27 insertions(+), 176 deletions(-) diff --git a/src/gfx_api.rs b/src/gfx_api.rs index abf53af2..018ba13b 100644 --- a/src/gfx_api.rs +++ b/src/gfx_api.rs @@ -265,16 +265,7 @@ pub trait GfxFramebuffer: Debug { clear: Option<&Color>, ) -> Result, GfxError>; - fn copy_to_shm( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError>; + fn copy_to_shm(self: Rc, shm: &[Cell]) -> Result<(), GfxError>; fn format(&self) -> &'static Format; } diff --git a/src/gfx_apis/gl/renderer/framebuffer.rs b/src/gfx_apis/gl/renderer/framebuffer.rs index 33071aa9..071f3a29 100644 --- a/src/gfx_apis/gl/renderer/framebuffer.rs +++ b/src/gfx_apis/gl/renderer/framebuffer.rs @@ -34,29 +34,21 @@ impl Debug for Framebuffer { } impl Framebuffer { - pub fn copy_to_shm( - &self, - x: i32, - y: i32, - width: i32, - height: i32, - format: &Format, - shm: &[Cell], - ) -> Result<(), RenderError> { + pub fn copy_to_shm(&self, shm: &[Cell]) -> Result<(), RenderError> { + let format = self.gl.rb.format; let Some(shm_info) = &format.shm_info else { return Err(RenderError::UnsupportedShmFormat(format.name)); }; let gles = self.ctx.ctx.dpy.gles; - let y = self.gl.height - y - height; let _ = self.ctx.ctx.with_current(|| { unsafe { (gles.glBindFramebuffer)(GL_FRAMEBUFFER, self.gl.fbo); (gles.glViewport)(0, 0, self.gl.width, self.gl.height); (gles.glReadnPixels)( - x, - y, - width, - height, + 0, + 0, + self.gl.width, + self.gl.height, shm_info.gl_format as _, shm_info.gl_type as _, shm.len() as _, @@ -112,19 +104,8 @@ impl GfxFramebuffer for Framebuffer { self.render(acquire_sync, ops, clear).map_err(|e| e.into()) } - fn copy_to_shm( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - _stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError> { - (*self) - .copy_to_shm(x, y, width, height, format, shm) - .map_err(|e| e.into()) + fn copy_to_shm(self: Rc, shm: &[Cell]) -> Result<(), GfxError> { + (*self).copy_to_shm(shm).map_err(|e| e.into()) } fn format(&self) -> &'static Format { diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs index fd8287c1..1193acd9 100644 --- a/src/gfx_apis/vulkan.rs +++ b/src/gfx_apis/vulkan.rs @@ -173,16 +173,6 @@ pub enum VulkanError { InvalidStride, #[error("Shm stride and height do not match buffer size")] InvalidBufferSize, - #[error("The shm parameters are invalid x={x}, y={y}, width={width}, height={height}, stride={stride}")] - InvalidShmParameters { - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - }, - #[error(transparent)] - GfxError(GfxError), #[error("Buffer format {0} is not supported for shm buffers in Vulkan context")] UnsupportedShmFormat(&'static str), #[error("Only BO_USE_RENDERING and BO_USE_WRITE are supported")] diff --git a/src/gfx_apis/vulkan/image.rs b/src/gfx_apis/vulkan/image.rs index 082ea2a8..b005c7bc 100644 --- a/src/gfx_apis/vulkan/image.rs +++ b/src/gfx_apis/vulkan/image.rs @@ -510,18 +510,9 @@ impl GfxFramebuffer for VulkanImage { .map_err(|e| e.into()) } - fn copy_to_shm( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError> { + fn copy_to_shm(self: Rc, shm: &[Cell]) -> Result<(), GfxError> { self.renderer - .read_pixels(&self, x, y, width, height, stride, format, shm) + .read_all_pixels(&self, shm) .map_err(|e| e.into()) } diff --git a/src/gfx_apis/vulkan/renderer.rs b/src/gfx_apis/vulkan/renderer.rs index cc225064..e3feb025 100644 --- a/src/gfx_apis/vulkan/renderer.rs +++ b/src/gfx_apis/vulkan/renderer.rs @@ -2,10 +2,10 @@ use { crate::{ async_engine::{AsyncEngine, SpawnedFuture}, cpu_worker::PendingJob, - format::{Format, XRGB8888}, + format::XRGB8888, gfx_api::{ - AcquireSync, BufferResv, BufferResvUser, GfxApiOpt, GfxFormat, GfxFramebuffer, - GfxTexture, GfxWriteModifier, ReleaseSync, SyncFile, + AcquireSync, BufferResv, BufferResvUser, GfxApiOpt, GfxFormat, GfxTexture, + GfxWriteModifier, ReleaseSync, SyncFile, }, gfx_apis::vulkan::{ allocator::{VulkanAllocator, VulkanThreadedAllocator}, @@ -905,74 +905,20 @@ impl VulkanRenderer { frame.waiter.set(Some(future)); } - pub fn read_pixels( - self: &Rc, - tex: &Rc, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - dst: &[Cell], - ) -> Result<(), VulkanError> { - if x < 0 || y < 0 || width <= 0 || height <= 0 || stride <= 0 { - return Err(VulkanError::InvalidShmParameters { - x, - y, - width, - height, - stride, - }); - } - let width = width as u32; - let height = height as u32; - let stride = stride as u32; - if x == 0 && y == 0 && width == tex.width && height == tex.height && format == tex.format { - return self.read_all_pixels(tex, stride, dst); - } - let tmp_tex = self.create_shm_texture( - format, - width as i32, - height as i32, - stride as i32, - &[], - true, - None, - )?; - (&*tmp_tex as &dyn GfxFramebuffer) - .copy_texture( - AcquireSync::None, - ReleaseSync::None, - &(tex.clone() as _), - None, - AcquireSync::None, - ReleaseSync::None, - x, - y, - ) - .map_err(VulkanError::GfxError)?; - self.read_all_pixels(&tmp_tex, stride, dst) - } - - fn read_all_pixels( + pub(super) fn read_all_pixels( self: &Rc, tex: &VulkanImage, - stride: u32, dst: &[Cell], ) -> Result<(), VulkanError> { let Some(shm_info) = &tex.format.shm_info else { return Err(VulkanError::UnsupportedShmFormat(tex.format.name)); }; - if stride < tex.width * shm_info.bpp || stride % shm_info.bpp != 0 { - return Err(VulkanError::InvalidStride); - } - let size = stride as u64 * tex.height as u64; + let size = tex.stride as u64 * tex.height as u64; if size != dst.len() as u64 { return Err(VulkanError::InvalidBufferSize); } let region = BufferImageCopy::default() - .buffer_row_length(stride / shm_info.bpp) + .buffer_row_length(tex.stride / shm_info.bpp) .buffer_image_height(tex.height) .image_subresource(ImageSubresourceLayers { aspect_mask: ImageAspectFlags::COLOR, diff --git a/src/it/test_gfx_api.rs b/src/it/test_gfx_api.rs index 0ec0cb4e..e6a0de03 100644 --- a/src/it/test_gfx_api.rs +++ b/src/it/test_gfx_api.rs @@ -214,44 +214,17 @@ struct TestDmaBufGfxImage { } impl TestGfxImage { - fn read_pixels( - &self, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError> { - assert!(x >= 0); - assert!(y >= 0); - assert!(width >= 0); - assert!(height >= 0); - assert!(stride >= 0); - assert!(x + width <= self.width()); - assert!(y + height <= self.height()); - assert!(stride >= width * 4); - let size = (stride * height) as usize; - assert!(shm.len() >= size); - let copy = |src_stride: i32, src_format: &Format, mut src: *const u8, mut dst: *mut u8| unsafe { - src = src.add((y * src_stride + x * 4) as usize); - for _ in 0..height { - ptr::copy_nonoverlapping(src, dst, (width * 4) as usize); - if !src_format.has_alpha && format.has_alpha { - for i in 0..width { - *dst.add((i * 4 + 3) as usize) = 255; - } - } - src = src.add(src_stride as usize); - dst = dst.add(stride as usize); - } + fn read_pixels(&self, shm: &[Cell]) -> Result<(), GfxError> { + let copy = |height: i32, stride: i32, src: *const u8, dst: *mut u8| unsafe { + let size = (height * stride) as usize; + assert!(shm.len() >= size); + ptr::copy_nonoverlapping(src, dst, size); }; match self { TestGfxImage::Shm(s) => { copy( + s.height, s.stride, - s.format, s.data.borrow().as_ptr(), shm.as_ptr() as _, ); @@ -260,8 +233,8 @@ impl TestGfxImage { let map = d.bo.clone().map_read().map_err(TestGfxError::MapDmaBuf)?; unsafe { copy( + d.buf.height, map.stride(), - d.buf.format, map.data().as_ptr(), shm.as_ptr() as _, ); @@ -575,19 +548,8 @@ impl GfxFramebuffer for TestGfxFb { Ok(None) } - fn copy_to_shm( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError> { - self.img - .deref() - .read_pixels(x, y, width, height, stride, format, shm) + fn copy_to_shm(self: Rc, shm: &[Cell]) -> Result<(), GfxError> { + self.img.deref().read_pixels(shm) } fn format(&self) -> &'static Format { diff --git a/src/state.rs b/src/state.rs index fbf862f0..75a13882 100644 --- a/src/state.rs +++ b/src/state.rs @@ -1045,17 +1045,7 @@ impl State { scale, ) .map_err(ShmScreencopyError::CopyToTemporary)?; - let acc = mem.access(|mem| { - fb.copy_to_shm( - 0, - 0, - capture.rect.width(), - capture.rect.height(), - stride, - format, - mem, - ) - }); + let acc = mem.access(|mem| fb.copy_to_shm(mem)); match acc { Ok(res) => res.map_err(ShmScreencopyError::ReadPixels), Err(e) => { From 028d0ed44c7c8c23f671a4ce9c1860d8a5f72a41 Mon Sep 17 00:00:00 2001 From: Julian Orth Date: Sun, 6 Oct 2024 15:11:00 +0200 Subject: [PATCH 7/7] gfx: implement async shm downloads --- src/gfx_api.rs | 21 +++- src/gfx_apis/gl/renderer/context.rs | 7 +- src/gfx_apis/gl/renderer/framebuffer.rs | 35 +++++- src/gfx_apis/vulkan.rs | 19 ++- src/gfx_apis/vulkan/image.rs | 45 +++++-- src/gfx_apis/vulkan/renderer.rs | 148 +----------------------- src/ifs/zwlr_screencopy_frame_v1.rs | 20 ++++ src/ifs/zwlr_screencopy_manager_v1.rs | 1 + src/it/test_gfx_api.rs | 36 ++++-- src/state.rs | 41 ++++--- src/tree/output.rs | 19 ++- 11 files changed, 194 insertions(+), 198 deletions(-) diff --git a/src/gfx_api.rs b/src/gfx_api.rs index 018ba13b..2af944fc 100644 --- a/src/gfx_api.rs +++ b/src/gfx_api.rs @@ -265,11 +265,23 @@ pub trait GfxFramebuffer: Debug { clear: Option<&Color>, ) -> Result, GfxError>; - fn copy_to_shm(self: Rc, shm: &[Cell]) -> Result<(), GfxError>; - fn format(&self) -> &'static Format; } +pub trait GfxInternalFramebuffer: GfxFramebuffer { + fn into_fb(self: Rc) -> Rc; + + fn staging_size(&self) -> usize; + + fn download( + self: Rc, + staging: &Rc, + callback: Rc, + mem: Rc, + damage: Region, + ) -> Result, GfxError>; +} + impl dyn GfxFramebuffer { pub fn clear( &self, @@ -593,13 +605,14 @@ pub trait GfxContext: Debug { fn gfx_api(&self) -> GfxApi; - fn create_fb( + fn create_internal_fb( self: Rc, + cpu_worker: &Rc, width: i32, height: i32, stride: i32, format: &'static Format, - ) -> Result, GfxError>; + ) -> Result, GfxError>; fn sync_obj_ctx(&self) -> Option<&Rc>; diff --git a/src/gfx_apis/gl/renderer/context.rs b/src/gfx_apis/gl/renderer/context.rs index e0ab21bb..32cf2392 100644 --- a/src/gfx_apis/gl/renderer/context.rs +++ b/src/gfx_apis/gl/renderer/context.rs @@ -5,7 +5,7 @@ use { format::{Format, XRGB8888}, gfx_api::{ AsyncShmGfxTexture, BufferResvUser, GfxContext, GfxError, GfxFormat, GfxFramebuffer, - GfxImage, ResetStatus, ShmGfxTexture, + GfxImage, GfxInternalFramebuffer, ResetStatus, ShmGfxTexture, }, gfx_apis::gl::{ egl::{context::EglContext, display::EglDisplay, image::EglImage}, @@ -316,13 +316,14 @@ impl GfxContext for GlRenderContext { GfxApi::OpenGl } - fn create_fb( + fn create_internal_fb( self: Rc, + _cpu_worker: &Rc, width: i32, height: i32, _stride: i32, format: &'static Format, - ) -> Result, GfxError> { + ) -> Result, GfxError> { let fb = self.ctx.with_current(|| unsafe { GlRenderBuffer::new(&self.ctx, width, height, format)?.create_framebuffer() })?; diff --git a/src/gfx_apis/gl/renderer/framebuffer.rs b/src/gfx_apis/gl/renderer/framebuffer.rs index 071f3a29..bbe98c16 100644 --- a/src/gfx_apis/gl/renderer/framebuffer.rs +++ b/src/gfx_apis/gl/renderer/framebuffer.rs @@ -1,7 +1,11 @@ use { crate::{ format::Format, - gfx_api::{AcquireSync, GfxApiOpt, GfxError, GfxFramebuffer, ReleaseSync, SyncFile}, + gfx_api::{ + AcquireSync, AsyncShmGfxTextureCallback, GfxApiOpt, GfxError, GfxFramebuffer, + GfxInternalFramebuffer, GfxStagingBuffer, PendingShmTransfer, ReleaseSync, ShmMemory, + SyncFile, + }, gfx_apis::gl::{ gl::{ frame_buffer::GlFrameBuffer, @@ -13,6 +17,7 @@ use { sys::{GL_ONE, GL_ONE_MINUS_SRC_ALPHA}, RenderError, }, + rect::Region, theme::Color, }, std::{ @@ -104,11 +109,31 @@ impl GfxFramebuffer for Framebuffer { self.render(acquire_sync, ops, clear).map_err(|e| e.into()) } - fn copy_to_shm(self: Rc, shm: &[Cell]) -> Result<(), GfxError> { - (*self).copy_to_shm(shm).map_err(|e| e.into()) - } - fn format(&self) -> &'static Format { self.gl.rb.format } } + +impl GfxInternalFramebuffer for Framebuffer { + fn into_fb(self: Rc) -> Rc { + self + } + + fn staging_size(&self) -> usize { + 0 + } + + fn download( + self: Rc, + _staging: &Rc, + _callback: Rc, + mem: Rc, + _damage: Region, + ) -> Result, GfxError> { + let mut res = Ok(()); + mem.access(&mut |mem| res = self.copy_to_shm(mem)) + .map_err(RenderError::AccessFailed)?; + res?; + Ok(None) + } +} diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs index 1193acd9..14e2ca82 100644 --- a/src/gfx_apis/vulkan.rs +++ b/src/gfx_apis/vulkan.rs @@ -23,7 +23,7 @@ use { cpu_worker::{jobs::read_write::ReadWriteJobError, CpuWorker}, format::Format, gfx_api::{ - AsyncShmGfxTexture, GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage, + AsyncShmGfxTexture, GfxContext, GfxError, GfxFormat, GfxImage, GfxInternalFramebuffer, GfxStagingBuffer, ResetStatus, ShmGfxTexture, StagingBufferUsecase, STAGING_DOWNLOAD, STAGING_UPLOAD, }, @@ -316,16 +316,23 @@ impl GfxContext for Context { GfxApi::Vulkan } - fn create_fb( + fn create_internal_fb( self: Rc, + cpu_worker: &Rc, width: i32, height: i32, stride: i32, format: &'static Format, - ) -> Result, GfxError> { - let fb = self - .0 - .create_shm_texture(format, width, height, stride, &[], true, None)?; + ) -> Result, GfxError> { + let fb = self.0.create_shm_texture( + format, + width, + height, + stride, + &[], + true, + Some(cpu_worker), + )?; Ok(fb) } diff --git a/src/gfx_apis/vulkan/image.rs b/src/gfx_apis/vulkan/image.rs index b005c7bc..43fcb376 100644 --- a/src/gfx_apis/vulkan/image.rs +++ b/src/gfx_apis/vulkan/image.rs @@ -4,8 +4,8 @@ use { gfx_api::{ AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback, AsyncShmGfxTextureTransferCancellable, GfxApiOpt, GfxError, GfxFramebuffer, GfxImage, - GfxStagingBuffer, GfxTexture, PendingShmTransfer, ReleaseSync, ShmGfxTexture, - ShmMemory, SyncFile, + GfxInternalFramebuffer, GfxStagingBuffer, GfxTexture, PendingShmTransfer, ReleaseSync, + ShmGfxTexture, ShmMemory, SyncFile, }, gfx_apis::vulkan::{ allocator::VulkanAllocation, device::VulkanDevice, format::VulkanModifierLimits, @@ -510,17 +510,46 @@ impl GfxFramebuffer for VulkanImage { .map_err(|e| e.into()) } - fn copy_to_shm(self: Rc, shm: &[Cell]) -> Result<(), GfxError> { - self.renderer - .read_all_pixels(&self, shm) - .map_err(|e| e.into()) - } - fn format(&self) -> &'static Format { self.format } } +impl GfxInternalFramebuffer for VulkanImage { + fn into_fb(self: Rc) -> Rc { + self + } + + fn staging_size(&self) -> usize { + let VulkanImageMemory::Internal(shm) = &self.ty else { + unreachable!(); + }; + shm.size as _ + } + + fn download( + self: Rc, + staging: &Rc, + callback: Rc, + mem: Rc, + damage: Region, + ) -> Result, GfxError> { + let VulkanImageMemory::Internal(shm) = &self.ty else { + unreachable!(); + }; + let staging = staging.clone().into_vk(&self.renderer.device.device); + let pending = shm.async_transfer( + &self, + staging, + &mem, + damage, + callback, + TransferType::Download, + )?; + Ok(pending) + } +} + impl GfxTexture for VulkanImage { fn size(&self) -> (i32, i32) { (self.width as _, self.height as _) diff --git a/src/gfx_apis/vulkan/renderer.rs b/src/gfx_apis/vulkan/renderer.rs index e3feb025..053877ad 100644 --- a/src/gfx_apis/vulkan/renderer.rs +++ b/src/gfx_apis/vulkan/renderer.rs @@ -32,11 +32,10 @@ use { ash::{ vk, vk::{ - AccessFlags2, AttachmentLoadOp, AttachmentStoreOp, BufferImageCopy, - BufferMemoryBarrier2, ClearColorValue, ClearValue, CommandBuffer, - CommandBufferBeginInfo, CommandBufferSubmitInfo, CommandBufferUsageFlags, - CopyImageInfo2, DependencyInfo, DependencyInfoKHR, DescriptorImageInfo, DescriptorType, - Extent2D, Extent3D, Fence, ImageAspectFlags, ImageCopy2, ImageLayout, + AccessFlags2, AttachmentLoadOp, AttachmentStoreOp, ClearColorValue, ClearValue, + CommandBuffer, CommandBufferBeginInfo, CommandBufferSubmitInfo, + CommandBufferUsageFlags, CopyImageInfo2, DependencyInfoKHR, DescriptorImageInfo, + DescriptorType, Extent2D, Extent3D, ImageAspectFlags, ImageCopy2, ImageLayout, ImageMemoryBarrier2, ImageSubresourceLayers, ImageSubresourceRange, PipelineBindPoint, PipelineStageFlags2, Rect2D, RenderingAttachmentInfo, RenderingInfo, SemaphoreSubmitInfo, SemaphoreSubmitInfoKHR, ShaderStageFlags, SubmitInfo2, Viewport, @@ -49,7 +48,7 @@ use { std::{ cell::{Cell, RefCell}, fmt::{Debug, Formatter}, - mem, ptr, + mem, rc::Rc, slice, }, @@ -905,143 +904,6 @@ impl VulkanRenderer { frame.waiter.set(Some(future)); } - pub(super) fn read_all_pixels( - self: &Rc, - tex: &VulkanImage, - dst: &[Cell], - ) -> Result<(), VulkanError> { - let Some(shm_info) = &tex.format.shm_info else { - return Err(VulkanError::UnsupportedShmFormat(tex.format.name)); - }; - let size = tex.stride as u64 * tex.height as u64; - if size != dst.len() as u64 { - return Err(VulkanError::InvalidBufferSize); - } - let region = BufferImageCopy::default() - .buffer_row_length(tex.stride / shm_info.bpp) - .buffer_image_height(tex.height) - .image_subresource(ImageSubresourceLayers { - aspect_mask: ImageAspectFlags::COLOR, - mip_level: 0, - base_array_layer: 0, - layer_count: 1, - }) - .image_extent(Extent3D { - width: tex.width, - height: tex.height, - depth: 1, - }); - let staging = - self.device - .create_staging_buffer(&self.allocator, size, false, true, true)?; - let initial_tex_barrier; - let initial_buffer_barrier = BufferMemoryBarrier2::default() - .buffer(staging.buffer) - .offset(0) - .size(staging.size) - .dst_access_mask(AccessFlags2::TRANSFER_WRITE) - .dst_stage_mask(PipelineStageFlags2::TRANSFER); - let mut initial_barriers = DependencyInfo::default() - .buffer_memory_barriers(slice::from_ref(&initial_buffer_barrier)); - if tex.bridge.is_none() { - initial_tex_barrier = image_barrier() - .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) - .dst_queue_family_index(self.device.graphics_queue_idx) - .image(tex.image) - .old_layout(ImageLayout::GENERAL) - .new_layout(ImageLayout::TRANSFER_SRC_OPTIMAL) - .dst_access_mask(AccessFlags2::TRANSFER_READ) - .dst_stage_mask(PipelineStageFlags2::TRANSFER); - initial_barriers = - initial_barriers.image_memory_barriers(slice::from_ref(&initial_tex_barrier)); - } - let final_tex_barrier; - let final_buffer_barrier = BufferMemoryBarrier2::default() - .buffer(staging.buffer) - .offset(0) - .size(staging.size) - .src_access_mask(AccessFlags2::TRANSFER_WRITE) - .src_stage_mask(PipelineStageFlags2::TRANSFER) - .dst_access_mask(AccessFlags2::HOST_READ) - .dst_stage_mask(PipelineStageFlags2::HOST); - let mut final_barriers = DependencyInfo::default() - .buffer_memory_barriers(slice::from_ref(&final_buffer_barrier)); - if tex.bridge.is_none() { - final_tex_barrier = image_barrier() - .src_queue_family_index(self.device.graphics_queue_idx) - .dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) - .image(tex.image) - .old_layout(ImageLayout::TRANSFER_SRC_OPTIMAL) - .new_layout(ImageLayout::GENERAL) - .src_access_mask(AccessFlags2::TRANSFER_READ) - .src_stage_mask(PipelineStageFlags2::TRANSFER); - final_barriers = - final_barriers.image_memory_barriers(slice::from_ref(&final_tex_barrier)); - } - let buf = self.gfx_command_buffers.allocate()?; - let mut semaphores = vec![]; - let mut semaphore_infos = vec![]; - if let VulkanImageMemory::DmaBuf(buf) = &tex.ty { - for plane in &buf.template.dmabuf.planes { - let fd = dma_buf_export_sync_file(&plane.fd, DMA_BUF_SYNC_READ) - .map_err(VulkanError::IoctlExportSyncFile)?; - let semaphore = self.allocate_semaphore()?; - semaphore.import_sync_file(fd)?; - let semaphore_info = SemaphoreSubmitInfo::default() - .semaphore(semaphore.semaphore) - .stage_mask(PipelineStageFlags2::TOP_OF_PIPE); - semaphores.push(semaphore); - semaphore_infos.push(semaphore_info); - } - } - let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(buf.buffer); - let submit_info = SubmitInfo2::default() - .wait_semaphore_infos(&semaphore_infos) - .command_buffer_infos(slice::from_ref(&command_buffer_info)); - let begin_info = - CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT); - unsafe { - self.device - .device - .begin_command_buffer(buf.buffer, &begin_info) - .map_err(VulkanError::BeginCommandBuffer)?; - self.device - .device - .cmd_pipeline_barrier2(buf.buffer, &initial_barriers); - self.device.device.cmd_copy_image_to_buffer( - buf.buffer, - tex.image, - ImageLayout::TRANSFER_SRC_OPTIMAL, - staging.buffer, - &[region], - ); - self.device - .device - .cmd_pipeline_barrier2(buf.buffer, &final_barriers); - self.device - .device - .end_command_buffer(buf.buffer) - .map_err(VulkanError::EndCommandBuffer)?; - self.device - .device - .queue_submit2( - self.device.graphics_queue, - slice::from_ref(&submit_info), - Fence::null(), - ) - .map_err(VulkanError::Submit)?; - } - self.block(); - self.gfx_command_buffers.buffers.push(buf); - for semaphore in semaphores { - self.wait_semaphores.push(semaphore); - } - staging.download(|mem, size| unsafe { - ptr::copy_nonoverlapping(mem, dst.as_ptr() as _, size); - })?; - Ok(()) - } - pub fn execute( self: &Rc, fb: &VulkanImage, diff --git a/src/ifs/zwlr_screencopy_frame_v1.rs b/src/ifs/zwlr_screencopy_frame_v1.rs index 44afe8c5..41183bd8 100644 --- a/src/ifs/zwlr_screencopy_frame_v1.rs +++ b/src/ifs/zwlr_screencopy_frame_v1.rs @@ -2,6 +2,7 @@ use { crate::{ client::{Client, ClientError}, format::XRGB8888, + gfx_api::{AsyncShmGfxTextureCallback, GfxError, PendingShmTransfer}, ifs::{ wl_buffer::{WlBuffer, WlBufferError, WlBufferStorage}, wl_output::OutputGlobalOpt, @@ -9,6 +10,7 @@ use { leaks::Tracker, object::{Object, Version}, rect::Rect, + utils::errorfmt::ErrorFmt, wire::{zwlr_screencopy_frame_v1::*, WlBufferId, ZwlrScreencopyFrameV1Id}, }, std::{cell::Cell, ops::Deref, rc::Rc}, @@ -29,6 +31,7 @@ pub struct ZwlrScreencopyFrameV1 { pub with_damage: Cell, pub buffer: Cell>>, pub version: Version, + pub pending: Cell>, } impl ZwlrScreencopyFrameV1 { @@ -132,6 +135,7 @@ impl ZwlrScreencopyFrameV1 { node.screencopies.remove(&(self.client.id, self.id)); node.screencast_changed(); } + self.pending.take(); } } @@ -153,6 +157,22 @@ impl ZwlrScreencopyFrameV1RequestHandler for ZwlrScreencopyFrameV1 { } } +impl AsyncShmGfxTextureCallback for ZwlrScreencopyFrameV1 { + fn completed(self: Rc, res: Result<(), GfxError>) { + self.pending.take(); + match res { + Ok(_) => { + let now = self.client.state.now(); + self.send_ready(now.0.tv_sec as _, now.0.tv_nsec as _); + } + Err(e) => { + log::warn!("Could not perform shm screencopy: {}", ErrorFmt(e)); + self.send_failed(); + } + } + } +} + object_base! { self = ZwlrScreencopyFrameV1; version = self.version; diff --git a/src/ifs/zwlr_screencopy_manager_v1.rs b/src/ifs/zwlr_screencopy_manager_v1.rs index ae99a4b4..66520274 100644 --- a/src/ifs/zwlr_screencopy_manager_v1.rs +++ b/src/ifs/zwlr_screencopy_manager_v1.rs @@ -130,6 +130,7 @@ impl ZwlrScreencopyManagerV1 { with_damage: Cell::new(false), buffer: Cell::new(None), version: self.version, + pending: Default::default(), }); track!(self.client, frame); self.client.add_client_obj(&frame)?; diff --git a/src/it/test_gfx_api.rs b/src/it/test_gfx_api.rs index e6a0de03..87d33a81 100644 --- a/src/it/test_gfx_api.rs +++ b/src/it/test_gfx_api.rs @@ -6,8 +6,8 @@ use { gfx_api::{ AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback, CopyTexture, FillRect, FramebufferRect, GfxApiOpt, GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage, - GfxStagingBuffer, GfxTexture, GfxWriteModifier, PendingShmTransfer, ReleaseSync, - ResetStatus, ShmGfxTexture, ShmMemory, SyncFile, + GfxInternalFramebuffer, GfxStagingBuffer, GfxTexture, GfxWriteModifier, + PendingShmTransfer, ReleaseSync, ResetStatus, ShmGfxTexture, ShmMemory, SyncFile, }, rect::{Rect, Region}, theme::Color, @@ -165,13 +165,14 @@ impl GfxContext for TestGfxCtx { GfxApi::OpenGl } - fn create_fb( + fn create_internal_fb( self: Rc, + _cpu_worker: &Rc, width: i32, height: i32, stride: i32, format: &'static Format, - ) -> Result, GfxError> { + ) -> Result, GfxError> { assert!(stride >= width * 4); Ok(Rc::new(TestGfxFb { img: Rc::new(TestGfxImage::Shm(TestShmGfxImage { @@ -548,15 +549,34 @@ impl GfxFramebuffer for TestGfxFb { Ok(None) } - fn copy_to_shm(self: Rc, shm: &[Cell]) -> Result<(), GfxError> { - self.img.deref().read_pixels(shm) - } - fn format(&self) -> &'static Format { &ARGB8888 } } +impl GfxInternalFramebuffer for TestGfxFb { + fn into_fb(self: Rc) -> Rc { + self + } + + fn staging_size(&self) -> usize { + 0 + } + + fn download( + self: Rc, + _staging: &Rc, + _callback: Rc, + mem: Rc, + _damage: Region, + ) -> Result, GfxError> { + let mut res = Ok(()); + mem.access(&mut |mem| res = self.img.deref().read_pixels(mem)) + .map_err(TestGfxError::AccessFailed)?; + res.map(|_| None) + } +} + impl dyn GfxTexture { fn as_native(&self) -> &TestGfxImage { self.as_any() diff --git a/src/state.rs b/src/state.rs index 75a13882..f9222f65 100644 --- a/src/state.rs +++ b/src/state.rs @@ -27,8 +27,8 @@ use { forker::ForkerProxy, format::Format, gfx_api::{ - AcquireSync, BufferResv, GfxContext, GfxError, GfxFramebuffer, GfxTexture, ReleaseSync, - SampleRect, SyncFile, + AcquireSync, BufferResv, GfxContext, GfxError, GfxFramebuffer, GfxTexture, + PendingShmTransfer, ReleaseSync, SampleRect, SyncFile, STAGING_DOWNLOAD, }, gfx_apis::create_gfx_context, globals::{Globals, GlobalsError, RemovableWaylandGlobal, WaylandGlobal}, @@ -62,7 +62,7 @@ use { io_uring::IoUring, leaks::Tracker, logger::Logger, - rect::Rect, + rect::{Rect, Region}, renderer::Renderer, scale::Scale, security_context_acceptor::SecurityContextAcceptors, @@ -1014,25 +1014,32 @@ impl State { x_off: i32, y_off: i32, size: Option<(i32, i32)>, - capture: &ZwlrScreencopyFrameV1, - mem: &ClientMemOffset, + capture: &Rc, + mem: &Rc, stride: i32, format: &'static Format, transform: Transform, scale: Scale, - ) -> Result<(), ShmScreencopyError> { + ) -> Result, ShmScreencopyError> { let Some(ctx) = self.render_ctx.get() else { return Err(ShmScreencopyError::NoRenderContext); }; let fb = ctx - .create_fb(capture.rect.width(), capture.rect.height(), stride, format) + .clone() + .create_internal_fb( + &self.cpu_worker, + capture.rect.width(), + capture.rect.height(), + stride, + format, + ) .map_err(ShmScreencopyError::CreateTemporaryFb)?; self.perform_screencopy( src, None, acquire_sync, ReleaseSync::None, - &fb, + &fb.clone().into_fb(), AcquireSync::Unnecessary, ReleaseSync::None, transform, @@ -1045,14 +1052,16 @@ impl State { scale, ) .map_err(ShmScreencopyError::CopyToTemporary)?; - let acc = mem.access(|mem| fb.copy_to_shm(mem)); - match acc { - Ok(res) => res.map_err(ShmScreencopyError::ReadPixels), - Err(e) => { - capture.client.error(e); - Ok(()) - } - } + let staging = ctx.create_staging_buffer(fb.staging_size(), STAGING_DOWNLOAD); + let pending = fb + .download( + &staging, + capture.clone(), + mem.clone(), + Region::new2(capture.rect.at_point(0, 0)), + ) + .map_err(ShmScreencopyError::ReadPixels)?; + Ok(pending) } pub fn create_seat(self: &Rc, name: &str) -> Rc { diff --git a/src/tree/output.rs b/src/tree/output.rs index d0dffce6..92f85598 100644 --- a/src/tree/output.rs +++ b/src/tree/output.rs @@ -267,6 +267,7 @@ impl OutputNode { capture.send_failed(); continue; } + let mut ready = true; if let Some(storage) = wl_buffer.storage.borrow_mut().deref() { match storage { WlBufferStorage::Shm { mem, stride } => { @@ -284,10 +285,16 @@ impl OutputNode { self.global.persistent.transform.get(), self.global.persistent.scale.get(), ); - if let Err(e) = res { - log::warn!("Could not perform shm screencopy: {}", ErrorFmt(e)); - capture.send_failed(); - continue; + match res { + Ok(p) => { + ready = p.is_none(); + capture.pending.set(p); + } + Err(e) => { + log::warn!("Could not perform shm screencopy: {}", ErrorFmt(e)); + capture.send_failed(); + continue; + } } } WlBufferStorage::Dmabuf { fb, .. } => { @@ -327,7 +334,9 @@ impl OutputNode { if capture.with_damage.get() { capture.send_damage(); } - capture.send_ready(now.0.tv_sec as _, now.0.tv_nsec as _); + if ready { + capture.send_ready(now.0.tv_sec as _, now.0.tv_nsec as _); + } } self.screencast_changed(); }