diff --git a/src/gfx_api.rs b/src/gfx_api.rs index a8214101..2af944fc 100644 --- a/src/gfx_api.rs +++ b/src/gfx_api.rs @@ -265,20 +265,23 @@ pub trait GfxFramebuffer: Debug { clear: Option<&Color>, ) -> Result, GfxError>; - fn copy_to_shm( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError>; - fn format(&self) -> &'static Format; } +pub trait GfxInternalFramebuffer: GfxFramebuffer { + fn into_fb(self: Rc) -> Rc; + + fn staging_size(&self) -> usize; + + fn download( + self: Rc, + staging: &Rc, + callback: Rc, + mem: Rc, + damage: Region, + ) -> Result, GfxError>; +} + impl dyn GfxFramebuffer { pub fn clear( &self, @@ -481,16 +484,6 @@ pub trait GfxTexture: Debug { fn size(&self) -> (i32, i32); fn as_any(&self) -> &dyn Any; fn into_any(self: Rc) -> Rc; - fn read_pixels( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError>; fn dmabuf(&self) -> Option<&DmaBuf>; fn format(&self) -> &'static Format; } @@ -503,12 +496,23 @@ pub trait AsyncShmGfxTextureCallback { fn completed(self: Rc, res: Result<(), GfxError>); } -pub trait AsyncShmGfxTextureUploadCancellable { +bitflags! { + StagingBufferUsecase: u32; + STAGING_UPLOAD = 1 << 0, + STAGING_DOWNLOAD = 1 << 1, +} + +pub trait GfxStagingBuffer { + fn size(&self) -> usize; + fn into_any(self: Rc) -> Rc; +} + +pub trait AsyncShmGfxTextureTransferCancellable { fn cancel(&self, id: u64); } -pub struct PendingShmUpload { - cancel: Rc, +pub struct PendingShmTransfer { + cancel: Rc, id: u64, } @@ -539,12 +543,17 @@ impl ShmMemory for Vec> { } pub trait AsyncShmGfxTexture: GfxTexture { + fn staging_size(&self) -> usize { + 0 + } + fn async_upload( self: Rc, + staging: &Rc, callback: Rc, mem: Rc, damage: Region, - ) -> Result, GfxError>; + ) -> Result, GfxError>; fn sync_upload(self: Rc, shm: &[Cell], damage: Region) -> Result<(), GfxError>; @@ -596,15 +605,35 @@ pub trait GfxContext: Debug { fn gfx_api(&self) -> GfxApi; - fn create_fb( + fn create_internal_fb( self: Rc, + cpu_worker: &Rc, width: i32, height: i32, stride: i32, format: &'static Format, - ) -> Result, GfxError>; + ) -> Result, GfxError>; fn sync_obj_ctx(&self) -> Option<&Rc>; + + fn create_staging_buffer( + &self, + size: usize, + usecase: StagingBufferUsecase, + ) -> Rc { + let _ = usecase; + struct Dummy(usize); + impl GfxStagingBuffer for Dummy { + fn size(&self) -> usize { + self.0 + } + + fn into_any(self: Rc) -> Rc { + self + } + } + Rc::new(Dummy(size)) + } } #[derive(Clone, Debug)] @@ -671,13 +700,13 @@ pub fn cross_intersect_formats( res } -impl PendingShmUpload { - pub fn new(cancel: Rc, id: u64) -> Self { +impl PendingShmTransfer { + pub fn new(cancel: Rc, id: u64) -> Self { Self { cancel, id } } } -impl Drop for PendingShmUpload { +impl Drop for PendingShmTransfer { fn drop(&mut self) { self.cancel.cancel(self.id); } diff --git a/src/gfx_apis/gl.rs b/src/gfx_apis/gl.rs index dd45501a..284e8131 100644 --- a/src/gfx_apis/gl.rs +++ b/src/gfx_apis/gl.rs @@ -185,8 +185,6 @@ enum RenderError { ExternalUnsupported, #[error("OpenGL context does not support any formats")] NoSupportedFormats, - #[error("Cannot convert a shm texture into a framebuffer")] - ShmTextureToFb, #[error("Could not create EGLSyncKHR")] CreateEglSync, #[error("Could not destroy EGLSyncKHR")] diff --git a/src/gfx_apis/gl/renderer/context.rs b/src/gfx_apis/gl/renderer/context.rs index e0ab21bb..32cf2392 100644 --- a/src/gfx_apis/gl/renderer/context.rs +++ b/src/gfx_apis/gl/renderer/context.rs @@ -5,7 +5,7 @@ use { format::{Format, XRGB8888}, gfx_api::{ AsyncShmGfxTexture, BufferResvUser, GfxContext, GfxError, GfxFormat, GfxFramebuffer, - GfxImage, ResetStatus, ShmGfxTexture, + GfxImage, GfxInternalFramebuffer, ResetStatus, ShmGfxTexture, }, gfx_apis::gl::{ egl::{context::EglContext, display::EglDisplay, image::EglImage}, @@ -316,13 +316,14 @@ impl GfxContext for GlRenderContext { GfxApi::OpenGl } - fn create_fb( + fn create_internal_fb( self: Rc, + _cpu_worker: &Rc, width: i32, height: i32, _stride: i32, format: &'static Format, - ) -> Result, GfxError> { + ) -> Result, GfxError> { let fb = self.ctx.with_current(|| unsafe { GlRenderBuffer::new(&self.ctx, width, height, format)?.create_framebuffer() })?; diff --git a/src/gfx_apis/gl/renderer/framebuffer.rs b/src/gfx_apis/gl/renderer/framebuffer.rs index 33071aa9..bbe98c16 100644 --- a/src/gfx_apis/gl/renderer/framebuffer.rs +++ b/src/gfx_apis/gl/renderer/framebuffer.rs @@ -1,7 +1,11 @@ use { crate::{ format::Format, - gfx_api::{AcquireSync, GfxApiOpt, GfxError, GfxFramebuffer, ReleaseSync, SyncFile}, + gfx_api::{ + AcquireSync, AsyncShmGfxTextureCallback, GfxApiOpt, GfxError, GfxFramebuffer, + GfxInternalFramebuffer, GfxStagingBuffer, PendingShmTransfer, ReleaseSync, ShmMemory, + SyncFile, + }, gfx_apis::gl::{ gl::{ frame_buffer::GlFrameBuffer, @@ -13,6 +17,7 @@ use { sys::{GL_ONE, GL_ONE_MINUS_SRC_ALPHA}, RenderError, }, + rect::Region, theme::Color, }, std::{ @@ -34,29 +39,21 @@ impl Debug for Framebuffer { } impl Framebuffer { - pub fn copy_to_shm( - &self, - x: i32, - y: i32, - width: i32, - height: i32, - format: &Format, - shm: &[Cell], - ) -> Result<(), RenderError> { + pub fn copy_to_shm(&self, shm: &[Cell]) -> Result<(), RenderError> { + let format = self.gl.rb.format; let Some(shm_info) = &format.shm_info else { return Err(RenderError::UnsupportedShmFormat(format.name)); }; let gles = self.ctx.ctx.dpy.gles; - let y = self.gl.height - y - height; let _ = self.ctx.ctx.with_current(|| { unsafe { (gles.glBindFramebuffer)(GL_FRAMEBUFFER, self.gl.fbo); (gles.glViewport)(0, 0, self.gl.width, self.gl.height); (gles.glReadnPixels)( - x, - y, - width, - height, + 0, + 0, + self.gl.width, + self.gl.height, shm_info.gl_format as _, shm_info.gl_type as _, shm.len() as _, @@ -112,22 +109,31 @@ impl GfxFramebuffer for Framebuffer { self.render(acquire_sync, ops, clear).map_err(|e| e.into()) } - fn copy_to_shm( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - _stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError> { - (*self) - .copy_to_shm(x, y, width, height, format, shm) - .map_err(|e| e.into()) - } - fn format(&self) -> &'static Format { self.gl.rb.format } } + +impl GfxInternalFramebuffer for Framebuffer { + fn into_fb(self: Rc) -> Rc { + self + } + + fn staging_size(&self) -> usize { + 0 + } + + fn download( + self: Rc, + _staging: &Rc, + _callback: Rc, + mem: Rc, + _damage: Region, + ) -> Result, GfxError> { + let mut res = Ok(()); + mem.access(&mut |mem| res = self.copy_to_shm(mem)) + .map_err(RenderError::AccessFailed)?; + res?; + Ok(None) + } +} diff --git a/src/gfx_apis/gl/renderer/texture.rs b/src/gfx_apis/gl/renderer/texture.rs index 7add2b78..a4172059 100644 --- a/src/gfx_apis/gl/renderer/texture.rs +++ b/src/gfx_apis/gl/renderer/texture.rs @@ -2,12 +2,12 @@ use { crate::{ format::Format, gfx_api::{ - AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxError, GfxTexture, PendingShmUpload, - ShmGfxTexture, ShmMemory, + AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxError, GfxStagingBuffer, GfxTexture, + PendingShmTransfer, ShmGfxTexture, ShmMemory, }, gfx_apis::gl::{ gl::texture::GlTexture, - renderer::{context::GlRenderContext, framebuffer::Framebuffer}, + renderer::context::GlRenderContext, sys::{ GLint, GL_CLAMP_TO_EDGE, GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_TEXTURE_WRAP_T, GL_UNPACK_ROW_LENGTH_EXT, @@ -45,13 +45,6 @@ impl Texture { pub fn height(&self) -> i32 { self.gl.height } - - pub fn to_framebuffer(&self) -> Result, RenderError> { - match &self.gl.img { - Some(img) => self.ctx.image_to_fb(img), - _ => Err(RenderError::ShmTextureToFb), - } - } } impl GfxTexture for Texture { @@ -67,21 +60,6 @@ impl GfxTexture for Texture { self } - fn read_pixels( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - _stride: i32, - format: &Format, - shm: &[Cell], - ) -> Result<(), GfxError> { - self.to_framebuffer()? - .copy_to_shm(x, y, width, height, format, shm) - .map_err(|e| e.into()) - } - fn dmabuf(&self) -> Option<&DmaBuf> { self.gl.img.as_ref().map(|i| &i.dmabuf) } @@ -100,10 +78,11 @@ impl ShmGfxTexture for Texture { impl AsyncShmGfxTexture for Texture { fn async_upload( self: Rc, + _staging: &Rc, _callback: Rc, mem: Rc, _damage: Region, - ) -> Result, GfxError> { + ) -> Result, GfxError> { let mut res = Ok(()); mem.access(&mut |data| { res = self.clone().sync_upload(data, Region::default()); diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs index 4562377b..14e2ca82 100644 --- a/src/gfx_apis/vulkan.rs +++ b/src/gfx_apis/vulkan.rs @@ -14,6 +14,7 @@ mod semaphore; mod shaders; mod shm_image; mod staging; +mod transfer; use { crate::{ @@ -22,8 +23,9 @@ use { cpu_worker::{jobs::read_write::ReadWriteJobError, CpuWorker}, format::Format, gfx_api::{ - AsyncShmGfxTexture, GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage, - ResetStatus, ShmGfxTexture, + AsyncShmGfxTexture, GfxContext, GfxError, GfxFormat, GfxImage, GfxInternalFramebuffer, + GfxStagingBuffer, ResetStatus, ShmGfxTexture, StagingBufferUsecase, STAGING_DOWNLOAD, + STAGING_UPLOAD, }, gfx_apis::vulkan::{ image::VulkanImageMemory, instance::VulkanInstance, renderer::VulkanRenderer, @@ -171,16 +173,6 @@ pub enum VulkanError { InvalidStride, #[error("Shm stride and height do not match buffer size")] InvalidBufferSize, - #[error("The shm parameters are invalid x={x}, y={y}, width={width}, height={height}, stride={stride}")] - InvalidShmParameters { - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - }, - #[error(transparent)] - GfxError(GfxError), #[error("Buffer format {0} is not supported for shm buffers in Vulkan context")] UnsupportedShmFormat(&'static str), #[error("Only BO_USE_RENDERING and BO_USE_WRITE are supported")] @@ -203,6 +195,16 @@ pub enum VulkanError { AsyncCopyToStaging(#[source] ReadWriteJobError), #[error("The async shm texture is busy")] AsyncCopyBusy, + #[error("The staging buffer is busy")] + StagingBufferBusy, + #[error("The staging buffer does not support uploads")] + StagingBufferNoUpload, + #[error("The staging buffer does not support downloads")] + StagingBufferNoDownload, + #[error("Image contents are undefined")] + UndefinedContents, + #[error("The framebuffer is being used by the transfer queue")] + BusyInTransfer, } impl From for GfxError { @@ -314,22 +316,41 @@ impl GfxContext for Context { GfxApi::Vulkan } - fn create_fb( + fn create_internal_fb( self: Rc, + cpu_worker: &Rc, width: i32, height: i32, stride: i32, format: &'static Format, - ) -> Result, GfxError> { - let fb = self - .0 - .create_shm_texture(format, width, height, stride, &[], true, None)?; + ) -> Result, GfxError> { + let fb = self.0.create_shm_texture( + format, + width, + height, + stride, + &[], + true, + Some(cpu_worker), + )?; Ok(fb) } fn sync_obj_ctx(&self) -> Option<&Rc> { Some(&self.0.device.sync_ctx) } + + fn create_staging_buffer( + &self, + size: usize, + usecase: StagingBufferUsecase, + ) -> Rc { + let upload = usecase.contains(STAGING_UPLOAD); + let download = usecase.contains(STAGING_DOWNLOAD); + self.0 + .device + .create_staging_shell(size as u64, upload, download) + } } impl Drop for Context { diff --git a/src/gfx_apis/vulkan/image.rs b/src/gfx_apis/vulkan/image.rs index 6826c2f6..43fcb376 100644 --- a/src/gfx_apis/vulkan/image.rs +++ b/src/gfx_apis/vulkan/image.rs @@ -3,12 +3,14 @@ use { format::Format, gfx_api::{ AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback, - AsyncShmGfxTextureUploadCancellable, GfxApiOpt, GfxError, GfxFramebuffer, GfxImage, - GfxTexture, PendingShmUpload, ReleaseSync, ShmGfxTexture, ShmMemory, SyncFile, + AsyncShmGfxTextureTransferCancellable, GfxApiOpt, GfxError, GfxFramebuffer, GfxImage, + GfxInternalFramebuffer, GfxStagingBuffer, GfxTexture, PendingShmTransfer, ReleaseSync, + ShmGfxTexture, ShmMemory, SyncFile, }, gfx_apis::vulkan::{ allocator::VulkanAllocation, device::VulkanDevice, format::VulkanModifierLimits, - renderer::VulkanRenderer, shm_image::VulkanShmImage, VulkanError, + renderer::VulkanRenderer, shm_image::VulkanShmImage, transfer::TransferType, + VulkanError, }, rect::Region, theme::Color, @@ -508,26 +510,46 @@ impl GfxFramebuffer for VulkanImage { .map_err(|e| e.into()) } - fn copy_to_shm( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError> { - self.renderer - .read_pixels(&self, x, y, width, height, stride, format, shm) - .map_err(|e| e.into()) - } - fn format(&self) -> &'static Format { self.format } } +impl GfxInternalFramebuffer for VulkanImage { + fn into_fb(self: Rc) -> Rc { + self + } + + fn staging_size(&self) -> usize { + let VulkanImageMemory::Internal(shm) = &self.ty else { + unreachable!(); + }; + shm.size as _ + } + + fn download( + self: Rc, + staging: &Rc, + callback: Rc, + mem: Rc, + damage: Region, + ) -> Result, GfxError> { + let VulkanImageMemory::Internal(shm) = &self.ty else { + unreachable!(); + }; + let staging = staging.clone().into_vk(&self.renderer.device.device); + let pending = shm.async_transfer( + &self, + staging, + &mem, + damage, + callback, + TransferType::Download, + )?; + Ok(pending) + } +} + impl GfxTexture for VulkanImage { fn size(&self) -> (i32, i32) { (self.width as _, self.height as _) @@ -541,21 +563,6 @@ impl GfxTexture for VulkanImage { self } - fn read_pixels( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError> { - self.renderer - .read_pixels(&self, x, y, width, height, stride, format, shm) - .map_err(|e| e.into()) - } - fn dmabuf(&self) -> Option<&DmaBuf> { match &self.ty { VulkanImageMemory::DmaBuf(b) => Some(&b.template.dmabuf), @@ -575,16 +582,26 @@ impl ShmGfxTexture for VulkanImage { } impl AsyncShmGfxTexture for VulkanImage { - fn async_upload( - self: Rc, - callback: Rc, - mem: Rc, - damage: Region, - ) -> Result, GfxError> { + fn staging_size(&self) -> usize { let VulkanImageMemory::Internal(shm) = &self.ty else { unreachable!(); }; - let pending = shm.async_upload(&self, &mem, damage, callback)?; + shm.size as _ + } + + fn async_upload( + self: Rc, + staging: &Rc, + callback: Rc, + mem: Rc, + damage: Region, + ) -> Result, GfxError> { + let VulkanImageMemory::Internal(shm) = &self.ty else { + unreachable!(); + }; + let staging = staging.clone().into_vk(&self.renderer.device.device); + let pending = + shm.async_transfer(&self, staging, &mem, damage, callback, TransferType::Upload)?; Ok(pending) } @@ -617,7 +634,7 @@ impl AsyncShmGfxTexture for VulkanImage { } } -impl AsyncShmGfxTextureUploadCancellable for VulkanImage { +impl AsyncShmGfxTextureTransferCancellable for VulkanImage { fn cancel(&self, id: u64) { let VulkanImageMemory::Internal(shm) = &self.ty else { unreachable!(); diff --git a/src/gfx_apis/vulkan/renderer.rs b/src/gfx_apis/vulkan/renderer.rs index 10592eba..053877ad 100644 --- a/src/gfx_apis/vulkan/renderer.rs +++ b/src/gfx_apis/vulkan/renderer.rs @@ -2,10 +2,10 @@ use { crate::{ async_engine::{AsyncEngine, SpawnedFuture}, cpu_worker::PendingJob, - format::{Format, XRGB8888}, + format::XRGB8888, gfx_api::{ - AcquireSync, BufferResv, BufferResvUser, GfxApiOpt, GfxFormat, GfxFramebuffer, - GfxTexture, GfxWriteModifier, ReleaseSync, SyncFile, + AcquireSync, BufferResv, BufferResvUser, GfxApiOpt, GfxFormat, GfxTexture, + GfxWriteModifier, ReleaseSync, SyncFile, }, gfx_apis::vulkan::{ allocator::{VulkanAllocator, VulkanThreadedAllocator}, @@ -32,11 +32,10 @@ use { ash::{ vk, vk::{ - AccessFlags2, AttachmentLoadOp, AttachmentStoreOp, BufferImageCopy, - BufferMemoryBarrier2, ClearColorValue, ClearValue, CommandBuffer, - CommandBufferBeginInfo, CommandBufferSubmitInfo, CommandBufferUsageFlags, - CopyImageInfo2, DependencyInfo, DependencyInfoKHR, DescriptorImageInfo, DescriptorType, - Extent2D, Extent3D, Fence, ImageAspectFlags, ImageCopy2, ImageLayout, + AccessFlags2, AttachmentLoadOp, AttachmentStoreOp, ClearColorValue, ClearValue, + CommandBuffer, CommandBufferBeginInfo, CommandBufferSubmitInfo, + CommandBufferUsageFlags, CopyImageInfo2, DependencyInfoKHR, DescriptorImageInfo, + DescriptorType, Extent2D, Extent3D, ImageAspectFlags, ImageCopy2, ImageLayout, ImageMemoryBarrier2, ImageSubresourceLayers, ImageSubresourceRange, PipelineBindPoint, PipelineStageFlags2, Rect2D, RenderingAttachmentInfo, RenderingInfo, SemaphoreSubmitInfo, SemaphoreSubmitInfoKHR, ShaderStageFlags, SubmitInfo2, Viewport, @@ -49,7 +48,7 @@ use { std::{ cell::{Cell, RefCell}, fmt::{Debug, Formatter}, - mem, ptr, + mem, rc::Rc, slice, }, @@ -337,38 +336,69 @@ impl VulkanRenderer { } } - fn initial_barriers(&self, buf: CommandBuffer, fb: &VulkanImage) { + fn initial_barriers(&self, buf: CommandBuffer, fb: &VulkanImage) -> Result<(), VulkanError> { zone!("initial_barriers"); let mut memory = self.memory.borrow_mut(); let memory = &mut *memory; memory.image_barriers.clear(); - let mut fb_image_memory_barrier = image_barrier() - .image(fb.image) - .new_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL) - .dst_access_mask( - AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ, - ) - .dst_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT); - if fb.bridge.is_some() { - fb_image_memory_barrier = fb_image_memory_barrier - .src_access_mask(AccessFlags2::TRANSFER_READ) - .src_stage_mask(PipelineStageFlags2::TRANSFER) - .old_layout(if fb.is_undefined.get() { - ImageLayout::UNDEFINED - } else { - ImageLayout::TRANSFER_SRC_OPTIMAL - }); - } else { - fb_image_memory_barrier = fb_image_memory_barrier - .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) - .dst_queue_family_index(self.device.graphics_queue_idx) - .old_layout(if fb.is_undefined.get() { - ImageLayout::UNDEFINED - } else { - ImageLayout::GENERAL - }); + let mut need_fb_barrier = true; + if let VulkanImageMemory::Internal(..) = &fb.ty { + need_fb_barrier = fb.is_undefined.get() + || (self.device.distinct_transfer_queue_family_idx.is_some() + && fb.queue_state.get().acquire(QueueFamily::Gfx) + != QueueTransfer::Unnecessary); + } + if need_fb_barrier { + let mut fb_image_memory_barrier = image_barrier() + .image(fb.image) + .new_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL) + .dst_access_mask( + AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ, + ) + .dst_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT); + if fb.bridge.is_some() { + fb_image_memory_barrier = fb_image_memory_barrier + .src_access_mask(AccessFlags2::TRANSFER_READ) + .src_stage_mask(PipelineStageFlags2::TRANSFER) + .old_layout(if fb.is_undefined.get() { + ImageLayout::UNDEFINED + } else { + ImageLayout::TRANSFER_SRC_OPTIMAL + }); + } else if let VulkanImageMemory::Internal(..) = &fb.ty { + let mut queue_transfer = QueueTransfer::Unnecessary; + if self.device.distinct_transfer_queue_family_idx.is_some() { + queue_transfer = fb.queue_state.get().acquire(QueueFamily::Gfx); + } + match queue_transfer { + QueueTransfer::Unnecessary => { + fb_image_memory_barrier = + fb_image_memory_barrier.old_layout(ImageLayout::UNDEFINED); + } + QueueTransfer::Possible => { + if let Some(transfer_queue_idx) = + self.device.distinct_transfer_queue_family_idx + { + fb_image_memory_barrier = fb_image_memory_barrier + .src_queue_family_index(transfer_queue_idx) + .dst_queue_family_index(self.device.graphics_queue_idx) + .old_layout(ImageLayout::TRANSFER_SRC_OPTIMAL); + } + } + QueueTransfer::Impossible => return Err(VulkanError::BusyInTransfer), + } + } else { + fb_image_memory_barrier = fb_image_memory_barrier + .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) + .dst_queue_family_index(self.device.graphics_queue_idx) + .old_layout(if fb.is_undefined.get() { + ImageLayout::UNDEFINED + } else { + ImageLayout::GENERAL + }); + } + memory.image_barriers.push(fb_image_memory_barrier); } - memory.image_barriers.push(fb_image_memory_barrier); for img in &memory.dmabuf_sample { let image_memory_barrier = image_barrier() .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) @@ -397,6 +427,7 @@ impl VulkanRenderer { unsafe { self.device.device.cmd_pipeline_barrier2(buf, &dep_info); } + Ok(()) } fn begin_rendering(&self, buf: CommandBuffer, fb: &VulkanImage, clear: Option<&Color>) { @@ -643,26 +674,28 @@ impl VulkanRenderer { let mut memory = self.memory.borrow_mut(); let memory = &mut *memory; memory.image_barriers.clear(); - let mut fb_image_memory_barrier = image_barrier() - .src_queue_family_index(self.device.graphics_queue_idx) - .dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) - .new_layout(ImageLayout::GENERAL); - if let Some(bridge) = &fb.bridge { - fb_image_memory_barrier = fb_image_memory_barrier - .image(bridge.dmabuf_image) - .old_layout(ImageLayout::TRANSFER_DST_OPTIMAL) - .src_access_mask(AccessFlags2::TRANSFER_WRITE) - .src_stage_mask(PipelineStageFlags2::TRANSFER); - } else { - fb_image_memory_barrier = fb_image_memory_barrier - .image(fb.image) - .old_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL) - .src_access_mask( - AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ, - ) - .src_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT); + if let VulkanImageMemory::DmaBuf(..) = fb.ty { + let mut fb_image_memory_barrier = image_barrier() + .src_queue_family_index(self.device.graphics_queue_idx) + .dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) + .new_layout(ImageLayout::GENERAL); + if let Some(bridge) = &fb.bridge { + fb_image_memory_barrier = fb_image_memory_barrier + .image(bridge.dmabuf_image) + .old_layout(ImageLayout::TRANSFER_DST_OPTIMAL) + .src_access_mask(AccessFlags2::TRANSFER_WRITE) + .src_stage_mask(PipelineStageFlags2::TRANSFER); + } else { + fb_image_memory_barrier = fb_image_memory_barrier + .image(fb.image) + .old_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL) + .src_access_mask( + AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ, + ) + .src_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT); + } + memory.image_barriers.push(fb_image_memory_barrier); } - memory.image_barriers.push(fb_image_memory_barrier); for img in &memory.dmabuf_sample { let image_memory_barrier = image_barrier() .src_queue_family_index(self.device.graphics_queue_idx) @@ -833,6 +866,10 @@ impl VulkanRenderer { fn store_layouts(&self, fb: &VulkanImage) { fb.is_undefined.set(false); + fb.contents_are_undefined.set(false); + fb.queue_state.set(QueueState::Acquired { + family: QueueFamily::Gfx, + }); let memory = self.memory.borrow(); for img in &*memory.queue_transfer { img.queue_state.set(QueueState::Acquired { @@ -867,197 +904,6 @@ impl VulkanRenderer { frame.waiter.set(Some(future)); } - pub fn read_pixels( - self: &Rc, - tex: &Rc, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - dst: &[Cell], - ) -> Result<(), VulkanError> { - if x < 0 || y < 0 || width <= 0 || height <= 0 || stride <= 0 { - return Err(VulkanError::InvalidShmParameters { - x, - y, - width, - height, - stride, - }); - } - let width = width as u32; - let height = height as u32; - let stride = stride as u32; - if x == 0 && y == 0 && width == tex.width && height == tex.height && format == tex.format { - return self.read_all_pixels(tex, stride, dst); - } - let tmp_tex = self.create_shm_texture( - format, - width as i32, - height as i32, - stride as i32, - &[], - true, - None, - )?; - (&*tmp_tex as &dyn GfxFramebuffer) - .copy_texture( - AcquireSync::None, - ReleaseSync::None, - &(tex.clone() as _), - None, - AcquireSync::None, - ReleaseSync::None, - x, - y, - ) - .map_err(VulkanError::GfxError)?; - self.read_all_pixels(&tmp_tex, stride, dst) - } - - fn read_all_pixels( - self: &Rc, - tex: &VulkanImage, - stride: u32, - dst: &[Cell], - ) -> Result<(), VulkanError> { - let Some(shm_info) = &tex.format.shm_info else { - return Err(VulkanError::UnsupportedShmFormat(tex.format.name)); - }; - if stride < tex.width * shm_info.bpp || stride % shm_info.bpp != 0 { - return Err(VulkanError::InvalidStride); - } - let size = stride as u64 * tex.height as u64; - if size != dst.len() as u64 { - return Err(VulkanError::InvalidBufferSize); - } - let region = BufferImageCopy::default() - .buffer_row_length(stride / shm_info.bpp) - .buffer_image_height(tex.height) - .image_subresource(ImageSubresourceLayers { - aspect_mask: ImageAspectFlags::COLOR, - mip_level: 0, - base_array_layer: 0, - layer_count: 1, - }) - .image_extent(Extent3D { - width: tex.width, - height: tex.height, - depth: 1, - }); - let staging = - self.device - .create_staging_buffer(&self.allocator, size, false, true, true)?; - let initial_tex_barrier; - let initial_buffer_barrier = BufferMemoryBarrier2::default() - .buffer(staging.buffer) - .offset(0) - .size(staging.size) - .dst_access_mask(AccessFlags2::TRANSFER_WRITE) - .dst_stage_mask(PipelineStageFlags2::TRANSFER); - let mut initial_barriers = DependencyInfo::default() - .buffer_memory_barriers(slice::from_ref(&initial_buffer_barrier)); - if tex.bridge.is_none() { - initial_tex_barrier = image_barrier() - .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) - .dst_queue_family_index(self.device.graphics_queue_idx) - .image(tex.image) - .old_layout(ImageLayout::GENERAL) - .new_layout(ImageLayout::TRANSFER_SRC_OPTIMAL) - .dst_access_mask(AccessFlags2::TRANSFER_READ) - .dst_stage_mask(PipelineStageFlags2::TRANSFER); - initial_barriers = - initial_barriers.image_memory_barriers(slice::from_ref(&initial_tex_barrier)); - } - let final_tex_barrier; - let final_buffer_barrier = BufferMemoryBarrier2::default() - .buffer(staging.buffer) - .offset(0) - .size(staging.size) - .src_access_mask(AccessFlags2::TRANSFER_WRITE) - .src_stage_mask(PipelineStageFlags2::TRANSFER) - .dst_access_mask(AccessFlags2::HOST_READ) - .dst_stage_mask(PipelineStageFlags2::HOST); - let mut final_barriers = DependencyInfo::default() - .buffer_memory_barriers(slice::from_ref(&final_buffer_barrier)); - if tex.bridge.is_none() { - final_tex_barrier = image_barrier() - .src_queue_family_index(self.device.graphics_queue_idx) - .dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) - .image(tex.image) - .old_layout(ImageLayout::TRANSFER_SRC_OPTIMAL) - .new_layout(ImageLayout::GENERAL) - .src_access_mask(AccessFlags2::TRANSFER_READ) - .src_stage_mask(PipelineStageFlags2::TRANSFER); - final_barriers = - final_barriers.image_memory_barriers(slice::from_ref(&final_tex_barrier)); - } - let buf = self.gfx_command_buffers.allocate()?; - let mut semaphores = vec![]; - let mut semaphore_infos = vec![]; - if let VulkanImageMemory::DmaBuf(buf) = &tex.ty { - for plane in &buf.template.dmabuf.planes { - let fd = dma_buf_export_sync_file(&plane.fd, DMA_BUF_SYNC_READ) - .map_err(VulkanError::IoctlExportSyncFile)?; - let semaphore = self.allocate_semaphore()?; - semaphore.import_sync_file(fd)?; - let semaphore_info = SemaphoreSubmitInfo::default() - .semaphore(semaphore.semaphore) - .stage_mask(PipelineStageFlags2::TOP_OF_PIPE); - semaphores.push(semaphore); - semaphore_infos.push(semaphore_info); - } - } - let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(buf.buffer); - let submit_info = SubmitInfo2::default() - .wait_semaphore_infos(&semaphore_infos) - .command_buffer_infos(slice::from_ref(&command_buffer_info)); - let begin_info = - CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT); - unsafe { - self.device - .device - .begin_command_buffer(buf.buffer, &begin_info) - .map_err(VulkanError::BeginCommandBuffer)?; - self.device - .device - .cmd_pipeline_barrier2(buf.buffer, &initial_barriers); - self.device.device.cmd_copy_image_to_buffer( - buf.buffer, - tex.image, - ImageLayout::TRANSFER_SRC_OPTIMAL, - staging.buffer, - &[region], - ); - self.device - .device - .cmd_pipeline_barrier2(buf.buffer, &final_barriers); - self.device - .device - .end_command_buffer(buf.buffer) - .map_err(VulkanError::EndCommandBuffer)?; - self.device - .device - .queue_submit2( - self.device.graphics_queue, - slice::from_ref(&submit_info), - Fence::null(), - ) - .map_err(VulkanError::Submit)?; - } - self.block(); - self.gfx_command_buffers.buffers.push(buf); - for semaphore in semaphores { - self.wait_semaphores.push(semaphore); - } - staging.download(|mem, size| unsafe { - ptr::copy_nonoverlapping(mem, dst.as_ptr() as _, size); - })?; - Ok(()) - } - pub fn execute( self: &Rc, fb: &VulkanImage, @@ -1101,7 +947,7 @@ impl VulkanRenderer { let buf = self.gfx_command_buffers.allocate()?; self.collect_memory(opts); self.begin_command_buffer(buf.buffer)?; - self.initial_barriers(buf.buffer, fb); + self.initial_barriers(buf.buffer, fb)?; self.begin_rendering(buf.buffer, fb, clear); self.set_viewport(buf.buffer, fb); self.record_draws(buf.buffer, fb, opts)?; diff --git a/src/gfx_apis/vulkan/shm_image.rs b/src/gfx_apis/vulkan/shm_image.rs index ced7ea0a..86155e2b 100644 --- a/src/gfx_apis/vulkan/shm_image.rs +++ b/src/gfx_apis/vulkan/shm_image.rs @@ -1,46 +1,32 @@ use { crate::{ - cpu_worker::{ - jobs::{ - img_copy::ImgCopyWork, - read_write::{ReadWriteJobError, ReadWriteWork}, - }, - CpuJob, CpuWork, CpuWorker, - }, + cpu_worker::CpuWorker, format::{Format, FormatShmInfo}, - gfx_api::{ - AsyncShmGfxTextureCallback, PendingShmUpload, ShmMemory, ShmMemoryBacking, SyncFile, - }, + gfx_api::SyncFile, gfx_apis::vulkan::{ allocator::VulkanAllocation, command::VulkanCommandBuffer, fence::VulkanFence, - image::{QueueFamily, QueueState, QueueTransfer, VulkanImage, VulkanImageMemory}, + image::{QueueFamily, QueueState, VulkanImage, VulkanImageMemory}, renderer::{image_barrier, VulkanRenderer}, staging::VulkanStagingBuffer, + transfer::{TransferType, VulkanShmImageAsyncData}, VulkanError, }, - rect::{Rect, Region}, - utils::{clonecell::CloneCell, errorfmt::ErrorFmt, on_drop::OnDrop}, + rect::Rect, + utils::{errorfmt::ErrorFmt, on_drop::OnDrop}, }, - arrayvec::ArrayVec, ash::vk::{ AccessFlags2, BufferImageCopy2, BufferMemoryBarrier2, CommandBufferBeginInfo, - CommandBufferSubmitInfo, CommandBufferUsageFlags, CopyBufferToImageInfo2, DependencyInfo, - DependencyInfoKHR, DeviceSize, Extent3D, ImageAspectFlags, ImageCreateInfo, ImageLayout, - ImageSubresourceLayers, ImageSubresourceRange, ImageTiling, ImageType, ImageUsageFlags, - ImageViewCreateInfo, ImageViewType, Offset3D, PipelineStageFlags2, SampleCountFlags, - SharingMode, SubmitInfo2, + CommandBufferSubmitInfo, CommandBufferUsageFlags, CopyBufferToImageInfo2, + CopyImageToBufferInfo2, DependencyInfoKHR, DeviceSize, Extent3D, ImageAspectFlags, + ImageCreateInfo, ImageLayout, ImageSubresourceLayers, ImageSubresourceRange, ImageTiling, + ImageType, ImageUsageFlags, ImageViewCreateInfo, ImageViewType, Offset3D, + PipelineStageFlags2, SampleCountFlags, SharingMode, SubmitInfo2, }, gpu_alloc::UsageFlags, isnt::std_1::primitive::IsntSliceExt, - std::{ - cell::{Cell, RefCell}, - ptr, - rc::Rc, - slice, - }, - uapi::OwnedFd, + std::{cell::Cell, ptr, rc::Rc, slice}, }; pub struct VulkanShmImage { @@ -51,19 +37,6 @@ pub struct VulkanShmImage { pub(super) async_data: Option, } -pub struct VulkanShmImageAsyncData { - pub(super) busy: Cell, - pub(super) io_job: Cell>>, - pub(super) copy_job: Cell>>, - pub(super) staging: CloneCell>>, - pub(super) callback: Cell>>, - pub(super) callback_id: Cell, - pub(super) regions: RefCell>>, - pub(super) cpu: Rc, - pub(super) last_sample: Cell>, - pub(super) data_copied: Cell, -} - impl VulkanShmImage { pub fn upload( &self, @@ -164,7 +137,7 @@ impl VulkanShmImage { } })?; let Some((cmd, fence, sync_file, point)) = - self.submit_buffer_to_image_copy(img, &staging, cpy, false)? + self.submit_buffer_image_copy(img, &staging, cpy, false, TransferType::Upload)? else { return Ok(()); }; @@ -176,12 +149,13 @@ impl VulkanShmImage { Ok(()) } - fn submit_buffer_to_image_copy( + pub(super) fn submit_buffer_image_copy( &self, img: &Rc, staging: &VulkanStagingBuffer, regions: &[BufferImageCopy2], use_transfer_queue: bool, + tt: TransferType, ) -> Result, Rc, SyncFile, u64)>, VulkanError> { let memory_barrier = |sam, ssm, dam, dsm| { @@ -209,18 +183,30 @@ impl VulkanShmImage { .old_layout(if img.is_undefined.get() { ImageLayout::UNDEFINED } else { - ImageLayout::SHADER_READ_ONLY_OPTIMAL + match tt { + TransferType::Upload => ImageLayout::SHADER_READ_ONLY_OPTIMAL, + TransferType::Download => ImageLayout::COLOR_ATTACHMENT_OPTIMAL, + } }) - .new_layout(ImageLayout::TRANSFER_DST_OPTIMAL); + .new_layout(match tt { + TransferType::Upload => ImageLayout::TRANSFER_DST_OPTIMAL, + TransferType::Download => ImageLayout::TRANSFER_SRC_OPTIMAL, + }); if transfer_queue_family_idx == img.renderer.device.graphics_queue_idx { initial_image_barrier = initial_image_barrier .src_access_mask(AccessFlags2::SHADER_SAMPLED_READ) .src_stage_mask(PipelineStageFlags2::FRAGMENT_SHADER) } let initial_buffer_barrier = memory_barrier( - AccessFlags2::HOST_WRITE, + match tt { + TransferType::Upload => AccessFlags2::HOST_WRITE, + TransferType::Download => AccessFlags2::HOST_READ, + }, PipelineStageFlags2::HOST, - AccessFlags2::TRANSFER_READ, + match tt { + TransferType::Upload => AccessFlags2::TRANSFER_READ, + TransferType::Download => AccessFlags2::TRANSFER_WRITE, + }, PipelineStageFlags2::TRANSFER, ); let initial_dep_info = DependencyInfoKHR::default() @@ -230,29 +216,42 @@ impl VulkanShmImage { .image(img.image) .src_queue_family_index(transfer_queue_family_idx) .dst_queue_family_index(img.renderer.device.graphics_queue_idx) - .src_access_mask(AccessFlags2::TRANSFER_WRITE) + .src_access_mask(match tt { + TransferType::Upload => AccessFlags2::TRANSFER_WRITE, + TransferType::Download => AccessFlags2::TRANSFER_READ, + }) .src_stage_mask(PipelineStageFlags2::TRANSFER) - .old_layout(ImageLayout::TRANSFER_DST_OPTIMAL) - .new_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL); + .old_layout(match tt { + TransferType::Upload => ImageLayout::TRANSFER_DST_OPTIMAL, + TransferType::Download => ImageLayout::TRANSFER_SRC_OPTIMAL, + }) + .new_layout(match tt { + TransferType::Upload => ImageLayout::SHADER_READ_ONLY_OPTIMAL, + TransferType::Download => ImageLayout::COLOR_ATTACHMENT_OPTIMAL, + }); if transfer_queue_family_idx == img.renderer.device.graphics_queue_idx { final_image_barrier = final_image_barrier - .dst_access_mask(AccessFlags2::SHADER_SAMPLED_READ) + .dst_access_mask(match tt { + TransferType::Upload => AccessFlags2::SHADER_SAMPLED_READ, + TransferType::Download => AccessFlags2::COLOR_ATTACHMENT_WRITE, + }) .dst_stage_mask(PipelineStageFlags2::FRAGMENT_SHADER); } let final_buffer_barrier = memory_barrier( - AccessFlags2::TRANSFER_READ, + match tt { + TransferType::Upload => AccessFlags2::TRANSFER_READ, + TransferType::Download => AccessFlags2::TRANSFER_WRITE, + }, PipelineStageFlags2::TRANSFER, - AccessFlags2::HOST_WRITE, + match tt { + TransferType::Upload => AccessFlags2::HOST_WRITE, + TransferType::Download => AccessFlags2::HOST_READ, + }, PipelineStageFlags2::HOST, ); let final_dep_info = DependencyInfoKHR::default() .buffer_memory_barriers(slice::from_ref(&final_buffer_barrier)) .image_memory_barriers(slice::from_ref(&final_image_barrier)); - let cpy_info = CopyBufferToImageInfo2::default() - .src_buffer(staging.buffer) - .dst_image(img.image) - .dst_image_layout(ImageLayout::TRANSFER_DST_OPTIMAL) - .regions(regions); let cmd = match &img.renderer.transfer_command_buffers { Some(b) if use_transfer_queue => b.allocate()?, _ => img.renderer.gfx_command_buffers.allocate()?, @@ -268,7 +267,24 @@ impl VulkanShmImage { dev.begin_command_buffer(cmd.buffer, &begin_info) .map_err(VulkanError::BeginCommandBuffer)?; dev.cmd_pipeline_barrier2(cmd.buffer, &initial_dep_info); - dev.cmd_copy_buffer_to_image2(cmd.buffer, &cpy_info); + match tt { + TransferType::Upload => { + let cpy_info = CopyBufferToImageInfo2::default() + .src_buffer(staging.buffer) + .dst_image(img.image) + .dst_image_layout(ImageLayout::TRANSFER_DST_OPTIMAL) + .regions(regions); + dev.cmd_copy_buffer_to_image2(cmd.buffer, &cpy_info); + } + TransferType::Download => { + let cpy_info = CopyImageToBufferInfo2::default() + .dst_buffer(staging.buffer) + .src_image(img.image) + .src_image_layout(ImageLayout::TRANSFER_SRC_OPTIMAL) + .regions(regions); + dev.cmd_copy_image_to_buffer2(cmd.buffer, &cpy_info); + } + } dev.cmd_pipeline_barrier2(cmd.buffer, &final_dep_info); dev.end_command_buffer(cmd.buffer) .map_err(VulkanError::EndCommandBuffer)?; @@ -282,8 +298,10 @@ impl VulkanShmImage { ) .map_err(VulkanError::Submit)?; } - img.is_undefined.set(false); - img.contents_are_undefined.set(false); + if tt == TransferType::Upload { + img.is_undefined.set(false); + img.contents_are_undefined.set(false); + } let release_sync_file = match release_fence.export_sync_file() { Ok(s) => s, Err(e) => { @@ -317,466 +335,6 @@ async fn await_upload( img.renderer.pending_submits.remove(&id); } -impl VulkanShmImageAsyncData { - fn complete(&self, result: Result<(), VulkanError>) { - self.busy.set(false); - if let Some(cb) = self.callback.take() { - cb.completed(result.map_err(|e| e.into())); - } - } -} - -impl VulkanShmImage { - pub fn async_upload( - &self, - img: &Rc, - client_mem: &Rc, - damage: Region, - callback: Rc, - ) -> Result, VulkanError> { - let data = self.async_data.as_ref().unwrap(); - let res = self.try_async_upload(img, data, client_mem, damage); - match res { - Ok(()) => { - let id = img.renderer.allocate_point(); - data.callback_id.set(id); - data.callback.set(Some(callback)); - Ok(Some(PendingShmUpload::new(img.clone(), id))) - } - Err(e) => Err(e), - } - } - - fn try_async_upload( - &self, - img: &Rc, - data: &VulkanShmImageAsyncData, - client_mem: &Rc, - mut damage: Region, - ) -> Result<(), VulkanError> { - if data.busy.get() { - return Err(VulkanError::AsyncCopyBusy); - } - if self.size > client_mem.len() as u64 { - return Err(VulkanError::InvalidBufferSize); - } - data.busy.set(true); - data.data_copied.set(false); - if img.contents_are_undefined.get() { - damage = Region::new2(Rect::new_sized(0, 0, img.width as _, img.height as _).unwrap()); - } - - let copies = &mut *data.regions.borrow_mut(); - copies.clear(); - - let mut copy = |x, y, width, height| { - let buffer_offset = (y as u32 * img.stride + x as u32 * self.shm_info.bpp) as u64; - let copy = BufferImageCopy2::default() - .buffer_offset(buffer_offset) - .image_offset(Offset3D { x, y, z: 0 }) - .image_extent(Extent3D { - width, - height, - depth: 1, - }) - .image_subresource(ImageSubresourceLayers { - aspect_mask: ImageAspectFlags::COLOR, - mip_level: 0, - base_array_layer: 0, - layer_count: 1, - }) - .buffer_image_height(img.height) - .buffer_row_length(img.stride / self.shm_info.bpp); - copies.push(copy); - }; - let (width_mask, height_mask) = img.renderer.device.transfer_granularity_mask; - let width_mask = width_mask as i32; - let height_mask = height_mask as i32; - for damage in damage.rects() { - if damage.x2() < 0 || damage.y2() < 0 { - continue; - } - let x1 = damage.x1().max(0) & !width_mask; - let y1 = damage.y1().max(0) & !height_mask; - let x2 = ((damage.x2() + width_mask) & !width_mask).min(img.width as i32); - let y2 = ((damage.y2() + height_mask) & !height_mask).min(img.height as i32); - let Some(damage) = Rect::new(x1, y1, x2, y2) else { - continue; - }; - if damage.is_empty() { - continue; - } - copy( - damage.x1(), - damage.y1(), - damage.width() as u32, - damage.height() as u32, - ); - } - - self.async_release_from_gfx_queue(img, data)?; - - if let Some(staging) = data.staging.get() { - return self.async_upload_initiate_copy(img, data, &staging, copies, client_mem); - } - - let img2 = img.clone(); - let client_mem = client_mem.clone(); - img.renderer.device.create_shm_staging( - &img.renderer, - &data.cpu, - self.size, - true, - false, - move |res| { - let VulkanImageMemory::Internal(shm) = &img2.ty else { - unreachable!(); - }; - if let Err(e) = shm.async_upload_after_allocation(&img2, &client_mem, res) { - shm.async_data.as_ref().unwrap().complete(Err(e)); - } - }, - ) - } - - fn async_release_from_gfx_queue( - &self, - img: &Rc, - data: &VulkanShmImageAsyncData, - ) -> Result<(), VulkanError> { - img.renderer.check_defunct()?; - let Some(transfer_queue_idx) = img.renderer.device.distinct_transfer_queue_family_idx - else { - let Some(sync_file) = data.last_sample.take() else { - img.queue_state.set(QueueState::Released { - to: QueueFamily::Transfer, - }); - return Ok(()); - }; - let id = img.renderer.allocate_point(); - let pending = img.renderer.eng.spawn( - "await_transfer_to_transfer", - await_gfx_queue_release(id, img.clone(), None, None, sync_file), - ); - img.renderer.pending_submits.set(id, pending); - img.queue_state.set(QueueState::Releasing); - return Ok(()); - }; - let mut barriers = ArrayVec::<_, 2>::new(); - match img.queue_state.get() { - QueueState::Acquired { family } => { - assert_eq!(family, QueueFamily::Gfx); - } - QueueState::Releasing => { - unreachable!(); - } - QueueState::Released { to } => { - assert_eq!(to, QueueFamily::Gfx); - let barrier = image_barrier() - .image(img.image) - .src_queue_family_index(transfer_queue_idx) - .dst_queue_family_index(img.renderer.device.graphics_queue_idx) - .dst_stage_mask(PipelineStageFlags2::ALL_COMMANDS) - .old_layout(ImageLayout::TRANSFER_DST_OPTIMAL) - .new_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL); - barriers.push(barrier); - } - } - let barrier = image_barrier() - .image(img.image) - .src_queue_family_index(img.renderer.device.graphics_queue_idx) - .dst_queue_family_index(transfer_queue_idx) - .src_access_mask(AccessFlags2::SHADER_SAMPLED_READ) - .src_stage_mask(PipelineStageFlags2::ALL_COMMANDS) - .old_layout(if img.is_undefined.get() { - ImageLayout::UNDEFINED - } else { - ImageLayout::SHADER_READ_ONLY_OPTIMAL - }) - .new_layout(ImageLayout::TRANSFER_DST_OPTIMAL); - barriers.push(barrier); - let dep_info = DependencyInfo::default().image_memory_barriers(&barriers); - let release_fence = img.renderer.device.create_fence()?; - let dev = &img.renderer.device.device; - let begin_info = - CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT); - let cmd = img.renderer.gfx_command_buffers.allocate()?; - let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd.buffer); - let submit_info = - SubmitInfo2::default().command_buffer_infos(slice::from_ref(&command_buffer_info)); - unsafe { - dev.begin_command_buffer(cmd.buffer, &begin_info) - .map_err(VulkanError::BeginCommandBuffer)?; - dev.cmd_pipeline_barrier2(cmd.buffer, &dep_info); - dev.end_command_buffer(cmd.buffer) - .map_err(VulkanError::EndCommandBuffer)?; - dev.queue_submit2( - img.renderer.device.graphics_queue, - slice::from_ref(&submit_info), - release_fence.fence, - ) - .map_err(VulkanError::Submit)?; - } - let sync_file = release_fence.export_sync_file()?; - let id = img.renderer.allocate_point(); - let pending = img.renderer.eng.spawn( - "await_transfer_to_transfer", - await_gfx_queue_release(id, img.clone(), Some(cmd), Some(release_fence), sync_file), - ); - img.renderer.pending_submits.set(id, pending); - img.queue_state.set(QueueState::Releasing); - Ok(()) - } - - fn async_upload_after_allocation( - &self, - img: &Rc, - client_mem: &Rc, - res: Result, - ) -> Result<(), VulkanError> { - let staging = Rc::new(res?); - let data = self.async_data.as_ref().unwrap(); - data.staging.set(Some(staging.clone())); - let copies = &*data.regions.borrow(); - self.async_upload_initiate_copy(img, data, &staging, copies, client_mem) - } - - fn async_upload_initiate_copy( - &self, - img: &Rc, - data: &VulkanShmImageAsyncData, - staging: &VulkanStagingBuffer, - copies: &[BufferImageCopy2], - client_mem: &Rc, - ) -> Result<(), VulkanError> { - img.renderer.check_defunct()?; - - let id = img.renderer.allocate_point(); - let pending; - match client_mem.safe_access() { - ShmMemoryBacking::Ptr(ptr) => { - let mut job = data.copy_job.take().unwrap_or_else(|| { - Box::new(CopyUploadJob { - img: None, - id, - _mem: None, - work: unsafe { ImgCopyWork::new() }, - }) - }); - job.id = id; - job.img = Some(img.clone()); - job._mem = Some(client_mem.clone()); - job.work.src = ptr as _; - job.work.dst = staging.allocation.mem.unwrap(); - job.work.width = img.width as _; - job.work.stride = img.stride as _; - job.work.bpp = self.shm_info.bpp as _; - job.work.rects.clear(); - for copy in copies { - job.work.rects.push( - Rect::new_sized( - copy.image_offset.x as _, - copy.image_offset.y as _, - copy.image_extent.width as _, - copy.image_extent.height as _, - ) - .unwrap(), - ); - } - pending = data.cpu.submit(job); - } - ShmMemoryBacking::Fd(fd, offset) => { - let mut min_offset = client_mem.len() as u64; - let mut max_offset = 0; - for copy in copies { - min_offset = min_offset.min(copy.buffer_offset); - let len = img.stride * (copy.image_extent.height - 1) - + copy.image_extent.width * self.shm_info.bpp; - max_offset = max_offset.max(copy.buffer_offset + len as u64); - } - let mut job = data.io_job.take().unwrap_or_else(|| { - Box::new(IoUploadJob { - img: None, - id, - _mem: None, - work: unsafe { ReadWriteWork::new() }, - fd: None, - }) - }); - job.id = id; - job.img = Some(img.clone()); - job._mem = Some(client_mem.clone()); - job.fd = Some(fd.clone()); - unsafe { - let config = job.work.config(); - config.fd = fd.raw(); - config.offset = offset + min_offset as usize; - config.ptr = staging.allocation.mem.unwrap().add(min_offset as _); - config.len = max_offset.saturating_sub(min_offset) as usize; - config.write = false; - } - pending = data.cpu.submit(job); - } - } - - img.renderer.pending_cpu_jobs.set(id, pending); - - Ok(()) - } - - fn async_upload_copy_buffer_to_image( - &self, - img: &Rc, - data: &VulkanShmImageAsyncData, - ) -> Result<(), VulkanError> { - if !data.data_copied.get() { - return Ok(()); - } - if img.queue_state.get().acquire(QueueFamily::Transfer) == QueueTransfer::Impossible { - return Ok(()); - } - img.renderer.check_defunct()?; - let regions = &*data.regions.borrow(); - let staging = data.staging.get().unwrap(); - staging.upload(|_, _| ())?; - let Some((cmd, fence, sync_file, point)) = - self.submit_buffer_to_image_copy(img, &staging, regions, true)? - else { - return Ok(()); - }; - img.queue_state.set(QueueState::Releasing); - let future = img.renderer.eng.spawn( - "await async upload", - await_async_upload(point, img.clone(), cmd, fence, sync_file), - ); - img.renderer.pending_submits.set(point, future); - Ok(()) - } -} - -pub(super) struct IoUploadJob { - img: Option>, - id: u64, - _mem: Option>, - fd: Option>, - work: ReadWriteWork, -} - -pub(super) struct CopyUploadJob { - img: Option>, - id: u64, - _mem: Option>, - work: ImgCopyWork, -} - -impl CpuJob for IoUploadJob { - fn work(&mut self) -> &mut dyn CpuWork { - &mut self.work - } - - fn completed(mut self: Box) { - self._mem = None; - self.fd = None; - let img = self.img.take().unwrap(); - let res = self.work.config().result.take().unwrap(); - complete_async_upload(&img, self.id, res, |data| data.io_job.set(Some(self))); - } -} - -impl CpuJob for CopyUploadJob { - fn work(&mut self) -> &mut dyn CpuWork { - &mut self.work - } - - fn completed(mut self: Box) { - self._mem = None; - let img = self.img.take().unwrap(); - complete_async_upload(&img, self.id, Ok(()), |data| data.copy_job.set(Some(self))); - } -} - -fn complete_async_upload( - img: &Rc, - id: u64, - res: Result<(), ReadWriteJobError>, - store: impl FnOnce(&VulkanShmImageAsyncData), -) { - img.renderer.pending_cpu_jobs.remove(&id); - let VulkanImageMemory::Internal(shm) = &img.ty else { - unreachable!(); - }; - let data = shm.async_data.as_ref().unwrap(); - store(data); - if let Err(e) = res { - data.complete(Err(VulkanError::AsyncCopyToStaging(e))); - } - data.data_copied.set(true); - if let Err(e) = shm.async_upload_copy_buffer_to_image(img, data) { - data.complete(Err(e)); - } -} - -async fn await_gfx_queue_release( - id: u64, - img: Rc, - buf: Option>, - _fence: Option>, - sync_file: SyncFile, -) { - let res = img.renderer.ring.readable(&sync_file.0).await; - if let Err(e) = res { - log::error!( - "Could not wait for sync file to become readable: {}", - ErrorFmt(e) - ); - img.renderer.block(); - } - if let Some(buf) = buf { - img.renderer.gfx_command_buffers.buffers.push(buf); - } - img.renderer.pending_submits.remove(&id); - img.queue_state.set(QueueState::Released { - to: QueueFamily::Transfer, - }); - let VulkanImageMemory::Internal(shm) = &img.ty else { - unreachable!(); - }; - let data = shm.async_data.as_ref().unwrap(); - if let Err(e) = shm.async_upload_copy_buffer_to_image(&img, data) { - data.complete(Err(e)); - } -} - -async fn await_async_upload( - id: u64, - img: Rc, - buf: Rc, - _fence: Rc, - sync_file: SyncFile, -) { - let res = img.renderer.ring.readable(&sync_file.0).await; - if let Err(e) = res { - log::error!( - "Could not wait for sync file to become readable: {}", - ErrorFmt(e) - ); - img.renderer.block(); - } - match &img.renderer.transfer_command_buffers { - Some(b) => b.buffers.push(buf), - None => img.renderer.gfx_command_buffers.buffers.push(buf), - } - img.queue_state.set(QueueState::Released { - to: QueueFamily::Gfx, - }); - img.renderer.pending_submits.remove(&id); - let VulkanImageMemory::Internal(shm) = &img.ty else { - unreachable!(); - }; - let data = shm.async_data.as_ref().unwrap(); - data.complete(Ok(())); -} - impl VulkanRenderer { pub fn create_shm_texture( self: &Rc, @@ -868,6 +426,7 @@ impl VulkanRenderer { io_job: Default::default(), copy_job: Default::default(), staging: Default::default(), + client_mem: Default::default(), callback: Default::default(), callback_id: Cell::new(0), regions: Default::default(), diff --git a/src/gfx_apis/vulkan/staging.rs b/src/gfx_apis/vulkan/staging.rs index 5a3faeac..d8fa685b 100644 --- a/src/gfx_apis/vulkan/staging.rs +++ b/src/gfx_apis/vulkan/staging.rs @@ -1,17 +1,24 @@ use { crate::{ cpu_worker::CpuWorker, + gfx_api::GfxStagingBuffer, gfx_apis::vulkan::{ allocator::{VulkanAllocation, VulkanAllocator}, device::VulkanDevice, renderer::VulkanRenderer, VulkanError, }, - utils::on_drop::{OnDrop, OnDrop2}, + utils::{ + clonecell::CloneCell, + on_drop::{OnDrop, OnDrop2}, + }, + }, + ash::{ + vk::{Buffer, BufferCreateInfo, BufferUsageFlags, MappedMemoryRange}, + Device, }, - ash::vk::{Buffer, BufferCreateInfo, BufferUsageFlags, MappedMemoryRange}, gpu_alloc::UsageFlags, - std::rc::Rc, + std::{any::Any, cell::Cell, rc::Rc}, }; pub struct VulkanStagingBuffer { @@ -22,6 +29,22 @@ pub struct VulkanStagingBuffer { } impl VulkanDevice { + pub(super) fn create_staging_shell( + self: &Rc, + size: u64, + upload: bool, + download: bool, + ) -> Rc { + Rc::new(VulkanStagingShell { + device: self.clone(), + staging: Default::default(), + size, + download, + upload, + busy: Cell::new(false), + }) + } + pub(super) fn create_staging_buffer( self: &Rc, allocator: &Rc, @@ -51,17 +74,15 @@ impl VulkanDevice { }) } - pub(super) fn create_shm_staging( + pub(super) fn fill_staging_shell( self: &Rc, renderer: &Rc, cpu: &Rc, - size: u64, - upload: bool, - download: bool, - cb: impl FnOnce(Result) + 'static, + shell: Rc, + cb: impl FnOnce(Result, VulkanError>) + 'static, ) -> Result<(), VulkanError> { - let (vk_usage, usage) = get_usage(upload, download, false); - let buffer = self.create_buffer(size, vk_usage)?; + let (vk_usage, usage) = get_usage(shell.upload, shell.download, false); + let buffer = self.create_buffer(shell.size, vk_usage)?; let memory_requirements = unsafe { self.device.get_buffer_memory_requirements(buffer) }; let slf = self.clone(); let destroy_buffer = @@ -77,12 +98,14 @@ impl VulkanDevice { res.map_err(VulkanError::BindBufferMemory)?; } destroy_buffer.forget(); - Ok(VulkanStagingBuffer { + let buffer = Rc::new(VulkanStagingBuffer { device: slf.clone(), allocation, buffer, - size, - }) + size: shell.size, + }); + shell.staging.set(Some(buffer.clone())); + Ok(buffer) }; renderer.shm_allocator.async_alloc( renderer, @@ -159,3 +182,43 @@ impl Drop for VulkanStagingBuffer { } } } + +pub(super) struct VulkanStagingShell { + pub(super) device: Rc, + pub(super) staging: CloneCell>>, + pub(super) size: u64, + pub(super) download: bool, + pub(super) upload: bool, + pub(super) busy: Cell, +} + +impl GfxStagingBuffer for VulkanStagingShell { + fn size(&self) -> usize { + self.size as _ + } + + fn into_any(self: Rc) -> Rc { + self + } +} + +impl VulkanStagingShell { + fn assert_device(&self, device: &Device) { + assert_eq!( + self.device.device.handle(), + device.handle(), + "Mixed vulkan device use" + ); + } +} + +impl dyn GfxStagingBuffer { + pub(super) fn into_vk(self: Rc, device: &Device) -> Rc { + let shell: Rc = self + .into_any() + .downcast() + .expect("Non-vulkan staging buffer passed into vulkan"); + shell.assert_device(device); + shell + } +} diff --git a/src/gfx_apis/vulkan/transfer.rs b/src/gfx_apis/vulkan/transfer.rs new file mode 100644 index 00000000..9062df13 --- /dev/null +++ b/src/gfx_apis/vulkan/transfer.rs @@ -0,0 +1,684 @@ +use { + crate::{ + cpu_worker::{ + jobs::{ + img_copy::ImgCopyWork, + read_write::{ReadWriteJobError, ReadWriteWork}, + }, + CpuJob, CpuWork, CpuWorker, + }, + gfx_api::{ + AsyncShmGfxTextureCallback, PendingShmTransfer, ShmMemory, ShmMemoryBacking, SyncFile, + }, + gfx_apis::vulkan::{ + command::VulkanCommandBuffer, + fence::VulkanFence, + image::{QueueFamily, QueueState, QueueTransfer, VulkanImage, VulkanImageMemory}, + renderer::image_barrier, + shm_image::VulkanShmImage, + staging::{VulkanStagingBuffer, VulkanStagingShell}, + VulkanError, + }, + rect::{Rect, Region}, + utils::{clonecell::CloneCell, errorfmt::ErrorFmt}, + }, + arrayvec::ArrayVec, + ash::vk::{ + AccessFlags2, BufferImageCopy2, CommandBufferBeginInfo, CommandBufferSubmitInfo, + CommandBufferUsageFlags, DependencyInfo, Extent3D, ImageAspectFlags, ImageLayout, + ImageSubresourceLayers, Offset3D, PipelineStageFlags2, SubmitInfo2, + }, + std::{ + cell::{Cell, RefCell}, + rc::Rc, + slice, + }, + uapi::OwnedFd, +}; + +pub struct VulkanShmImageAsyncData { + pub(super) busy: Cell, + pub(super) io_job: Cell>>, + pub(super) copy_job: Cell>>, + pub(super) staging: CloneCell>>, + pub(super) client_mem: CloneCell>>, + pub(super) callback: Cell>>, + pub(super) callback_id: Cell, + pub(super) regions: RefCell>>, + pub(super) cpu: Rc, + pub(super) last_sample: Cell>, + pub(super) data_copied: Cell, +} + +impl VulkanShmImageAsyncData { + fn complete(&self, result: Result<(), VulkanError>) { + self.busy.set(false); + self.staging.take().unwrap().busy.set(false); + self.client_mem.take(); + if let Some(cb) = self.callback.take() { + cb.completed(result.map_err(|e| e.into())); + } + } +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(super) enum TransferType { + Upload, + Download, +} + +impl VulkanShmImage { + pub fn async_transfer( + &self, + img: &Rc, + staging: Rc, + client_mem: &Rc, + damage: Region, + callback: Rc, + tt: TransferType, + ) -> Result, VulkanError> { + let data = self.async_data.as_ref().unwrap(); + let res = self.try_async_transfer(img, staging, data, client_mem, damage, tt); + match res { + Ok(()) => { + let id = img.renderer.allocate_point(); + data.callback_id.set(id); + data.callback.set(Some(callback)); + Ok(Some(PendingShmTransfer::new(img.clone(), id))) + } + Err(e) => Err(e), + } + } + + fn try_async_transfer( + &self, + img: &Rc, + staging: Rc, + data: &VulkanShmImageAsyncData, + client_mem: &Rc, + mut damage: Region, + tt: TransferType, + ) -> Result<(), VulkanError> { + if data.busy.get() { + return Err(VulkanError::AsyncCopyBusy); + } + if staging.busy.get() { + return Err(VulkanError::StagingBufferBusy); + } + match tt { + TransferType::Upload => { + if !staging.upload { + return Err(VulkanError::StagingBufferNoUpload); + } + } + TransferType::Download => { + if !staging.download { + return Err(VulkanError::StagingBufferNoDownload); + } + } + } + if self.size > client_mem.len() as u64 { + return Err(VulkanError::InvalidBufferSize); + } + data.busy.set(true); + data.data_copied.set(false); + staging.busy.set(true); + data.staging.set(Some(staging.clone())); + data.client_mem.set(Some(client_mem.clone())); + if img.contents_are_undefined.get() { + if tt == TransferType::Download { + return Err(VulkanError::UndefinedContents); + } + damage = Region::new2(Rect::new_sized(0, 0, img.width as _, img.height as _).unwrap()); + } + + let copies = &mut *data.regions.borrow_mut(); + copies.clear(); + + let mut copy = |x, y, width, height| { + let buffer_offset = (y as u32 * img.stride + x as u32 * self.shm_info.bpp) as u64; + let copy = BufferImageCopy2::default() + .buffer_offset(buffer_offset) + .image_offset(Offset3D { x, y, z: 0 }) + .image_extent(Extent3D { + width, + height, + depth: 1, + }) + .image_subresource(ImageSubresourceLayers { + aspect_mask: ImageAspectFlags::COLOR, + mip_level: 0, + base_array_layer: 0, + layer_count: 1, + }) + .buffer_image_height(img.height) + .buffer_row_length(img.stride / self.shm_info.bpp); + copies.push(copy); + }; + let (width_mask, height_mask) = img.renderer.device.transfer_granularity_mask; + let width_mask = width_mask as i32; + let height_mask = height_mask as i32; + for damage in damage.rects() { + if damage.x2() < 0 || damage.y2() < 0 { + continue; + } + let x1 = damage.x1().max(0) & !width_mask; + let y1 = damage.y1().max(0) & !height_mask; + let x2 = ((damage.x2() + width_mask) & !width_mask).min(img.width as i32); + let y2 = ((damage.y2() + height_mask) & !height_mask).min(img.height as i32); + let Some(damage) = Rect::new(x1, y1, x2, y2) else { + continue; + }; + if damage.is_empty() { + continue; + } + copy( + damage.x1(), + damage.y1(), + damage.width() as u32, + damage.height() as u32, + ); + } + + self.async_release_from_gfx_queue(img, data, tt)?; + + if let Some(staging) = staging.staging.get() { + return match tt { + TransferType::Upload => self + .async_transfer_initiate_host_copy(img, data, &staging, copies, client_mem, tt), + TransferType::Download => { + self.async_download_copy_image_to_buffer(img, &staging, copies) + } + }; + } + + let img2 = img.clone(); + let client_mem = client_mem.clone(); + img.renderer + .device + .fill_staging_shell(&img.renderer, &data.cpu, staging, move |res| { + let VulkanImageMemory::Internal(shm) = &img2.ty else { + unreachable!(); + }; + if let Err(e) = shm.async_transfer_after_allocation(&img2, &client_mem, res, tt) { + shm.async_data.as_ref().unwrap().complete(Err(e)); + } + }) + } + + fn async_release_from_gfx_queue( + &self, + img: &Rc, + data: &VulkanShmImageAsyncData, + tt: TransferType, + ) -> Result<(), VulkanError> { + img.renderer.check_defunct()?; + let Some(transfer_queue_idx) = img.renderer.device.distinct_transfer_queue_family_idx + else { + let Some(sync_file) = data.last_sample.take() else { + img.queue_state.set(QueueState::Released { + to: QueueFamily::Transfer, + }); + return Ok(()); + }; + let id = img.renderer.allocate_point(); + let pending = img.renderer.eng.spawn( + "await_transfer_to_transfer", + await_gfx_queue_release(id, img.clone(), None, None, sync_file, tt), + ); + img.renderer.pending_submits.set(id, pending); + img.queue_state.set(QueueState::Releasing); + return Ok(()); + }; + let (gfx_access_mask, gfx_layout, transfer_layout) = match tt { + TransferType::Upload => ( + AccessFlags2::SHADER_SAMPLED_READ, + ImageLayout::SHADER_READ_ONLY_OPTIMAL, + ImageLayout::TRANSFER_DST_OPTIMAL, + ), + TransferType::Download => ( + AccessFlags2::COLOR_ATTACHMENT_WRITE, + ImageLayout::COLOR_ATTACHMENT_OPTIMAL, + ImageLayout::TRANSFER_SRC_OPTIMAL, + ), + }; + let mut barriers = ArrayVec::<_, 2>::new(); + match img.queue_state.get() { + QueueState::Acquired { family } => { + assert_eq!(family, QueueFamily::Gfx); + } + QueueState::Releasing => { + unreachable!(); + } + QueueState::Released { to } => { + assert_eq!(to, QueueFamily::Gfx); + let barrier = image_barrier() + .image(img.image) + .src_queue_family_index(transfer_queue_idx) + .dst_queue_family_index(img.renderer.device.graphics_queue_idx) + .dst_stage_mask(PipelineStageFlags2::ALL_COMMANDS) + .old_layout(transfer_layout) + .new_layout(gfx_layout); + barriers.push(barrier); + } + } + let barrier = image_barrier() + .image(img.image) + .src_queue_family_index(img.renderer.device.graphics_queue_idx) + .dst_queue_family_index(transfer_queue_idx) + .src_access_mask(gfx_access_mask) + .src_stage_mask(PipelineStageFlags2::ALL_COMMANDS) + .old_layout(if img.is_undefined.get() { + ImageLayout::UNDEFINED + } else { + gfx_layout + }) + .new_layout(transfer_layout); + barriers.push(barrier); + let dep_info = DependencyInfo::default().image_memory_barriers(&barriers); + let release_fence = img.renderer.device.create_fence()?; + let dev = &img.renderer.device.device; + let begin_info = + CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT); + let cmd = img.renderer.gfx_command_buffers.allocate()?; + let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd.buffer); + let submit_info = + SubmitInfo2::default().command_buffer_infos(slice::from_ref(&command_buffer_info)); + unsafe { + dev.begin_command_buffer(cmd.buffer, &begin_info) + .map_err(VulkanError::BeginCommandBuffer)?; + dev.cmd_pipeline_barrier2(cmd.buffer, &dep_info); + dev.end_command_buffer(cmd.buffer) + .map_err(VulkanError::EndCommandBuffer)?; + dev.queue_submit2( + img.renderer.device.graphics_queue, + slice::from_ref(&submit_info), + release_fence.fence, + ) + .map_err(VulkanError::Submit)?; + } + let sync_file = release_fence.export_sync_file()?; + let id = img.renderer.allocate_point(); + let pending = img.renderer.eng.spawn( + "await_transfer_to_transfer", + await_gfx_queue_release( + id, + img.clone(), + Some(cmd), + Some(release_fence), + sync_file, + tt, + ), + ); + img.renderer.pending_submits.set(id, pending); + img.queue_state.set(QueueState::Releasing); + Ok(()) + } + + fn async_transfer_after_allocation( + &self, + img: &Rc, + client_mem: &Rc, + res: Result, VulkanError>, + tt: TransferType, + ) -> Result<(), VulkanError> { + let staging = res?; + let data = self.async_data.as_ref().unwrap(); + let copies = &*data.regions.borrow(); + match tt { + TransferType::Upload => { + self.async_transfer_initiate_host_copy(img, data, &staging, copies, client_mem, tt) + } + TransferType::Download => { + self.async_download_copy_image_to_buffer(img, &staging, copies) + } + } + } + + pub(super) fn async_transfer_initiate_host_copy( + &self, + img: &Rc, + data: &VulkanShmImageAsyncData, + staging: &VulkanStagingBuffer, + copies: &[BufferImageCopy2], + client_mem: &Rc, + tt: TransferType, + ) -> Result<(), VulkanError> { + img.renderer.check_defunct()?; + + if tt == TransferType::Download { + staging.download(|_, _| ())?; + } + let id = img.renderer.allocate_point(); + let pending; + match client_mem.safe_access() { + ShmMemoryBacking::Ptr(ptr) => { + let mut job = data.copy_job.take().unwrap_or_else(|| { + Box::new(CopyTransferJob { + img: None, + id, + _mem: None, + work: unsafe { ImgCopyWork::new() }, + tt, + }) + }); + job.id = id; + job.img = Some(img.clone()); + job._mem = Some(client_mem.clone()); + job.tt = tt; + match tt { + TransferType::Upload => { + job.work.src = ptr as _; + job.work.dst = staging.allocation.mem.unwrap(); + } + TransferType::Download => { + job.work.src = staging.allocation.mem.unwrap(); + job.work.dst = ptr as _; + } + } + job.work.width = img.width as _; + job.work.stride = img.stride as _; + job.work.bpp = self.shm_info.bpp as _; + job.work.rects.clear(); + for copy in copies { + job.work.rects.push( + Rect::new_sized( + copy.image_offset.x as _, + copy.image_offset.y as _, + copy.image_extent.width as _, + copy.image_extent.height as _, + ) + .unwrap(), + ); + } + pending = data.cpu.submit(job); + } + ShmMemoryBacking::Fd(fd, offset) => { + let mut min_offset = client_mem.len() as u64; + let mut max_offset = 0; + for copy in copies { + min_offset = min_offset.min(copy.buffer_offset); + let len = img.stride * (copy.image_extent.height - 1) + + copy.image_extent.width * self.shm_info.bpp; + max_offset = max_offset.max(copy.buffer_offset + len as u64); + } + let mut job = data.io_job.take().unwrap_or_else(|| { + Box::new(IoTransferJob { + img: None, + id, + _mem: None, + work: unsafe { ReadWriteWork::new() }, + fd: None, + tt, + }) + }); + job.id = id; + job.img = Some(img.clone()); + job._mem = Some(client_mem.clone()); + job.fd = Some(fd.clone()); + job.tt = tt; + unsafe { + let config = job.work.config(); + config.fd = fd.raw(); + config.offset = offset + min_offset as usize; + config.ptr = staging.allocation.mem.unwrap().add(min_offset as _); + config.len = max_offset.saturating_sub(min_offset) as usize; + config.write = tt == TransferType::Download; + } + pending = data.cpu.submit(job); + } + } + + img.renderer.pending_cpu_jobs.set(id, pending); + + Ok(()) + } + + fn async_upload_copy_buffer_to_image( + &self, + img: &Rc, + data: &VulkanShmImageAsyncData, + ) -> Result<(), VulkanError> { + if !data.data_copied.get() { + return Ok(()); + } + if img.queue_state.get().acquire(QueueFamily::Transfer) == QueueTransfer::Impossible { + return Ok(()); + } + img.renderer.check_defunct()?; + let regions = &*data.regions.borrow(); + let staging = data.staging.get().unwrap().staging.get().unwrap(); + staging.upload(|_, _| ())?; + let Some((cmd, fence, sync_file, point)) = + self.submit_buffer_image_copy(img, &staging, regions, true, TransferType::Upload)? + else { + return Ok(()); + }; + img.queue_state.set(QueueState::Releasing); + let future = img.renderer.eng.spawn( + "await async upload", + await_async_transfer_release_to_gfx( + point, + img.clone(), + cmd, + fence, + sync_file, + TransferType::Upload, + ), + ); + img.renderer.pending_submits.set(point, future); + Ok(()) + } + + fn async_download_copy_image_to_buffer( + &self, + img: &Rc, + staging: &VulkanStagingBuffer, + copies: &[BufferImageCopy2], + ) -> Result<(), VulkanError> { + if img.queue_state.get().acquire(QueueFamily::Transfer) == QueueTransfer::Impossible { + return Ok(()); + } + img.renderer.check_defunct()?; + let Some((cmd, fence, sync_file, point)) = + self.submit_buffer_image_copy(img, &staging, copies, true, TransferType::Download)? + else { + img.queue_state.set(QueueState::Released { + to: QueueFamily::Gfx, + }); + let data = self.async_data.as_ref().unwrap(); + let client_mem = data.client_mem.get().unwrap(); + return self.async_transfer_initiate_host_copy( + &img, + data, + &staging, + copies, + &client_mem, + TransferType::Download, + ); + }; + img.queue_state.set(QueueState::Releasing); + let future = img.renderer.eng.spawn( + "await async image to buffer copy", + await_async_transfer_release_to_gfx( + point, + img.clone(), + cmd, + fence, + sync_file, + TransferType::Download, + ), + ); + img.renderer.pending_submits.set(point, future); + Ok(()) + } +} + +pub(super) struct IoTransferJob { + img: Option>, + id: u64, + _mem: Option>, + fd: Option>, + work: ReadWriteWork, + tt: TransferType, +} + +pub(super) struct CopyTransferJob { + img: Option>, + id: u64, + _mem: Option>, + work: ImgCopyWork, + tt: TransferType, +} + +impl CpuJob for IoTransferJob { + fn work(&mut self) -> &mut dyn CpuWork { + &mut self.work + } + + fn completed(mut self: Box) { + self._mem = None; + self.fd = None; + let img = self.img.take().unwrap(); + let res = self.work.config().result.take().unwrap(); + complete_async_host_copy(&img, self.id, res, self.tt, |data| { + data.io_job.set(Some(self)) + }); + } +} + +impl CpuJob for CopyTransferJob { + fn work(&mut self) -> &mut dyn CpuWork { + &mut self.work + } + + fn completed(mut self: Box) { + self._mem = None; + let img = self.img.take().unwrap(); + complete_async_host_copy(&img, self.id, Ok(()), self.tt, |data| { + data.copy_job.set(Some(self)) + }); + } +} + +fn complete_async_host_copy( + img: &Rc, + id: u64, + res: Result<(), ReadWriteJobError>, + tt: TransferType, + store: impl FnOnce(&VulkanShmImageAsyncData), +) { + img.renderer.pending_cpu_jobs.remove(&id); + let VulkanImageMemory::Internal(shm) = &img.ty else { + unreachable!(); + }; + let data = shm.async_data.as_ref().unwrap(); + store(data); + if let Err(e) = res { + data.complete(Err(VulkanError::AsyncCopyToStaging(e))); + } + data.data_copied.set(true); + match tt { + TransferType::Upload => { + let res = shm.async_upload_copy_buffer_to_image(img, data); + if let Err(e) = res { + data.complete(Err(e)); + } + } + TransferType::Download => data.complete(Ok(())), + } +} + +async fn await_gfx_queue_release( + id: u64, + img: Rc, + buf: Option>, + _fence: Option>, + sync_file: SyncFile, + tt: TransferType, +) { + let res = img.renderer.ring.readable(&sync_file.0).await; + if let Err(e) = res { + log::error!( + "Could not wait for sync file to become readable: {}", + ErrorFmt(e) + ); + img.renderer.block(); + } + if let Some(buf) = buf { + img.renderer.gfx_command_buffers.buffers.push(buf); + } + img.renderer.pending_submits.remove(&id); + img.queue_state.set(QueueState::Released { + to: QueueFamily::Transfer, + }); + let VulkanImageMemory::Internal(shm) = &img.ty else { + unreachable!(); + }; + let data = shm.async_data.as_ref().unwrap(); + let res = match tt { + TransferType::Upload => shm.async_upload_copy_buffer_to_image(&img, data), + TransferType::Download => match data.staging.get().unwrap().staging.get() { + Some(staging) => { + let copies = &*data.regions.borrow(); + shm.async_download_copy_image_to_buffer(&img, &staging, copies) + } + None => Ok(()), + }, + }; + if let Err(e) = res { + data.complete(Err(e)); + } +} + +pub async fn await_async_transfer_release_to_gfx( + id: u64, + img: Rc, + buf: Rc, + _fence: Rc, + sync_file: SyncFile, + tt: TransferType, +) { + let res = img.renderer.ring.readable(&sync_file.0).await; + if let Err(e) = res { + log::error!( + "Could not wait for sync file to become readable: {}", + ErrorFmt(e) + ); + img.renderer.block(); + } + match &img.renderer.transfer_command_buffers { + Some(b) => b.buffers.push(buf), + None => img.renderer.gfx_command_buffers.buffers.push(buf), + } + img.queue_state.set(QueueState::Released { + to: QueueFamily::Gfx, + }); + img.renderer.pending_submits.remove(&id); + let VulkanImageMemory::Internal(shm) = &img.ty else { + unreachable!(); + }; + let data = shm.async_data.as_ref().unwrap(); + match tt { + TransferType::Upload => { + data.complete(Ok(())); + } + TransferType::Download => { + let data = shm.async_data.as_ref().unwrap(); + let staging = data.staging.get().unwrap().staging.get().unwrap(); + let client_mem = data.client_mem.get().unwrap(); + let copies = &*data.regions.borrow(); + let res = shm.async_transfer_initiate_host_copy( + &img, + data, + &staging, + copies, + &client_mem, + tt, + ); + if let Err(e) = res { + data.complete(Err(e)); + } + } + } +} diff --git a/src/ifs/wl_buffer.rs b/src/ifs/wl_buffer.rs index 3068d572..d60de1c9 100644 --- a/src/ifs/wl_buffer.rs +++ b/src/ifs/wl_buffer.rs @@ -175,6 +175,7 @@ impl WlBuffer { WlBufferStorage::Shm { .. } => { return match surface { Some(s) => { + s.shm_staging.take(); s.shm_textures.back().tex.take(); s.shm_textures.front().tex.take().is_some() } diff --git a/src/ifs/wl_surface.rs b/src/ifs/wl_surface.rs index 61df4dd0..f8559cb2 100644 --- a/src/ifs/wl_surface.rs +++ b/src/ifs/wl_surface.rs @@ -23,8 +23,8 @@ use { drm_feedback::DrmFeedback, fixed::Fixed, gfx_api::{ - AsyncShmGfxTexture, BufferResv, BufferResvUser, GfxError, ReleaseSync, SampleRect, - SyncFile, + AsyncShmGfxTexture, BufferResv, BufferResvUser, GfxError, GfxStagingBuffer, + ReleaseSync, SampleRect, SyncFile, }, ifs::{ wl_buffer::WlBuffer, @@ -276,6 +276,7 @@ pub struct WlSurface { pub buffer_abs_pos: Cell, pub need_extents_update: Cell, pub buffer: CloneCell>>, + pub shm_staging: CloneCell>>, pub shm_textures: DoubleBuffered, pub buf_x: NumCell, pub buf_y: NumCell, @@ -593,6 +594,7 @@ impl WlSurface { buffer_abs_pos: Cell::new(Default::default()), need_extents_update: Default::default(), buffer: Default::default(), + shm_staging: Default::default(), shm_textures: DoubleBuffered::new(DamageQueue::new().map(|damage| SurfaceShmTexture { tex: Default::default(), damage, @@ -1327,6 +1329,7 @@ impl WlSurface { } pub fn reset_shm_textures(&self) { + self.shm_staging.take(); for tex in &*self.shm_textures { tex.tex.take(); tex.damage.clear(); diff --git a/src/ifs/wl_surface/commit_timeline.rs b/src/ifs/wl_surface/commit_timeline.rs index 544d98a6..04e64f7a 100644 --- a/src/ifs/wl_surface/commit_timeline.rs +++ b/src/ifs/wl_surface/commit_timeline.rs @@ -1,6 +1,6 @@ use { crate::{ - gfx_api::{AsyncShmGfxTextureCallback, GfxError, PendingShmUpload}, + gfx_api::{AsyncShmGfxTextureCallback, GfxError, PendingShmTransfer, STAGING_UPLOAD}, ifs::{ wl_buffer::WlBufferStorage, wl_surface::{PendingState, WlSurface, WlSurfaceError}, @@ -310,7 +310,7 @@ enum EntryKind { enum ShmUploadState { None, Todo(Rc>), - Scheduled(#[expect(dead_code)] SmallVec<[PendingShmUpload; 1]>), + Scheduled(#[expect(dead_code)] SmallVec<[PendingShmTransfer; 1]>), } struct Commit { @@ -395,7 +395,7 @@ fn schedule_async_uploads( node_ref: &Rc>, surface: &WlSurface, pending: &PendingState, - uploads: &mut SmallVec<[PendingShmUpload; 1]>, + uploads: &mut SmallVec<[PendingShmTransfer; 1]>, ) -> Result<(), WlSurfaceError> { if let Some(pending) = schedule_async_upload(node_ref, surface, pending)? { uploads.push(pending); @@ -412,7 +412,7 @@ fn schedule_async_upload( node_ref: &Rc>, surface: &WlSurface, pending: &PendingState, -) -> Result, WlSurfaceError> { +) -> Result, WlSurfaceError> { let Some(Some(buf)) = &pending.buffer else { return Ok(None); }; @@ -459,8 +459,28 @@ fn schedule_async_upload( back_tex } }; + let mut staging_opt = surface.shm_staging.get(); + if let Some(staging) = &staging_opt { + if staging.size() != back_tex.staging_size() { + staging_opt = None; + } + } + let staging = match staging_opt { + Some(s) => s, + None => { + let s = surface + .client + .state + .render_ctx + .get() + .ok_or(WlSurfaceError::NoRenderContext)? + .create_staging_buffer(back_tex.staging_size(), STAGING_UPLOAD); + surface.shm_staging.set(Some(s.clone())); + s + } + }; back_tex - .async_upload(node_ref.clone(), mem.clone(), back.damage.get()) + .async_upload(&staging, node_ref.clone(), mem.clone(), back.damage.get()) .map_err(WlSurfaceError::PrepareAsyncUpload) } diff --git a/src/ifs/zwlr_screencopy_frame_v1.rs b/src/ifs/zwlr_screencopy_frame_v1.rs index 44afe8c5..41183bd8 100644 --- a/src/ifs/zwlr_screencopy_frame_v1.rs +++ b/src/ifs/zwlr_screencopy_frame_v1.rs @@ -2,6 +2,7 @@ use { crate::{ client::{Client, ClientError}, format::XRGB8888, + gfx_api::{AsyncShmGfxTextureCallback, GfxError, PendingShmTransfer}, ifs::{ wl_buffer::{WlBuffer, WlBufferError, WlBufferStorage}, wl_output::OutputGlobalOpt, @@ -9,6 +10,7 @@ use { leaks::Tracker, object::{Object, Version}, rect::Rect, + utils::errorfmt::ErrorFmt, wire::{zwlr_screencopy_frame_v1::*, WlBufferId, ZwlrScreencopyFrameV1Id}, }, std::{cell::Cell, ops::Deref, rc::Rc}, @@ -29,6 +31,7 @@ pub struct ZwlrScreencopyFrameV1 { pub with_damage: Cell, pub buffer: Cell>>, pub version: Version, + pub pending: Cell>, } impl ZwlrScreencopyFrameV1 { @@ -132,6 +135,7 @@ impl ZwlrScreencopyFrameV1 { node.screencopies.remove(&(self.client.id, self.id)); node.screencast_changed(); } + self.pending.take(); } } @@ -153,6 +157,22 @@ impl ZwlrScreencopyFrameV1RequestHandler for ZwlrScreencopyFrameV1 { } } +impl AsyncShmGfxTextureCallback for ZwlrScreencopyFrameV1 { + fn completed(self: Rc, res: Result<(), GfxError>) { + self.pending.take(); + match res { + Ok(_) => { + let now = self.client.state.now(); + self.send_ready(now.0.tv_sec as _, now.0.tv_nsec as _); + } + Err(e) => { + log::warn!("Could not perform shm screencopy: {}", ErrorFmt(e)); + self.send_failed(); + } + } + } +} + object_base! { self = ZwlrScreencopyFrameV1; version = self.version; diff --git a/src/ifs/zwlr_screencopy_manager_v1.rs b/src/ifs/zwlr_screencopy_manager_v1.rs index ae99a4b4..66520274 100644 --- a/src/ifs/zwlr_screencopy_manager_v1.rs +++ b/src/ifs/zwlr_screencopy_manager_v1.rs @@ -130,6 +130,7 @@ impl ZwlrScreencopyManagerV1 { with_damage: Cell::new(false), buffer: Cell::new(None), version: self.version, + pending: Default::default(), }); track!(self.client, frame); self.client.add_client_obj(&frame)?; diff --git a/src/it/test_gfx_api.rs b/src/it/test_gfx_api.rs index 95c52864..87d33a81 100644 --- a/src/it/test_gfx_api.rs +++ b/src/it/test_gfx_api.rs @@ -6,8 +6,8 @@ use { gfx_api::{ AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback, CopyTexture, FillRect, FramebufferRect, GfxApiOpt, GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage, - GfxTexture, GfxWriteModifier, PendingShmUpload, ReleaseSync, ResetStatus, - ShmGfxTexture, ShmMemory, SyncFile, + GfxInternalFramebuffer, GfxStagingBuffer, GfxTexture, GfxWriteModifier, + PendingShmTransfer, ReleaseSync, ResetStatus, ShmGfxTexture, ShmMemory, SyncFile, }, rect::{Rect, Region}, theme::Color, @@ -165,13 +165,14 @@ impl GfxContext for TestGfxCtx { GfxApi::OpenGl } - fn create_fb( + fn create_internal_fb( self: Rc, + _cpu_worker: &Rc, width: i32, height: i32, stride: i32, format: &'static Format, - ) -> Result, GfxError> { + ) -> Result, GfxError> { assert!(stride >= width * 4); Ok(Rc::new(TestGfxFb { img: Rc::new(TestGfxImage::Shm(TestShmGfxImage { @@ -214,44 +215,17 @@ struct TestDmaBufGfxImage { } impl TestGfxImage { - fn read_pixels( - &self, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError> { - assert!(x >= 0); - assert!(y >= 0); - assert!(width >= 0); - assert!(height >= 0); - assert!(stride >= 0); - assert!(x + width <= self.width()); - assert!(y + height <= self.height()); - assert!(stride >= width * 4); - let size = (stride * height) as usize; - assert!(shm.len() >= size); - let copy = |src_stride: i32, src_format: &Format, mut src: *const u8, mut dst: *mut u8| unsafe { - src = src.add((y * src_stride + x * 4) as usize); - for _ in 0..height { - ptr::copy_nonoverlapping(src, dst, (width * 4) as usize); - if !src_format.has_alpha && format.has_alpha { - for i in 0..width { - *dst.add((i * 4 + 3) as usize) = 255; - } - } - src = src.add(src_stride as usize); - dst = dst.add(stride as usize); - } + fn read_pixels(&self, shm: &[Cell]) -> Result<(), GfxError> { + let copy = |height: i32, stride: i32, src: *const u8, dst: *mut u8| unsafe { + let size = (height * stride) as usize; + assert!(shm.len() >= size); + ptr::copy_nonoverlapping(src, dst, size); }; match self { TestGfxImage::Shm(s) => { copy( + s.height, s.stride, - s.format, s.data.borrow().as_ptr(), shm.as_ptr() as _, ); @@ -260,8 +234,8 @@ impl TestGfxImage { let map = d.bo.clone().map_read().map_err(TestGfxError::MapDmaBuf)?; unsafe { copy( + d.buf.height, map.stride(), - d.buf.format, map.data().as_ptr(), shm.as_ptr() as _, ); @@ -300,20 +274,6 @@ impl GfxTexture for TestGfxImage { self } - fn read_pixels( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError> { - self.deref() - .read_pixels(x, y, width, height, stride, format, shm) - } - fn dmabuf(&self) -> Option<&DmaBuf> { match self { TestGfxImage::Shm(_) => None, @@ -335,10 +295,11 @@ impl ShmGfxTexture for TestGfxImage { impl AsyncShmGfxTexture for TestGfxImage { fn async_upload( self: Rc, + _staging: &Rc, _callback: Rc, mem: Rc, _damage: Region, - ) -> Result, GfxError> { + ) -> Result, GfxError> { let mut res = Ok(()); mem.access(&mut |d| { res = self.clone().sync_upload(d, Region::default()); @@ -588,26 +549,34 @@ impl GfxFramebuffer for TestGfxFb { Ok(None) } - fn copy_to_shm( - self: Rc, - x: i32, - y: i32, - width: i32, - height: i32, - stride: i32, - format: &'static Format, - shm: &[Cell], - ) -> Result<(), GfxError> { - self.img - .deref() - .read_pixels(x, y, width, height, stride, format, shm) - } - fn format(&self) -> &'static Format { &ARGB8888 } } +impl GfxInternalFramebuffer for TestGfxFb { + fn into_fb(self: Rc) -> Rc { + self + } + + fn staging_size(&self) -> usize { + 0 + } + + fn download( + self: Rc, + _staging: &Rc, + _callback: Rc, + mem: Rc, + _damage: Region, + ) -> Result, GfxError> { + let mut res = Ok(()); + mem.access(&mut |mem| res = self.img.deref().read_pixels(mem)) + .map_err(TestGfxError::AccessFailed)?; + res.map(|_| None) + } +} + impl dyn GfxTexture { fn as_native(&self) -> &TestGfxImage { self.as_any() diff --git a/src/state.rs b/src/state.rs index 1871084b..f9222f65 100644 --- a/src/state.rs +++ b/src/state.rs @@ -27,8 +27,8 @@ use { forker::ForkerProxy, format::Format, gfx_api::{ - AcquireSync, BufferResv, GfxContext, GfxError, GfxFramebuffer, GfxTexture, ReleaseSync, - SampleRect, SyncFile, + AcquireSync, BufferResv, GfxContext, GfxError, GfxFramebuffer, GfxTexture, + PendingShmTransfer, ReleaseSync, SampleRect, SyncFile, STAGING_DOWNLOAD, }, gfx_apis::create_gfx_context, globals::{Globals, GlobalsError, RemovableWaylandGlobal, WaylandGlobal}, @@ -62,7 +62,7 @@ use { io_uring::IoUring, leaks::Tracker, logger::Logger, - rect::Rect, + rect::{Rect, Region}, renderer::Renderer, scale::Scale, security_context_acceptor::SecurityContextAcceptors, @@ -1006,17 +1006,6 @@ impl State { ) } - fn have_hardware_cursor(&self) -> bool { - if let Some(group) = self.cursor_user_group_hardware_cursor.get() { - if let Some(user) = group.active() { - if user.get().is_some() { - return true; - } - } - } - false - } - pub fn perform_shm_screencopy( &self, src: &Rc, @@ -1025,80 +1014,54 @@ impl State { x_off: i32, y_off: i32, size: Option<(i32, i32)>, - capture: &ZwlrScreencopyFrameV1, - mem: &ClientMemOffset, + capture: &Rc, + mem: &Rc, stride: i32, format: &'static Format, transform: Transform, scale: Scale, - ) -> Result<(), ShmScreencopyError> { - let (src_width, src_height) = src.size(); - let mut needs_copy = capture.rect.x1() < x_off - || capture.rect.x2() > x_off + src_width - || capture.rect.y1() < y_off - || capture.rect.y2() > y_off + src_height - || self.have_hardware_cursor(); - if let Some((target_width, target_height)) = size { - if (target_width, target_height) != (src_width, src_height) { - needs_copy = true; - } - } - let acc = if needs_copy { - let Some(ctx) = self.render_ctx.get() else { - return Err(ShmScreencopyError::NoRenderContext); - }; - let fb = ctx - .create_fb(capture.rect.width(), capture.rect.height(), stride, format) - .map_err(ShmScreencopyError::CreateTemporaryFb)?; - self.perform_screencopy( - src, - None, - acquire_sync, - ReleaseSync::None, - &fb, - AcquireSync::Unnecessary, - ReleaseSync::None, - transform, - position, - true, - x_off - capture.rect.x1(), - y_off - capture.rect.y1(), - size, - transform, - scale, - ) - .map_err(ShmScreencopyError::CopyToTemporary)?; - mem.access(|mem| { - fb.copy_to_shm( - 0, - 0, - capture.rect.width(), - capture.rect.height(), - stride, - format, - mem, - ) - }) - } else { - mem.access(|mem| { - src.clone().read_pixels( - capture.rect.x1() - x_off, - capture.rect.y1() - y_off, - capture.rect.width(), - capture.rect.height(), - stride, - format, - mem, - ) - }) + ) -> Result, ShmScreencopyError> { + let Some(ctx) = self.render_ctx.get() else { + return Err(ShmScreencopyError::NoRenderContext); }; - match acc { - Ok(res) => res.map_err(ShmScreencopyError::ReadPixels), - Err(e) => { - capture.client.error(e); - Ok(()) - } - } + let fb = ctx + .clone() + .create_internal_fb( + &self.cpu_worker, + capture.rect.width(), + capture.rect.height(), + stride, + format, + ) + .map_err(ShmScreencopyError::CreateTemporaryFb)?; + self.perform_screencopy( + src, + None, + acquire_sync, + ReleaseSync::None, + &fb.clone().into_fb(), + AcquireSync::Unnecessary, + ReleaseSync::None, + transform, + position, + true, + x_off - capture.rect.x1(), + y_off - capture.rect.y1(), + size, + transform, + scale, + ) + .map_err(ShmScreencopyError::CopyToTemporary)?; + let staging = ctx.create_staging_buffer(fb.staging_size(), STAGING_DOWNLOAD); + let pending = fb + .download( + &staging, + capture.clone(), + mem.clone(), + Region::new2(capture.rect.at_point(0, 0)), + ) + .map_err(ShmScreencopyError::ReadPixels)?; + Ok(pending) } pub fn create_seat(self: &Rc, name: &str) -> Rc { diff --git a/src/text.rs b/src/text.rs index 7eed4e21..0a9408d4 100644 --- a/src/text.rs +++ b/src/text.rs @@ -3,8 +3,8 @@ use { cpu_worker::{AsyncCpuWork, CpuJob, CpuWork, CpuWorker, PendingJob}, format::ARGB8888, gfx_api::{ - AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxContext, GfxError, GfxTexture, - PendingShmUpload, + AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxContext, GfxError, GfxStagingBuffer, + GfxTexture, PendingShmTransfer, STAGING_UPLOAD, }, pango::{ consts::{ @@ -302,9 +302,10 @@ impl Drop for TextTexture { struct Shared { cpu_worker: Rc, ctx: Rc, + staging: CloneCell>>, textures: DoubleBuffered, pending_render: Cell>, - pending_upload: Cell>, + pending_upload: Cell>, render_job: Cell>>, result: Cell>>, waiter: Cell>>, @@ -367,6 +368,7 @@ impl TextTexture { let data = Rc::new(Shared { cpu_worker: cpu_worker.clone(), ctx: ctx.clone(), + staging: Default::default(), textures: Default::default(), pending_render: Default::default(), pending_upload: Default::default(), @@ -528,9 +530,22 @@ impl CpuJob for RenderJob { } } }; + let mut staging_opt = data.staging.take(); + if let Some(staging) = &staging_opt { + if staging.size() != tex.staging_size() { + staging_opt = None; + } + } + let staging = match staging_opt { + Some(s) => s, + None => data + .ctx + .create_staging_buffer(tex.staging_size(), STAGING_UPLOAD), + }; let pending = tex .clone() .async_upload( + &staging, data.clone(), Rc::new(rt.data), Region::new2(Rect::new_sized_unchecked(0, 0, rt.width, rt.height)), @@ -538,6 +553,7 @@ impl CpuJob for RenderJob { .map_err(TextError::Upload); if pending.is_ok() { data.textures.back().tex.set(Some(tex)); + data.staging.set(Some(staging)); } match pending { Ok(Some(p)) => data.pending_upload.set(Some(p)), diff --git a/src/tree/output.rs b/src/tree/output.rs index d0dffce6..92f85598 100644 --- a/src/tree/output.rs +++ b/src/tree/output.rs @@ -267,6 +267,7 @@ impl OutputNode { capture.send_failed(); continue; } + let mut ready = true; if let Some(storage) = wl_buffer.storage.borrow_mut().deref() { match storage { WlBufferStorage::Shm { mem, stride } => { @@ -284,10 +285,16 @@ impl OutputNode { self.global.persistent.transform.get(), self.global.persistent.scale.get(), ); - if let Err(e) = res { - log::warn!("Could not perform shm screencopy: {}", ErrorFmt(e)); - capture.send_failed(); - continue; + match res { + Ok(p) => { + ready = p.is_none(); + capture.pending.set(p); + } + Err(e) => { + log::warn!("Could not perform shm screencopy: {}", ErrorFmt(e)); + capture.send_failed(); + continue; + } } } WlBufferStorage::Dmabuf { fb, .. } => { @@ -327,7 +334,9 @@ impl OutputNode { if capture.with_damage.get() { capture.send_damage(); } - capture.send_ready(now.0.tv_sec as _, now.0.tv_nsec as _); + if ready { + capture.send_ready(now.0.tv_sec as _, now.0.tv_nsec as _); + } } self.screencast_changed(); }