From a3d3a62af36ac11732aefe46ef23b454647c0a3a Mon Sep 17 00:00:00 2001 From: Julian Orth Date: Tue, 30 Sep 2025 18:52:57 +0200 Subject: [PATCH] vulkan: import wl_shm buffers as udmabuf --- src/clientmem.rs | 18 +- src/compositor.rs | 1 + src/gfx_api.rs | 34 +++ src/gfx_apis/vulkan.rs | 23 ++- src/gfx_apis/vulkan/dmabuf_buffer.rs | 130 ++++++++++++ src/gfx_apis/vulkan/image.rs | 16 +- src/gfx_apis/vulkan/shm_image.rs | 24 ++- src/gfx_apis/vulkan/transfer.rs | 194 +++++++++++++----- .../ext_image_copy_capture_frame_v1.rs | 2 +- src/ifs/wl_buffer.rs | 108 +++++++++- src/ifs/wl_surface/commit_timeline.rs | 33 ++- src/state.rs | 24 ++- src/tree/output.rs | 2 +- src/udmabuf.rs | 35 +++- 14 files changed, 545 insertions(+), 99 deletions(-) create mode 100644 src/gfx_apis/vulkan/dmabuf_buffer.rs diff --git a/src/clientmem.rs b/src/clientmem.rs index f8ee8239..69f9c530 100644 --- a/src/clientmem.rs +++ b/src/clientmem.rs @@ -3,7 +3,7 @@ use { client::Client, cpu_worker::{AsyncCpuWork, CpuJob, CpuWork, CpuWorker}, gfx_api::{ShmMemory, ShmMemoryBacking}, - utils::vec_ext::VecExt, + utils::{page_size::page_size, vec_ext::VecExt}, }, std::{ cell::Cell, @@ -18,6 +18,7 @@ use { uapi::{ OwnedFd, c::{self, raise}, + ftruncate, }, }; @@ -76,10 +77,12 @@ impl ClientMem { flags: c::c_int, ) -> Result { let mut sigbus_impossible = false; + let mut real_size = None; if let Ok(seals) = uapi::fcntl_get_seals(fd.raw()) && seals & c::F_SEAL_SHRINK != 0 && let Ok(stat) = uapi::fstat(fd.raw()) { + real_size = Some(stat.st_size as usize); sigbus_impossible = stat.st_size as u64 >= len as u64; } if !sigbus_impossible && let Some(client) = client { @@ -89,6 +92,12 @@ impl ClientMem { client.id, ); } + let len = len.next_multiple_of(page_size()); + if let Some(real_size) = real_size + && real_size < len + { + let _ = ftruncate(fd.raw(), len as _); + } let data = if len == 0 { &mut [][..] } else { @@ -126,23 +135,20 @@ impl ClientMem { } } - #[expect(dead_code)] pub fn fd(&self) -> &Rc { &self.fd } - pub fn sigbus_impossible(&self) -> bool { + pub fn is_sealed_memfd(&self) -> bool { self.sigbus_impossible } } impl ClientMemOffset { - #[expect(dead_code)] pub fn pool(&self) -> &ClientMem { &self.mem } - #[expect(dead_code)] pub fn offset(&self) -> usize { self.offset } @@ -299,7 +305,7 @@ impl ShmMemory for ClientMemOffset { } fn safe_access(&self) -> ShmMemoryBacking { - match self.mem.sigbus_impossible() { + match self.mem.is_sealed_memfd() { true => ShmMemoryBacking::Ptr(self.data), false => ShmMemoryBacking::Fd(self.mem.fd.deref().clone(), self.offset), } diff --git a/src/compositor.rs b/src/compositor.rs index f8e36ced..4835f429 100644 --- a/src/compositor.rs +++ b/src/compositor.rs @@ -360,6 +360,7 @@ fn start_compositor2( xdg_surface_configure_events: Default::default(), workspace_display_order: Cell::new(WorkspaceDisplayOrder::Manual), outputs_without_hc: Default::default(), + udmabuf: Default::default(), }); state.tracker.register(ClientId::from_raw(0)); create_dummy_output(&state); diff --git a/src/gfx_api.rs b/src/gfx_api.rs index 3472c41c..4245f813 100644 --- a/src/gfx_api.rs +++ b/src/gfx_api.rs @@ -665,6 +665,8 @@ pub trait GfxStagingBuffer: Any { fn size(&self) -> usize; } +pub trait GfxBuffer: Any {} + pub trait AsyncShmGfxTextureTransferCancellable { fn cancel(&self, id: u64); } @@ -713,6 +715,22 @@ pub trait AsyncShmGfxTexture: GfxTexture { damage: Region, ) -> Result, GfxError>; + fn async_upload_from_buffer( + self: Rc, + buf: &Rc, + callback: Rc, + damage: Region, + ) -> Result, GfxError> { + let _ = buf; + let _ = callback; + let _ = damage; + + #[derive(Debug, Error)] + #[error("Host buffers are not supported")] + struct E; + Err(GfxError(Box::new(E))) + } + fn sync_upload(self: Rc, shm: &[Cell], damage: Region) -> Result<(), GfxError>; fn compatible_with( @@ -800,6 +818,22 @@ pub trait GfxContext: Debug { fn supports_invalid_modifier(&self) -> bool { false } + + fn create_dmabuf_buffer( + &self, + dmabuf: &Rc, + offset: usize, + size: usize, + ) -> Result, GfxError> { + let _ = dmabuf; + let _ = offset; + let _ = size; + + #[derive(Debug, Error)] + #[error("Host buffers are not supported")] + struct E; + Err(GfxError(Box::new(E))) + } } #[derive(Clone, Debug)] diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs index 3088ad28..3b1114bb 100644 --- a/src/gfx_apis/vulkan.rs +++ b/src/gfx_apis/vulkan.rs @@ -6,6 +6,7 @@ mod command; mod descriptor; mod descriptor_buffer; mod device; +mod dmabuf_buffer; mod eotfs; mod fence; mod format; @@ -27,9 +28,9 @@ use { cpu_worker::{CpuWorker, jobs::read_write::ReadWriteJobError}, format::Format, gfx_api::{ - AsyncShmGfxTexture, GfxBlendBuffer, GfxContext, GfxError, GfxFormat, GfxImage, - GfxInternalFramebuffer, GfxStagingBuffer, GfxTexture, ResetStatus, STAGING_DOWNLOAD, - STAGING_UPLOAD, ShmGfxTexture, StagingBufferUsecase, + AsyncShmGfxTexture, GfxBlendBuffer, GfxBuffer, GfxContext, GfxError, GfxFormat, + GfxImage, GfxInternalFramebuffer, GfxStagingBuffer, GfxTexture, ResetStatus, + STAGING_DOWNLOAD, STAGING_UPLOAD, ShmGfxTexture, StagingBufferUsecase, }, gfx_apis::vulkan::{ image::VulkanImageMemory, instance::VulkanInstance, renderer::VulkanRenderer, @@ -57,7 +58,7 @@ use { sync::Arc, }, thiserror::Error, - uapi::c::dev_t, + uapi::{OwnedFd, c::dev_t}, }; #[derive(Debug, Error)] @@ -394,6 +395,20 @@ impl GfxContext for Context { fn supports_color_management(&self) -> bool { self.0.device.descriptor_buffer.is_some() } + + fn create_dmabuf_buffer( + &self, + dmabuf: &Rc, + offset: usize, + size: usize, + ) -> Result, GfxError> { + self.0.check_defunct()?; + let buffer = self + .0 + .device + .create_dmabuf_buffer(dmabuf, offset as u64, size as u64)?; + Ok(buffer) + } } impl Drop for Context { diff --git a/src/gfx_apis/vulkan/dmabuf_buffer.rs b/src/gfx_apis/vulkan/dmabuf_buffer.rs new file mode 100644 index 00000000..c2912151 --- /dev/null +++ b/src/gfx_apis/vulkan/dmabuf_buffer.rs @@ -0,0 +1,130 @@ +use { + crate::{ + gfx_api::GfxBuffer, + gfx_apis::vulkan::{VulkanError, device::VulkanDevice}, + utils::on_drop::OnDrop, + }, + ash::{ + Device, + vk::{ + self, BufferCreateInfo, BufferUsageFlags, ExternalMemoryBufferCreateInfo, + ExternalMemoryHandleTypeFlags, ImportMemoryFdInfoKHR, MemoryAllocateInfo, + MemoryFdPropertiesKHR, MemoryPropertyFlags, + }, + }, + std::{any::Any, rc::Rc}, + uapi::OwnedFd, +}; + +pub struct VulkanDmabufBuffer { + pub(super) device: Rc, + pub(super) size: u64, + pub(super) offset: u64, + pub(super) buffer: vk::Buffer, + pub(super) memory: vk::DeviceMemory, +} + +impl VulkanDevice { + pub fn create_dmabuf_buffer( + self: &Rc, + dmabuf: &Rc, + offset: u64, + size: u64, + ) -> Result, VulkanError> { + let mut memory_fd_properties = MemoryFdPropertiesKHR::default(); + unsafe { + self.external_memory_fd + .get_memory_fd_properties( + ExternalMemoryHandleTypeFlags::DMA_BUF_EXT, + dmabuf.raw(), + &mut memory_fd_properties, + ) + .map_err(VulkanError::MemoryFdProperties)? + } + let buffer = { + let mut external_info = ExternalMemoryBufferCreateInfo::default() + .handle_types(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let create_info = BufferCreateInfo::default() + .size(size) + .usage(BufferUsageFlags::TRANSFER_SRC) + .push_next(&mut external_info); + unsafe { + self.device + .create_buffer(&create_info, None) + .map_err(VulkanError::CreateBuffer)? + } + }; + let destroy_buffer = OnDrop(|| unsafe { self.device.destroy_buffer(buffer, None) }); + let requirements = unsafe { self.device.get_buffer_memory_requirements(buffer) }; + let memory_type = self.find_memory_type( + MemoryPropertyFlags::HOST_VISIBLE, + requirements.memory_type_bits & memory_fd_properties.memory_type_bits, + ); + let Some(memory_type) = memory_type else { + return Err(VulkanError::MemoryType); + }; + let fd = + uapi::fcntl_dupfd_cloexec(dmabuf.raw(), 0).map_err(|e| VulkanError::Dupfd(e.into()))?; + let memory = { + let mut import_info = ImportMemoryFdInfoKHR::default() + .fd(fd.raw()) + .handle_type(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let allocate_info = MemoryAllocateInfo::default() + .allocation_size(requirements.size) + .memory_type_index(memory_type) + .push_next(&mut import_info); + unsafe { + self.device + .allocate_memory(&allocate_info, None) + .map_err(VulkanError::AllocateMemory)? + } + }; + fd.unwrap(); + let free_memory = OnDrop(|| unsafe { self.device.free_memory(memory, None) }); + unsafe { + self.device + .bind_buffer_memory(buffer, memory, 0) + .map_err(VulkanError::BindBufferMemory)?; + } + free_memory.forget(); + destroy_buffer.forget(); + Ok(Rc::new(VulkanDmabufBuffer { + device: self.clone(), + size, + offset, + buffer, + memory, + })) + } +} + +impl Drop for VulkanDmabufBuffer { + fn drop(&mut self) { + unsafe { + self.device.device.free_memory(self.memory, None); + self.device.device.destroy_buffer(self.buffer, None); + } + } +} + +impl VulkanDmabufBuffer { + fn assert_device(&self, device: &Device) { + assert_eq!( + self.device.device.handle(), + device.handle(), + "Mixed vulkan device use" + ); + } +} + +impl GfxBuffer for VulkanDmabufBuffer {} + +impl dyn GfxBuffer { + pub(super) fn into_vk(self: Rc, device: &Device) -> Rc { + let buffer: Rc = (self as Rc) + .downcast() + .expect("Non-vulkan buffer passed into vulkan"); + buffer.assert_device(device); + buffer + } +} diff --git a/src/gfx_apis/vulkan/image.rs b/src/gfx_apis/vulkan/image.rs index 4216e84e..72e576ce 100644 --- a/src/gfx_apis/vulkan/image.rs +++ b/src/gfx_apis/vulkan/image.rs @@ -4,7 +4,7 @@ use { format::Format, gfx_api::{ AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback, - AsyncShmGfxTextureTransferCancellable, GfxApiOpt, GfxBlendBuffer, GfxError, + AsyncShmGfxTextureTransferCancellable, GfxApiOpt, GfxBlendBuffer, GfxBuffer, GfxError, GfxFramebuffer, GfxImage, GfxInternalFramebuffer, GfxStagingBuffer, GfxTexture, PendingShmTransfer, ReleaseSync, ShmGfxTexture, ShmMemory, SyncFile, }, @@ -672,6 +672,20 @@ impl AsyncShmGfxTexture for VulkanImage { Ok(pending) } + fn async_upload_from_buffer( + self: Rc, + buf: &Rc, + callback: Rc, + damage: Region, + ) -> Result, GfxError> { + let VulkanImageMemory::Internal(shm) = &self.ty else { + unreachable!(); + }; + let buf = buf.clone().into_vk(&self.renderer.device.device); + let pending = shm.async_transfer2(&self, buf, damage, callback)?; + Ok(pending) + } + fn sync_upload(self: Rc, mem: &[Cell], damage: Region) -> Result<(), GfxError> { let VulkanImageMemory::Internal(shm) = &self.ty else { unreachable!(); diff --git a/src/gfx_apis/vulkan/shm_image.rs b/src/gfx_apis/vulkan/shm_image.rs index 33f0a9b6..567d33f8 100644 --- a/src/gfx_apis/vulkan/shm_image.rs +++ b/src/gfx_apis/vulkan/shm_image.rs @@ -17,7 +17,7 @@ use { utils::{errorfmt::ErrorFmt, on_drop::OnDrop}, }, ash::vk::{ - AccessFlags2, BufferImageCopy2, BufferMemoryBarrier2, CommandBufferBeginInfo, + AccessFlags2, Buffer, BufferImageCopy2, BufferMemoryBarrier2, CommandBufferBeginInfo, CommandBufferSubmitInfo, CommandBufferUsageFlags, CopyBufferToImageInfo2, CopyImageToBufferInfo2, DependencyInfoKHR, DeviceSize, Extent3D, ImageAspectFlags, ImageCreateInfo, ImageLayout, ImageSubresourceLayers, ImageSubresourceRange, ImageTiling, @@ -136,8 +136,14 @@ impl VulkanShmImage { ptr::copy_nonoverlapping(buf, mem, total_size as usize); } })?; - let (cmd, fence, sync_file, point) = - self.submit_buffer_image_copy(img, &staging, cpy, false, TransferType::Upload)?; + let (cmd, fence, sync_file, point) = self.submit_buffer_image_copy( + img, + staging.buffer, + staging.size, + cpy, + false, + TransferType::Upload, + )?; let future = img.renderer.eng.spawn( "await upload", await_upload(point, img.clone(), cmd, sync_file, fence, staging), @@ -149,7 +155,8 @@ impl VulkanShmImage { pub(super) fn submit_buffer_image_copy( &self, img: &Rc, - staging: &VulkanStagingBuffer, + buffer: Buffer, + size: DeviceSize, regions: &[BufferImageCopy2], use_transfer_queue: bool, tt: TransferType, @@ -164,9 +171,9 @@ impl VulkanShmImage { > { let memory_barrier = |sam, ssm, dam, dsm| { BufferMemoryBarrier2::default() - .buffer(staging.buffer) + .buffer(buffer) .offset(0) - .size(staging.size) + .size(size) .src_access_mask(sam) .src_stage_mask(ssm) .dst_access_mask(dam) @@ -274,7 +281,7 @@ impl VulkanShmImage { match tt { TransferType::Upload => { let cpy_info = CopyBufferToImageInfo2::default() - .src_buffer(staging.buffer) + .src_buffer(buffer) .dst_image(img.image) .dst_image_layout(ImageLayout::TRANSFER_DST_OPTIMAL) .regions(regions); @@ -282,7 +289,7 @@ impl VulkanShmImage { } TransferType::Download => { let cpy_info = CopyImageToBufferInfo2::default() - .dst_buffer(staging.buffer) + .dst_buffer(buffer) .src_image(img.image) .src_image_layout(ImageLayout::TRANSFER_SRC_OPTIMAL) .regions(regions); @@ -432,6 +439,7 @@ impl VulkanRenderer { io_job: Default::default(), copy_job: Default::default(), staging: Default::default(), + buffer: Default::default(), client_mem: Default::default(), callback: Default::default(), callback_id: Cell::new(0), diff --git a/src/gfx_apis/vulkan/transfer.rs b/src/gfx_apis/vulkan/transfer.rs index dcce432c..ce358905 100644 --- a/src/gfx_apis/vulkan/transfer.rs +++ b/src/gfx_apis/vulkan/transfer.rs @@ -13,6 +13,7 @@ use { gfx_apis::vulkan::{ VulkanError, command::VulkanCommandBuffer, + dmabuf_buffer::VulkanDmabufBuffer, fence::VulkanFence, image::{QueueFamily, QueueState, QueueTransfer, VulkanImage, VulkanImageMemory}, renderer::image_barrier, @@ -29,7 +30,7 @@ use { ImageSubresourceLayers, Offset3D, PipelineStageFlags2, SubmitInfo2, }, std::{ - cell::{Cell, RefCell}, + cell::{Cell, RefCell, RefMut}, rc::Rc, slice, }, @@ -41,6 +42,7 @@ pub struct VulkanShmImageAsyncData { pub(super) io_job: Cell>>, pub(super) copy_job: Cell>>, pub(super) staging: CloneCell>>, + pub(super) buffer: CloneCell>>, pub(super) client_mem: CloneCell>>, pub(super) callback: Cell>>, pub(super) callback_id: Cell, @@ -53,7 +55,10 @@ pub struct VulkanShmImageAsyncData { impl VulkanShmImageAsyncData { fn complete(&self, result: Result<(), VulkanError>) { self.busy.set(false); - self.staging.take().unwrap().busy.set(false); + if let Some(staging) = self.staging.take() { + staging.busy.set(false); + } + self.buffer.take(); self.client_mem.take(); if let Some(cb) = self.callback.take() { cb.completed(result.map_err(|e| e.into())); @@ -68,6 +73,43 @@ pub(super) enum TransferType { } impl VulkanShmImage { + pub fn async_transfer2( + &self, + img: &Rc, + buffer: Rc, + damage: Region, + callback: Rc, + ) -> Result, VulkanError> { + self.async_transfer_(img, damage, callback, |data, damage| { + self.try_async_transfer2(img, buffer, data, damage) + }) + } + + fn try_async_transfer2( + &self, + img: &Rc, + buffer: Rc, + data: &VulkanShmImageAsyncData, + mut damage: Region, + ) -> Result<(), VulkanError> { + if data.busy.get() { + return Err(VulkanError::AsyncCopyBusy); + } + if self.size > buffer.size { + return Err(VulkanError::InvalidBufferSize); + } + data.busy.set(true); + data.data_copied.set(true); + data.buffer.set(Some(buffer.clone())); + if img.contents_are_undefined.get() { + damage = Region::new(Rect::new_sized(0, 0, img.width as _, img.height as _).unwrap()); + } + self.calculate_copies(img, data, damage, buffer.offset); + self.async_release_from_gfx_queue(img, data, TransferType::Upload)?; + self.async_upload_copy_buffer_to_image(img, data)?; + Ok(()) + } + pub fn async_transfer( &self, img: &Rc, @@ -76,12 +118,24 @@ impl VulkanShmImage { damage: Region, callback: Rc, tt: TransferType, + ) -> Result, VulkanError> { + self.async_transfer_(img, damage, callback, |data, damage| { + self.try_async_transfer(img, staging, data, client_mem, damage, tt) + }) + } + + fn async_transfer_( + &self, + img: &Rc, + damage: Region, + callback: Rc, + f: impl FnOnce(&VulkanShmImageAsyncData, Region) -> Result<(), VulkanError>, ) -> Result, VulkanError> { if damage.is_empty() { return Ok(None); } let data = self.async_data.as_ref().unwrap(); - let res = self.try_async_transfer(img, staging, data, client_mem, damage, tt); + let res = f(data, damage); match res { Ok(()) => { let id = img.renderer.allocate_point(); @@ -135,53 +189,7 @@ impl VulkanShmImage { damage = Region::new(Rect::new_sized(0, 0, img.width as _, img.height as _).unwrap()); } - let copies = &mut *data.regions.borrow_mut(); - copies.clear(); - - let mut copy = |x, y, width, height| { - let buffer_offset = (y as u32 * img.stride + x as u32 * self.shm_info.bpp) as u64; - let copy = BufferImageCopy2::default() - .buffer_offset(buffer_offset) - .image_offset(Offset3D { x, y, z: 0 }) - .image_extent(Extent3D { - width, - height, - depth: 1, - }) - .image_subresource(ImageSubresourceLayers { - aspect_mask: ImageAspectFlags::COLOR, - mip_level: 0, - base_array_layer: 0, - layer_count: 1, - }) - .buffer_image_height(img.height) - .buffer_row_length(img.stride / self.shm_info.bpp); - copies.push(copy); - }; - let (width_mask, height_mask) = img.renderer.device.transfer_granularity_mask; - let width_mask = width_mask as i32; - let height_mask = height_mask as i32; - for damage in damage.rects() { - if damage.x2() < 0 || damage.y2() < 0 { - continue; - } - let x1 = damage.x1().max(0) & !width_mask; - let y1 = damage.y1().max(0) & !height_mask; - let x2 = ((damage.x2() + width_mask) & !width_mask).min(img.width as i32); - let y2 = ((damage.y2() + height_mask) & !height_mask).min(img.height as i32); - let Some(damage) = Rect::new(x1, y1, x2, y2) else { - continue; - }; - if damage.is_empty() { - continue; - } - copy( - damage.x1(), - damage.y1(), - damage.width() as u32, - damage.height() as u32, - ); - } + let copies = &mut *self.calculate_copies(img, data, damage, 0); self.async_release_from_gfx_queue(img, data, tt)?; @@ -209,6 +217,67 @@ impl VulkanShmImage { }) } + fn calculate_copies<'a>( + &self, + img: &Rc, + data: &'a VulkanShmImageAsyncData, + damage: Region, + extra_offset: u64, + ) -> RefMut<'a, Vec>> { + let mut copies_ref = data.regions.borrow_mut(); + let copies = &mut *copies_ref; + copies.clear(); + let mut copy = |x, y, width, height| { + let buffer_offset = (y as u32 * img.stride + x as u32 * self.shm_info.bpp) as u64; + let copy = BufferImageCopy2::default() + .buffer_offset(buffer_offset + extra_offset) + .image_offset(Offset3D { x, y, z: 0 }) + .image_extent(Extent3D { + width, + height, + depth: 1, + }) + .image_subresource(ImageSubresourceLayers { + aspect_mask: ImageAspectFlags::COLOR, + mip_level: 0, + base_array_layer: 0, + layer_count: 1, + }) + .buffer_image_height(img.height) + .buffer_row_length(img.stride / self.shm_info.bpp); + copies.push(copy); + }; + let (width_mask, height_mask) = img.renderer.device.transfer_granularity_mask; + let width_mask = width_mask as i32; + let height_mask = height_mask as i32; + for damage in damage.rects() { + if damage.x2() < 0 || damage.y2() < 0 { + continue; + } + let x1 = damage.x1().max(0); + let y1 = damage.y1().max(0); + let x2 = damage.x2().min(img.width as i32); + let y2 = damage.y2().min(img.height as i32); + let x1 = x1 & !width_mask; + let y1 = y1 & !height_mask; + let x2 = ((x2 + width_mask) & !width_mask).min(img.width as i32); + let y2 = ((y2 + height_mask) & !height_mask).min(img.height as i32); + let Some(damage) = Rect::new(x1, y1, x2, y2) else { + continue; + }; + if damage.is_empty() { + continue; + } + copy( + damage.x1(), + damage.y1(), + damage.width() as u32, + damage.height() as u32, + ); + } + copies_ref + } + fn async_release_from_gfx_queue( &self, img: &Rc, @@ -451,10 +520,19 @@ impl VulkanShmImage { } img.renderer.check_defunct()?; let regions = &*data.regions.borrow(); - let staging = data.staging.get().unwrap().staging.get().unwrap(); - staging.upload(|_, _| ())?; + let (buffer, size) = match data.staging.get() { + Some(s) => { + let staging = s.staging.get().unwrap(); + staging.upload(|_, _| ())?; + (staging.buffer, staging.size) + } + _ => { + let host_buffer = data.buffer.get().unwrap(); + (host_buffer.buffer, host_buffer.size) + } + }; let (cmd, fence, sync_file, point) = - self.submit_buffer_image_copy(img, &staging, regions, true, TransferType::Upload)?; + self.submit_buffer_image_copy(img, buffer, size, regions, true, TransferType::Upload)?; img.queue_state.set(QueueState::Releasing); let future = img.renderer.eng.spawn( "await async upload", @@ -481,8 +559,14 @@ impl VulkanShmImage { return Ok(()); } img.renderer.check_defunct()?; - let (cmd, fence, sync_file, point) = - self.submit_buffer_image_copy(img, &staging, copies, true, TransferType::Download)?; + let (cmd, fence, sync_file, point) = self.submit_buffer_image_copy( + img, + staging.buffer, + staging.size, + copies, + true, + TransferType::Download, + )?; img.queue_state.set(QueueState::Releasing); let future = img.renderer.eng.spawn( "await async image to buffer copy", diff --git a/src/ifs/ext_image_copy/ext_image_copy_capture_frame_v1.rs b/src/ifs/ext_image_copy/ext_image_copy_capture_frame_v1.rs index 79452e38..5f47052b 100644 --- a/src/ifs/ext_image_copy/ext_image_copy_capture_frame_v1.rs +++ b/src/ifs/ext_image_copy/ext_image_copy_capture_frame_v1.rs @@ -101,7 +101,7 @@ impl ExtImageCopyCaptureFrameV1 { let mut shm_bridge = self.session.shm_bridge.take(); let mut shm_staging = self.session.shm_staging.take(); match storage { - WlBufferStorage::Shm { mem, stride } => { + WlBufferStorage::Shm { mem, stride, .. } => { if let Some(b) = &shm_bridge && (b.physical_size() != buffer.rect.size() || b.format() != buffer.format diff --git a/src/ifs/wl_buffer.rs b/src/ifs/wl_buffer.rs index e53a98fb..f7897269 100644 --- a/src/ifs/wl_buffer.rs +++ b/src/ifs/wl_buffer.rs @@ -3,12 +3,12 @@ use { client::{Client, ClientError}, clientmem::{ClientMem, ClientMemError, ClientMemOffset}, format::{ARGB8888, Format}, - gfx_api::{GfxError, GfxFramebuffer, GfxImage, GfxTexture}, + gfx_api::{GfxBuffer, GfxContext, GfxError, GfxFramebuffer, GfxImage, GfxTexture}, ifs::wl_surface::WlSurface, leaks::Tracker, object::{Object, Version}, rect::{Rect, Region}, - utils::errorfmt::ErrorFmt, + utils::{errorfmt::ErrorFmt, page_size::page_size}, video::dmabuf::DmaBuf, wire::{WlBufferId, wl_buffer::*}, }, @@ -17,12 +17,14 @@ use { rc::Rc, }, thiserror::Error, + uapi::OwnedFd, }; pub enum WlBufferStorage { Shm { mem: Rc, stride: i32, + dmabuf_buffer_params: DmabufBufferParams, }, Dmabuf { img: Rc, @@ -31,6 +33,16 @@ pub enum WlBufferStorage { }, } +pub struct DmabufBufferParams { + size: usize, + udmabuf: Option>, + udmabuf_offset: usize, + udmabuf_size: usize, + udmabuf_impossible: bool, + host_buffer: Option>, + host_buffer_impossible: bool, +} + pub struct WlBuffer { pub id: WlBufferId, destroyed: Cell, @@ -118,7 +130,19 @@ impl WlBuffer { format, dmabuf: None, render_ctx_version: Cell::new(client.state.render_ctx_version.get()), - storage: RefCell::new(Some(WlBufferStorage::Shm { mem, stride })), + storage: RefCell::new(Some(WlBufferStorage::Shm { + dmabuf_buffer_params: DmabufBufferParams { + size: bytes as usize, + udmabuf: None, + udmabuf_offset: 0, + udmabuf_size: 0, + udmabuf_impossible: !mem.pool().is_sealed_memfd(), + host_buffer: None, + host_buffer_impossible: !mem.pool().is_sealed_memfd(), + }, + mem, + stride, + })), shm: true, width, height, @@ -169,7 +193,18 @@ impl WlBuffer { return false; }; let had_texture = match s { - WlBufferStorage::Shm { .. } => { + WlBufferStorage::Shm { + mem, + dmabuf_buffer_params: + DmabufBufferParams { + host_buffer, + host_buffer_impossible, + .. + }, + .. + } => { + host_buffer.take(); + *host_buffer_impossible = !mem.pool().is_sealed_memfd(); return match surface { Some(s) => { s.shm_staging.take(); @@ -224,6 +259,69 @@ impl WlBuffer { } } + pub fn get_gfx_buffer( + self: &Rc, + ctx: &Rc, + mem: &Rc, + dmabuf_buffer_params: &mut DmabufBufferParams, + ) -> Result>, GfxError> { + let DmabufBufferParams { + size, + udmabuf, + udmabuf_offset, + udmabuf_size, + udmabuf_impossible, + host_buffer, + host_buffer_impossible, + } = dmabuf_buffer_params; + if let Some(hb) = host_buffer { + return Ok(Some(hb.clone())); + } + if *host_buffer_impossible { + return Ok(None); + } + let udmabuf = 'udmabuf: { + if let Some(b) = udmabuf { + break 'udmabuf b.clone(); + } + if *udmabuf_impossible { + return Ok(None); + } + let Some(dev) = self.client.state.udmabuf() else { + return Ok(None); + }; + let mask = page_size() - 1; + let offset = mem.offset() & mask; + let base = mem.offset() & !mask; + let end = (mem.offset() + *size + mask) & !mask; + let len = end - base; + match dev.create_dmabuf_from_memfd(mem.pool().fd(), base, len) { + Ok(b) => { + let b = Rc::new(b); + *udmabuf_offset = offset; + *udmabuf_size = len; + *udmabuf = Some(b.clone()); + b + } + Err(e) => { + *udmabuf_impossible = true; + log::debug!("Could not create udmabuf: {}", ErrorFmt(e)); + return Ok(None); + } + } + }; + let hb = match ctx.create_dmabuf_buffer(&udmabuf, *udmabuf_offset, *udmabuf_size) { + Ok(hb) => hb, + Err(e) => { + *host_buffer_impossible = true; + log::debug!("Could not create gfx host buffer: {}", ErrorFmt(e)); + return Ok(None); + } + }; + *host_buffer = Some(hb.clone()); + Ok(Some(hb)) + } + fn update_texture(&self, surface: &WlSurface, sync_shm: bool) -> Result<(), WlBufferError> { let storage = &mut *self.storage.borrow_mut(); let storage = match storage { @@ -231,7 +329,7 @@ impl WlBuffer { _ => return Ok(()), }; match storage { - WlBufferStorage::Shm { mem, stride } => { + WlBufferStorage::Shm { mem, stride, .. } => { if sync_shm && let Some(ctx) = self.client.state.render_ctx.get() { let tex = ctx.async_shmem_texture( self.format, diff --git a/src/ifs/wl_surface/commit_timeline.rs b/src/ifs/wl_surface/commit_timeline.rs index 88c52951..ef18bb04 100644 --- a/src/ifs/wl_surface/commit_timeline.rs +++ b/src/ifs/wl_surface/commit_timeline.rs @@ -14,6 +14,7 @@ use { utils::{ clonecell::CloneCell, copyhashmap::CopyHashMap, + errorfmt::ErrorFmt, hash_map_ext::HashMapExt, linkedlist::{LinkedList, LinkedNode, NodeRef}, numcell::NumCell, @@ -599,7 +600,12 @@ fn schedule_async_upload( let Some(Some(buf)) = &pending.buffer else { return Ok(None); }; - let Some(WlBufferStorage::Shm { mem, stride, .. }) = &*buf.storage.borrow() else { + let Some(WlBufferStorage::Shm { + mem, + stride, + dmabuf_buffer_params, + }) = &mut *buf.storage.borrow_mut() + else { return Ok(None); }; let back = surface.shm_textures.back(); @@ -613,6 +619,11 @@ fn schedule_async_upload( back.damage.clear(); back.damage.damage(slice::from_ref(&buf.rect)); }; + let state = &surface.client.state; + let ctx = state + .render_ctx + .get() + .ok_or(WlSurfaceError::NoRenderContext)?; let back_tex = match back_tex_opt { Some(b) => { if pending.damage_full || pending.surface_damage.is_not_empty() { @@ -624,12 +635,8 @@ fn schedule_async_upload( } None => { damage_full(); - let state = &surface.client.state; - let ctx = state - .render_ctx - .get() - .ok_or(WlSurfaceError::NoRenderContext)?; let back_tex = ctx + .clone() .async_shmem_texture( buf.format, buf.rect.width(), @@ -642,6 +649,20 @@ fn schedule_async_upload( back_tex } }; + match buf.get_gfx_buffer(&ctx, mem, dmabuf_buffer_params) { + Ok(Some(hb)) => { + return back_tex + .async_upload_from_buffer(&hb, node_ref.clone(), back.damage.get()) + .map_err(WlSurfaceError::PrepareAsyncUpload); + } + Ok(None) => {} + Err(e) => { + log::error!( + "Could not create GPU mapping of host buffer: {}", + ErrorFmt(e), + ); + } + } let mut staging_opt = surface.shm_staging.get(); if let Some(staging) = &staging_opt && staging.size() != back_tex.staging_size() diff --git a/src/state.rs b/src/state.rs index 3f3d2fef..204995d3 100644 --- a/src/state.rs +++ b/src/state.rs @@ -99,6 +99,7 @@ use { TearingMode, ToplevelData, ToplevelNode, ToplevelNodeBase, VrrMode, WorkspaceNode, generic_node_visitor, }, + udmabuf::{Udmabuf, UdmabufError}, utils::{ activation_token::ActivationToken, asyncevent::AsyncEvent, @@ -111,6 +112,7 @@ use { hash_map_ext::HashMapExt, linkedlist::LinkedList, numcell::NumCell, + oserror::OsError, queue::AsyncQueue, refcounted::RefCounted, run_toplevel::RunToplevel, @@ -150,7 +152,7 @@ use { time::Duration, }, thiserror::Error, - uapi::OwnedFd, + uapi::{OwnedFd, c}, }; pub struct State { @@ -288,6 +290,7 @@ pub struct State { pub xdg_surface_configure_events: AsyncQueue, pub workspace_display_order: Cell, pub outputs_without_hc: NumCell, + pub udmabuf: CloneCell>>>, } // impl Drop for State { @@ -1565,6 +1568,25 @@ impl State { found_tree.clear(); node } + + pub fn udmabuf(&self) -> Option> { + if let Some(u) = self.udmabuf.get() { + return u; + } + match Udmabuf::new() { + Ok(u) => { + let u = Rc::new(u); + self.udmabuf.set(Some(Some(u.clone()))); + Some(u) + } + Err(UdmabufError::Open(OsError(c::EPERM))) => None, + Err(e) => { + log::error!("Could not create udmabuf device: {}", ErrorFmt(e)); + self.udmabuf.set(Some(None)); + None + } + } + } } #[derive(Debug, Error)] diff --git a/src/tree/output.rs b/src/tree/output.rs index d5d298e8..96851473 100644 --- a/src/tree/output.rs +++ b/src/tree/output.rs @@ -367,7 +367,7 @@ impl OutputNode { let mut ready = true; if let Some(storage) = wl_buffer.storage.borrow_mut().deref() { match storage { - WlBufferStorage::Shm { mem, stride } => { + WlBufferStorage::Shm { mem, stride, .. } => { let res = self.state.perform_shm_screencopy( tex, cd, diff --git a/src/udmabuf.rs b/src/udmabuf.rs index d2c55056..d2dc2487 100644 --- a/src/udmabuf.rs +++ b/src/udmabuf.rs @@ -63,6 +63,26 @@ impl Udmabuf { }; Ok(Self { fd }) } + + pub fn create_dmabuf_from_memfd( + &self, + memfd: &OwnedFd, + offset: usize, + size: usize, + ) -> Result { + let mut cmd = udmabuf_create { + memfd: memfd.raw() as u32, + flags: UDMABUF_FLAGS_CLOEXEC, + offset: offset as u64, + size: size as u64, + }; + let dmabuf = unsafe { ioctl(self.fd.raw(), UDMABUF_CREATE, &mut cmd) }; + let dmabuf = match map_err!(dmabuf) { + Ok(d) => OwnedFd::new(d), + Err(e) => return Err(UdmabufError::CreateDmabuf(e.into())), + }; + Ok(dmabuf) + } } impl Allocator for Udmabuf { @@ -104,17 +124,7 @@ impl Allocator for Udmabuf { if let Err(e) = uapi::fcntl_add_seals(memfd.raw(), F_SEAL_SHRINK) { return Err(UdmabufError::Seal(e.into()).into()); } - let mut cmd = udmabuf_create { - memfd: memfd.raw() as u32, - flags: 0, - offset: 0, - size: size as u64, - }; - let dmabuf = unsafe { ioctl(self.fd.raw(), UDMABUF_CREATE, &mut cmd) }; - let dmabuf = match map_err!(dmabuf) { - Ok(d) => OwnedFd::new(d), - Err(e) => return Err(UdmabufError::CreateDmabuf(e.into()).into()), - }; + let dmabuf = self.create_dmabuf_from_memfd(&memfd, 0, size as _)?; let mut planes = PlaneVec::new(); planes.push(DmaBufPlane { offset: 0, @@ -260,6 +270,7 @@ impl From for AllocatorError { } #[repr(C)] +#[derive(Debug)] struct udmabuf_create { memfd: u32, flags: u32, @@ -267,4 +278,6 @@ struct udmabuf_create { size: u64, } +const UDMABUF_FLAGS_CLOEXEC: u32 = 0x01; + const UDMABUF_CREATE: IoctlNumber = _IOW::(b'u' as u64, 0x42) as IoctlNumber;