Merge pull request #621 from mahkoh/jorth/external-memory-host-newer

vulkan: wl_shm performance improvements

src/clientmem.rs

@@ -3,7 +3,7 @@ use {
         client::Client,
         cpu_worker::{AsyncCpuWork, CpuJob, CpuWork, CpuWorker},
         gfx_api::{ShmMemory, ShmMemoryBacking},
-        utils::vec_ext::VecExt,
+        utils::{page_size::page_size, vec_ext::VecExt},
     },
     std::{
         cell::Cell,
@@ -18,6 +18,7 @@ use {
     uapi::{
         OwnedFd,
         c::{self, raise},
+        ftruncate,
     },
 };
 
@@ -76,10 +77,12 @@ impl ClientMem {
         flags: c::c_int,
     ) -> Result<Self, ClientMemError> {
         let mut sigbus_impossible = false;
+        let mut real_size = None;
         if let Ok(seals) = uapi::fcntl_get_seals(fd.raw())
             && seals & c::F_SEAL_SHRINK != 0
             && let Ok(stat) = uapi::fstat(fd.raw())
         {
+            real_size = Some(stat.st_size as usize);
             sigbus_impossible = stat.st_size as u64 >= len as u64;
         }
         if !sigbus_impossible && let Some(client) = client {
@@ -89,6 +92,12 @@ impl ClientMem {
                 client.id,
             );
         }
+        let len = len.next_multiple_of(page_size());
+        if let Some(real_size) = real_size
+            && real_size < len
+        {
+            let _ = ftruncate(fd.raw(), len as _);
+        }
         let data = if len == 0 {
             &mut [][..]
         } else {
@@ -126,23 +135,20 @@ impl ClientMem {
         }
     }
 
-    #[expect(dead_code)]
     pub fn fd(&self) -> &Rc<OwnedFd> {
         &self.fd
     }
 
-    pub fn sigbus_impossible(&self) -> bool {
+    pub fn is_sealed_memfd(&self) -> bool {
         self.sigbus_impossible
     }
 }
 
 impl ClientMemOffset {
-    #[expect(dead_code)]
     pub fn pool(&self) -> &ClientMem {
         &self.mem
     }
 
-    #[expect(dead_code)]
     pub fn offset(&self) -> usize {
         self.offset
     }
@@ -299,7 +305,7 @@ impl ShmMemory for ClientMemOffset {
     }
 
     fn safe_access(&self) -> ShmMemoryBacking {
-        match self.mem.sigbus_impossible() {
+        match self.mem.is_sealed_memfd() {
             true => ShmMemoryBacking::Ptr(self.data),
             false => ShmMemoryBacking::Fd(self.mem.fd.deref().clone(), self.offset),
         }

src/compositor.rs

@@ -360,6 +360,7 @@ fn start_compositor2(
         xdg_surface_configure_events: Default::default(),
         workspace_display_order: Cell::new(WorkspaceDisplayOrder::Manual),
         outputs_without_hc: Default::default(),
+        udmabuf: Default::default(),
     });
     state.tracker.register(ClientId::from_raw(0));
     create_dummy_output(&state);

src/gfx_api.rs

@@ -665,6 +665,8 @@ pub trait GfxStagingBuffer: Any {
     fn size(&self) -> usize;
 }
 
+pub trait GfxBuffer: Any {}
+
 pub trait AsyncShmGfxTextureTransferCancellable {
     fn cancel(&self, id: u64);
 }
@@ -713,6 +715,22 @@ pub trait AsyncShmGfxTexture: GfxTexture {
         damage: Region,
     ) -> Result<Option<PendingShmTransfer>, GfxError>;
 
+    fn async_upload_from_buffer(
+        self: Rc<Self>,
+        buf: &Rc<dyn GfxBuffer>,
+        callback: Rc<dyn AsyncShmGfxTextureCallback>,
+        damage: Region,
+    ) -> Result<Option<PendingShmTransfer>, GfxError> {
+        let _ = buf;
+        let _ = callback;
+        let _ = damage;
+
+        #[derive(Debug, Error)]
+        #[error("Host buffers are not supported")]
+        struct E;
+        Err(GfxError(Box::new(E)))
+    }
+
     fn sync_upload(self: Rc<Self>, shm: &[Cell<u8>], damage: Region) -> Result<(), GfxError>;
 
     fn compatible_with(
@@ -800,6 +818,22 @@ pub trait GfxContext: Debug {
     fn supports_invalid_modifier(&self) -> bool {
         false
     }
+
+    fn create_dmabuf_buffer(
+        &self,
+        dmabuf: &Rc<OwnedFd>,
+        offset: usize,
+        size: usize,
+    ) -> Result<Rc<dyn GfxBuffer>, GfxError> {
+        let _ = dmabuf;
+        let _ = offset;
+        let _ = size;
+
+        #[derive(Debug, Error)]
+        #[error("Host buffers are not supported")]
+        struct E;
+        Err(GfxError(Box::new(E)))
+    }
 }
 
 #[derive(Clone, Debug)]

src/gfx_apis/vulkan.rs

@@ -6,6 +6,7 @@ mod command;
 mod descriptor;
 mod descriptor_buffer;
 mod device;
+mod dmabuf_buffer;
 mod eotfs;
 mod fence;
 mod format;
@@ -27,9 +28,9 @@ use {
         cpu_worker::{CpuWorker, jobs::read_write::ReadWriteJobError},
         format::Format,
         gfx_api::{
-            AsyncShmGfxTexture, GfxBlendBuffer, GfxContext, GfxError, GfxFormat, GfxImage,
+            AsyncShmGfxTexture, GfxBlendBuffer, GfxBuffer, GfxContext, GfxError, GfxFormat,
-            GfxInternalFramebuffer, GfxStagingBuffer, GfxTexture, ResetStatus, STAGING_DOWNLOAD,
+            GfxImage, GfxInternalFramebuffer, GfxStagingBuffer, GfxTexture, ResetStatus,
-            STAGING_UPLOAD, ShmGfxTexture, StagingBufferUsecase,
+            STAGING_DOWNLOAD, STAGING_UPLOAD, ShmGfxTexture, StagingBufferUsecase,
         },
         gfx_apis::vulkan::{
             image::VulkanImageMemory, instance::VulkanInstance, renderer::VulkanRenderer,
@@ -57,7 +58,7 @@ use {
         sync::Arc,
     },
     thiserror::Error,
-    uapi::c::dev_t,
+    uapi::{OwnedFd, c::dev_t},
 };
 
 #[derive(Debug, Error)]
@@ -394,6 +395,20 @@ impl GfxContext for Context {
     fn supports_color_management(&self) -> bool {
         self.0.device.descriptor_buffer.is_some()
     }
+
+    fn create_dmabuf_buffer(
+        &self,
+        dmabuf: &Rc<OwnedFd>,
+        offset: usize,
+        size: usize,
+    ) -> Result<Rc<dyn GfxBuffer>, GfxError> {
+        self.0.check_defunct()?;
+        let buffer = self
+            .0
+            .device
+            .create_dmabuf_buffer(dmabuf, offset as u64, size as u64)?;
+        Ok(buffer)
+    }
 }
 
 impl Drop for Context {

src/gfx_apis/vulkan/allocator.rs

@@ -4,7 +4,7 @@ use {
         gfx_apis::vulkan::{
             VulkanError, device::VulkanDevice, instance::API_VERSION, renderer::VulkanRenderer,
         },
-        utils::{numcell::NumCell, ptr_ext::MutPtrExt},
+        utils::{numcell::NumCell, page_size::page_size, ptr_ext::MutPtrExt},
     },
     ash::{
         Device,
@@ -304,6 +304,19 @@ impl CpuWork for AllocWork {
             self.usage,
             self.map,
         );
+        if self.usage.contains(UsageFlags::UPLOAD)
+            && let Ok(r) = &r
+            && let Some(ptr) = r.ptr
+        {
+            let page_size = page_size() as u64;
+            let mut offset = 0;
+            while offset < r.block.size() {
+                unsafe {
+                    *ptr.add(offset as usize) = 0;
+                }
+                offset += page_size;
+            }
+        }
         self.res = Some(r);
         None
     }

src/gfx_apis/vulkan/dmabuf_buffer.rs

@@ -0,0 +1,130 @@
+use {
+    crate::{
+        gfx_api::GfxBuffer,
+        gfx_apis::vulkan::{VulkanError, device::VulkanDevice},
+        utils::on_drop::OnDrop,
+    },
+    ash::{
+        Device,
+        vk::{
+            self, BufferCreateInfo, BufferUsageFlags, ExternalMemoryBufferCreateInfo,
+            ExternalMemoryHandleTypeFlags, ImportMemoryFdInfoKHR, MemoryAllocateInfo,
+            MemoryFdPropertiesKHR, MemoryPropertyFlags,
+        },
+    },
+    std::{any::Any, rc::Rc},
+    uapi::OwnedFd,
+};
+
+pub struct VulkanDmabufBuffer {
+    pub(super) device: Rc<VulkanDevice>,
+    pub(super) size: u64,
+    pub(super) offset: u64,
+    pub(super) buffer: vk::Buffer,
+    pub(super) memory: vk::DeviceMemory,
+}
+
+impl VulkanDevice {
+    pub fn create_dmabuf_buffer(
+        self: &Rc<Self>,
+        dmabuf: &Rc<OwnedFd>,
+        offset: u64,
+        size: u64,
+    ) -> Result<Rc<VulkanDmabufBuffer>, VulkanError> {
+        let mut memory_fd_properties = MemoryFdPropertiesKHR::default();
+        unsafe {
+            self.external_memory_fd
+                .get_memory_fd_properties(
+                    ExternalMemoryHandleTypeFlags::DMA_BUF_EXT,
+                    dmabuf.raw(),
+                    &mut memory_fd_properties,
+                )
+                .map_err(VulkanError::MemoryFdProperties)?
+        }
+        let buffer = {
+            let mut external_info = ExternalMemoryBufferCreateInfo::default()
+                .handle_types(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT);
+            let create_info = BufferCreateInfo::default()
+                .size(size)
+                .usage(BufferUsageFlags::TRANSFER_SRC)
+                .push_next(&mut external_info);
+            unsafe {
+                self.device
+                    .create_buffer(&create_info, None)
+                    .map_err(VulkanError::CreateBuffer)?
+            }
+        };
+        let destroy_buffer = OnDrop(|| unsafe { self.device.destroy_buffer(buffer, None) });
+        let requirements = unsafe { self.device.get_buffer_memory_requirements(buffer) };
+        let memory_type = self.find_memory_type(
+            MemoryPropertyFlags::HOST_VISIBLE,
+            requirements.memory_type_bits & memory_fd_properties.memory_type_bits,
+        );
+        let Some(memory_type) = memory_type else {
+            return Err(VulkanError::MemoryType);
+        };
+        let fd =
+            uapi::fcntl_dupfd_cloexec(dmabuf.raw(), 0).map_err(|e| VulkanError::Dupfd(e.into()))?;
+        let memory = {
+            let mut import_info = ImportMemoryFdInfoKHR::default()
+                .fd(fd.raw())
+                .handle_type(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT);
+            let allocate_info = MemoryAllocateInfo::default()
+                .allocation_size(requirements.size)
+                .memory_type_index(memory_type)
+                .push_next(&mut import_info);
+            unsafe {
+                self.device
+                    .allocate_memory(&allocate_info, None)
+                    .map_err(VulkanError::AllocateMemory)?
+            }
+        };
+        fd.unwrap();
+        let free_memory = OnDrop(|| unsafe { self.device.free_memory(memory, None) });
+        unsafe {
+            self.device
+                .bind_buffer_memory(buffer, memory, 0)
+                .map_err(VulkanError::BindBufferMemory)?;
+        }
+        free_memory.forget();
+        destroy_buffer.forget();
+        Ok(Rc::new(VulkanDmabufBuffer {
+            device: self.clone(),
+            size,
+            offset,
+            buffer,
+            memory,
+        }))
+    }
+}
+
+impl Drop for VulkanDmabufBuffer {
+    fn drop(&mut self) {
+        unsafe {
+            self.device.device.free_memory(self.memory, None);
+            self.device.device.destroy_buffer(self.buffer, None);
+        }
+    }
+}
+
+impl VulkanDmabufBuffer {
+    fn assert_device(&self, device: &Device) {
+        assert_eq!(
+            self.device.device.handle(),
+            device.handle(),
+            "Mixed vulkan device use"
+        );
+    }
+}
+
+impl GfxBuffer for VulkanDmabufBuffer {}
+
+impl dyn GfxBuffer {
+    pub(super) fn into_vk(self: Rc<Self>, device: &Device) -> Rc<VulkanDmabufBuffer> {
+        let buffer: Rc<VulkanDmabufBuffer> = (self as Rc<dyn Any>)
+            .downcast()
+            .expect("Non-vulkan buffer passed into vulkan");
+        buffer.assert_device(device);
+        buffer
+    }
+}

src/gfx_apis/vulkan/image.rs

@@ -4,7 +4,7 @@ use {
         format::Format,
         gfx_api::{
             AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback,
-            AsyncShmGfxTextureTransferCancellable, GfxApiOpt, GfxBlendBuffer, GfxError,
+            AsyncShmGfxTextureTransferCancellable, GfxApiOpt, GfxBlendBuffer, GfxBuffer, GfxError,
             GfxFramebuffer, GfxImage, GfxInternalFramebuffer, GfxStagingBuffer, GfxTexture,
             PendingShmTransfer, ReleaseSync, ShmGfxTexture, ShmMemory, SyncFile,
         },
@@ -672,6 +672,20 @@ impl AsyncShmGfxTexture for VulkanImage {
         Ok(pending)
     }
 
+    fn async_upload_from_buffer(
+        self: Rc<Self>,
+        buf: &Rc<dyn GfxBuffer>,
+        callback: Rc<dyn AsyncShmGfxTextureCallback>,
+        damage: Region,
+    ) -> Result<Option<PendingShmTransfer>, GfxError> {
+        let VulkanImageMemory::Internal(shm) = &self.ty else {
+            unreachable!();
+        };
+        let buf = buf.clone().into_vk(&self.renderer.device.device);
+        let pending = shm.async_transfer2(&self, buf, damage, callback)?;
+        Ok(pending)
+    }
+
     fn sync_upload(self: Rc<Self>, mem: &[Cell<u8>], damage: Region) -> Result<(), GfxError> {
         let VulkanImageMemory::Internal(shm) = &self.ty else {
             unreachable!();

src/gfx_apis/vulkan/shm_image.rs

@@ -17,7 +17,7 @@ use {
         utils::{errorfmt::ErrorFmt, on_drop::OnDrop},
     },
     ash::vk::{
-        AccessFlags2, BufferImageCopy2, BufferMemoryBarrier2, CommandBufferBeginInfo,
+        AccessFlags2, Buffer, BufferImageCopy2, BufferMemoryBarrier2, CommandBufferBeginInfo,
         CommandBufferSubmitInfo, CommandBufferUsageFlags, CopyBufferToImageInfo2,
         CopyImageToBufferInfo2, DependencyInfoKHR, DeviceSize, Extent3D, ImageAspectFlags,
         ImageCreateInfo, ImageLayout, ImageSubresourceLayers, ImageSubresourceRange, ImageTiling,
@@ -136,8 +136,14 @@ impl VulkanShmImage {
                 ptr::copy_nonoverlapping(buf, mem, total_size as usize);
             }
         })?;
-        let (cmd, fence, sync_file, point) =
+        let (cmd, fence, sync_file, point) = self.submit_buffer_image_copy(
-            self.submit_buffer_image_copy(img, &staging, cpy, false, TransferType::Upload)?;
+            img,
+            staging.buffer,
+            staging.size,
+            cpy,
+            false,
+            TransferType::Upload,
+        )?;
         let future = img.renderer.eng.spawn(
             "await upload",
             await_upload(point, img.clone(), cmd, sync_file, fence, staging),
@@ -149,7 +155,8 @@ impl VulkanShmImage {
     pub(super) fn submit_buffer_image_copy(
         &self,
         img: &Rc<VulkanImage>,
-        staging: &VulkanStagingBuffer,
+        buffer: Buffer,
+        size: DeviceSize,
         regions: &[BufferImageCopy2],
         use_transfer_queue: bool,
         tt: TransferType,
@@ -164,9 +171,9 @@ impl VulkanShmImage {
     > {
         let memory_barrier = |sam, ssm, dam, dsm| {
             BufferMemoryBarrier2::default()
-                .buffer(staging.buffer)
+                .buffer(buffer)
                 .offset(0)
-                .size(staging.size)
+                .size(size)
                 .src_access_mask(sam)
                 .src_stage_mask(ssm)
                 .dst_access_mask(dam)
@@ -274,7 +281,7 @@ impl VulkanShmImage {
             match tt {
                 TransferType::Upload => {
                     let cpy_info = CopyBufferToImageInfo2::default()
-                        .src_buffer(staging.buffer)
+                        .src_buffer(buffer)
                         .dst_image(img.image)
                         .dst_image_layout(ImageLayout::TRANSFER_DST_OPTIMAL)
                         .regions(regions);
@@ -282,7 +289,7 @@ impl VulkanShmImage {
                 }
                 TransferType::Download => {
                     let cpy_info = CopyImageToBufferInfo2::default()
-                        .dst_buffer(staging.buffer)
+                        .dst_buffer(buffer)
                         .src_image(img.image)
                         .src_image_layout(ImageLayout::TRANSFER_SRC_OPTIMAL)
                         .regions(regions);
@@ -432,6 +439,7 @@ impl VulkanRenderer {
                 io_job: Default::default(),
                 copy_job: Default::default(),
                 staging: Default::default(),
+                buffer: Default::default(),
                 client_mem: Default::default(),
                 callback: Default::default(),
                 callback_id: Cell::new(0),

src/gfx_apis/vulkan/transfer.rs

@@ -13,6 +13,7 @@ use {
         gfx_apis::vulkan::{
             VulkanError,
             command::VulkanCommandBuffer,
+            dmabuf_buffer::VulkanDmabufBuffer,
             fence::VulkanFence,
             image::{QueueFamily, QueueState, QueueTransfer, VulkanImage, VulkanImageMemory},
             renderer::image_barrier,
@@ -29,7 +30,7 @@ use {
         ImageSubresourceLayers, Offset3D, PipelineStageFlags2, SubmitInfo2,
     },
     std::{
-        cell::{Cell, RefCell},
+        cell::{Cell, RefCell, RefMut},
         rc::Rc,
         slice,
     },
@@ -41,6 +42,7 @@ pub struct VulkanShmImageAsyncData {
     pub(super) io_job: Cell<Option<Box<IoTransferJob>>>,
     pub(super) copy_job: Cell<Option<Box<CopyTransferJob>>>,
     pub(super) staging: CloneCell<Option<Rc<VulkanStagingShell>>>,
+    pub(super) buffer: CloneCell<Option<Rc<VulkanDmabufBuffer>>>,
     pub(super) client_mem: CloneCell<Option<Rc<dyn ShmMemory>>>,
     pub(super) callback: Cell<Option<Rc<dyn AsyncShmGfxTextureCallback>>>,
     pub(super) callback_id: Cell<u64>,
@@ -53,7 +55,10 @@ pub struct VulkanShmImageAsyncData {
 impl VulkanShmImageAsyncData {
     fn complete(&self, result: Result<(), VulkanError>) {
         self.busy.set(false);
-        self.staging.take().unwrap().busy.set(false);
+        if let Some(staging) = self.staging.take() {
+            staging.busy.set(false);
+        }
+        self.buffer.take();
         self.client_mem.take();
         if let Some(cb) = self.callback.take() {
             cb.completed(result.map_err(|e| e.into()));
@@ -68,6 +73,43 @@ pub(super) enum TransferType {
 }
 
 impl VulkanShmImage {
+    pub fn async_transfer2(
+        &self,
+        img: &Rc<VulkanImage>,
+        buffer: Rc<VulkanDmabufBuffer>,
+        damage: Region,
+        callback: Rc<dyn AsyncShmGfxTextureCallback>,
+    ) -> Result<Option<PendingShmTransfer>, VulkanError> {
+        self.async_transfer_(img, damage, callback, |data, damage| {
+            self.try_async_transfer2(img, buffer, data, damage)
+        })
+    }
+
+    fn try_async_transfer2(
+        &self,
+        img: &Rc<VulkanImage>,
+        buffer: Rc<VulkanDmabufBuffer>,
+        data: &VulkanShmImageAsyncData,
+        mut damage: Region,
+    ) -> Result<(), VulkanError> {
+        if data.busy.get() {
+            return Err(VulkanError::AsyncCopyBusy);
+        }
+        if self.size > buffer.size {
+            return Err(VulkanError::InvalidBufferSize);
+        }
+        data.busy.set(true);
+        data.data_copied.set(true);
+        data.buffer.set(Some(buffer.clone()));
+        if img.contents_are_undefined.get() {
+            damage = Region::new(Rect::new_sized(0, 0, img.width as _, img.height as _).unwrap());
+        }
+        self.calculate_copies(img, data, damage, buffer.offset);
+        self.async_release_from_gfx_queue(img, data, TransferType::Upload)?;
+        self.async_upload_copy_buffer_to_image(img, data)?;
+        Ok(())
+    }
+
     pub fn async_transfer(
         &self,
         img: &Rc<VulkanImage>,
@@ -76,12 +118,24 @@ impl VulkanShmImage {
         damage: Region,
         callback: Rc<dyn AsyncShmGfxTextureCallback>,
         tt: TransferType,
+    ) -> Result<Option<PendingShmTransfer>, VulkanError> {
+        self.async_transfer_(img, damage, callback, |data, damage| {
+            self.try_async_transfer(img, staging, data, client_mem, damage, tt)
+        })
+    }
+
+    fn async_transfer_(
+        &self,
+        img: &Rc<VulkanImage>,
+        damage: Region,
+        callback: Rc<dyn AsyncShmGfxTextureCallback>,
+        f: impl FnOnce(&VulkanShmImageAsyncData, Region) -> Result<(), VulkanError>,
     ) -> Result<Option<PendingShmTransfer>, VulkanError> {
         if damage.is_empty() {
             return Ok(None);
         }
         let data = self.async_data.as_ref().unwrap();
-        let res = self.try_async_transfer(img, staging, data, client_mem, damage, tt);
+        let res = f(data, damage);
         match res {
             Ok(()) => {
                 let id = img.renderer.allocate_point();
@@ -135,53 +189,7 @@ impl VulkanShmImage {
             damage = Region::new(Rect::new_sized(0, 0, img.width as _, img.height as _).unwrap());
         }
 
-        let copies = &mut *data.regions.borrow_mut();
+        let copies = &mut *self.calculate_copies(img, data, damage, 0);
-        copies.clear();
-
-        let mut copy = |x, y, width, height| {
-            let buffer_offset = (y as u32 * img.stride + x as u32 * self.shm_info.bpp) as u64;
-            let copy = BufferImageCopy2::default()
-                .buffer_offset(buffer_offset)
-                .image_offset(Offset3D { x, y, z: 0 })
-                .image_extent(Extent3D {
-                    width,
-                    height,
-                    depth: 1,
-                })
-                .image_subresource(ImageSubresourceLayers {
-                    aspect_mask: ImageAspectFlags::COLOR,
-                    mip_level: 0,
-                    base_array_layer: 0,
-                    layer_count: 1,
-                })
-                .buffer_image_height(img.height)
-                .buffer_row_length(img.stride / self.shm_info.bpp);
-            copies.push(copy);
-        };
-        let (width_mask, height_mask) = img.renderer.device.transfer_granularity_mask;
-        let width_mask = width_mask as i32;
-        let height_mask = height_mask as i32;
-        for damage in damage.rects() {
-            if damage.x2() < 0 || damage.y2() < 0 {
-                continue;
-            }
-            let x1 = damage.x1().max(0) & !width_mask;
-            let y1 = damage.y1().max(0) & !height_mask;
-            let x2 = ((damage.x2() + width_mask) & !width_mask).min(img.width as i32);
-            let y2 = ((damage.y2() + height_mask) & !height_mask).min(img.height as i32);
-            let Some(damage) = Rect::new(x1, y1, x2, y2) else {
-                continue;
-            };
-            if damage.is_empty() {
-                continue;
-            }
-            copy(
-                damage.x1(),
-                damage.y1(),
-                damage.width() as u32,
-                damage.height() as u32,
-            );
-        }
 
         self.async_release_from_gfx_queue(img, data, tt)?;
 
@@ -209,6 +217,67 @@ impl VulkanShmImage {
             })
     }
 
+    fn calculate_copies<'a>(
+        &self,
+        img: &Rc<VulkanImage>,
+        data: &'a VulkanShmImageAsyncData,
+        damage: Region,
+        extra_offset: u64,
+    ) -> RefMut<'a, Vec<BufferImageCopy2<'static>>> {
+        let mut copies_ref = data.regions.borrow_mut();
+        let copies = &mut *copies_ref;
+        copies.clear();
+        let mut copy = |x, y, width, height| {
+            let buffer_offset = (y as u32 * img.stride + x as u32 * self.shm_info.bpp) as u64;
+            let copy = BufferImageCopy2::default()
+                .buffer_offset(buffer_offset + extra_offset)
+                .image_offset(Offset3D { x, y, z: 0 })
+                .image_extent(Extent3D {
+                    width,
+                    height,
+                    depth: 1,
+                })
+                .image_subresource(ImageSubresourceLayers {
+                    aspect_mask: ImageAspectFlags::COLOR,
+                    mip_level: 0,
+                    base_array_layer: 0,
+                    layer_count: 1,
+                })
+                .buffer_image_height(img.height)
+                .buffer_row_length(img.stride / self.shm_info.bpp);
+            copies.push(copy);
+        };
+        let (width_mask, height_mask) = img.renderer.device.transfer_granularity_mask;
+        let width_mask = width_mask as i32;
+        let height_mask = height_mask as i32;
+        for damage in damage.rects() {
+            if damage.x2() < 0 || damage.y2() < 0 {
+                continue;
+            }
+            let x1 = damage.x1().max(0);
+            let y1 = damage.y1().max(0);
+            let x2 = damage.x2().min(img.width as i32);
+            let y2 = damage.y2().min(img.height as i32);
+            let x1 = x1 & !width_mask;
+            let y1 = y1 & !height_mask;
+            let x2 = ((x2 + width_mask) & !width_mask).min(img.width as i32);
+            let y2 = ((y2 + height_mask) & !height_mask).min(img.height as i32);
+            let Some(damage) = Rect::new(x1, y1, x2, y2) else {
+                continue;
+            };
+            if damage.is_empty() {
+                continue;
+            }
+            copy(
+                damage.x1(),
+                damage.y1(),
+                damage.width() as u32,
+                damage.height() as u32,
+            );
+        }
+        copies_ref
+    }
+
     fn async_release_from_gfx_queue(
         &self,
         img: &Rc<VulkanImage>,
@@ -451,10 +520,19 @@ impl VulkanShmImage {
         }
         img.renderer.check_defunct()?;
         let regions = &*data.regions.borrow();
-        let staging = data.staging.get().unwrap().staging.get().unwrap();
+        let (buffer, size) = match data.staging.get() {
-        staging.upload(|_, _| ())?;
+            Some(s) => {
+                let staging = s.staging.get().unwrap();
+                staging.upload(|_, _| ())?;
+                (staging.buffer, staging.size)
+            }
+            _ => {
+                let host_buffer = data.buffer.get().unwrap();
+                (host_buffer.buffer, host_buffer.size)
+            }
+        };
         let (cmd, fence, sync_file, point) =
-            self.submit_buffer_image_copy(img, &staging, regions, true, TransferType::Upload)?;
+            self.submit_buffer_image_copy(img, buffer, size, regions, true, TransferType::Upload)?;
         img.queue_state.set(QueueState::Releasing);
         let future = img.renderer.eng.spawn(
             "await async upload",
@@ -481,8 +559,14 @@ impl VulkanShmImage {
             return Ok(());
         }
         img.renderer.check_defunct()?;
-        let (cmd, fence, sync_file, point) =
+        let (cmd, fence, sync_file, point) = self.submit_buffer_image_copy(
-            self.submit_buffer_image_copy(img, &staging, copies, true, TransferType::Download)?;
+            img,
+            staging.buffer,
+            staging.size,
+            copies,
+            true,
+            TransferType::Download,
+        )?;
         img.queue_state.set(QueueState::Releasing);
         let future = img.renderer.eng.spawn(
             "await async image to buffer copy",

src/ifs/ext_image_copy/ext_image_copy_capture_frame_v1.rs

@@ -101,7 +101,7 @@ impl ExtImageCopyCaptureFrameV1 {
         let mut shm_bridge = self.session.shm_bridge.take();
         let mut shm_staging = self.session.shm_staging.take();
         match storage {
-            WlBufferStorage::Shm { mem, stride } => {
+            WlBufferStorage::Shm { mem, stride, .. } => {
                 if let Some(b) = &shm_bridge
                     && (b.physical_size() != buffer.rect.size()
                         || b.format() != buffer.format

src/ifs/wl_buffer.rs

@@ -3,12 +3,12 @@ use {
         client::{Client, ClientError},
         clientmem::{ClientMem, ClientMemError, ClientMemOffset},
         format::{ARGB8888, Format},
-        gfx_api::{GfxError, GfxFramebuffer, GfxImage, GfxTexture},
+        gfx_api::{GfxBuffer, GfxContext, GfxError, GfxFramebuffer, GfxImage, GfxTexture},
         ifs::wl_surface::WlSurface,
         leaks::Tracker,
         object::{Object, Version},
         rect::{Rect, Region},
-        utils::errorfmt::ErrorFmt,
+        utils::{errorfmt::ErrorFmt, page_size::page_size},
         video::dmabuf::DmaBuf,
         wire::{WlBufferId, wl_buffer::*},
     },
@@ -17,12 +17,14 @@ use {
         rc::Rc,
     },
     thiserror::Error,
+    uapi::OwnedFd,
 };
 
 pub enum WlBufferStorage {
     Shm {
         mem: Rc<ClientMemOffset>,
         stride: i32,
+        dmabuf_buffer_params: DmabufBufferParams,
     },
     Dmabuf {
         img: Rc<dyn GfxImage>,
@@ -31,6 +33,16 @@ pub enum WlBufferStorage {
     },
 }
 
+pub struct DmabufBufferParams {
+    size: usize,
+    udmabuf: Option<Rc<OwnedFd>>,
+    udmabuf_offset: usize,
+    udmabuf_size: usize,
+    udmabuf_impossible: bool,
+    host_buffer: Option<Rc<dyn GfxBuffer>>,
+    host_buffer_impossible: bool,
+}
+
 pub struct WlBuffer {
     pub id: WlBufferId,
     destroyed: Cell<bool>,
@@ -118,7 +130,19 @@ impl WlBuffer {
             format,
             dmabuf: None,
             render_ctx_version: Cell::new(client.state.render_ctx_version.get()),
-            storage: RefCell::new(Some(WlBufferStorage::Shm { mem, stride })),
+            storage: RefCell::new(Some(WlBufferStorage::Shm {
+                dmabuf_buffer_params: DmabufBufferParams {
+                    size: bytes as usize,
+                    udmabuf: None,
+                    udmabuf_offset: 0,
+                    udmabuf_size: 0,
+                    udmabuf_impossible: !mem.pool().is_sealed_memfd(),
+                    host_buffer: None,
+                    host_buffer_impossible: !mem.pool().is_sealed_memfd(),
+                },
+                mem,
+                stride,
+            })),
             shm: true,
             width,
             height,
@@ -169,7 +193,18 @@ impl WlBuffer {
             return false;
         };
         let had_texture = match s {
-            WlBufferStorage::Shm { .. } => {
+            WlBufferStorage::Shm {
+                mem,
+                dmabuf_buffer_params:
+                    DmabufBufferParams {
+                        host_buffer,
+                        host_buffer_impossible,
+                        ..
+                    },
+                ..
+            } => {
+                host_buffer.take();
+                *host_buffer_impossible = !mem.pool().is_sealed_memfd();
                 return match surface {
                     Some(s) => {
                         s.shm_staging.take();
@@ -224,6 +259,69 @@ impl WlBuffer {
         }
     }
 
+    pub fn get_gfx_buffer(
+        self: &Rc<Self>,
+        ctx: &Rc<dyn GfxContext>,
+        mem: &Rc<ClientMemOffset>,
+        dmabuf_buffer_params: &mut DmabufBufferParams,
+    ) -> Result<Option<Rc<dyn GfxBuffer>>, GfxError> {
+        let DmabufBufferParams {
+            size,
+            udmabuf,
+            udmabuf_offset,
+            udmabuf_size,
+            udmabuf_impossible,
+            host_buffer,
+            host_buffer_impossible,
+        } = dmabuf_buffer_params;
+        if let Some(hb) = host_buffer {
+            return Ok(Some(hb.clone()));
+        }
+        if *host_buffer_impossible {
+            return Ok(None);
+        }
+        let udmabuf = 'udmabuf: {
+            if let Some(b) = udmabuf {
+                break 'udmabuf b.clone();
+            }
+            if *udmabuf_impossible {
+                return Ok(None);
+            }
+            let Some(dev) = self.client.state.udmabuf() else {
+                return Ok(None);
+            };
+            let mask = page_size() - 1;
+            let offset = mem.offset() & mask;
+            let base = mem.offset() & !mask;
+            let end = (mem.offset() + *size + mask) & !mask;
+            let len = end - base;
+            match dev.create_dmabuf_from_memfd(mem.pool().fd(), base, len) {
+                Ok(b) => {
+                    let b = Rc::new(b);
+                    *udmabuf_offset = offset;
+                    *udmabuf_size = len;
+                    *udmabuf = Some(b.clone());
+                    b
+                }
+                Err(e) => {
+                    *udmabuf_impossible = true;
+                    log::debug!("Could not create udmabuf: {}", ErrorFmt(e));
+                    return Ok(None);
+                }
+            }
+        };
+        let hb = match ctx.create_dmabuf_buffer(&udmabuf, *udmabuf_offset, *udmabuf_size) {
+            Ok(hb) => hb,
+            Err(e) => {
+                *host_buffer_impossible = true;
+                log::debug!("Could not create gfx host buffer: {}", ErrorFmt(e));
+                return Ok(None);
+            }
+        };
+        *host_buffer = Some(hb.clone());
+        Ok(Some(hb))
+    }
+
     fn update_texture(&self, surface: &WlSurface, sync_shm: bool) -> Result<(), WlBufferError> {
         let storage = &mut *self.storage.borrow_mut();
         let storage = match storage {
@@ -231,7 +329,7 @@ impl WlBuffer {
             _ => return Ok(()),
         };
         match storage {
-            WlBufferStorage::Shm { mem, stride } => {
+            WlBufferStorage::Shm { mem, stride, .. } => {
                 if sync_shm && let Some(ctx) = self.client.state.render_ctx.get() {
                     let tex = ctx.async_shmem_texture(
                         self.format,

src/ifs/wl_surface/commit_timeline.rs

@@ -14,6 +14,7 @@ use {
         utils::{
             clonecell::CloneCell,
             copyhashmap::CopyHashMap,
+            errorfmt::ErrorFmt,
             hash_map_ext::HashMapExt,
             linkedlist::{LinkedList, LinkedNode, NodeRef},
             numcell::NumCell,
@@ -599,7 +600,12 @@ fn schedule_async_upload(
     let Some(Some(buf)) = &pending.buffer else {
         return Ok(None);
     };
-    let Some(WlBufferStorage::Shm { mem, stride, .. }) = &*buf.storage.borrow() else {
+    let Some(WlBufferStorage::Shm {
+        mem,
+        stride,
+        dmabuf_buffer_params,
+    }) = &mut *buf.storage.borrow_mut()
+    else {
         return Ok(None);
     };
     let back = surface.shm_textures.back();
@@ -613,6 +619,11 @@ fn schedule_async_upload(
         back.damage.clear();
         back.damage.damage(slice::from_ref(&buf.rect));
     };
+    let state = &surface.client.state;
+    let ctx = state
+        .render_ctx
+        .get()
+        .ok_or(WlSurfaceError::NoRenderContext)?;
     let back_tex = match back_tex_opt {
         Some(b) => {
             if pending.damage_full || pending.surface_damage.is_not_empty() {
@@ -624,12 +635,8 @@ fn schedule_async_upload(
         }
         None => {
             damage_full();
-            let state = &surface.client.state;
-            let ctx = state
-                .render_ctx
-                .get()
-                .ok_or(WlSurfaceError::NoRenderContext)?;
             let back_tex = ctx
+                .clone()
                 .async_shmem_texture(
                     buf.format,
                     buf.rect.width(),
@@ -642,6 +649,20 @@ fn schedule_async_upload(
             back_tex
         }
     };
+    match buf.get_gfx_buffer(&ctx, mem, dmabuf_buffer_params) {
+        Ok(Some(hb)) => {
+            return back_tex
+                .async_upload_from_buffer(&hb, node_ref.clone(), back.damage.get())
+                .map_err(WlSurfaceError::PrepareAsyncUpload);
+        }
+        Ok(None) => {}
+        Err(e) => {
+            log::error!(
+                "Could not create GPU mapping of host buffer: {}",
+                ErrorFmt(e),
+            );
+        }
+    }
     let mut staging_opt = surface.shm_staging.get();
     if let Some(staging) = &staging_opt
         && staging.size() != back_tex.staging_size()

src/state.rs

@@ -99,6 +99,7 @@ use {
             TearingMode, ToplevelData, ToplevelNode, ToplevelNodeBase, VrrMode, WorkspaceNode,
             generic_node_visitor,
         },
+        udmabuf::{Udmabuf, UdmabufError},
         utils::{
             activation_token::ActivationToken,
             asyncevent::AsyncEvent,
@@ -111,6 +112,7 @@ use {
             hash_map_ext::HashMapExt,
             linkedlist::LinkedList,
             numcell::NumCell,
+            oserror::OsError,
             queue::AsyncQueue,
             refcounted::RefCounted,
             run_toplevel::RunToplevel,
@@ -150,7 +152,7 @@ use {
         time::Duration,
     },
     thiserror::Error,
-    uapi::OwnedFd,
+    uapi::{OwnedFd, c},
 };
 
 pub struct State {
@@ -288,6 +290,7 @@ pub struct State {
     pub xdg_surface_configure_events: AsyncQueue<XdgSurfaceConfigureEvent>,
     pub workspace_display_order: Cell<WorkspaceDisplayOrder>,
     pub outputs_without_hc: NumCell<usize>,
+    pub udmabuf: CloneCell<Option<Option<Rc<Udmabuf>>>>,
 }
 
 // impl Drop for State {
@@ -1565,6 +1568,25 @@ impl State {
         found_tree.clear();
         node
     }
+
+    pub fn udmabuf(&self) -> Option<Rc<Udmabuf>> {
+        if let Some(u) = self.udmabuf.get() {
+            return u;
+        }
+        match Udmabuf::new() {
+            Ok(u) => {
+                let u = Rc::new(u);
+                self.udmabuf.set(Some(Some(u.clone())));
+                Some(u)
+            }
+            Err(UdmabufError::Open(OsError(c::EPERM))) => None,
+            Err(e) => {
+                log::error!("Could not create udmabuf device: {}", ErrorFmt(e));
+                self.udmabuf.set(Some(None));
+                None
+            }
+        }
+    }
 }
 
 #[derive(Debug, Error)]

src/tree/output.rs

@@ -367,7 +367,7 @@ impl OutputNode {
             let mut ready = true;
             if let Some(storage) = wl_buffer.storage.borrow_mut().deref() {
                 match storage {
-                    WlBufferStorage::Shm { mem, stride } => {
+                    WlBufferStorage::Shm { mem, stride, .. } => {
                         let res = self.state.perform_shm_screencopy(
                             tex,
                             cd,

src/udmabuf.rs

@@ -63,6 +63,26 @@ impl Udmabuf {
         };
         Ok(Self { fd })
     }
+
+    pub fn create_dmabuf_from_memfd(
+        &self,
+        memfd: &OwnedFd,
+        offset: usize,
+        size: usize,
+    ) -> Result<OwnedFd, UdmabufError> {
+        let mut cmd = udmabuf_create {
+            memfd: memfd.raw() as u32,
+            flags: UDMABUF_FLAGS_CLOEXEC,
+            offset: offset as u64,
+            size: size as u64,
+        };
+        let dmabuf = unsafe { ioctl(self.fd.raw(), UDMABUF_CREATE, &mut cmd) };
+        let dmabuf = match map_err!(dmabuf) {
+            Ok(d) => OwnedFd::new(d),
+            Err(e) => return Err(UdmabufError::CreateDmabuf(e.into())),
+        };
+        Ok(dmabuf)
+    }
 }
 
 impl Allocator for Udmabuf {
@@ -104,17 +124,7 @@ impl Allocator for Udmabuf {
         if let Err(e) = uapi::fcntl_add_seals(memfd.raw(), F_SEAL_SHRINK) {
             return Err(UdmabufError::Seal(e.into()).into());
         }
-        let mut cmd = udmabuf_create {
+        let dmabuf = self.create_dmabuf_from_memfd(&memfd, 0, size as _)?;
-            memfd: memfd.raw() as u32,
-            flags: 0,
-            offset: 0,
-            size: size as u64,
-        };
-        let dmabuf = unsafe { ioctl(self.fd.raw(), UDMABUF_CREATE, &mut cmd) };
-        let dmabuf = match map_err!(dmabuf) {
-            Ok(d) => OwnedFd::new(d),
-            Err(e) => return Err(UdmabufError::CreateDmabuf(e.into()).into()),
-        };
         let mut planes = PlaneVec::new();
         planes.push(DmaBufPlane {
             offset: 0,
@@ -260,6 +270,7 @@ impl From<UdmabufError> for AllocatorError {
 }
 
 #[repr(C)]
+#[derive(Debug)]
 struct udmabuf_create {
     memfd: u32,
     flags: u32,
@@ -267,4 +278,6 @@ struct udmabuf_create {
     size: u64,
 }
 
+const UDMABUF_FLAGS_CLOEXEC: u32 = 0x01;
+
 const UDMABUF_CREATE: IoctlNumber = _IOW::<udmabuf_create>(b'u' as u64, 0x42) as IoctlNumber;