use { crate::{ async_engine::{AsyncEngine, SpawnedFuture}, eventfd_cache::EventfdCache, format::{FORMATS, Format}, gfx_api::FdSync, io_uring::IoUring, utils::{ clonecell::CloneCell, errorfmt::ErrorFmt, numcell::NumCell, oserror::{OsError, OsErrorExt2}, queue::AsyncQueue, stack::Stack, }, video::{ LINEAR_MODIFIER, LINEAR_STRIDE_ALIGN, Modifier, dmabuf::{DmaBuf, DmaBufIds, DmaBufPlane, PlaneVec}, drm::{DrmError, syncobj::SyncobjCtx}, }, vulkan_core::{ self, VULKAN_API_VERSION, VulkanCoreError, VulkanCoreInstance, device::VulkanDeviceInf, map_extension_properties, timeline_semaphore::VulkanDeviceTimelineSemaphoreExt, }, }, ahash::AHashMap, ash::{ Device, ext::{ external_memory_dma_buf, image_drm_format_modifier, physical_device_drm, queue_family_foreign, }, khr::{external_fence_fd, external_memory_fd, external_semaphore_fd}, vk::{ self, BindImageMemoryInfo, BindImagePlaneMemoryInfo, BufferCopy2, BufferCreateInfo, BufferImageCopy2, BufferUsageFlags, CommandBuffer, CommandBufferAllocateInfo, CommandPoolCreateFlags, CommandPoolCreateInfo, DeviceCreateInfo, DeviceMemory, DeviceQueueCreateInfo, DrmFormatModifierPropertiesEXT, DrmFormatModifierPropertiesListEXT, ExportMemoryAllocateInfo, Extent3D, ExternalBufferProperties, ExternalFenceFeatureFlags, ExternalFenceHandleTypeFlags, ExternalFenceProperties, ExternalImageFormatPropertiesKHR, ExternalMemoryBufferCreateInfo, ExternalMemoryBufferCreateInfoKHR, ExternalMemoryFeatureFlags, ExternalMemoryHandleTypeFlags, ExternalMemoryImageCreateInfo, ExternalSemaphoreFeatureFlags, ExternalSemaphoreHandleTypeFlags, ExternalSemaphoreProperties, FormatFeatureFlags, FormatProperties2, ImageAspectFlags, ImageBlit2, ImageCopy2, ImageCreateFlags, ImageCreateInfo, ImageDrmFormatModifierExplicitCreateInfoEXT, ImageFormatProperties2, ImageLayout, ImageMemoryRequirementsInfo2, ImagePlaneMemoryRequirementsInfo, ImageTiling, ImageType, ImageUsageFlags, ImportMemoryFdInfoKHR, ImportSemaphoreFdInfoKHR, MemoryAllocateInfo, MemoryDedicatedAllocateInfo, MemoryFdPropertiesKHR, MemoryGetFdInfoKHR, MemoryPropertyFlags, MemoryRequirements2, MemoryType, PhysicalDevice, PhysicalDeviceDrmPropertiesEXT, PhysicalDeviceExternalBufferInfo, PhysicalDeviceExternalFenceInfo, PhysicalDeviceExternalImageFormatInfoKHR, PhysicalDeviceExternalSemaphoreInfo, PhysicalDeviceFeatures2, PhysicalDeviceImageDrmFormatModifierInfoEXT, PhysicalDeviceImageFormatInfo2, PhysicalDeviceProperties2, PhysicalDeviceSynchronization2Features, PhysicalDeviceTimelineSemaphoreFeatures, Queue, QueueFlags, SampleCountFlags, SemaphoreCreateInfo, SemaphoreImportFlags, SharingMode, SubresourceLayout, }, }, bstr::ByteSlice, isnt::std_1::collections::IsntHashMapExt, linearize::{Linearize, LinearizeExt, StaticCopyMap, StaticMap, static_map}, log::Level, run_on_drop::on_drop, std::{ cell::{Cell, RefCell}, ffi::CStr, fmt::{Debug, Formatter}, ops::Deref, rc::Rc, slice, }, thiserror::Error, uapi::{AsUstr, OwnedFd, c}, vk::{Buffer, CommandPool, Image, Semaphore}, }; mod execute; mod queue_allocation; mod registry; pub use registry::CopyDeviceRegistry; use queue_allocation::{QueueIndex, QueueToAllocate, allocate_queues}; #[derive(Debug, Error)] pub enum CopyDeviceError { #[error(transparent)] Core(#[from] VulkanCoreError), #[error("Could not create a semaphore")] CreateSemaphore(#[source] vk::Result), #[error("Could not dup a sync file")] DupSyncFile(#[source] OsError), #[error("Could not dup a dma buf")] DupDmaBuf(#[source] OsError), #[error("Could not import a sync file")] ImportSyncFile(#[source] vk::Result), #[error("Could not submit the copy")] SubmitCopy(#[source] vk::Result), #[error("Could not enumerate the physical devices")] EnumeratePhysicalDevice(#[source] vk::Result), #[error("Could not find a corresponding vulkan device")] NoVulkanDevice, #[error("Device does not support vulkan 1.3")] NoVulkan13, #[error("Device does not support the synchronization2 feature")] NoSynchronization2, #[error("Device does not support the device extension {}", .0.as_ustr().as_bytes().as_bstr())] MissingDeviceExtensions(&'static CStr), #[error("Device does not support importing sync files")] NoSyncFileImport, #[error("Device does not support exporting sync files")] NoSyncFileExport, #[error("Device does not support importing dma bufs as buffers")] NoDmaBufBufferImport, #[error("Device does not have a graphics queue family")] NoGfxQueueFamily, #[error("Could not create the device")] CreateDevice(#[source] vk::Result), #[error("Could not create a command pool")] CreateCommandPool(#[source] vk::Result), #[error("Could not create a command buffer")] CreateCommandBuffer(#[source] vk::Result), #[error("Copy source and destination must have the same size")] NotSameSize, #[error("Copy source has a non-positive size")] NonPositiveSize, #[error("The size calculation overflowed")] SizeOverflow, #[error("The format and/or modifier is not supported")] UnsupportedFormat, #[error("the image is too large")] TooLarge, #[error("Copy source has an incorrect number of planes")] WrongNumberOfPlanes, #[error("Could not create a buffer")] CreateBuffer(#[source] vk::Result), #[error("Device returned an unexpected required buffer size")] UnexpectedBufferSize, #[error("Could not query memory fd properties")] GetMemoryFdProperties(#[source] vk::Result), #[error("Could not find a memory type for import")] NoMemoryTypeForImport, #[error("Could not import memory")] ImportMemory(#[source] vk::Result), #[error("Could not bind buffer memory")] BindBufferMemory(#[source] vk::Result), #[error("Could not bind image memory")] BindImageMemory(#[source] vk::Result), #[error("Could not create an image")] CreateImage(#[source] vk::Result), #[error("Could not begin a command buffer")] BeginCommandBuffer(#[source] vk::Result), #[error("Could not end a command buffer")] EndCommandBuffer(#[source] vk::Result), #[error("The previous copy is still executing")] Busy, #[error("The device does not support dmabuf export")] NoDmabufExport, #[error("Could not find a memory type for import")] NoMemoryTypeForAllocation, #[error("Could not allocate memory")] AllocateMemory(#[source] vk::Result), #[error("Could not export a dmabuf")] ExportDmabuf(#[source] vk::Result), #[error("Both buffers are off device")] BothOffDevice, #[error("Cannot blit between these formats")] BlitNotSupported, #[error("Could not create syncobj ctx")] CreateSyncobjCtx(#[source] DrmError), } type Keyed = StaticMap; type KeyedCopy = StaticCopyMap; pub struct PhysicalCopyDevice { ring: Rc, eng: Rc, eventfd_cache: Rc, sync_ctx: Rc, instance: VulkanCoreInstance, physical_device: PhysicalDevice, support: AHashMap>>, queues_to_allocate: Vec, queues: KeyedCopy, supports_dmabuf_export: bool, supports_timeline_opaque_export: bool, memory_types: Vec, rects: RefCell>, buffer_copy_2: RefCell>>, buffer_image_copy_2: RefCell>>, image_copy_2: RefCell>>, image_blit_2: RefCell>>, } pub struct CopyDevice { _tasks: Vec>, dev: Rc, timeline_semaphore: Option>, } struct CopyDeviceInner { phy: Rc, dev: Device, unique_pools: Vec, pools: Keyed, queues: KeyedCopy, external_semaphore_fd: external_semaphore_fd::Device, external_fence_fd: external_fence_fd::Device, external_memory_fd: external_memory_fd::Device, semaphores: Stack, fences: Stack>, submissions: Keyed>, } #[derive(Default)] struct PendingSubmissions { task_has_pending: Cell, pending: AsyncQueue, } pub struct CopyDeviceCopy { inner: Rc, dev: Rc, } struct CopyDeviceCopyInner { dev: Rc, busy_id: NumCell, busy: CloneCell>, width: u32, height: u32, command_buffer: CommandBuffer, tt: TransferType, ty: CopyDeviceCopyType, } enum CopyDeviceCopyType { BufferToBuffer { src: VulkanBuffer, dst: VulkanBuffer, stride: u32, bpp: u32, }, BufferToImage { buf: VulkanBuffer, buf_format: &'static Format, buf_stride: u32, img: VulkanImage, }, ImageToBuffer { img: VulkanImage, buf: VulkanBuffer, buf_format: &'static Format, buf_stride: u32, }, ImageToImage { src: VulkanImage, dst: VulkanImage, }, Blit { src: VulkanImage, dst: VulkanImage, }, } struct Pending { dev: Rc, busy_id: u64, sync: Option, copy: Rc, semaphore: Option, vulkan_sync: VulkanSync, } struct VulkanSemaphore { dev: Rc, semaphore: Semaphore, } type VulkanFence = vulkan_core::fence::VulkanFence; type VulkanTimelineSemaphore = vulkan_core::timeline_semaphore::VulkanTimelineSemaphore; type VulkanSync = vulkan_core::sync::VulkanSync; struct VulkanBuffer { dev: Rc, buf: Buffer, mem: DeviceMemory, } struct VulkanImage { dev: Rc, img: Image, mem: PlaneVec, } #[derive(Copy, Clone)] pub struct CopyDeviceSupport { pub modifier: Modifier, pub planes: usize, pub max_width: u32, pub max_height: u32, pub blit: bool, } pub struct CopyDeviceBuffer { device: Rc, memory: DeviceMemory, dmabuf: DmaBuf, } #[derive(Copy, Clone, Debug, Linearize)] enum TransferType { Blit, Intra, Download, Upload, } #[derive(Copy, Clone, Debug, Linearize)] enum Dir { Src, Dst, } struct ClassifiedDmabuf<'a> { fd_props: PlaneVec>, on_device: bool, buffer_possible: bool, format: &'a CopyDeviceSupport, } const DEVICE_EXTENSIONS: [&CStr; 6] = [ external_semaphore_fd::NAME, external_fence_fd::NAME, external_memory_fd::NAME, external_memory_dma_buf::NAME, image_drm_format_modifier::NAME, queue_family_foreign::NAME, ]; impl PhysicalCopyDevice { fn new( ring: &Rc, eng: &Rc, eventfd_cache: &Rc, dev: c::dev_t, ) -> Result, CopyDeviceError> { let core_instance = VulkanCoreInstance::new(Level::Debug)?; let instance = &core_instance.instance; let physical_device; let device_extensions; let device_properties; let supports_dmabuf_export; 'find_device: { let devices = unsafe { instance .enumerate_physical_devices() .map_err(CopyDeviceError::EnumeratePhysicalDevice)? }; 'outer: for phy in devices { let res = unsafe { instance.enumerate_device_extension_properties(phy) }; let exts = match res { Ok(res) => map_extension_properties(res), Err(e) => { log::error!( "Could not enumerate extensions of physical device: {}", ErrorFmt(e), ); continue; } }; if exts.not_contains_key(physical_device_drm::NAME) { continue 'outer; } let mut drm_props = PhysicalDeviceDrmPropertiesEXT::default(); let mut props = PhysicalDeviceProperties2::default().push_next(&mut drm_props); unsafe { instance.get_physical_device_properties2(phy, &mut props); } let props = props.properties; let major = uapi::major(dev) as i64; let minor = uapi::minor(dev) as i64; let matches = (drm_props.has_primary == vk::TRUE && drm_props.primary_major == major && drm_props.primary_minor == minor) || (drm_props.has_render == vk::TRUE && drm_props.render_major == major && drm_props.render_minor == minor); if matches { physical_device = phy; device_extensions = exts; device_properties = props; break 'find_device; } } return Err(CopyDeviceError::NoVulkanDevice); } let sync_ctx = SyncobjCtx::from_dev_t(dev) .map(Rc::new) .map_err(CopyDeviceError::CreateSyncobjCtx)?; if device_properties.api_version < VULKAN_API_VERSION { return Err(CopyDeviceError::NoVulkan13); } for ext in DEVICE_EXTENSIONS { if device_extensions.not_contains_key(ext) { return Err(CopyDeviceError::MissingDeviceExtensions(ext)); } } { let mut synchronization2_features = PhysicalDeviceSynchronization2Features::default(); let mut physical_device_features = PhysicalDeviceFeatures2::default().push_next(&mut synchronization2_features); unsafe { instance .get_physical_device_features2(physical_device, &mut physical_device_features); } if synchronization2_features.synchronization2 != vk::TRUE { return Err(CopyDeviceError::NoSynchronization2); } } { let info = PhysicalDeviceExternalSemaphoreInfo::default() .handle_type(ExternalSemaphoreHandleTypeFlags::SYNC_FD); let mut props = ExternalSemaphoreProperties::default(); unsafe { instance.get_physical_device_external_semaphore_properties( physical_device, &info, &mut props, ); } let supported = props .external_semaphore_features .contains(ExternalSemaphoreFeatureFlags::IMPORTABLE); if !supported { return Err(CopyDeviceError::NoSyncFileImport); } } { let info = PhysicalDeviceExternalFenceInfo::default() .handle_type(ExternalFenceHandleTypeFlags::SYNC_FD); let mut props = ExternalFenceProperties::default(); unsafe { instance.get_physical_device_external_fence_properties( physical_device, &info, &mut props, ); } let supported = props .external_fence_features .contains(ExternalFenceFeatureFlags::EXPORTABLE); if !supported { return Err(CopyDeviceError::NoSyncFileExport); } } { let info = PhysicalDeviceExternalBufferInfo::default() .handle_type(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT) .usage(BufferUsageFlags::TRANSFER_SRC | BufferUsageFlags::TRANSFER_DST); let mut props = ExternalBufferProperties::default(); unsafe { instance.get_physical_device_external_buffer_properties( physical_device, &info, &mut props, ); } let features = props.external_memory_properties.external_memory_features; let supported = features.contains(ExternalMemoryFeatureFlags::IMPORTABLE); if !supported { return Err(CopyDeviceError::NoDmaBufBufferImport); } supports_dmabuf_export = features.contains(ExternalMemoryFeatureFlags::EXPORTABLE); } let (queues_to_allocate, queue_indices) = { let families = unsafe { instance.get_physical_device_queue_family_properties(physical_device) }; let mut transfer_only = None; let mut compute_only = None; let mut gfx = None; for (idx, family) in families.iter().enumerate() { let idx = idx as u32; let g = family.min_image_transfer_granularity; let g = (g.width.wrapping_sub(1), g.height.wrapping_sub(1)); if g.0 == u32::MAX || g.1 == u32::MAX { continue; } let count = family.queue_count; if count == 0 { continue; } let v = (idx, g, count); let flags = family.queue_flags; if flags.contains(QueueFlags::GRAPHICS) { if gfx.is_none() { gfx = Some(v); } } else if flags.contains(QueueFlags::COMPUTE) { if compute_only.is_none() { compute_only = Some(v); } } else if flags.contains(QueueFlags::TRANSFER) { if transfer_only.is_none() { transfer_only = Some(v); } } } let gfx = gfx.ok_or(CopyDeviceError::NoGfxQueueFamily)?; allocate_queues(gfx, compute_only, transfer_only) }; let mut support = AHashMap::default(); for format in FORMATS { let mut list = vec![]; for attach in [false, true] { let mut modifiers = DrmFormatModifierPropertiesListEXT::default(); if attach { modifiers = modifiers.drm_format_modifier_properties(&mut list); } let mut out = FormatProperties2::default().push_next(&mut modifiers); unsafe { instance.get_physical_device_format_properties2( physical_device, format.vk_format, &mut out, ); } if !attach { list = vec![ DrmFormatModifierPropertiesEXT::default(); modifiers.drm_format_modifier_count as usize ]; } } let mut format_support = StaticMap::<_, Vec<_>>::default(); for modifier in list { for dir in Dir::variants() { let format_feature_flags = match dir { Dir::Src => FormatFeatureFlags::TRANSFER_SRC, Dir::Dst => FormatFeatureFlags::TRANSFER_DST, }; let blit_feature_flags = match dir { Dir::Src => FormatFeatureFlags::BLIT_SRC, Dir::Dst => FormatFeatureFlags::BLIT_DST, }; let image_usage_flags = match dir { Dir::Src => ImageUsageFlags::TRANSFER_SRC, Dir::Dst => ImageUsageFlags::TRANSFER_DST, }; let image_features = modifier.drm_format_modifier_tiling_features; if !image_features.contains(format_feature_flags) { continue; } let supports_blit = image_features.contains(blit_feature_flags); let mut modifier_info = PhysicalDeviceImageDrmFormatModifierInfoEXT::default() .drm_format_modifier(modifier.drm_format_modifier); let mut external_memory_info = PhysicalDeviceExternalImageFormatInfoKHR::default() .handle_type(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); let info = PhysicalDeviceImageFormatInfo2::default() .format(format.vk_format) .ty(ImageType::TYPE_2D) .tiling(ImageTiling::DRM_FORMAT_MODIFIER_EXT) .usage(image_usage_flags) .push_next(&mut external_memory_info) .push_next(&mut modifier_info); let mut external_memory_prop = ExternalImageFormatPropertiesKHR::default(); let mut prop = ImageFormatProperties2::default().push_next(&mut external_memory_prop); let res = unsafe { instance.get_physical_device_image_format_properties2( physical_device, &info, &mut prop, ) }; if res.is_err() { continue; } let prop = prop.image_format_properties; let memory_features = external_memory_prop .external_memory_properties .external_memory_features; if !memory_features.contains(ExternalMemoryFeatureFlags::IMPORTABLE) { continue; } let me = prop.max_extent; if me.width > 0 && me.height > 0 && me.depth > 0 { format_support[dir].push(CopyDeviceSupport { modifier: modifier.drm_format_modifier, planes: modifier.drm_format_modifier_plane_count as usize, max_width: me.width, max_height: me.height, blit: supports_blit, }); } } } support.insert(format.drm, format_support); } let memory_info = unsafe { instance.get_physical_device_memory_properties(physical_device) }; let features = core_instance.get_features(physical_device); let supports_timeline_opaque_export = core_instance.supports_timeline_opaque_export(physical_device, &features); let dev = Rc::new(PhysicalCopyDevice { ring: ring.clone(), eng: eng.clone(), eventfd_cache: eventfd_cache.clone(), sync_ctx, instance: core_instance, physical_device, support, queues_to_allocate, queues: queue_indices, supports_dmabuf_export, supports_timeline_opaque_export, memory_types: memory_info.memory_types_as_slice().to_vec(), rects: Default::default(), buffer_copy_2: Default::default(), image_blit_2: Default::default(), image_copy_2: Default::default(), buffer_image_copy_2: Default::default(), }); Ok(dev) } pub fn src_support(&self, format: &Format) -> &[CopyDeviceSupport] { self.support(format, Dir::Src) } pub fn dst_support(&self, format: &Format) -> &[CopyDeviceSupport] { self.support(format, Dir::Dst) } fn support(&self, format: &Format, dir: Dir) -> &[CopyDeviceSupport] { self.support .get(&format.drm) .map(|s| s[dir].as_slice()) .unwrap_or_default() } pub fn create_device(self: &Rc) -> Result, CopyDeviceError> { let instance = &self.instance.instance; let device = { let priorities = [1.0; TransferType::LENGTH]; let queue_create_info: Vec<_> = self .queues_to_allocate .iter() .map(|q| { DeviceQueueCreateInfo::default() .queue_family_index(q.family) .queue_priorities(&priorities[..q.num]) }) .collect(); let extensions = DEVICE_EXTENSIONS.map(|e| e.as_ptr()); let mut semaphore_features = PhysicalDeviceTimelineSemaphoreFeatures::default() .timeline_semaphore(self.supports_timeline_opaque_export); let mut synchronization2_features = PhysicalDeviceSynchronization2Features::default().synchronization2(true); let info = DeviceCreateInfo::default() .queue_create_infos(&queue_create_info) .enabled_extension_names(&extensions) .push_next(&mut semaphore_features) .push_next(&mut synchronization2_features); unsafe { instance .create_device(self.physical_device, &info, None) .map_err(CopyDeviceError::CreateDevice)? } }; let destroy_device = on_drop(|| unsafe { device.destroy_device(None) }); let external_semaphore_fd = external_semaphore_fd::Device::new(instance, &device); let external_fence_fd = external_fence_fd::Device::new(instance, &device); let external_memory_fd = external_memory_fd::Device::new(instance, &device); let queues = self.queues.map_values(|idx| unsafe { device.get_device_queue(idx.family, idx.idx_within_family) }); let mut unique_pools = vec![]; let mut destroy_pools = vec![]; for q in &self.queues_to_allocate { let info = CommandPoolCreateInfo::default() .queue_family_index(q.family) .flags(CommandPoolCreateFlags::RESET_COMMAND_BUFFER); let pool = unsafe { device .create_command_pool(&info, None) .map_err(CopyDeviceError::CreateCommandPool)? }; unique_pools.push(pool); let device = &device; let destroy_pool = on_drop(move || unsafe { device.destroy_command_pool(pool, None) }); destroy_pools.push(destroy_pool); } let pools: StaticMap = static_map! { tt => unique_pools[self.queues[tt].allocate_idx] }; let submissions_list: Vec>> = self .queues_to_allocate .iter() .map(|q| vec![Default::default(); q.num]) .collect(); let submissions = self .queues .into_static_map() .map_values(|q| submissions_list[q.allocate_idx][q.idx_within_family as usize].clone()); destroy_pools.into_iter().for_each(|v| v.forget()); destroy_device.forget(); let dev = Rc::new(CopyDeviceInner { phy: self.clone(), dev: device, unique_pools, pools, queues, external_semaphore_fd, external_fence_fd, external_memory_fd, semaphores: Default::default(), fences: Default::default(), submissions, }); let mut tasks = vec![]; for submissions in submissions_list.iter().flatten().cloned() { let future = wait_for_submissions(submissions, dev.clone(), self.ring.clone()); let task = self.eng.spawn("copy-device-await-pending", future); tasks.push(task); } let queue = Rc::new(CopyDevice { timeline_semaphore: dev.create_timeline_semaphore_or_log(), dev, _tasks: tasks, }); Ok(queue) } } async fn wait_for_submissions( submissions: Rc, dev: Rc, ring: Rc, ) { loop { submissions.task_has_pending.set(false); let pending = submissions.pending.pop().await; submissions.task_has_pending.set(true); if let Some(sync) = &pending.sync && let Err(e) = sync.try_signaled(&ring).await { log::warn!( "Could not wait for sync file to become readable: {}", ErrorFmt(e), ); dev.wait_idle(); } pending.vulkan_sync.handle_validation(); } } impl CopyDevice { fn classify_dmabuf( &self, buf: &DmaBuf, dir: Dir, ) -> Result, CopyDeviceError> { if buf.width <= 0 || buf.height <= 0 { return Err(CopyDeviceError::NonPositiveSize); } let width = buf.width as u32; let height = buf.height as u32; let Some(format) = self .dev .phy .support(buf.format, dir) .iter() .find(|s| s.modifier == buf.modifier) else { return Err(CopyDeviceError::UnsupportedFormat); }; if width > format.max_width || height > format.max_height { return Err(CopyDeviceError::TooLarge); } if buf.planes.len() != format.planes { return Err(CopyDeviceError::WrongNumberOfPlanes); } let mut fd_props = PlaneVec::new(); for plane in &buf.planes { let mut props = MemoryFdPropertiesKHR::default(); unsafe { self.dev .external_memory_fd .get_memory_fd_properties( ExternalMemoryHandleTypeFlags::DMA_BUF_EXT, plane.fd.raw(), &mut props, ) .map_err(CopyDeviceError::GetMemoryFdProperties)?; } fd_props.push(props); if buf.is_one_file() { break; } } let mut on_device = true; for prop in &fd_props { let mut plane_on_device = false; for (idx, ty) in self.dev.phy.memory_types.iter().enumerate() { if prop.memory_type_bits & (1 << idx) != 0 && ty .property_flags .contains(MemoryPropertyFlags::DEVICE_LOCAL) { plane_on_device = true; break; } } if !plane_on_device { on_device = false; break; } } let buffer_possible = buf.modifier == LINEAR_MODIFIER && buf.planes.len() == 1 && buf.planes[0].stride % buf.format.bpp == 0 && width <= buf.planes[0].stride / buf.format.bpp; Ok(ClassifiedDmabuf { fd_props, on_device, buffer_possible, format, }) } fn import_buffer( &self, tt: TransferType, class: &ClassifiedDmabuf, buf: &DmaBuf, dir: Dir, ) -> Result { assert!(class.buffer_possible); let height = buf.height as u32; let plane = &buf.planes[0]; let queue_family = self.dev.phy.queues[tt].family; let buffer_size = plane.stride as u64 * height as u64; let buffer = { let buffer_usage_flags = match dir { Dir::Src => BufferUsageFlags::TRANSFER_SRC, Dir::Dst => BufferUsageFlags::TRANSFER_DST, }; let mut external_info = ExternalMemoryBufferCreateInfoKHR::default() .handle_types(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); let info = BufferCreateInfo::default() .size(buffer_size) .usage(buffer_usage_flags) .queue_family_indices(slice::from_ref(&queue_family)) .push_next(&mut external_info); unsafe { self.dev .dev .create_buffer(&info, None) .map_err(CopyDeviceError::CreateBuffer)? } }; let destroy_buffer = on_drop(|| unsafe { self.dev.dev.destroy_buffer(buffer, None) }); let memory = { let out = unsafe { self.dev.dev.get_buffer_memory_requirements(buffer) }; if out.size > buffer_size { return Err(CopyDeviceError::UnexpectedBufferSize); } let memory_type_bits = class.fd_props[0].memory_type_bits & out.memory_type_bits; if memory_type_bits == 0 { return Err(CopyDeviceError::NoMemoryTypeForImport); } let fd = uapi::fcntl_dupfd_cloexec(plane.fd.raw(), 0) .map_os_err(CopyDeviceError::DupDmaBuf)?; let mut dedicated_allocation = MemoryDedicatedAllocateInfo::default().buffer(buffer); let mut external_memory = ImportMemoryFdInfoKHR::default() .handle_type(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT) .fd(fd.raw()); let allocate_info = MemoryAllocateInfo::default() .allocation_size(out.size) .memory_type_index(memory_type_bits.trailing_zeros() as _) .push_next(&mut external_memory) .push_next(&mut dedicated_allocation); let memory = unsafe { self.dev .dev .allocate_memory(&allocate_info, None) .map_err(CopyDeviceError::ImportMemory)? }; let _ = fd.unwrap(); memory }; let free_memory = on_drop(|| unsafe { self.dev.dev.free_memory(memory, None) }); unsafe { self.dev .dev .bind_buffer_memory(buffer, memory, 0) .map_err(CopyDeviceError::BindBufferMemory)?; } free_memory.forget(); destroy_buffer.forget(); Ok(VulkanBuffer { dev: self.dev.clone(), buf: buffer, mem: memory, }) } fn import_image( &self, tt: TransferType, class: &ClassifiedDmabuf, buf: &DmaBuf, dir: Dir, ) -> Result { let dev = &self.dev.dev; let disjoint = buf.is_disjoint(); let queue_family = self.dev.phy.queues[tt].family; let image = { let image_create_flags = match disjoint { true => ImageCreateFlags::DISJOINT, false => ImageCreateFlags::empty(), }; let image_usage_flags = match dir { Dir::Src => ImageUsageFlags::TRANSFER_SRC, Dir::Dst => ImageUsageFlags::TRANSFER_DST, }; let plane_layouts: PlaneVec<_> = buf .planes .iter() .map(|p| SubresourceLayout { offset: p.offset as _, row_pitch: p.stride as _, size: 0, array_pitch: 0, depth_pitch: 0, }) .collect(); let mut mod_info = ImageDrmFormatModifierExplicitCreateInfoEXT::default() .drm_format_modifier(buf.modifier) .plane_layouts(&plane_layouts); let mut memory_image_create_info = ExternalMemoryImageCreateInfo::default() .handle_types(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); let info = ImageCreateInfo::default() .flags(image_create_flags) .image_type(ImageType::TYPE_2D) .format(buf.format.vk_format) .extent(Extent3D { width: buf.width as _, height: buf.height as _, depth: 1, }) .mip_levels(1) .array_layers(1) .samples(SampleCountFlags::TYPE_1) .tiling(ImageTiling::DRM_FORMAT_MODIFIER_EXT) .usage(image_usage_flags) .sharing_mode(SharingMode::EXCLUSIVE) .queue_family_indices(slice::from_ref(&queue_family)) .initial_layout(ImageLayout::UNDEFINED) .push_next(&mut mod_info) .push_next(&mut memory_image_create_info); unsafe { dev.create_image(&info, None) .map_err(CopyDeviceError::CreateImage)? } }; let destroy_image = on_drop(|| unsafe { dev.destroy_image(image, None) }); let mut memories = PlaneVec::new(); let mut free_memories = PlaneVec::new(); { let num_device_memories = match disjoint { true => buf.planes.len(), false => 1, }; let mut bind_image_plane_memory_infos = PlaneVec::new(); for plane_idx in 0..num_device_memories { let dma_buf_plane = &buf.planes[plane_idx]; let mut image_memory_requirements_info = ImageMemoryRequirementsInfo2::default().image(image); let mut image_plane_memory_requirements_info; if disjoint { let plane_aspect = match plane_idx { 0 => ImageAspectFlags::MEMORY_PLANE_0_EXT, 1 => ImageAspectFlags::MEMORY_PLANE_1_EXT, 2 => ImageAspectFlags::MEMORY_PLANE_2_EXT, 3 => ImageAspectFlags::MEMORY_PLANE_3_EXT, _ => unreachable!(), }; image_plane_memory_requirements_info = ImagePlaneMemoryRequirementsInfo::default().plane_aspect(plane_aspect); image_memory_requirements_info = image_memory_requirements_info .push_next(&mut image_plane_memory_requirements_info); bind_image_plane_memory_infos .push(BindImagePlaneMemoryInfo::default().plane_aspect(plane_aspect)); } let mut memory_requirements = MemoryRequirements2::default(); unsafe { dev.get_image_memory_requirements2( &image_memory_requirements_info, &mut memory_requirements, ); } let memory_type_bits = memory_requirements.memory_requirements.memory_type_bits & class.fd_props[plane_idx].memory_type_bits; if memory_type_bits == 0 { return Err(CopyDeviceError::NoMemoryTypeForImport); } let fd = uapi::fcntl_dupfd_cloexec(dma_buf_plane.fd.raw(), 0) .map_os_err(CopyDeviceError::DupDmaBuf)?; let mut memory_dedicated_allocate_info = MemoryDedicatedAllocateInfo::default().image(image); let mut import_memory_fd_info = ImportMemoryFdInfoKHR::default() .fd(fd.raw()) .handle_type(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); let memory_allocate_info = MemoryAllocateInfo::default() .allocation_size(memory_requirements.memory_requirements.size) .memory_type_index(memory_type_bits.trailing_zeros() as _) .push_next(&mut import_memory_fd_info) .push_next(&mut memory_dedicated_allocate_info); let device_memory = unsafe { dev.allocate_memory(&memory_allocate_info, None) .map_err(CopyDeviceError::ImportMemory)? }; let _ = fd.unwrap(); memories.push(device_memory); free_memories.push(on_drop(move || unsafe { dev.free_memory(device_memory, None) })); } let mut bind_image_memory_infos = PlaneVec::new(); let mut bind_image_plane_memory_infos = bind_image_plane_memory_infos.iter_mut(); for mem in memories.iter().copied() { let mut info = BindImageMemoryInfo::default().image(image).memory(mem); if disjoint { info = info.push_next(bind_image_plane_memory_infos.next().unwrap()); } bind_image_memory_infos.push(info); } unsafe { dev.bind_image_memory2(&bind_image_memory_infos) .map_err(CopyDeviceError::BindImageMemory)?; } } free_memories.into_iter().for_each(|f| f.forget()); destroy_image.forget(); Ok(VulkanImage { dev: self.dev.clone(), img: image, mem: memories, }) } pub fn create_copy( self: &Rc, src: &DmaBuf, dst: &DmaBuf, ) -> Result { if (dst.width, dst.height) != (src.width, src.height) { return Err(CopyDeviceError::NotSameSize); } let src_class = self.classify_dmabuf(src, Dir::Src)?; let dst_class = self.classify_dmabuf(dst, Dir::Dst)?; let blit = src.format != dst.format; if blit && (!src_class.format.blit || !dst_class.format.blit) { return Err(CopyDeviceError::BlitNotSupported); } let tt = match (src_class.on_device, dst_class.on_device) { (false, false) => return Err(CopyDeviceError::BothOffDevice), _ if blit => TransferType::Blit, (false, true) => TransferType::Upload, (true, false) => TransferType::Download, (true, true) => TransferType::Intra, }; let dev = &self.dev.dev; let command_buffer = { let info = CommandBufferAllocateInfo::default() .command_pool(self.dev.pools[tt]) .command_buffer_count(1); let mut buf = unsafe { dev.allocate_command_buffers(&info) .map_err(CopyDeviceError::CreateCommandBuffer)? }; assert_eq!(buf.len(), 1); buf.pop().unwrap() }; let free_command_buffer = on_drop(|| unsafe { dev.free_command_buffers(self.dev.pools[tt], &[command_buffer]) }); let ty = if blit { CopyDeviceCopyType::Blit { src: self.import_image(tt, &src_class, src, Dir::Src)?, dst: self.import_image(tt, &dst_class, dst, Dir::Dst)?, } } else if !src_class.buffer_possible && !dst_class.buffer_possible { CopyDeviceCopyType::ImageToImage { src: self.import_image(tt, &src_class, src, Dir::Src)?, dst: self.import_image(tt, &dst_class, dst, Dir::Dst)?, } } else if src_class.buffer_possible && dst_class.buffer_possible && src.planes[0].stride == dst.planes[0].stride { CopyDeviceCopyType::BufferToBuffer { src: self.import_buffer(tt, &src_class, src, Dir::Src)?, dst: self.import_buffer(tt, &dst_class, dst, Dir::Dst)?, stride: src.planes[0].stride, bpp: src.format.bpp, } } else if src_class.buffer_possible { CopyDeviceCopyType::BufferToImage { buf: self.import_buffer(tt, &src_class, src, Dir::Src)?, buf_format: src.format, buf_stride: src.planes[0].stride, img: self.import_image(tt, &dst_class, dst, Dir::Dst)?, } } else { CopyDeviceCopyType::ImageToBuffer { img: self.import_image(tt, &src_class, src, Dir::Src)?, buf: self.import_buffer(tt, &dst_class, dst, Dir::Dst)?, buf_format: dst.format, buf_stride: dst.planes[0].stride, } }; free_command_buffer.forget(); Ok(CopyDeviceCopy { inner: Rc::new(CopyDeviceCopyInner { dev: self.dev.clone(), busy_id: Default::default(), busy: Default::default(), width: src.width as _, height: src.height as _, command_buffer, tt, ty, }), dev: self.clone(), }) } pub fn create_buffer( &self, dma_buf_ids: &DmaBufIds, width: i32, height: i32, format: &'static Format, ) -> Result { if !self.dev.phy.supports_dmabuf_export { return Err(CopyDeviceError::NoDmabufExport); } if width <= 0 || height <= 0 { return Err(CopyDeviceError::NonPositiveSize); } let stride = width as u32 * format.bpp as u32; let Some(stride) = stride.checked_next_multiple_of(LINEAR_STRIDE_ALIGN as u32) else { return Err(CopyDeviceError::SizeOverflow); }; let Some(size) = (stride as u64).checked_mul(height as u64) else { return Err(CopyDeviceError::SizeOverflow); }; let dev = &self.dev.dev; let buffer = { let mut external_info = ExternalMemoryBufferCreateInfo::default() .handle_types(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); let info = BufferCreateInfo::default() .size(size) .usage(BufferUsageFlags::TRANSFER_SRC | BufferUsageFlags::TRANSFER_DST) .sharing_mode(SharingMode::EXCLUSIVE) .push_next(&mut external_info); unsafe { dev.create_buffer(&info, None) .map_err(CopyDeviceError::CreateBuffer)? } }; let _destroy_buffer = on_drop(|| unsafe { dev.destroy_buffer(buffer, None) }); let memory = { let memory_requirements = unsafe { dev.get_buffer_memory_requirements(buffer) }; let required_flags = MemoryPropertyFlags::DEVICE_LOCAL | MemoryPropertyFlags::HOST_VISIBLE; let index = 'index: { for (idx, ty) in self.dev.phy.memory_types.iter().enumerate() { if memory_requirements.memory_type_bits & (1 << idx) != 0 && ty.property_flags.contains(required_flags) { break 'index idx; } } return Err(CopyDeviceError::NoMemoryTypeForAllocation); }; let mut dedicated_allocation = MemoryDedicatedAllocateInfo::default().buffer(buffer); let mut external_memory = ExportMemoryAllocateInfo::default() .handle_types(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); let info = MemoryAllocateInfo::default() .allocation_size(memory_requirements.size) .memory_type_index(index as _) .push_next(&mut external_memory) .push_next(&mut dedicated_allocation); unsafe { dev.allocate_memory(&info, None) .map_err(CopyDeviceError::AllocateMemory)? } }; let free_memory = on_drop(|| unsafe { dev.free_memory(memory, None) }); let fd = { let info = MemoryGetFdInfoKHR::default() .memory(memory) .handle_type(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); unsafe { self.dev .external_memory_fd .get_memory_fd(&info) .map_err(CopyDeviceError::ExportDmabuf)? } }; let fd = Rc::new(OwnedFd::new(fd)); let mut dmabuf = DmaBuf { id: dma_buf_ids.next(), width, height, format, modifier: LINEAR_MODIFIER, planes: Default::default(), is_disjoint: Default::default(), }; dmabuf.planes.push(DmaBufPlane { offset: 0, stride, fd, }); free_memory.forget(); Ok(CopyDeviceBuffer { device: self.dev.clone(), memory, dmabuf, }) } } impl CopyDeviceInner { fn wait_idle(&self) { log::warn!("Blocking"); let res = unsafe { self.dev.device_wait_idle() }; if let Err(e) = res { log::error!("Could not wait for device idle: {}", ErrorFmt(e)); log::error!("This is unsound."); } for submissions in self.submissions.values() { submissions.pending.clear(); } } fn create_semaphore(self: &Rc) -> Result { let create_info = SemaphoreCreateInfo::default(); let semaphore = unsafe { self.dev .create_semaphore(&create_info, None) .map_err(CopyDeviceError::CreateSemaphore)? }; Ok(VulkanSemaphore { dev: self.clone(), semaphore, }) } } impl VulkanSemaphore { fn import(&self, sync_file: &OwnedFd) -> Result<(), CopyDeviceError> { let fd = uapi::fcntl_dupfd_cloexec(sync_file.raw(), 0) .map_os_err(CopyDeviceError::DupSyncFile)?; let info = ImportSemaphoreFdInfoKHR::default() .flags(SemaphoreImportFlags::TEMPORARY) .semaphore(self.semaphore) .handle_type(ExternalSemaphoreHandleTypeFlags::SYNC_FD) .fd(fd.raw()); unsafe { self.dev .external_semaphore_fd .import_semaphore_fd(&info) .map_err(CopyDeviceError::ImportSyncFile)?; } let _ = fd.unwrap(); Ok(()) } } impl Drop for VulkanSemaphore { fn drop(&mut self) { unsafe { self.dev.dev.destroy_semaphore(self.semaphore, None); } } } impl Drop for CopyDeviceCopyInner { fn drop(&mut self) { unsafe { self.dev.dev.free_command_buffers( self.dev.pools[self.tt], slice::from_ref(&self.command_buffer), ); } } } impl Drop for CopyDeviceInner { fn drop(&mut self) { unsafe { for &pool in &self.unique_pools { self.dev.destroy_command_pool(pool, None); } self.dev.destroy_device(None); } } } impl Drop for CopyDevice { fn drop(&mut self) { let dev = &self.dev; let has_pending = dev .submissions .values() .any(|s| s.task_has_pending.get() || s.pending.is_not_empty()); if has_pending { dev.wait_idle(); } dev.semaphores.take(); dev.fences.take(); } } impl Drop for Pending { fn drop(&mut self) { if let Some(v) = self.semaphore.take() { self.dev.semaphores.push(v); } if self.copy.busy_id.get() == self.busy_id { self.copy.busy.take(); } } } impl CopyDeviceBuffer { pub fn dmabuf(&self) -> &DmaBuf { &self.dmabuf } } impl Drop for CopyDeviceBuffer { fn drop(&mut self) { unsafe { self.device.dev.free_memory(self.memory, None); } } } impl Debug for CopyDeviceBuffer { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("CopyDeviceBuffer").finish_non_exhaustive() } } impl Debug for PhysicalCopyDevice { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("PhysicalCopyDevice").finish_non_exhaustive() } } impl Debug for CopyDevice { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("CopyDevice").finish_non_exhaustive() } } impl Debug for CopyDeviceCopy { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("CopyDeviceCopy").finish_non_exhaustive() } } impl Drop for VulkanBuffer { fn drop(&mut self) { let dev = &self.dev.dev; unsafe { dev.destroy_buffer(self.buf, None); dev.free_memory(self.mem, None); } } } impl Drop for VulkanImage { fn drop(&mut self) { let dev = &self.dev.dev; unsafe { dev.destroy_image(self.img, None); for &mem in &self.mem { dev.free_memory(mem, None); } } } } impl Deref for CopyDevice { type Target = Rc; fn deref(&self) -> &Self::Target { &self.dev.phy } } impl VulkanDeviceInf for CopyDeviceInner { fn instance(&self) -> &VulkanCoreInstance { &self.phy.instance } fn device(&self) -> &Device { &self.dev } fn external_fence_fd(&self) -> &external_fence_fd::Device { &self.external_fence_fd } fn external_semaphore_fd(&self) -> &external_semaphore_fd::Device { &self.external_semaphore_fd } fn supports_timeline_opaque_export(&self) -> bool { self.phy.supports_timeline_opaque_export } fn sync_ctx(&self) -> Option<&Rc> { Some(&self.phy.sync_ctx) } fn eventfd_cache(&self) -> &Rc { &self.phy.eventfd_cache } }