diff --git a/src/backend.rs b/src/backend.rs index bf07449d..531e858a 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -9,7 +9,7 @@ use { drm_feedback::DrmFeedback, fixed::Fixed, format::Format, - gfx_api::{GfxApi, GfxFramebuffer, SyncFile}, + gfx_api::{FdSync, GfxApi, GfxFramebuffer}, ifs::{ wl_output::OutputId, wl_seat::{ @@ -186,7 +186,7 @@ pub trait HardwareCursorUpdate { fn set_enabled(&mut self, enabled: bool); fn get_buffer(&self) -> Rc; fn set_position(&mut self, x: i32, y: i32); - fn swap_buffer(&mut self, sync_file: Option); + fn swap_buffer(&mut self, sync: Option); fn size(&self) -> (i32, i32); } diff --git a/src/backends/metal/allocator.rs b/src/backends/metal/allocator.rs index 63e24ad7..956de322 100644 --- a/src/backends/metal/allocator.rs +++ b/src/backends/metal/allocator.rs @@ -11,8 +11,8 @@ use { }, format::Format, gfx_api::{ - AcquireSync, GfxBlendBuffer, GfxError, GfxFormat, GfxFramebuffer, GfxTexture, - GfxWriteModifier, ReleaseSync, SyncFile, needs_render_usage, + AcquireSync, FdSync, GfxBlendBuffer, GfxError, GfxFormat, GfxFramebuffer, GfxTexture, + GfxWriteModifier, ReleaseSync, needs_render_usage, }, rect::{DamageQueue, Rect, Region}, udmabuf::{Udmabuf, UdmabufError}, @@ -120,15 +120,15 @@ pub enum RenderBufferError { #[derive(Default)] pub struct RenderBufferCopy { - pub render_block: Option, - pub present_block: Option, + pub render_block: Option, + pub present_block: Option, } impl RenderBufferCopy { - pub fn for_both(sf: Option) -> Self { + pub fn for_both(sync: Option) -> Self { Self { - render_block: sf.clone(), - present_block: sf, + render_block: sync.clone(), + present_block: sync, } } } @@ -138,12 +138,12 @@ impl RenderBuffer { &self, cd: &Rc, region: Option<&Region>, - sync_file: Option, + sync: Option, ) -> Result { match &self.prime { RenderBufferPrime::None => Ok(RenderBufferCopy { render_block: None, - present_block: sync_file, + present_block: sync, }), RenderBufferPrime::Sampling { dev_render_tex, @@ -157,7 +157,7 @@ impl RenderBuffer { dev_render_tex, cd, None, - AcquireSync::from_sync_file(sync_file), + AcquireSync::from_fd_sync(sync), ReleaseSync::None, 0, 0, @@ -175,7 +175,7 @@ impl RenderBuffer { .. } => { let render_block = render_copy - .execute(sync_file.as_ref(), region) + .execute(sync.as_ref(), region) .map_err(RenderBufferError::CopyRenderToUdmabuf)?; let present_block = dev_copy .execute(render_block.as_ref(), region) @@ -189,7 +189,7 @@ impl RenderBuffer { | RenderBufferPrime::CopyDirectPush { render_copy: copy, .. } => copy - .execute(sync_file.as_ref(), region) + .execute(sync.as_ref(), region) .map_err(RenderBufferError::CopyRenderToDev) .map(RenderBufferCopy::for_both), } @@ -201,8 +201,8 @@ impl RenderBuffer { self.damage_queue.damage(&[rect]); } - pub fn clear(&self, cd: &Rc) -> Result, RenderBufferError> { - let sync_file = match &self.prime { + pub fn clear(&self, cd: &Rc) -> Result, RenderBufferError> { + let sync = match &self.prime { RenderBufferPrime::None => { self.render .fb @@ -222,14 +222,14 @@ impl RenderBuffer { self.copy_to_dev(cd, None, sf)?.present_block } }; - Ok(sync_file) + Ok(sync) } pub fn copy_to_new( &self, new: &Self, cd: &Rc, - ) -> Result, RenderBufferError> { + ) -> Result, RenderBufferError> { let old = self; if (old.width, old.height) != (new.width, new.height) { diff --git a/src/backends/metal/present.rs b/src/backends/metal/present.rs index 722faef2..bf211e29 100644 --- a/src/backends/metal/present.rs +++ b/src/backends/metal/present.rs @@ -240,13 +240,8 @@ impl MetalConnector { // current PresentFb if present_fb is None, potentially mutating the fb that is // currently being scanned out, which would render such a wait absurd. self.perform_screencopies(&present_fb, &node, &cd); - if let Some(sync_file) = self.cursor_sync_file.take() - && let Err(e) = self.state.ring.readable(&sync_file).await - { - log::error!( - "Could not wait for cursor sync file to complete: {}", - ErrorFmt(e) - ); + if let Some(sync) = self.cursor_sync.take() { + sync.signaled(&self.state.ring, "cursor").await; } self.await_present_fb(present_fb.as_mut(), PresentFbWait::Scanout) .await; @@ -353,19 +348,14 @@ impl MetalConnector { W::Render => &mut fb.copy.render_block, W::Scanout => &mut fb.copy.present_block, }; - let Some(sync_file) = field.take() else { + let Some(sync) = field.take() else { return; }; - if let Err(e) = self.state.ring.readable(&sync_file).await { - let name = match wait { - W::Render => "render", - W::Scanout => "scanout", - }; - log::error!( - "Could not wait for primary {name} sync file to complete: {}", - ErrorFmt(e), - ); - } + let name = match wait { + W::Render => "render", + W::Scanout => "scanout", + }; + sync.signaled(&self.state.ring, name).await; } fn try_async_flip(&self) -> bool { @@ -554,13 +544,13 @@ impl MetalConnector { self.state.present_hardware_cursor(node, &mut c); let swap_buffers = c.cursor_swap_buffer.is_some(); self.cursor_swap_buffer.set(swap_buffers); - if let Some(sf) = c.cursor_swap_buffer.take() { - let sf = c + if let Some(sync) = c.cursor_swap_buffer.take() { + let sync = c .cursor_buffer - .copy_to_dev(cd, None, sf) + .copy_to_dev(cd, None, sync) .map_err(MetalError::CopyToDev)? .present_block; - self.cursor_sync_file.set(sf); + self.cursor_sync.set(sync); } let mut cursor_changed = false; cursor_changed |= self.cursor_enabled.replace(c.cursor_enabled) != c.cursor_enabled; @@ -886,13 +876,13 @@ impl MetalConnector { tex = buffer.render.tex.clone(); } Some(dsd) => { - let sf = match &dsd.acquire_sync { + let sync = match &dsd.acquire_sync { AcquireSync::None => None, AcquireSync::Implicit => None, - AcquireSync::SyncFile { sync_file } => Some(sync_file.clone()), + AcquireSync::FdSync(sync) => Some(sync.clone()), AcquireSync::Unnecessary => None, }; - copy = RenderBufferCopy::for_both(sf); + copy = RenderBufferCopy::for_both(sync); fb = dsd.fb.clone(); tex = dsd.tex.clone(); } diff --git a/src/backends/metal/transaction.rs b/src/backends/metal/transaction.rs index 79e6d00e..8863a8a1 100644 --- a/src/backends/metal/transaction.rs +++ b/src/backends/metal/transaction.rs @@ -242,7 +242,7 @@ impl MetalDeviceTransaction { let mut unused_crtcs = BinarySearchMap::<_, _, SIZE>::new(); let mut unused_planes = BinarySearchMap::<_, _, SIZE>::new(); let mut crtc_planes = BinarySearchMap::<_, _, SIZE>::new(); - let mut sync_files = vec![]; + let mut syncs = vec![]; let slf = &mut self.common; for (_, crtc) in &mut slf.crtcs { crtc_planes.insert(crtc.obj.id, CrtcPlanes::default()); @@ -615,7 +615,7 @@ impl MetalDeviceTransaction { new_buffer.clear(&cd) }; match res { - Ok(sf) => sync_files.extend(sf), + Ok(sf) => syncs.extend(sf), Err(e) => { log::warn!("Could not clear new buffer: {}", ErrorFmt(e)); } @@ -653,7 +653,7 @@ impl MetalDeviceTransaction { match res { Ok(sf) => { buffer.locked.set(true); - sync_files.extend(sf); + syncs.extend(sf); } Err(e) => { log::error!( @@ -753,19 +753,8 @@ impl MetalDeviceTransaction { plane.new = DrmPlaneState::default(); } } - for sf in sync_files { - let mut pollfd = c::pollfd { - fd: sf.0.raw(), - events: c::POLLIN, - revents: 0, - }; - let res = uapi::poll(slice::from_mut(&mut pollfd), -1); - if let Err(e) = res { - log::warn!( - "Could not wait for sync file to become readable: {}", - ErrorFmt(e) - ); - } + for sync in syncs { + sync.signaled_blocking("transaction"); } Ok(MetalDeviceTransactionWithDrmState { common: self.common, diff --git a/src/backends/metal/video.rs b/src/backends/metal/video.rs index 8b70038d..a15681e0 100644 --- a/src/backends/metal/video.rs +++ b/src/backends/metal/video.rs @@ -26,7 +26,7 @@ use { drm_feedback::DrmFeedback, edid::{CtaDataBlock, Descriptor, EdidExtension}, format::{Format, XRGB8888}, - gfx_api::{GfxApi, GfxContext, GfxFramebuffer, SyncFile}, + gfx_api::{FdSync, GfxApi, GfxContext, GfxFramebuffer}, ifs::{ wl_output::OutputId, wp_presentation_feedback::{KIND_HW_COMPLETION, KIND_VSYNC, KIND_ZERO_COPY}, @@ -534,7 +534,7 @@ pub struct MetalConnector { pub cursor_enabled: Cell, pub cursor_buffers: CloneCell>>, pub cursor_swap_buffer: Cell, - pub cursor_sync_file: CloneCell>, + pub cursor_sync: CloneCell>, pub drm_feedback: CloneCell>>, pub scanout_buffers: RefCell>, @@ -565,7 +565,7 @@ pub struct MetalHardwareCursor { } pub struct MetalHardwareCursorChange<'a> { - pub cursor_swap_buffer: Option>, + pub cursor_swap_buffer: Option>, pub cursor_enabled: bool, pub cursor_x: i32, pub cursor_y: i32, @@ -603,8 +603,8 @@ impl HardwareCursorUpdate for MetalHardwareCursorChange<'_> { self.cursor_y = y; } - fn swap_buffer(&mut self, sync_file: Option) { - self.cursor_swap_buffer = Some(sync_file); + fn swap_buffer(&mut self, sync: Option) { + self.cursor_swap_buffer = Some(sync); } fn size(&self) -> (i32, i32) { @@ -1127,7 +1127,7 @@ fn create_connector( cursor_changed: Cell::new(false), cursor_damage: Cell::new(false), cursor_swap_buffer: Cell::new(false), - cursor_sync_file: Default::default(), + cursor_sync: Default::default(), drm_feedback: Default::default(), scanout_buffers: Default::default(), active_framebuffer: Default::default(), diff --git a/src/cli/screenshot.rs b/src/cli/screenshot.rs index 7402453b..8e8894e6 100644 --- a/src/cli/screenshot.rs +++ b/src/cli/screenshot.rs @@ -2,6 +2,7 @@ use { crate::{ allocator::{Allocator, AllocatorError, BufferUsage, MappedBuffer}, cli::{GlobalArgs, ScreenshotArgs, ScreenshotFormat}, + eventfd_cache::EventfdCache, format::XRGB8888, gfx_apis, tools::tool_client::{Handle, ToolClient, with_tool_client}, @@ -109,7 +110,8 @@ async fn run(screenshot: Rc) { } }; let format = screenshot.args.format; - let data = match buf_to_bytes(drm_dev.as_ref(), &buf, format) { + let eventfd_cache = EventfdCache::new(&tc.ring, &tc.eng); + let data = match buf_to_bytes(&eventfd_cache, drm_dev.as_ref(), &buf, format) { Ok(d) => d, Err(e) => fatal!("{}", ErrorFmt(e)), }; @@ -159,6 +161,7 @@ fn map( } pub fn buf_to_bytes( + eventfd_cache: &Rc, drm_dev: Option<&Rc>, buf: &DmaBuf, format: ScreenshotFormat, @@ -174,7 +177,7 @@ pub fn buf_to_bytes( .map_err(ScreenshotError::CreateGbmDevice) }); let vulkan = Box::new(move || { - gfx_apis::create_vulkan_allocator(&drm()?) + gfx_apis::create_vulkan_allocator(&drm()?, eventfd_cache) .map_err(ScreenshotError::CreateVulkanAllocator) }); allocators.push(vulkan); diff --git a/src/compositor.rs b/src/compositor.rs index 312ccbba..fa6e9835 100644 --- a/src/compositor.rs +++ b/src/compositor.rs @@ -26,6 +26,7 @@ use { damage::{DamageVisualizer, visualize_damage}, dbus::Dbus, ei::ei_client::EiClients, + eventfd_cache::EventfdCache, forker, format::XRGB8888, gfx_api::GfxApi, @@ -213,6 +214,7 @@ fn start_compositor2( let cpu_worker = Rc::new(CpuWorker::new(&ring, &engine)?); let color_manager = ColorManager::new(); let crit_ids = Rc::new(CritMatcherIds::default()); + let eventfd_cache = EventfdCache::new(&ring, &engine); let state = Rc::new(State { kb_ctx, backend: CloneCell::new(Rc::new(DummyBackend)), @@ -372,8 +374,9 @@ fn start_compositor2( outputs_without_hc: Default::default(), udmabuf: Default::default(), gfx_ctx_changed: Default::default(), - copy_device_registry: Rc::new(CopyDeviceRegistry::new(&ring, &engine)), + copy_device_registry: Rc::new(CopyDeviceRegistry::new(&ring, &engine, &eventfd_cache)), supports_presentation_feedback: Default::default(), + eventfd_cache, }); state.tracker.register(ClientId::from_raw(0)); create_dummy_output(&state); diff --git a/src/copy_device.rs b/src/copy_device.rs index effbbe1b..9313ed03 100644 --- a/src/copy_device.rs +++ b/src/copy_device.rs @@ -1,20 +1,24 @@ use { crate::{ async_engine::{AsyncEngine, SpawnedFuture}, + eventfd_cache::EventfdCache, format::{FORMATS, Format}, - gfx_api::SyncFile, + gfx_api::FdSync, io_uring::IoUring, rect::{Rect, Region}, utils::{ - clonecell::CloneCell, copyhashmap::CopyHashMap, errorfmt::ErrorFmt, queue::AsyncQueue, - stack::Stack, + clonecell::CloneCell, copyhashmap::CopyHashMap, errorfmt::ErrorFmt, numcell::NumCell, + oserror::OsError, queue::AsyncQueue, stack::Stack, }, video::{ LINEAR_MODIFIER, LINEAR_STRIDE_ALIGN, Modifier, dmabuf::{DmaBuf, DmaBufIds, DmaBufPlane, PlaneVec}, + drm::{NodeType, get_drm_nodes_from_dev, syncobj::SyncobjCtx}, }, vulkan_core::{ - VULKAN_API_VERSION, VulkanCoreError, VulkanCoreInstance, map_extension_properties, + self, VULKAN_API_VERSION, VulkanCoreError, VulkanCoreInstance, device::VulkanDeviceInf, + map_extension_properties, sync::VulkanDeviceSyncExt, + timeline_semaphore::VulkanDeviceTimelineSemaphoreExt, }, }, ahash::{AHashMap, AHashSet}, @@ -34,14 +38,13 @@ use { CommandPoolCreateInfo, CopyBufferInfo2, CopyBufferToImageInfo2, CopyImageInfo2, CopyImageToBufferInfo2, DependencyInfo, DeviceCreateInfo, DeviceMemory, DeviceQueueCreateInfo, DrmFormatModifierPropertiesEXT, - DrmFormatModifierPropertiesListEXT, ExportFenceCreateInfo, ExportMemoryAllocateInfo, - Extent3D, ExternalBufferProperties, ExternalFenceFeatureFlags, - ExternalFenceHandleTypeFlags, ExternalFenceProperties, - ExternalImageFormatPropertiesKHR, ExternalMemoryBufferCreateInfo, - ExternalMemoryBufferCreateInfoKHR, ExternalMemoryFeatureFlags, - ExternalMemoryHandleTypeFlags, ExternalMemoryImageCreateInfo, - ExternalSemaphoreFeatureFlags, ExternalSemaphoreHandleTypeFlags, - ExternalSemaphoreProperties, Fence, FenceCreateInfo, FenceGetFdInfoKHR, Filter, + DrmFormatModifierPropertiesListEXT, ExportMemoryAllocateInfo, Extent3D, + ExternalBufferProperties, ExternalFenceFeatureFlags, ExternalFenceHandleTypeFlags, + ExternalFenceProperties, ExternalImageFormatPropertiesKHR, + ExternalMemoryBufferCreateInfo, ExternalMemoryBufferCreateInfoKHR, + ExternalMemoryFeatureFlags, ExternalMemoryHandleTypeFlags, + ExternalMemoryImageCreateInfo, ExternalSemaphoreFeatureFlags, + ExternalSemaphoreHandleTypeFlags, ExternalSemaphoreProperties, Filter, FormatFeatureFlags, FormatProperties2, ImageAspectFlags, ImageBlit2, ImageCopy2, ImageCreateFlags, ImageCreateInfo, ImageDrmFormatModifierExplicitCreateInfoEXT, ImageFormatProperties2, ImageLayout, ImageMemoryBarrier2, ImageMemoryRequirementsInfo2, @@ -54,9 +57,10 @@ use { PhysicalDeviceExternalImageFormatInfoKHR, PhysicalDeviceExternalSemaphoreInfo, PhysicalDeviceFeatures2, PhysicalDeviceImageDrmFormatModifierInfoEXT, PhysicalDeviceImageFormatInfo2, PhysicalDeviceProperties2, - PhysicalDeviceSynchronization2Features, PipelineStageFlags2, QUEUE_FAMILY_FOREIGN_EXT, - Queue, QueueFlags, SampleCountFlags, SemaphoreCreateInfo, SemaphoreImportFlags, - SemaphoreSubmitInfo, SharingMode, SubmitInfo2, SubresourceLayout, WHOLE_SIZE, + PhysicalDeviceSynchronization2Features, PhysicalDeviceTimelineSemaphoreFeatures, + PipelineStageFlags2, QUEUE_FAMILY_FOREIGN_EXT, Queue, QueueFlags, SampleCountFlags, + SemaphoreCreateInfo, SemaphoreImportFlags, SemaphoreSubmitInfo, SharingMode, + SubmitInfo2, SubresourceLayout, WHOLE_SIZE, }, }, bstr::ByteSlice, @@ -84,16 +88,12 @@ pub enum CopyDeviceError { Core(#[from] VulkanCoreError), #[error("Could not create a semaphore")] CreateSemaphore(#[source] vk::Result), - #[error("Could not create a fence")] - CreateFence(#[source] vk::Result), #[error("Could not dup a sync file")] DupSyncFile(#[source] io::Error), #[error("Could not dup a dma buf")] DupDmaBuf(#[source] io::Error), #[error("Could not import a sync file")] ImportSyncFile(#[source] vk::Result), - #[error("Could not export a sync file")] - ExportSyncFile(#[source] vk::Result), #[error("Could not submit the copy")] SubmitCopy(#[source] vk::Result), #[error("Could not enumerate the physical devices")] @@ -166,6 +166,12 @@ pub enum CopyDeviceError { BothOffDevice, #[error("Cannot blit between these formats")] BlitNotSupported, + #[error("Could not get DRM nodes")] + GetDrmNodes(#[source] OsError), + #[error("Device has no device nodes")] + NoDeviceNodes, + #[error("Could not open device node")] + OpenDeviceNode(#[source] OsError), } type Keyed = StaticMap; @@ -174,12 +180,15 @@ type KeyedCopy = StaticCopyMap; pub struct PhysicalCopyDevice { ring: Rc, eng: Rc, + eventfd_cache: Rc, + sync_ctx: Rc, instance: VulkanCoreInstance, physical_device: PhysicalDevice, support: AHashMap>>, queues_to_allocate: Vec, queues: KeyedCopy, supports_dmabuf_export: bool, + supports_timeline_opaque_export: bool, memory_types: Vec, rects: RefCell>, buffer_copy_2: RefCell>>, @@ -205,6 +214,7 @@ struct QueueIndex { pub struct CopyDevice { _tasks: Vec>, dev: Rc, + timeline_semaphore: Option>, } struct CopyDeviceInner { @@ -217,7 +227,7 @@ struct CopyDeviceInner { external_fence_fd: external_fence_fd::Device, external_memory_fd: external_memory_fd::Device, semaphores: Stack, - fences: Stack, + fences: Stack>, submissions: Keyed>, } @@ -229,12 +239,13 @@ struct PendingSubmissions { pub struct CopyDeviceCopy { inner: Rc, - _dev: Rc, + dev: Rc, } struct CopyDeviceCopyInner { dev: Rc, - busy: CloneCell>, + busy_id: NumCell, + busy: CloneCell>, width: u32, height: u32, command_buffer: CommandBuffer, @@ -273,10 +284,11 @@ enum CopyDeviceCopyType { struct Pending { dev: Rc, - sync_file: Option, + busy_id: u64, + sync: Option, copy: Rc, semaphore: Option, - fence: Option, + vulkan_sync: VulkanSync, } struct VulkanSemaphore { @@ -284,10 +296,10 @@ struct VulkanSemaphore { semaphore: Semaphore, } -struct VulkanFence { - dev: Rc, - fence: Fence, -} +type VulkanFence = vulkan_core::fence::VulkanFence; +type VulkanTimelineSemaphore = + vulkan_core::timeline_semaphore::VulkanTimelineSemaphore; +type VulkanSync = vulkan_core::sync::VulkanSync; struct VulkanBuffer { dev: Rc, @@ -340,6 +352,7 @@ struct ClassifiedDmabuf<'a> { pub struct CopyDeviceRegistry { ring: Rc, eng: Rc, + eventfd_cache: Rc, devs: CopyHashMap>>, } @@ -356,6 +369,7 @@ impl PhysicalCopyDevice { fn new( ring: &Rc, eng: &Rc, + eventfd_cache: &Rc, dev: c::dev_t, ) -> Result, CopyDeviceError> { let core_instance = VulkanCoreInstance::new(Level::Debug)?; @@ -408,6 +422,16 @@ impl PhysicalCopyDevice { } return Err(CopyDeviceError::NoVulkanDevice); } + let nodes = get_drm_nodes_from_dev(uapi::major(dev), uapi::minor(dev)) + .map_err(CopyDeviceError::GetDrmNodes)?; + let path = nodes + .get(&NodeType::Render) + .or_else(|| nodes.get(&NodeType::Primary)) + .ok_or(CopyDeviceError::NoDeviceNodes)?; + let device_fd = uapi::open(path.as_c_str(), c::O_RDWR, 0) + .map(Rc::new) + .map_err(Into::into) + .map_err(CopyDeviceError::OpenDeviceNode)?; if device_properties.api_version < VULKAN_API_VERSION { return Err(CopyDeviceError::NoVulkan13); } @@ -610,15 +634,21 @@ impl PhysicalCopyDevice { } let memory_info = unsafe { instance.get_physical_device_memory_properties(physical_device) }; + let features = core_instance.get_features(physical_device); + let supports_timeline_opaque_export = + core_instance.supports_timeline_opaque_export(physical_device, &features); let dev = Rc::new(PhysicalCopyDevice { ring: ring.clone(), eng: eng.clone(), + eventfd_cache: eventfd_cache.clone(), + sync_ctx: Rc::new(SyncobjCtx::new(&device_fd)), instance: core_instance, physical_device, support, queues_to_allocate, queues: queue_indices, supports_dmabuf_export, + supports_timeline_opaque_export, memory_types: memory_info.memory_types_as_slice().to_vec(), rects: Default::default(), buffer_copy_2: Default::default(), @@ -658,11 +688,14 @@ impl PhysicalCopyDevice { }) .collect(); let extensions = DEVICE_EXTENSIONS.map(|e| e.as_ptr()); + let mut semaphore_features = PhysicalDeviceTimelineSemaphoreFeatures::default() + .timeline_semaphore(self.supports_timeline_opaque_export); let mut synchronization2_features = PhysicalDeviceSynchronization2Features::default().synchronization2(true); let info = DeviceCreateInfo::default() .queue_create_infos(&queue_create_info) .enabled_extension_names(&extensions) + .push_next(&mut semaphore_features) .push_next(&mut synchronization2_features); unsafe { instance @@ -726,7 +759,11 @@ impl PhysicalCopyDevice { let task = self.eng.spawn("copy-device-await-pending", future); tasks.push(task); } - let queue = Rc::new(CopyDevice { dev, _tasks: tasks }); + let queue = Rc::new(CopyDevice { + timeline_semaphore: dev.create_timeline_semaphore_or_log(), + dev, + _tasks: tasks, + }); Ok(queue) } } @@ -740,8 +777,8 @@ async fn wait_for_submissions( submissions.task_has_pending.set(false); let pending = submissions.pending.pop().await; submissions.task_has_pending.set(true); - if let Some(sync_file) = &pending.sync_file - && let Err(e) = ring.readable(sync_file).await + if let Some(sync) = &pending.sync + && let Err(e) = sync.try_signaled(&ring).await { log::warn!( "Could not wait for sync file to become readable: {}", @@ -749,6 +786,7 @@ async fn wait_for_submissions( ); dev.wait_idle(); } + pending.vulkan_sync.handle_validation(); } } @@ -1124,6 +1162,7 @@ impl CopyDevice { Ok(CopyDeviceCopy { inner: Rc::new(CopyDeviceCopyInner { dev: self.dev.clone(), + busy_id: Default::default(), busy: Default::default(), width: src.width as _, height: src.height as _, @@ -1131,7 +1170,7 @@ impl CopyDevice { tt, ty, }), - _dev: self.clone(), + dev: self.clone(), }) } @@ -1258,36 +1297,14 @@ impl CopyDeviceInner { semaphore, }) } - - fn create_fence(self: &Rc) -> Result { - let mut export_info = - ExportFenceCreateInfo::default().handle_types(ExternalFenceHandleTypeFlags::SYNC_FD); - let create_info = FenceCreateInfo::default().push_next(&mut export_info); - let fence = unsafe { - self.dev - .create_fence(&create_info, None) - .map_err(CopyDeviceError::CreateFence)? - }; - Ok(VulkanFence { - dev: self.clone(), - fence, - }) - } } impl CopyDeviceCopy { fn ensure_not_busy(&self) -> Result<(), CopyDeviceError> { let slf = &*self.inner; - let Some(busy) = slf.busy.get() else { - return Ok(()); - }; - let mut pollfd = c::pollfd { - fd: busy.raw(), - events: c::POLLIN, - revents: 0, - }; - let res = uapi::poll(slice::from_mut(&mut pollfd), 0); - if res != Ok(1) { + if let Some(sync) = slf.busy.get() + && sync.is_unsignaled() + { return Err(CopyDeviceError::Busy); } slf.busy.take(); @@ -1296,9 +1313,9 @@ impl CopyDeviceCopy { pub fn execute( &self, - sync_file: Option<&SyncFile>, + sync: Option<&FdSync>, region: Option<&Region>, - ) -> Result, CopyDeviceError> { + ) -> Result, CopyDeviceError> { self.ensure_not_busy()?; let slf = &*self.inner; let tt = slf.tt; @@ -1656,7 +1673,9 @@ impl CopyDeviceCopy { } let mut wait_semaphore = None; let mut wait_semaphores = ArrayVec::<_, 1>::new(); - if let Some(sync_file) = sync_file { + if let Some(sync) = sync + && let Some(sync_file) = sync.get_sync_file() + { let semaphore = match slf.dev.semaphores.pop() { Some(s) => s, _ => slf.dev.create_semaphore()?, @@ -1668,42 +1687,38 @@ impl CopyDeviceCopy { wait_semaphores.push(info); wait_semaphore = Some(semaphore); } - let signal_fence = match slf.dev.fences.pop() { - Some(s) => s, - _ => slf.dev.create_fence()?, - }; let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd); - let submit_info = SubmitInfo2::default() + let mut semaphore_submit_info = SemaphoreSubmitInfo::default(); + let mut submit_info = SubmitInfo2::default() .command_buffer_infos(slice::from_ref(&command_buffer_info)) .wait_semaphore_infos(&wait_semaphores); + let vulkan_sync = slf.dev.create_sync( + self.dev.timeline_semaphore.as_ref(), + &mut semaphore_submit_info, + &mut submit_info, + )?; unsafe { slf.dev .dev .queue_submit2( slf.dev.queues[tt], slice::from_ref(&submit_info), - signal_fence.fence, + vulkan_sync.fence(), ) .map_err(CopyDeviceError::SubmitCopy)?; } - let sync_file = match signal_fence.export() { - Ok(f) => f, - Err(e) => { - log::error!("Could not export signal fence: {}", ErrorFmt(e)); - slf.dev.wait_idle(); - None - } - }; - slf.busy.set(sync_file.clone()); + let sync = vulkan_sync.to_sync(|| slf.dev.wait_idle()); + slf.busy.set(sync.clone()); let pending = Pending { dev: slf.dev.clone(), - sync_file: sync_file.clone(), + busy_id: slf.busy_id.add_fetch(1), + sync: sync.clone(), copy: self.inner.clone(), semaphore: wait_semaphore, - fence: Some(signal_fence), + vulkan_sync, }; slf.dev.submissions[tt].pending.push(pending); - Ok(sync_file) + Ok(sync) } } @@ -1728,31 +1743,16 @@ impl VulkanSemaphore { } } -impl VulkanFence { - fn export(&self) -> Result, CopyDeviceError> { - let info = FenceGetFdInfoKHR::default() - .fence(self.fence) - .handle_type(ExternalFenceHandleTypeFlags::SYNC_FD); - let fd = unsafe { - self.dev - .external_fence_fd - .get_fence_fd(&info) - .map_err(CopyDeviceError::ExportSyncFile)? - }; - let fd = if fd == -1 { - None - } else { - Some(SyncFile(Rc::new(OwnedFd::new(fd)))) - }; - Ok(fd) - } -} - impl CopyDeviceRegistry { - pub fn new(ring: &Rc, eng: &Rc) -> Self { + pub fn new( + ring: &Rc, + eng: &Rc, + eventfd_cache: &Rc, + ) -> Self { Self { ring: ring.clone(), eng: eng.clone(), + eventfd_cache: eventfd_cache.clone(), devs: Default::default(), } } @@ -1765,7 +1765,7 @@ impl CopyDeviceRegistry { if let Some(dev) = self.devs.get(&dev) { return dev; } - match PhysicalCopyDevice::new(&self.ring, &self.eng, dev).map(Some) { + match PhysicalCopyDevice::new(&self.ring, &self.eng, &self.eventfd_cache, dev).map(Some) { Ok(cd) => { self.devs.set(dev, cd.clone()); cd @@ -1792,14 +1792,6 @@ impl Drop for VulkanSemaphore { } } -impl Drop for VulkanFence { - fn drop(&mut self) { - unsafe { - self.dev.dev.destroy_fence(self.fence, None); - } - } -} - impl Drop for CopyDeviceCopyInner { fn drop(&mut self) { unsafe { @@ -1842,10 +1834,7 @@ impl Drop for Pending { if let Some(v) = self.semaphore.take() { self.dev.semaphores.push(v); } - if let Some(v) = self.fence.take() { - self.dev.fences.push(v); - } - if self.copy.busy.get() == self.sync_file { + if self.copy.busy_id.get() == self.busy_id { self.copy.busy.take(); } } @@ -2037,3 +2026,33 @@ fn allocate_queues( }; (queues_to_allocate, queue_indices) } + +impl VulkanDeviceInf for CopyDeviceInner { + fn instance(&self) -> &VulkanCoreInstance { + &self.phy.instance + } + + fn device(&self) -> &Device { + &self.dev + } + + fn external_fence_fd(&self) -> &external_fence_fd::Device { + &self.external_fence_fd + } + + fn external_semaphore_fd(&self) -> &external_semaphore_fd::Device { + &self.external_semaphore_fd + } + + fn supports_timeline_opaque_export(&self) -> bool { + self.phy.supports_timeline_opaque_export + } + + fn sync_ctx(&self) -> &Rc { + &self.phy.sync_ctx + } + + fn eventfd_cache(&self) -> &Rc { + &self.phy.eventfd_cache + } +} diff --git a/src/cursor_user.rs b/src/cursor_user.rs index 3b783636..6d911048 100644 --- a/src/cursor_user.rs +++ b/src/cursor_user.rs @@ -514,8 +514,8 @@ impl CursorUser { &cd, ); match res { - Ok(sync_file) => { - hc.swap_buffer(sync_file); + Ok(sync) => { + hc.swap_buffer(sync); } Err(e) => { log::error!("Could not render hardware cursor: {}", ErrorFmt(e)); diff --git a/src/eventfd_cache.rs b/src/eventfd_cache.rs new file mode 100644 index 00000000..de9f2000 --- /dev/null +++ b/src/eventfd_cache.rs @@ -0,0 +1,154 @@ +use { + crate::{ + async_engine::{AsyncEngine, SpawnedFuture}, + io_uring::{IoUring, IoUringError}, + utils::{buf::Buf, errorfmt::ErrorFmt, oserror::OsError, queue::AsyncQueue, stack::Stack}, + }, + std::{cell::Cell, future::poll_fn, pin::Pin, rc::Rc, slice, task::Poll}, + thiserror::Error, + uapi::{OwnedFd, c}, +}; + +#[cfg(test)] +mod tests; + +#[derive(Debug, Error)] +pub enum EventfdError { + #[error("Could not create an eventfd")] + CreateEventfd(#[source] OsError), +} + +pub struct EventfdCache { + inner: Rc, + _task: SpawnedFuture<()>, +} + +struct Inner { + ring: Rc, + fds: Stack>, + recycle: AsyncQueue>, +} + +pub struct Eventfd { + cache: Rc, + pub fd: Rc, + signaled: Cell, +} + +impl EventfdCache { + pub fn new(ring: &Rc, eng: &Rc) -> Rc { + let inner = Rc::new(Inner { + ring: ring.clone(), + fds: Default::default(), + recycle: Default::default(), + }); + let task = eng.spawn("eventfd-cache", inner.clone().recycle()); + Rc::new(Self { inner, _task: task }) + } + + pub fn acquire(&self) -> Result { + let fd = match self.inner.fds.pop() { + Some(fd) => fd, + _ => uapi::eventfd(0, c::EFD_CLOEXEC) + .map(Rc::new) + .map_err(Into::into) + .map_err(EventfdError::CreateEventfd)?, + }; + Ok(Eventfd { + cache: self.inner.clone(), + fd, + signaled: Default::default(), + }) + } +} + +impl Eventfd { + pub fn is_signaled(&self) -> bool { + self.signaled.get() + } + + pub async fn signaled(&self) -> Result<(), IoUringError> { + if self.signaled.get() { + return Ok(()); + } + self.cache.ring.readable(&self.fd).await?; + self.signaled.set(true); + Ok(()) + } + + pub fn signaled_blocking(&self) -> Result<(), OsError> { + if self.signaled.get() { + return Ok(()); + } + let mut pollfd = c::pollfd { + fd: self.fd.raw(), + events: c::POLLIN, + revents: 0, + }; + uapi::poll(slice::from_mut(&mut pollfd), -1)?; + self.signaled.set(true); + Ok(()) + } +} + +impl Inner { + async fn recycle(self: Rc) { + let slf = &*self; + let mut fds = vec![]; + let mut bufs = vec![]; + let mut tasks = vec![]; + let mut todo = vec![]; + loop { + fds.clear(); + tasks.clear(); + todo.clear(); + slf.recycle.non_empty().await; + while let Some(fd) = slf.recycle.try_pop() { + fds.push(fd); + } + for (idx, fd) in fds.iter().enumerate() { + if idx >= bufs.len() { + bufs.push(Buf::new(size_of::())); + } + let fd = fd.clone(); + let buf = bufs[idx].clone(); + tasks.push(async move { slf.ring.read(&fd, buf).await }); + todo.push(idx); + } + poll_fn(|ctx| { + let mut i = 0; + while i < todo.len() { + let idx = todo[i]; + let task = unsafe { Pin::new_unchecked(&mut tasks[idx]) }; + if let Poll::Ready(res) = task.poll(ctx) { + todo.swap_remove(i); + match res { + Ok(_) => { + self.fds.push(fds[idx].clone()); + } + Err(e) => { + log::error!("Could not read from eventfd: {}", ErrorFmt(e)); + } + } + } else { + i += 1; + } + } + if todo.is_empty() { + Poll::Ready(()) + } else { + Poll::Pending + } + }) + .await; + } + } +} + +impl Drop for Eventfd { + fn drop(&mut self) { + if self.signaled.get() { + self.cache.recycle.push(self.fd.clone()); + } + } +} diff --git a/src/eventfd_cache/tests.rs b/src/eventfd_cache/tests.rs new file mode 100644 index 00000000..2c536f6d --- /dev/null +++ b/src/eventfd_cache/tests.rs @@ -0,0 +1,66 @@ +use { + crate::{ + async_engine::AsyncEngine, eventfd_cache::EventfdCache, io_uring::IoUring, utils::array, + }, + std::{rc::Rc, slice}, + uapi::c, +}; + +#[test] +fn test() { + let eng = AsyncEngine::new(); + let ring = IoUring::new(&eng, 32).unwrap(); + let cache = Rc::new(EventfdCache::new(&ring, &eng)); + const TOTAL: usize = 5; + let signaled = 3; + let fd1: [_; TOTAL] = array::from_fn(|_| cache.acquire().unwrap()); + let fd2: [_; TOTAL] = array::from_fn(|_| cache.acquire().unwrap()); + for fd in fd1.iter().chain(fd2.iter()) { + uapi::eventfd_write(fd.fd.raw(), 1).unwrap(); + let mut poll = c::pollfd { + fd: fd.fd.raw(), + events: c::POLLIN, + revents: 0, + }; + uapi::poll(slice::from_mut(&mut poll), 0).unwrap(); + assert_eq!(poll.revents, c::POLLIN); + } + assert_eq!(cache.inner.fds.len(), 0); + let ring2 = ring.clone(); + let cache2 = cache.clone(); + let _fut1 = eng.spawn("", async move { + for i in 0..signaled { + fd1[i].signaled().await.unwrap(); + } + drop(fd1); + let debouncer = ring2.debouncer(0); + while cache2.inner.fds.len() != signaled { + debouncer.debounce().await; + } + for i in 0..signaled { + fd2[i].signaled().await.unwrap(); + } + drop(fd2); + while cache2.inner.fds.len() != 2 * signaled { + debouncer.debounce().await; + } + ring2.stop(); + }); + let now_nsec = eng.now().nsec(); + let ring2 = ring.clone(); + let _fut2 = eng.spawn("", async move { + ring2.timeout(now_nsec + 1_000_000_000).await.unwrap(); + ring2.stop(); + }); + ring.run().unwrap(); + assert_eq!(cache.inner.fds.len(), 2 * signaled); + for fd in cache.inner.fds.take() { + let mut poll = c::pollfd { + fd: fd.raw(), + events: c::POLLIN, + revents: 0, + }; + uapi::poll(slice::from_mut(&mut poll), 0).unwrap(); + assert_eq!(poll.revents, 0); + } +} diff --git a/src/gfx_api.rs b/src/gfx_api.rs index 45f9d91f..f178b753 100644 --- a/src/gfx_api.rs +++ b/src/gfx_api.rs @@ -5,16 +5,22 @@ use { cpu_worker::CpuWorker, cursor::Cursor, damage::DamageVisualizer, + eventfd_cache::Eventfd, fixed::Fixed, format::Format, + io_uring::{IoUring, IoUringError}, rect::{Rect, Region}, renderer::{Renderer, renderer_base::RendererBase}, scale::Scale, state::State, theme::Color, tree::{Node, OutputNode, Transform}, - utils::clonecell::UnsafeCellCloneSafe, - video::{Modifier, dmabuf::DmaBuf, drm::syncobj::SyncobjCtx}, + utils::{clonecell::UnsafeCellCloneSafe, errorfmt::ErrorFmt}, + video::{ + Modifier, + dmabuf::DmaBuf, + drm::syncobj::{Syncobj, SyncobjCtx, SyncobjPoint}, + }, }, ahash::AHashMap, indexmap::{IndexMap, IndexSet}, @@ -22,16 +28,17 @@ use { linearize::Linearize, std::{ any::Any, - cell::Cell, + cell::{Cell, OnceCell}, error::Error, ffi::CString, fmt::{Debug, Formatter}, ops::Deref, rc::Rc, + slice, sync::atomic::{AtomicU64, Ordering::Relaxed}, }, thiserror::Error, - uapi::OwnedFd, + uapi::{OwnedFd, c}, }; #[derive(Copy, Clone, Debug, Hash, Eq, PartialEq, Linearize)] @@ -285,15 +292,24 @@ unsafe impl UnsafeCellCloneSafe for SyncFile {} pub enum AcquireSync { None, Implicit, - SyncFile { sync_file: SyncFile }, + FdSync(FdSync), Unnecessary, } impl AcquireSync { - pub fn from_sync_file(sync_file: Option) -> Self { - match sync_file { + pub fn from_fd_sync(sync: Option) -> Self { + match sync { None => Self::Unnecessary, - Some(sync_file) => Self::SyncFile { sync_file }, + Some(sync) => Self::FdSync(sync), + } + } + + pub fn get_sync_file(&self) -> Option<&SyncFile> { + match self { + Self::None => None, + Self::Implicit => None, + Self::FdSync(sync) => sync.get_sync_file(), + Self::Unnecessary => None, } } } @@ -310,7 +326,7 @@ impl Debug for AcquireSync { let name = match self { AcquireSync::None => "None", AcquireSync::Implicit => "Implicit", - AcquireSync::SyncFile { .. } => "SyncFile", + AcquireSync::FdSync(d) => return Debug::fmt(d, f), AcquireSync::Unnecessary => "Unnecessary", }; f.debug_struct(name).finish_non_exhaustive() @@ -318,7 +334,7 @@ impl Debug for AcquireSync { } pub trait BufferResv: Debug { - fn set_sync_file(&self, user: BufferResvUser, sync_file: &SyncFile); + fn set_sync(&self, user: BufferResvUser, sync: &FdSync); } #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] @@ -363,7 +379,7 @@ pub trait GfxFramebuffer: Debug { region: &Region, blend_buffer: Option<&Rc>, blend_cd: &Rc, - ) -> Result, GfxError>; + ) -> Result, GfxError>; fn format(&self) -> &'static Format; @@ -398,7 +414,7 @@ impl dyn GfxFramebuffer { clear_cd: &Rc, blend_buffer: Option<&Rc>, blend_cd: &Rc, - ) -> Result, GfxError> { + ) -> Result, GfxError> { self.clone().render_with_region( acquire_sync, release_sync, @@ -417,7 +433,7 @@ impl dyn GfxFramebuffer { acquire_sync: AcquireSync, release_sync: ReleaseSync, cd: &Rc, - ) -> Result, GfxError> { + ) -> Result, GfxError> { self.clear_with( acquire_sync, release_sync, @@ -434,7 +450,7 @@ impl dyn GfxFramebuffer { cd: &Rc, color: &Color, color_cd: &Rc, - ) -> Result, GfxError> { + ) -> Result, GfxError> { self.render( acquire_sync, release_sync, @@ -472,7 +488,7 @@ impl dyn GfxFramebuffer { release_sync: ReleaseSync, x: i32, y: i32, - ) -> Result, GfxError> { + ) -> Result, GfxError> { let mut ops = vec![]; let scale = Scale::from_int(1); let mut renderer = self.renderer_base(&mut ops, scale, Transform::None); @@ -516,7 +532,7 @@ impl dyn GfxFramebuffer { blend_buffer: Option<&Rc>, blend_cd: &Rc, f: &mut dyn FnMut(&mut RendererBase), - ) -> Result, GfxError> { + ) -> Result, GfxError> { let mut ops = vec![]; let mut renderer = self.renderer_base(&mut ops, scale, Transform::None); f(&mut renderer); @@ -569,7 +585,7 @@ impl dyn GfxFramebuffer { region: &Region, blend_buffer: Option<&Rc>, blend_cd: &Rc, - ) -> Result, GfxError> { + ) -> Result, GfxError> { self.clone().render_with_region( acquire_sync, release_sync, @@ -596,7 +612,7 @@ impl dyn GfxFramebuffer { fill_black_in_grace_period: bool, blend_buffer: Option<&Rc>, blend_cd: &Rc, - ) -> Result, GfxError> { + ) -> Result, GfxError> { self.render_node( acquire_sync, release_sync, @@ -631,7 +647,7 @@ impl dyn GfxFramebuffer { transform: Transform, blend_buffer: Option<&Rc>, blend_cd: &Rc, - ) -> Result, GfxError> { + ) -> Result, GfxError> { let pass = self.create_render_pass( node, state, @@ -664,7 +680,7 @@ impl dyn GfxFramebuffer { scale: Scale, transform: Transform, cd: &Rc, - ) -> Result, GfxError> { + ) -> Result, GfxError> { let mut ops = vec![]; let mut renderer = Renderer { base: self.renderer_base(&mut ops, scale, transform), @@ -1087,3 +1103,108 @@ pub fn renderer_base<'a>( pub fn logical_size(physical_size: (i32, i32), transform: Transform) -> (i32, i32) { transform.maybe_swap(physical_size) } + +pub struct ReservedSyncobjPoint { + pub ctx: Rc, + pub syncobj: Rc, + pub point: SyncobjPoint, + pub sync_file: OnceCell>, + pub signaled: Eventfd, +} + +impl Debug for ReservedSyncobjPoint { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReservedSyncobjPoint") + .field("syncobj", &self.syncobj.id()) + .field("point", &self.point) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Debug)] +pub enum FdSync { + SyncFile(SyncFile), + Syncobj(Rc), +} + +unsafe impl UnsafeCellCloneSafe for FdSync {} + +impl FdSync { + pub async fn try_signaled(&self, ring: &Rc) -> Result<(), IoUringError> { + match self { + FdSync::SyncFile(f) => ring.readable(&f.0).await.map(drop), + FdSync::Syncobj(obj) => obj.signaled.signaled().await, + } + } + + pub async fn signaled(&self, ring: &Rc, name: &str) { + if let Err(e) = self.try_signaled(ring).await { + log::error!( + "Could not wait for {name} sync to become signaled: {}", + ErrorFmt(e), + ); + } + } + + pub fn signaled_blocking(&self, name: &str) { + let res = match self { + FdSync::Syncobj(obj) => obj.signaled.signaled_blocking(), + FdSync::SyncFile(f) => { + let mut pollfd = c::pollfd { + fd: f.raw(), + events: c::POLLIN, + revents: 0, + }; + uapi::poll(slice::from_mut(&mut pollfd), -1) + .map(drop) + .map_err(Into::into) + } + }; + if let Err(e) = res { + log::error!( + "Could not wait for {name} sync to become signaled: {}", + ErrorFmt(e), + ); + } + } + + pub fn is_unsignaled(&self) -> bool { + !self.is_signaled() + } + + pub fn is_signaled(&self) -> bool { + match self { + FdSync::Syncobj(obj) => obj.signaled.is_signaled(), + FdSync::SyncFile(f) => { + let mut pollfd = c::pollfd { + fd: f.raw(), + events: c::POLLIN, + revents: 0, + }; + uapi::poll(slice::from_mut(&mut pollfd), 0) == Ok(1) + } + } + } + + pub fn get_sync_file(&self) -> Option<&SyncFile> { + match self { + FdSync::SyncFile(f) => Some(f), + FdSync::Syncobj(obj) => { + if obj.signaled.is_signaled() { + return None; + } + obj.sync_file + .get_or_init(|| { + match obj.ctx.export_sync_file_blocking(&obj.syncobj, obj.point) { + Ok(sf) => Some(sf), + Err(e) => { + log::error!("Could not export sync file: {}", ErrorFmt(e)); + None + } + } + }) + .as_ref() + } + } + } +} diff --git a/src/gfx_apis.rs b/src/gfx_apis.rs index f0d1aaa2..bb9b48a9 100644 --- a/src/gfx_apis.rs +++ b/src/gfx_apis.rs @@ -2,6 +2,7 @@ pub use vulkan::create_vulkan_allocator; use { crate::{ async_engine::AsyncEngine, + eventfd_cache::EventfdCache, gfx_api::{GfxApi, GfxContext, GfxError}, io_uring::IoUring, pr_caps::PrCapsThread, @@ -17,6 +18,7 @@ mod vulkan; pub fn create_gfx_context( eng: &Rc, ring: &Rc, + eventfd_cache: &Rc, drm: &Drm, api: GfxApi, caps_thread: Option<&PrCapsThread>, @@ -26,7 +28,8 @@ pub fn create_gfx_context( let mut last_err = None; for software in [false, true] { for api in apis { - let res = create_gfx_context_(eng, ring, drm, api, caps_thread, software); + let res = + create_gfx_context_(eng, ring, eventfd_cache, drm, api, caps_thread, software); match res { Ok(_) => { log::info!("Created a {api:?} renderer"); @@ -48,6 +51,7 @@ pub fn create_gfx_context( fn create_gfx_context_( eng: &Rc, ring: &Rc, + eventfd_cache: &Rc, drm: &Drm, api: GfxApi, caps_thread: Option<&PrCapsThread>, @@ -55,6 +59,8 @@ fn create_gfx_context_( ) -> Result, GfxError> { match api { GfxApi::OpenGl => gl::create_gfx_context(drm, software), - GfxApi::Vulkan => vulkan::create_graphics_context(eng, ring, drm, caps_thread, software), + GfxApi::Vulkan => { + vulkan::create_graphics_context(eng, ring, eventfd_cache, drm, caps_thread, software) + } } } diff --git a/src/gfx_apis/gl.rs b/src/gfx_apis/gl.rs index ff6f7aed..1afd8c9d 100644 --- a/src/gfx_apis/gl.rs +++ b/src/gfx_apis/gl.rs @@ -69,8 +69,8 @@ use { crate::{ cmm::cmm_eotf::Eotf, gfx_api::{ - AcquireSync, CopyTexture, FramebufferRect, GfxApiOpt, GfxContext, GfxError, GfxTexture, - ReleaseSync, SyncFile, + AcquireSync, CopyTexture, FdSync, FramebufferRect, GfxApiOpt, GfxContext, GfxError, + GfxTexture, ReleaseSync, SyncFile, }, gfx_apis::gl::{ egl::image::EglImage, @@ -221,7 +221,7 @@ struct GlFillRect { pub color: Color, } -fn run_ops(fb: &Framebuffer, ops: &[GfxApiOpt]) -> Option { +fn run_ops(fb: &Framebuffer, ops: &[GfxApiOpt]) -> Option { let mut state = fb.ctx.gl_state.borrow_mut(); let state = &mut *state; let mut fill_rect = state.fill_rect.take(); @@ -301,13 +301,14 @@ fn run_ops(fb: &Framebuffer, ops: &[GfxApiOpt]) -> Option { return None; } }; + let file = FdSync::SyncFile(file); let user = fb.ctx.buffer_resv_user; for op in ops { if let GfxApiOpt::CopyTexture(ct) = op && ct.release_sync == ReleaseSync::Explicit && let Some(resv) = &ct.buffer_resv { - resv.set_sync_file(user, &file); + resv.set_sync(user, &file); } } return Some(file); @@ -417,9 +418,8 @@ fn render_texture(ctx: &GlRenderContext, tex: &CopyTexture) { } fn handle_explicit_sync(ctx: &GlRenderContext, img: Option<&Rc>, sync: &AcquireSync) { - let sync_file = match sync { - AcquireSync::None | AcquireSync::Implicit | AcquireSync::Unnecessary => return, - AcquireSync::SyncFile { sync_file } => sync_file, + let Some(sync_file) = sync.get_sync_file() else { + return; }; let sync_file = match uapi::fcntl_dupfd_cloexec(sync_file.raw(), 0) { Ok(s) => s, diff --git a/src/gfx_apis/gl/renderer/framebuffer.rs b/src/gfx_apis/gl/renderer/framebuffer.rs index 7df3bee8..57d12729 100644 --- a/src/gfx_apis/gl/renderer/framebuffer.rs +++ b/src/gfx_apis/gl/renderer/framebuffer.rs @@ -6,9 +6,9 @@ use { }, format::Format, gfx_api::{ - AcquireSync, AsyncShmGfxTextureCallback, GfxApiOpt, GfxBlendBuffer, GfxError, + AcquireSync, AsyncShmGfxTextureCallback, FdSync, GfxApiOpt, GfxBlendBuffer, GfxError, GfxFramebuffer, GfxInternalFramebuffer, GfxStagingBuffer, PendingShmTransfer, - ReleaseSync, ShmMemory, SyncFile, + ReleaseSync, ShmMemory, }, gfx_apis::gl::{ RenderError, @@ -74,7 +74,7 @@ impl Framebuffer { acquire_sync: AcquireSync, ops: &[GfxApiOpt], clear: Option<&Color>, - ) -> Result, RenderError> { + ) -> Result, RenderError> { let gles = self.ctx.ctx.dpy.gles; self.ctx.ctx.with_current(|| { handle_explicit_sync(&self.ctx, self.gl.rb._img.as_ref(), &acquire_sync); @@ -88,13 +88,13 @@ impl Framebuffer { } (gles.glBlendFunc)(GL_ONE, GL_ONE_MINUS_SRC_ALPHA); } - let fd = run_ops(self, ops); - if fd.is_none() { + let sync = run_ops(self, ops); + if sync.is_none() { unsafe { (gles.glFinish)(); } } - Ok(fd) + Ok(sync) }) } } @@ -115,7 +115,7 @@ impl GfxFramebuffer for Framebuffer { _region: &Region, _blend_buffer: Option<&Rc>, _blend_cd: &Rc, - ) -> Result, GfxError> { + ) -> Result, GfxError> { (*self) .render(acquire_sync, ops, clear) .map_err(|e| e.into()) diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs index 2b3ec089..78a1f396 100644 --- a/src/gfx_apis/vulkan.rs +++ b/src/gfx_apis/vulkan.rs @@ -9,7 +9,6 @@ mod descriptor_buffer; mod device; mod dmabuf_buffer; mod eotfs; -mod fence; mod format; mod gpu_alloc_ash; mod image; @@ -28,6 +27,7 @@ use { allocator::{Allocator, AllocatorError}, async_engine::AsyncEngine, cpu_worker::{CpuWorker, jobs::read_write::ReadWriteJobError}, + eventfd_cache::EventfdCache, format::Format, gfx_api::{ AsyncShmGfxTexture, GfxApi, GfxBlendBuffer, GfxBuffer, GfxContext, GfxError, GfxFormat, @@ -35,7 +35,8 @@ use { STAGING_DOWNLOAD, STAGING_UPLOAD, ShmGfxTexture, StagingBufferUsecase, }, gfx_apis::vulkan::{ - image::VulkanImageMemory, instance::VulkanInstance, renderer::VulkanRenderer, + device::VulkanDevice, image::VulkanImageMemory, instance::VulkanInstance, + renderer::VulkanRenderer, }, io_uring::IoUring, pr_caps::PrCapsThread, @@ -46,7 +47,7 @@ use { drm::{Drm, DrmError, syncobj::SyncobjCtx}, gbm::GbmError, }, - vulkan_core::VulkanCoreError, + vulkan_core::{self, VulkanCoreError}, }, ahash::AHashMap, ash::vk, @@ -73,8 +74,6 @@ pub enum VulkanError { CreateDevice(#[source] vk::Result), #[error("Could not create a semaphore")] CreateSemaphore(#[source] vk::Result), - #[error("Could not create a fence")] - CreateFence(#[source] vk::Result), #[error("Could not create the buffer")] CreateBuffer(#[source] vk::Result), #[error("Could not create a shader module")] @@ -157,8 +156,6 @@ pub enum VulkanError { IoctlExportSyncFile(#[source] OsError), #[error("Could not import a sync file into a semaphore")] ImportSyncFile(#[source] vk::Result), - #[error("Could not export a sync file from a semaphore")] - ExportSyncFile(#[source] vk::Result), #[error("Could not fetch the render node of the device")] FetchRenderNode(#[source] DrmError), #[error("Device has no render node")] @@ -215,6 +212,10 @@ pub enum VulkanError { DmaBufBufferOffsetAlignment, } +type VulkanSync = vulkan_core::sync::VulkanSync; +type VulkanTimelineSemaphore = + vulkan_core::timeline_semaphore::VulkanTimelineSemaphore; + impl From for GfxError { fn from(value: VulkanError) -> Self { Self(Box::new(value)) @@ -224,6 +225,7 @@ impl From for GfxError { pub fn create_graphics_context( eng: &Rc, ring: &Rc, + eventfd_cache: &Rc, drm: &Drm, caps_thread: Option<&PrCapsThread>, software: bool, @@ -231,22 +233,25 @@ pub fn create_graphics_context( let instance = VulkanInstance::new(Level::Info)?; let device = 'device: { if let Some(t) = caps_thread { - match unsafe { t.run(|| instance.create_device(drm, true, software)) } { + match unsafe { t.run(|| instance.create_device(drm, eventfd_cache, true, software)) } { Ok(d) => break 'device d, Err(e) => { log::warn!("Could not create high-priority device: {}", ErrorFmt(e)); } } } - instance.create_device(drm, false, software)? + instance.create_device(drm, eventfd_cache, false, software)? }; let renderer = device.create_renderer(eng, ring)?; Ok(Rc::new(Context(renderer))) } -pub fn create_vulkan_allocator(drm: &Drm) -> Result, AllocatorError> { +pub fn create_vulkan_allocator( + drm: &Drm, + eventfd_cache: &Rc, +) -> Result, AllocatorError> { let instance = VulkanInstance::new(Level::Debug)?; - let device = instance.create_device(drm, false, false)?; + let device = instance.create_device(drm, eventfd_cache, false, false)?; let allocator = device.create_bo_allocator(drm)?; Ok(Rc::new(allocator)) } diff --git a/src/gfx_apis/vulkan/device.rs b/src/gfx_apis/vulkan/device.rs index 671b4d52..f65e8e06 100644 --- a/src/gfx_apis/vulkan/device.rs +++ b/src/gfx_apis/vulkan/device.rs @@ -1,6 +1,7 @@ use { crate::{ allocator::BufferObject, + eventfd_cache::EventfdCache, format::XRGB8888, gfx_apis::vulkan::{ VulkanError, @@ -14,7 +15,8 @@ use { gbm::{GBM_BO_USE_RENDERING, GbmDevice}, }, vulkan_core::{ - ApiVersionDisplay, Extensions, VULKAN_API_VERSION, map_extension_properties, + ApiVersionDisplay, Extensions, VULKAN_API_VERSION, VulkanCoreInstance, + device::VulkanDeviceInf, map_extension_properties, }, }, ahash::AHashMap, @@ -63,6 +65,7 @@ pub struct VulkanDevice { pub(super) sync_ctx: Rc, pub(super) instance: Rc, pub(super) device: Arc, + pub(super) eventfd_cache: Rc, pub(super) external_memory_fd: external_memory_fd::Device, pub(super) external_semaphore_fd: external_semaphore_fd::Device, pub(super) external_fence_fd: external_fence_fd::Device, @@ -85,6 +88,7 @@ pub struct VulkanDevice { pub(super) uniform_buffer_descriptor_size: usize, pub(super) lost: Cell, pub(super) fast_ram_access: bool, + pub(super) supports_timeline_opaque_export: bool, } impl Drop for VulkanDevice { @@ -337,6 +341,7 @@ impl VulkanInstance { pub fn create_device( self: &Rc, drm: &Drm, + eventfd_cache: &Rc, mut high_priority: bool, software: bool, ) -> Result, VulkanError> { @@ -379,6 +384,9 @@ impl VulkanInstance { } transfer_granularity_mask = (width_mask, height_mask); } + let features = self.get_features(phy_dev); + let supports_timeline_opaque_export = + self.supports_timeline_opaque_export(phy_dev, &features); if !self.supports_semaphore_import(phy_dev) { return Err(VulkanError::SyncFileImport); } @@ -389,8 +397,8 @@ impl VulkanInstance { if supports_descriptor_buffer { enabled_extensions.push(descriptor_buffer::NAME.as_ptr()); } - let mut semaphore_features = - PhysicalDeviceTimelineSemaphoreFeatures::default().timeline_semaphore(true); + let mut semaphore_features = PhysicalDeviceTimelineSemaphoreFeatures::default() + .timeline_semaphore(supports_timeline_opaque_export); let mut synchronization2_features = PhysicalDeviceSynchronization2Features::default().synchronization2(true); let mut dynamic_rendering_features = @@ -557,6 +565,7 @@ impl VulkanInstance { gbm: Rc::new(gbm), instance: self.clone(), device: Arc::new(device), + eventfd_cache: eventfd_cache.clone(), external_memory_fd, external_semaphore_fd, external_fence_fd, @@ -580,6 +589,7 @@ impl VulkanInstance { uniform_buffer_descriptor_size, lost: Cell::new(false), fast_ram_access, + supports_timeline_opaque_export, })) } } @@ -639,3 +649,33 @@ fn log_device( } } } + +impl VulkanDeviceInf for VulkanDevice { + fn instance(&self) -> &VulkanCoreInstance { + &self.instance + } + + fn device(&self) -> &Device { + &self.device + } + + fn external_fence_fd(&self) -> &external_fence_fd::Device { + &self.external_fence_fd + } + + fn external_semaphore_fd(&self) -> &external_semaphore_fd::Device { + &self.external_semaphore_fd + } + + fn supports_timeline_opaque_export(&self) -> bool { + self.supports_timeline_opaque_export + } + + fn sync_ctx(&self) -> &Rc { + &self.sync_ctx + } + + fn eventfd_cache(&self) -> &Rc { + &self.eventfd_cache + } +} diff --git a/src/gfx_apis/vulkan/image.rs b/src/gfx_apis/vulkan/image.rs index e4397d02..ee529fe2 100644 --- a/src/gfx_apis/vulkan/image.rs +++ b/src/gfx_apis/vulkan/image.rs @@ -4,9 +4,9 @@ use { format::Format, gfx_api::{ AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback, - AsyncShmGfxTextureTransferCancellable, GfxApiOpt, GfxBlendBuffer, GfxBuffer, GfxError, - GfxFramebuffer, GfxImage, GfxInternalFramebuffer, GfxStagingBuffer, GfxTexture, - PendingShmTransfer, ReleaseSync, ShmGfxTexture, ShmMemory, SyncFile, + AsyncShmGfxTextureTransferCancellable, FdSync, GfxApiOpt, GfxBlendBuffer, GfxBuffer, + GfxError, GfxFramebuffer, GfxImage, GfxInternalFramebuffer, GfxStagingBuffer, + GfxTexture, PendingShmTransfer, ReleaseSync, ShmGfxTexture, ShmMemory, }, gfx_apis::vulkan::{ VulkanError, allocator::VulkanAllocation, device::VulkanDevice, @@ -563,7 +563,7 @@ impl GfxFramebuffer for VulkanImage { region: &Region, blend_buffer: Option<&Rc>, blend_cd: &Rc, - ) -> Result, GfxError> { + ) -> Result, GfxError> { let mut blend_buffer = blend_buffer .map(|b| b.clone().into_vk(&self.renderer.device.device)) .transpose()?; diff --git a/src/gfx_apis/vulkan/renderer.rs b/src/gfx_apis/vulkan/renderer.rs index 44d11c2c..911fa6a8 100644 --- a/src/gfx_apis/vulkan/renderer.rs +++ b/src/gfx_apis/vulkan/renderer.rs @@ -8,11 +8,11 @@ use { }, cpu_worker::PendingJob, gfx_api::{ - AcquireSync, AlphaMode, BufferResv, BufferResvUser, GfxApiOpt, GfxBlendBuffer, - GfxFormat, GfxTexture, GfxWriteModifier, ReleaseSync, SyncFile, + AcquireSync, AlphaMode, BufferResv, BufferResvUser, FdSync, GfxApiOpt, GfxBlendBuffer, + GfxFormat, GfxTexture, GfxWriteModifier, ReleaseSync, }, gfx_apis::vulkan::{ - VulkanError, + VulkanError, VulkanSync, VulkanTimelineSemaphore, allocator::{VulkanAllocator, VulkanThreadedAllocator}, buffer_cache::{GenericBufferWriter, VulkanBuffer, VulkanBufferCache}, command::{VulkanCommandBuffer, VulkanCommandPool}, @@ -20,7 +20,6 @@ use { descriptor_buffer::VulkanDescriptorBufferWriter, device::VulkanDevice, eotfs::{EOTF_LINEAR, EotfExt, VulkanEotf}, - fence::VulkanFence, image::{QueueFamily, QueueState, QueueTransfer, VulkanImage, VulkanImageMemory}, pipeline::{PipelineCreateInfo, VulkanPipeline}, sampler::VulkanSampler, @@ -40,6 +39,9 @@ use { stack::Stack, }, video::dmabuf::{DMA_BUF_SYNC_READ, DMA_BUF_SYNC_WRITE, dma_buf_export_sync_file}, + vulkan_core::{ + sync::VulkanDeviceSyncExt, timeline_semaphore::VulkanDeviceTimelineSemaphoreExt, + }, }, ahash::AHashMap, arrayvec::ArrayVec, @@ -65,7 +67,7 @@ use { std::{ any::Any, borrow::Cow, - cell::{Cell, RefCell}, + cell::{Cell, LazyCell, RefCell}, collections::hash_map::Entry, fmt::{Debug, Formatter}, mem, @@ -113,6 +115,8 @@ pub struct VulkanRenderer { pub(super) blend_buffers: RefCell>>, pub(super) shader_buffer_cache: Rc, pub(super) uniform_buffer_cache: Rc, + pub(super) render_tls: Option>, + pub(super) transfer_tls: Option>, } pub(super) struct CachedCommandBuffers { @@ -162,8 +166,8 @@ pub(super) struct Memory { image_barriers: Vec>, wait_semaphores: Vec>, wait_semaphore_infos: Vec>, - release_fence: Option>, - release_sync_file: Option, + release_vulkan_sync: Option, + release_sync: Option, used_buffers: ArrayVec, paint_bounds: StaticMap>, paint_regions: StaticMap>, @@ -249,7 +253,7 @@ pub(super) struct PendingFrame { _textures: Vec, wait_semaphores: Cell>>, waiter: Cell>>, - _release_fence: Option>, + vulkan_sync: Option, _used_buffers: ArrayVec, } @@ -407,6 +411,8 @@ impl VulkanDevice { blend_buffers: Default::default(), shader_buffer_cache, uniform_buffer_cache, + render_tls: self.create_timeline_semaphore_or_log(), + transfer_tls: self.create_timeline_semaphore_or_log(), }); Ok(render) } @@ -1671,10 +1677,12 @@ impl VulkanRenderer { } } } - AcquireSync::SyncFile { sync_file } => { - let fd = uapi::fcntl_dupfd_cloexec(sync_file.raw(), 0) - .map_err(|e| VulkanError::Dupfd(e.into()))?; - import_sync_file(fd)?; + AcquireSync::FdSync(sync) => { + if let Some(sync_file) = sync.get_sync_file() { + let fd = uapi::fcntl_dupfd_cloexec(sync_file.raw(), 0) + .map_err(|e| VulkanError::Dupfd(e.into()))?; + import_sync_file(fd)?; + } } AcquireSync::Unnecessary => {} } @@ -1703,20 +1711,22 @@ impl VulkanRenderer { fn import_release_semaphore(&self, fb: &VulkanImage, fb_release_sync: ReleaseSync) { zone!("import_release_semaphore"); let memory = &mut *self.memory.borrow_mut(); - let sync_file = match memory.release_sync_file.as_ref() { - Some(sync_file) => sync_file, + let fd_sync = match memory.release_sync.as_ref() { + Some(sync) => sync, _ => return, }; + let sync_file = LazyCell::new(|| fd_sync.get_sync_file()); let import = |img: &VulkanImage, sync: ReleaseSync, resv: Option>, flag: u32| { if sync == ReleaseSync::None { return; } if let Some(resv) = resv { - resv.set_sync_file(self.buffer_resv_user, sync_file); + resv.set_sync(self.buffer_resv_user, &fd_sync); } else if sync == ReleaseSync::Implicit { if let VulkanImageMemory::DmaBuf(buf) = &img.ty - && let Err(e) = buf.template.dmabuf.import_sync_file(flag, sync_file) + && let Some(fd) = &*sync_file + && let Err(e) = buf.template.dmabuf.import_sync_file(flag, fd) { log::error!("Could not import sync file into dmabuf: {}", ErrorFmt(e)); log::warn!("Relying on implicit sync"); @@ -1736,14 +1746,14 @@ impl VulkanRenderer { && let VulkanImageMemory::Internal(shm) = &texture.tex.ty && let Some(data) = &shm.async_data { - data.last_gfx_use.set(Some(sync_file.clone())); + data.last_gfx_use.set(Some(fd_sync.clone())); } } if attach_async_shm_sync_file && let VulkanImageMemory::Internal(shm) = &fb.ty && let Some(data) = &shm.async_data { - data.last_gfx_use.set(Some(sync_file.clone())); + data.last_gfx_use.set(Some(fd_sync.clone())); } import(fb, fb_release_sync, None, DMA_BUF_SYNC_WRITE); } @@ -1751,33 +1761,29 @@ impl VulkanRenderer { fn submit(&self, buf: CommandBuffer) -> Result<(), VulkanError> { zone!("submit"); let mut memory = self.memory.borrow_mut(); - let release_fence = self.device.create_fence()?; + let mut semaphore_submit_info = SemaphoreSubmitInfo::default(); let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(buf); - let submit_info = SubmitInfo2::default() + let mut submit_info = SubmitInfo2::default() .wait_semaphore_infos(&memory.wait_semaphore_infos) .command_buffer_infos(slice::from_ref(&command_buffer_info)); + let vulkan_sync = self.device.create_sync( + self.render_tls.as_ref(), + &mut semaphore_submit_info, + &mut submit_info, + )?; unsafe { self.device .device .queue_submit2( self.device.graphics_queue, slice::from_ref(&submit_info), - release_fence.fence, + vulkan_sync.fence(), ) .inspect_err(self.device.idl()) .map_err(VulkanError::Submit)?; } - zone!("export_sync_file"); - let release_sync_file = match release_fence.export_sync_file() { - Ok(s) => s, - Err(e) => { - log::error!("Could not export sync file from fence: {}", ErrorFmt(e)); - self.block(); - None - } - }; - memory.release_fence = Some(release_fence); - memory.release_sync_file = release_sync_file; + memory.release_sync = vulkan_sync.to_sync(|| self.block()); + memory.release_vulkan_sync = Some(vulkan_sync); Ok(()) } @@ -1816,14 +1822,14 @@ impl VulkanRenderer { _textures: mem::take(&mut memory.textures), wait_semaphores: Cell::new(mem::take(&mut memory.wait_semaphores)), waiter: Cell::new(None), - _release_fence: memory.release_fence.take(), + vulkan_sync: memory.release_vulkan_sync.take(), _used_buffers: mem::take(&mut memory.used_buffers), }); self.pending_frames.set(frame.point, frame.clone()); let future = self.eng.spawn( "await release", await_release( - memory.release_sync_file.clone(), + memory.release_sync.clone(), self.ring.clone(), frame.clone(), self.clone(), @@ -1844,7 +1850,7 @@ impl VulkanRenderer { region: &Region, blend_buffer: Option>, blend_cd: &Rc, - ) -> Result, VulkanError> { + ) -> Result, VulkanError> { zone!("execute"); let res = self.try_execute( fb, @@ -1858,19 +1864,19 @@ impl VulkanRenderer { blend_buffer, blend_cd, ); - let sync_file = { + let sync = { let mut memory = self.memory.borrow_mut(); memory.textures.clear(); memory.dmabuf_sample.clear(); memory.queue_transfer.clear(); memory.wait_semaphores.clear(); - memory.release_fence.take(); + memory.release_vulkan_sync.take(); memory.used_buffers.clear(); memory.ops.clear(); memory.ops_tmp.clear(); - memory.release_sync_file.take() + memory.release_sync.take() }; - res.map(|_| sync_file) + res.map(|_| sync) } fn allocate_semaphore(&self) -> Result, VulkanError> { @@ -2187,13 +2193,13 @@ pub(super) fn image_barrier() -> ImageMemoryBarrier2<'static> { } async fn await_release( - sync_file: Option, + sync: Option, ring: Rc, frame: Rc, renderer: Rc, ) { - if let Some(sync_file) = sync_file - && let Err(e) = ring.readable(&sync_file).await + if let Some(sync) = sync + && let Err(e) = sync.try_signaled(&ring).await { log::error!( "Could not wait for release semaphore to be signaled: {}", @@ -2201,6 +2207,9 @@ async fn await_release( ); frame.renderer.block(); } + if let Some(vs) = &frame.vulkan_sync { + vs.handle_validation(); + } if let Some(buf) = frame.cmd.take() { frame.renderer.gfx_command_buffers.buffers.push(buf); } diff --git a/src/gfx_apis/vulkan/shm_image.rs b/src/gfx_apis/vulkan/shm_image.rs index 37ba16ff..06e347d7 100644 --- a/src/gfx_apis/vulkan/shm_image.rs +++ b/src/gfx_apis/vulkan/shm_image.rs @@ -2,12 +2,11 @@ use { crate::{ cpu_worker::CpuWorker, format::Format, - gfx_api::SyncFile, + gfx_api::FdSync, gfx_apis::vulkan::{ - VulkanError, + VulkanError, VulkanSync, allocator::VulkanAllocation, command::VulkanCommandBuffer, - fence::VulkanFence, image::{QueueFamily, QueueState, VulkanImage, VulkanImageMemory}, renderer::{VulkanRenderer, image_barrier}, staging::VulkanStagingBuffer, @@ -15,6 +14,7 @@ use { }, rect::Rect, utils::errorfmt::ErrorFmt, + vulkan_core::sync::VulkanDeviceSyncExt, }, ash::vk::{ AccessFlags2, Buffer, BufferImageCopy2, BufferMemoryBarrier2, CommandBufferBeginInfo, @@ -22,7 +22,8 @@ use { CopyImageToBufferInfo2, DependencyInfoKHR, DeviceSize, Extent3D, ImageAspectFlags, ImageCreateInfo, ImageLayout, ImageSubresourceLayers, ImageSubresourceRange, ImageTiling, ImageType, ImageUsageFlags, ImageViewCreateInfo, ImageViewType, Offset3D, - PipelineStageFlags2, QUEUE_FAMILY_FOREIGN_EXT, SampleCountFlags, SharingMode, SubmitInfo2, + PipelineStageFlags2, QUEUE_FAMILY_FOREIGN_EXT, SampleCountFlags, SemaphoreSubmitInfo, + SharingMode, SubmitInfo2, }, gpu_alloc::UsageFlags, isnt::std_1::primitive::IsntSliceExt, @@ -136,7 +137,7 @@ impl VulkanShmImage { ptr::copy_nonoverlapping(buf, mem, total_size as usize); } })?; - let (cmd, fence, sync_file, point) = self.submit_buffer_image_copy( + let (cmd, vulkan_sync, sync, point) = self.submit_buffer_image_copy( img, staging.buffer, staging.size, @@ -147,7 +148,7 @@ impl VulkanShmImage { )?; let future = img.renderer.eng.spawn( "await upload", - await_upload(point, img.clone(), cmd, sync_file, fence, staging), + await_upload(point, img.clone(), cmd, sync, vulkan_sync, staging), ); img.renderer.pending_submits.set(point, future); Ok(()) @@ -162,15 +163,7 @@ impl VulkanShmImage { use_transfer_queue: bool, tt: TransferType, foreign_buffer: bool, - ) -> Result< - ( - Rc, - Rc, - Option, - u64, - ), - VulkanError, - > { + ) -> Result<(Rc, VulkanSync, Option, u64), VulkanError> { let mut transfer_queue_family_idx = img.renderer.device.graphics_queue_idx; if use_transfer_queue && let Some(idx) = img.renderer.device.distinct_transfer_queue_family_idx @@ -277,11 +270,20 @@ impl VulkanShmImage { }; let dev = &img.renderer.device.device; let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd.buffer); - let submit_info = + let mut semaphore_submit_info = SemaphoreSubmitInfo::default(); + let mut submit_info = SubmitInfo2::default().command_buffer_infos(slice::from_ref(&command_buffer_info)); let begin_info = CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT); - let release_fence = img.renderer.device.create_fence()?; + let tls = match use_transfer_queue { + true => &img.renderer.transfer_tls, + false => &img.renderer.render_tls, + }; + let vulkan_sync = img.renderer.device.create_sync( + tls.as_ref(), + &mut semaphore_submit_info, + &mut submit_info, + )?; unsafe { dev.begin_command_buffer(cmd.buffer, &begin_info) .map_err(VulkanError::BeginCommandBuffer)?; @@ -313,7 +315,7 @@ impl VulkanShmImage { _ => img.renderer.device.graphics_queue, }, slice::from_ref(&submit_info), - release_fence.fence, + vulkan_sync.fence(), ) .inspect_err(img.renderer.device.idl()) .map_err(VulkanError::Submit)?; @@ -322,16 +324,9 @@ impl VulkanShmImage { img.is_undefined.set(false); img.contents_are_undefined.set(false); } - let release_sync_file = match release_fence.export_sync_file() { - Ok(s) => s, - Err(e) => { - log::error!("Could not export sync file from fence: {}", ErrorFmt(e)); - img.renderer.block(); - None - } - }; + let release_sync = vulkan_sync.to_sync(|| img.renderer.block()); let point = img.renderer.allocate_point(); - Ok((cmd, release_fence, release_sync_file, point)) + Ok((cmd, vulkan_sync, release_sync, point)) } } @@ -339,12 +334,12 @@ async fn await_upload( id: u64, img: Rc, buf: Rc, - sync_file: Option, - _fence: Rc, + sync: Option, + vulkan_sync: VulkanSync, _staging: VulkanStagingBuffer, ) { - if let Some(sync_file) = sync_file - && let Err(e) = img.renderer.ring.readable(&sync_file.0).await + if let Some(sync) = &sync + && let Err(e) = sync.try_signaled(&img.renderer.ring).await { log::error!( "Could not wait for sync file to become readable: {}", @@ -352,6 +347,7 @@ async fn await_upload( ); img.renderer.block(); } + vulkan_sync.handle_validation(); img.renderer.gfx_command_buffers.buffers.push(buf); img.renderer.pending_submits.remove(&id); } diff --git a/src/gfx_apis/vulkan/transfer.rs b/src/gfx_apis/vulkan/transfer.rs index eca55277..13d8bd93 100644 --- a/src/gfx_apis/vulkan/transfer.rs +++ b/src/gfx_apis/vulkan/transfer.rs @@ -8,13 +8,12 @@ use { }, }, gfx_api::{ - AsyncShmGfxTextureCallback, PendingShmTransfer, ShmMemory, ShmMemoryBacking, SyncFile, + AsyncShmGfxTextureCallback, FdSync, PendingShmTransfer, ShmMemory, ShmMemoryBacking, }, gfx_apis::vulkan::{ - VulkanError, + VulkanError, VulkanSync, command::VulkanCommandBuffer, dmabuf_buffer::{TRANSFER_QUEUE_BUFFER_ALIGNMENT, VulkanDmabufBuffer}, - fence::VulkanFence, image::{QueueFamily, QueueState, QueueTransfer, VulkanImage, VulkanImageMemory}, renderer::image_barrier, shm_image::VulkanShmImage, @@ -22,12 +21,13 @@ use { }, rect::{Rect, Region}, utils::{clonecell::CloneCell, errorfmt::ErrorFmt}, + vulkan_core::sync::VulkanDeviceSyncExt, }, arrayvec::ArrayVec, ash::vk::{ AccessFlags2, BufferImageCopy2, CommandBufferBeginInfo, CommandBufferSubmitInfo, CommandBufferUsageFlags, DependencyInfo, Extent3D, ImageAspectFlags, ImageLayout, - ImageSubresourceLayers, Offset3D, PipelineStageFlags2, SubmitInfo2, + ImageSubresourceLayers, Offset3D, PipelineStageFlags2, SemaphoreSubmitInfo, SubmitInfo2, }, std::{ cell::{Cell, RefCell, RefMut}, @@ -48,7 +48,7 @@ pub struct VulkanShmImageAsyncData { pub(super) callback_id: Cell, pub(super) regions: RefCell>>, pub(super) cpu: Rc, - pub(super) last_gfx_use: Cell>, + pub(super) last_gfx_use: Cell>, pub(super) data_copied: Cell, } @@ -314,7 +314,7 @@ impl VulkanShmImage { img.renderer.check_defunct()?; let Some(transfer_queue_idx) = img.renderer.device.distinct_transfer_queue_family_idx else { - let Some(sync_file) = data.last_gfx_use.take() else { + let Some(sync) = data.last_gfx_use.take() else { img.queue_state.set(QueueState::Released { to: QueueFamily::Transfer, }); @@ -323,7 +323,7 @@ impl VulkanShmImage { let id = img.renderer.allocate_point(); let pending = img.renderer.eng.spawn( "await_transfer_to_transfer", - await_gfx_queue_release(id, img.clone(), None, None, Some(sync_file), tt), + await_gfx_queue_release(id, img.clone(), None, None, Some(sync), tt), ); img.renderer.pending_submits.set(id, pending); img.queue_state.set(QueueState::Releasing); @@ -375,14 +375,19 @@ impl VulkanShmImage { .new_layout(transfer_layout); barriers.push(barrier); let dep_info = DependencyInfo::default().image_memory_barriers(&barriers); - let release_fence = img.renderer.device.create_fence()?; let dev = &img.renderer.device.device; let begin_info = CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT); let cmd = img.renderer.gfx_command_buffers.allocate()?; let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd.buffer); - let submit_info = + let mut semaphore_submit_info = SemaphoreSubmitInfo::default(); + let mut submit_info = SubmitInfo2::default().command_buffer_infos(slice::from_ref(&command_buffer_info)); + let vulkan_sync = img.renderer.device.create_sync( + img.renderer.render_tls.as_ref(), + &mut semaphore_submit_info, + &mut submit_info, + )?; unsafe { dev.begin_command_buffer(cmd.buffer, &begin_info) .map_err(VulkanError::BeginCommandBuffer)?; @@ -392,23 +397,16 @@ impl VulkanShmImage { dev.queue_submit2( img.renderer.device.graphics_queue, slice::from_ref(&submit_info), - release_fence.fence, + vulkan_sync.fence(), ) .inspect_err(img.renderer.device.idl()) .map_err(VulkanError::Submit)?; } - let sync_file = release_fence.export_sync_file()?; + let sync = vulkan_sync.to_sync(|| img.renderer.block()); let id = img.renderer.allocate_point(); let pending = img.renderer.eng.spawn( "await_transfer_to_transfer", - await_gfx_queue_release( - id, - img.clone(), - Some(cmd), - Some(release_fence), - sync_file, - tt, - ), + await_gfx_queue_release(id, img.clone(), Some(cmd), Some(vulkan_sync), sync, tt), ); img.renderer.pending_submits.set(id, pending); img.queue_state.set(QueueState::Releasing); @@ -555,7 +553,7 @@ impl VulkanShmImage { (host_buffer.buffer, host_buffer.size, true) } }; - let (cmd, fence, sync_file, point) = self.submit_buffer_image_copy( + let (cmd, vulkan_sync, sync, point) = self.submit_buffer_image_copy( img, buffer, size, @@ -571,8 +569,8 @@ impl VulkanShmImage { point, img.clone(), cmd, - fence, - sync_file, + vulkan_sync, + sync, TransferType::Upload, ), ); @@ -590,7 +588,7 @@ impl VulkanShmImage { return Ok(()); } img.renderer.check_defunct()?; - let (cmd, fence, sync_file, point) = self.submit_buffer_image_copy( + let (cmd, vulkan_sync, sync, point) = self.submit_buffer_image_copy( img, staging.buffer, staging.size, @@ -606,8 +604,8 @@ impl VulkanShmImage { point, img.clone(), cmd, - fence, - sync_file, + vulkan_sync, + sync, TransferType::Download, ), ); @@ -695,12 +693,12 @@ async fn await_gfx_queue_release( id: u64, img: Rc, buf: Option>, - _fence: Option>, - sync_file: Option, + vulkan_sync: Option, + sync: Option, tt: TransferType, ) { - if let Some(sync_file) = sync_file - && let Err(e) = img.renderer.ring.readable(&sync_file.0).await + if let Some(sync) = &sync + && let Err(e) = sync.try_signaled(&img.renderer.ring).await { log::error!( "Could not wait for sync file to become readable: {}", @@ -708,6 +706,9 @@ async fn await_gfx_queue_release( ); img.renderer.block(); } + if let Some(vs) = vulkan_sync { + vs.handle_validation(); + } if let Some(buf) = buf { img.renderer.gfx_command_buffers.buffers.push(buf); } @@ -738,12 +739,12 @@ pub async fn await_async_transfer_release_to_gfx( id: u64, img: Rc, buf: Rc, - _fence: Rc, - sync_file: Option, + vulkan_sync: VulkanSync, + sync: Option, tt: TransferType, ) { - if let Some(sync_file) = sync_file - && let Err(e) = img.renderer.ring.readable(&sync_file.0).await + if let Some(sync) = &sync + && let Err(e) = sync.try_signaled(&img.renderer.ring).await { log::error!( "Could not wait for sync file to become readable: {}", @@ -751,6 +752,7 @@ pub async fn await_async_transfer_release_to_gfx( ); img.renderer.block(); } + vulkan_sync.handle_validation(); match &img.renderer.transfer_command_buffers { Some(b) => b.buffers.push(buf), None => img.renderer.gfx_command_buffers.buffers.push(buf), diff --git a/src/ifs/ext_image_copy/ext_image_copy_capture_frame_v1.rs b/src/ifs/ext_image_copy/ext_image_copy_capture_frame_v1.rs index c48cfaa0..48fa9038 100644 --- a/src/ifs/ext_image_copy/ext_image_copy_capture_frame_v1.rs +++ b/src/ifs/ext_image_copy/ext_image_copy_capture_frame_v1.rs @@ -3,8 +3,8 @@ use { client::{Client, ClientError}, cmm::cmm_description::ColorDescription, gfx_api::{ - AcquireSync, AsyncShmGfxTextureCallback, BufferResv, GfxError, GfxFramebuffer, - GfxTexture, ReleaseSync, STAGING_DOWNLOAD, SyncFile, + AcquireSync, AsyncShmGfxTextureCallback, BufferResv, FdSync, GfxError, GfxFramebuffer, + GfxTexture, ReleaseSync, STAGING_DOWNLOAD, }, ifs::{ ext_image_capture_source_v1::ImageCaptureSource, @@ -78,7 +78,7 @@ impl ExtImageCopyCaptureFrameV1 { Rc, AcquireSync, ReleaseSync, - ) -> Result, GfxError>, + ) -> Result, GfxError>, ) -> Result<(), FrameFailureReason> { let Some(ctx) = self.client.state.render_ctx.get() else { return Err(FrameFailureReason::BufferConstraints); @@ -183,7 +183,7 @@ impl ExtImageCopyCaptureFrameV1 { Rc, AcquireSync, ReleaseSync, - ) -> Result, GfxError>, + ) -> Result, GfxError>, ) { match self.try_copy(on, size, f) { Ok(()) => self.session.status.set(FrameStatus::Captured), diff --git a/src/ifs/wl_surface.rs b/src/ifs/wl_surface.rs index 4089efe1..4d59924e 100644 --- a/src/ifs/wl_surface.rs +++ b/src/ifs/wl_surface.rs @@ -30,8 +30,8 @@ use { drm_feedback::DrmFeedback, fixed::Fixed, gfx_api::{ - AlphaMode, AsyncShmGfxTexture, BufferResv, BufferResvUser, GfxError, GfxStagingBuffer, - ReleaseSync, SampleRect, SyncFile, + AlphaMode, AsyncShmGfxTexture, BufferResv, BufferResvUser, FdSync, GfxError, + GfxStagingBuffer, ReleaseSync, SampleRect, }, ifs::{ color_management::wp_color_management_surface_feedback_v1::WpColorManagementSurfaceFeedbackV1, @@ -215,7 +215,7 @@ impl NodeVisitorBase for SurfaceSendPreferredColorDescription { pub struct SurfaceBuffer { pub buffer: AttachedBuffer, - sync_files: SmallMap, + syncs: SmallMap, pub release_sync: ReleaseSync, release: Option, _surface_release: SmallVec<[SurfaceRelease; 1]>, @@ -223,14 +223,16 @@ pub struct SurfaceBuffer { impl Drop for SurfaceBuffer { fn drop(&mut self) { - let sync_files = self.sync_files.take(); + let syncs = self.syncs.take(); if let Some(release) = &mut self.release { - release.signal(Some(&sync_files)); + release.signal(Some(&syncs)); return; } if let Some(dmabuf) = &self.buffer.buf.client_dmabuf { - for (_, sync_file) in &sync_files { - if let Err(e) = dmabuf.import_sync_file(DMA_BUF_SYNC_READ, sync_file) { + for (_, sync) in &syncs { + if let Some(sf) = sync.get_sync_file() + && let Err(e) = dmabuf.import_sync_file(DMA_BUF_SYNC_READ, &sf) + { log::error!("Could not import sync file: {}", ErrorFmt(e)); } } @@ -245,8 +247,8 @@ impl Debug for SurfaceBuffer { } impl BufferResv for SurfaceBuffer { - fn set_sync_file(&self, user: BufferResvUser, sync_file: &SyncFile) { - self.sync_files.insert(user, sync_file.clone()); + fn set_sync(&self, user: BufferResvUser, sync: &FdSync) { + self.syncs.insert(user, sync.clone()); } } @@ -1231,7 +1233,7 @@ impl WlSurface { }; let surface_buffer = SurfaceBuffer { buffer, - sync_files: Default::default(), + syncs: Default::default(), release_sync, release: pending.release_point.take(), _surface_release: mem::take(&mut pending.surface_release), @@ -2263,7 +2265,7 @@ pub struct SyncobjRelease { } impl SyncobjRelease { - fn signal(&mut self, sync_files: Option<&SmallVec<[(BufferResvUser, SyncFile); 1]>>) { + fn signal(&mut self, syncs: Option<&SmallVec<[(BufferResvUser, FdSync); 1]>>) { if !self.committed { return; } @@ -2278,10 +2280,14 @@ impl SyncobjRelease { log::error!("Cannot signal release point because there is no syncobj context"); return; }; - if let Some(sync_files) = sync_files - && sync_files.is_not_empty() + if let Some(syncs) = syncs + && syncs.is_not_empty() { - let res = ctx.import_sync_files(&syncobj, self.point, sync_files.iter().map(|f| &f.1)); + let res = ctx.import_sync_files( + &syncobj, + self.point, + syncs.iter().flat_map(|f| f.1.get_sync_file()), + ); match res { Ok(_) => return, Err(e) => { diff --git a/src/it/test_backend.rs b/src/it/test_backend.rs index a14527f9..42748ae0 100644 --- a/src/it/test_backend.rs +++ b/src/it/test_backend.rs @@ -211,7 +211,10 @@ impl TestBackend { }; } let udmabuf = ("udmabuf", constructor!(Udmabuf::new())); - let vulkan = ("Vulkan", constructor!(create_vk_allocator(), nomap)); + let vulkan = ( + "Vulkan", + constructor!(create_vk_allocator(&self.state), nomap), + ); let gbm = ("GBM", constructor!(create_gbm_allocator())); let allocators = match need_drm { true => [vulkan, gbm, udmabuf], @@ -243,9 +246,10 @@ fn create_gbm_allocator() -> Result { create_drm_allocator(|drm| GbmDevice::new(&drm).map_err(TestBackendError::CreateGbmDevice)) } -fn create_vk_allocator() -> Result, TestBackendError> { +fn create_vk_allocator(state: &State) -> Result, TestBackendError> { create_drm_allocator(|drm| { - create_vulkan_allocator(&drm).map_err(TestBackendError::CreateVulkanAllocator) + create_vulkan_allocator(&drm, &state.eventfd_cache) + .map_err(TestBackendError::CreateVulkanAllocator) }) } diff --git a/src/it/test_client.rs b/src/it/test_client.rs index bf6e544e..5252bda4 100644 --- a/src/it/test_client.rs +++ b/src/it/test_client.rs @@ -96,7 +96,12 @@ impl TestClient { pub async fn take_screenshot(&self, include_cursor: bool) -> Result, TestError> { let (dmabuf, dev) = self.jc.take_screenshot(include_cursor).await?; - let qoi = buf_to_bytes(dev.as_ref(), &dmabuf, ScreenshotFormat::Qoi)?; + let qoi = buf_to_bytes( + &self.run.state.eventfd_cache, + dev.as_ref(), + &dmabuf, + ScreenshotFormat::Qoi, + )?; Ok(qoi) } diff --git a/src/it/test_gfx_api.rs b/src/it/test_gfx_api.rs index 5f37a322..24d0fc7c 100644 --- a/src/it/test_gfx_api.rs +++ b/src/it/test_gfx_api.rs @@ -5,11 +5,11 @@ use { cpu_worker::CpuWorker, format::{ARGB8888, Format, XRGB8888}, gfx_api::{ - AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback, CopyTexture, FillRect, - FramebufferRect, GfxApi, GfxApiOpt, GfxBlendBuffer, GfxContext, GfxError, GfxFormat, - GfxFramebuffer, GfxImage, GfxInternalFramebuffer, GfxStagingBuffer, GfxTexture, - GfxWriteModifier, PendingShmTransfer, ReleaseSync, ResetStatus, ShmGfxTexture, - ShmMemory, SyncFile, + AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback, CopyTexture, FdSync, + FillRect, FramebufferRect, GfxApi, GfxApiOpt, GfxBlendBuffer, GfxContext, GfxError, + GfxFormat, GfxFramebuffer, GfxImage, GfxInternalFramebuffer, GfxStagingBuffer, + GfxTexture, GfxWriteModifier, PendingShmTransfer, ReleaseSync, ResetStatus, + ShmGfxTexture, ShmMemory, }, rect::{Rect, Region}, theme::Color, @@ -387,7 +387,7 @@ impl GfxFramebuffer for TestGfxFb { _region: &Region, _blend_buffer: Option<&Rc>, _blend_cd: &Rc, - ) -> Result, GfxError> { + ) -> Result, GfxError> { let fb_points = |width: i32, height: i32, rect: &FramebufferRect| { let points = rect.to_points(); let x1 = points[1][0]; diff --git a/src/main.rs b/src/main.rs index ad8c8db2..129978a5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -68,6 +68,7 @@ mod dbus; mod drm_feedback; mod edid; mod ei; +mod eventfd_cache; mod fixed; mod forker; mod format; diff --git a/src/portal.rs b/src/portal.rs index f3e57a5a..b2b9f62e 100644 --- a/src/portal.rs +++ b/src/portal.rs @@ -15,6 +15,7 @@ use { BUS_DEST, BUS_PATH, DBUS_NAME_FLAG_DO_NOT_QUEUE, DBUS_REQUEST_NAME_REPLY_PRIMARY_OWNER, Dbus, DbusSocket, }, + eventfd_cache::EventfdCache, forker::ForkerError, io_uring::IoUring, logger::Logger, @@ -242,9 +243,11 @@ async fn run_async( None } }; + let eventfd_cache = EventfdCache::new(&ring, &eng); let state = Rc::new(PortalState { xrd, ring, + eventfd_cache, eng, wheel, displays: Default::default(), @@ -324,6 +327,7 @@ async fn init_dbus_session(dbus: &Dbus, logger: Arc, path_sink: OwnedFd) struct PortalState { xrd: String, ring: Rc, + eventfd_cache: Rc, eng: Rc, wheel: Rc, displays: CopyHashMap>, diff --git a/src/portal/ptl_display.rs b/src/portal/ptl_display.rs index 2c59eeb4..a0223a40 100644 --- a/src/portal/ptl_display.rs +++ b/src/portal/ptl_display.rs @@ -187,6 +187,7 @@ impl UsrJayRenderCtxOwner for PortalDisplay { let ctx = match create_gfx_context( &self.state.eng, &self.state.ring, + &self.state.eventfd_cache, &drm, GfxApi::OpenGl, None, diff --git a/src/state.rs b/src/state.rs index eeaf2290..49c703e9 100644 --- a/src/state.rs +++ b/src/state.rs @@ -27,13 +27,14 @@ use { ei_acceptor::EiAcceptor, ei_client::{EiClient, EiClients}, }, + eventfd_cache::EventfdCache, fixed::Fixed, forker::ForkerProxy, format::Format, gfx_api::{ - AcquireSync, AlphaMode, BufferResv, GfxApi, GfxBlendBuffer, GfxContext, GfxError, - GfxFramebuffer, GfxTexture, PendingShmTransfer, ReleaseSync, STAGING_DOWNLOAD, - SampleRect, SyncFile, + AcquireSync, AlphaMode, BufferResv, FdSync, GfxApi, GfxBlendBuffer, GfxContext, + GfxError, GfxFramebuffer, GfxTexture, PendingShmTransfer, ReleaseSync, + STAGING_DOWNLOAD, SampleRect, }, gfx_apis::create_gfx_context, globals::{Globals, GlobalsError, RemovableWaylandGlobal, WaylandGlobal}, @@ -288,6 +289,7 @@ pub struct State { pub gfx_ctx_changed: EventSource, pub copy_device_registry: Rc, pub supports_presentation_feedback: Cell, + pub eventfd_cache: Rc, } // impl Drop for State { @@ -555,6 +557,7 @@ impl State { create_gfx_context( &self.eng, &self.ring, + &self.eventfd_cache, drm, api.unwrap_or(self.default_gfx_api.get()), self.caps_thread.as_ref(), @@ -1187,8 +1190,8 @@ impl State { render_hw_cursor: bool, blend_buffer: Option<&Rc>, blend_cd: &Rc, - ) -> Result, GfxError> { - let sync_file = fb.render_output( + ) -> Result, GfxError> { + let sync = fb.render_output( acquire_sync, release_sync, cd, @@ -1213,7 +1216,7 @@ impl State { 0, None, ); - Ok(sync_file) + Ok(sync) } pub fn perform_screencopy( @@ -1235,7 +1238,7 @@ impl State { size: Option<(i32, i32)>, transform: Transform, scale: Scale, - ) -> Result, GfxError> { + ) -> Result, GfxError> { let mut ops = vec![]; let mut renderer = Renderer { base: target.renderer_base(&mut ops, scale, target_transform), diff --git a/src/utils/stack.rs b/src/utils/stack.rs index f415a8c8..0ff7ee6f 100644 --- a/src/utils/stack.rs +++ b/src/utils/stack.rs @@ -42,4 +42,9 @@ impl Stack { pub fn take(&self) -> Vec { unsafe { mem::take(self.vec.get().deref_mut()) } } + + #[cfg_attr(not(test), expect(dead_code))] + pub fn len(&self) -> usize { + unsafe { self.vec.get().deref().len() } + } } diff --git a/src/video/drm.rs b/src/video/drm.rs index 20ae55a5..f70b5497 100644 --- a/src/video/drm.rs +++ b/src/video/drm.rs @@ -1,21 +1,33 @@ pub mod syncobj; mod sys; pub mod wait_for_syncobj; -pub use consts::*; use { crate::{ - utils::oserror::OsError, - video::drm::sys::{ - DRM_DISPLAY_MODE_LEN, DRM_MODE_ATOMIC_TEST_ONLY, DRM_MODE_FB_MODIFIERS, - DRM_MODE_OBJECT_BLOB, DRM_MODE_OBJECT_CONNECTOR, DRM_MODE_OBJECT_CRTC, - DRM_MODE_OBJECT_ENCODER, DRM_MODE_OBJECT_FB, DRM_MODE_OBJECT_PLANE, - DRM_MODE_OBJECT_PROPERTY, create_lease, drm_event, drm_event_vblank, gem_close, - get_cap, get_device_name_from_fd2, get_minor_name_from_fd, get_node_type_from_fd, - get_nodes, mode_addfb2, mode_atomic, mode_create_blob, mode_destroy_blob, - mode_get_resources, mode_getconnector, mode_getencoder, mode_getplane, - mode_getplaneresources, mode_getprobblob, mode_getproperty, mode_obj_getproperties, - mode_rmfb, prime_fd_to_handle, set_client_cap, + backend, + format::Format, + io_uring::{IoUring, IoUringError}, + utils::{ + buf::Buf, errorfmt::ErrorFmt, oserror::OsError, stack::Stack, syncqueue::SyncQueue, + vec_ext::VecExt, + }, + video::{ + INVALID_MODIFIER, Modifier, + dmabuf::DmaBuf, + drm::sys::{ + DRM_CAP_ATOMIC_ASYNC_PAGE_FLIP, DRM_CAP_CURSOR_HEIGHT, DRM_CAP_CURSOR_WIDTH, + DRM_DISPLAY_MODE_LEN, DRM_MODE_ATOMIC_TEST_ONLY, DRM_MODE_FB_MODIFIERS, + DRM_MODE_OBJECT_BLOB, DRM_MODE_OBJECT_CONNECTOR, DRM_MODE_OBJECT_CRTC, + DRM_MODE_OBJECT_ENCODER, DRM_MODE_OBJECT_FB, DRM_MODE_OBJECT_PLANE, + DRM_MODE_OBJECT_PROPERTY, FORMAT_BLOB_CURRENT, auth_magic, create_lease, drm_event, + drm_event_crtc_sequence, drm_event_vblank, drm_format_modifier, + drm_format_modifier_blob, drop_master, gem_close, get_cap, + get_device_name_from_fd2, get_minor_name_from_fd, get_node_type_from_fd, get_nodes, + get_version, mode_addfb2, mode_atomic, mode_create_blob, mode_destroy_blob, + mode_get_resources, mode_getconnector, mode_getencoder, mode_getplane, + mode_getplaneresources, mode_getprobblob, mode_getproperty, mode_obj_getproperties, + mode_rmfb, prime_fd_to_handle, queue_sequence, revoke_lease, set_client_cap, + }, }, }, ahash::AHashMap, @@ -32,26 +44,14 @@ use { thiserror::Error, uapi::{OwnedFd, Pod, Ustring, c}, }; - -use crate::{ - backend, - format::Format, - io_uring::{IoUring, IoUringError}, - utils::{buf::Buf, errorfmt::ErrorFmt, stack::Stack, syncqueue::SyncQueue, vec_ext::VecExt}, - video::{ - INVALID_MODIFIER, Modifier, - dmabuf::DmaBuf, - drm::sys::{ - DRM_CAP_ATOMIC_ASYNC_PAGE_FLIP, DRM_CAP_CURSOR_HEIGHT, DRM_CAP_CURSOR_WIDTH, - FORMAT_BLOB_CURRENT, auth_magic, drm_event_crtc_sequence, drm_format_modifier, - drm_format_modifier_blob, drop_master, get_version, queue_sequence, revoke_lease, - }, +pub use { + consts::*, + sys::{ + DRM_CLIENT_CAP_ATOMIC, DRM_MODE_ATOMIC_ALLOW_MODESET, DRM_MODE_ATOMIC_NONBLOCK, + DRM_MODE_PAGE_FLIP_ASYNC, DRM_MODE_PAGE_FLIP_EVENT, drm_mode_modeinfo, + get_drm_nodes_from_dev, }, }; -pub use sys::{ - DRM_CLIENT_CAP_ATOMIC, DRM_MODE_ATOMIC_ALLOW_MODESET, DRM_MODE_ATOMIC_NONBLOCK, - DRM_MODE_PAGE_FLIP_ASYNC, DRM_MODE_PAGE_FLIP_EVENT, drm_mode_modeinfo, -}; #[derive(Debug, Error)] pub enum DrmError { @@ -123,6 +123,8 @@ pub enum DrmError { CreateSyncobj(#[source] OsError), #[error("Could not export a syncobj")] ExportSyncobj(#[source] OsError), + #[error("Could not query a syncobj")] + QuerySyncobj(#[source] OsError), #[error("Could not register an eventfd with a syncobj")] RegisterEventfd(#[source] OsError), #[error("Could not create an eventfd")] @@ -139,6 +141,8 @@ pub enum DrmError { Merge(#[source] OsError), #[error("Could not import a sync file into a syncobj")] ImportSyncFile(#[source] OsError), + #[error("Could not export a sync file")] + ExportSyncFile(#[source] OsError), #[error("Could not create a lease")] CreateLease(#[source] OsError), #[error("Could not drop DRM master")] diff --git a/src/video/drm/syncobj.rs b/src/video/drm/syncobj.rs index 694000ff..160528fa 100644 --- a/src/video/drm/syncobj.rs +++ b/src/video/drm/syncobj.rs @@ -15,9 +15,10 @@ use { DRM_SYNCOBJ_CREATE_SIGNALED, DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE, DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_TIMELINE, DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE, - DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE, sync_ioc_merge, syncobj_create, + DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_TIMELINE, DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE, + DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT, sync_ioc_merge, syncobj_create, syncobj_destroy, syncobj_eventfd, syncobj_fd_to_handle, syncobj_handle_to_fd, - syncobj_signal, syncobj_transfer, + syncobj_query, syncobj_signal, syncobj_transfer, }, }, }, @@ -61,6 +62,10 @@ impl Syncobj { pub fn fd(&self) -> &Rc { &self.fd } + + pub fn id(&self) -> SyncobjId { + self.id + } } impl Drop for Syncobj { @@ -118,7 +123,7 @@ impl SyncobjCtx { pub fn create_syncobj(&self) -> Result { let handle = syncobj_create(self.inner.drm.raw(), 0).map_err(DrmError::CreateSyncobj)?; let handle = SyncobjHandle(handle); - let fd = syncobj_handle_to_fd(self.inner.drm.raw(), handle.0, 0); + let fd = syncobj_handle_to_fd(self.inner.drm.raw(), handle.0, 0, 0); if fd.is_err() { destroy(&self.inner.drm, handle); } @@ -138,6 +143,7 @@ impl SyncobjCtx { self.inner.drm.raw(), handle.0, DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE, + 0, ); destroy(&self.inner.drm, handle); fd.map_err(DrmError::ExportSyncobj) @@ -250,7 +256,46 @@ impl SyncobjCtx { } let dummy = self.get_dummy()?; import(0, self.get_handle(&dummy)?)?; - self.transfer(&dummy, SyncobjPoint(0), syncobj, point) + self.transfer(&dummy, SyncobjPoint(0), syncobj, point, 0) + } + + pub fn export_sync_file_blocking( + &self, + syncobj: &Syncobj, + point: SyncobjPoint, + ) -> Result { + let export = |flags: u32, handle: SyncobjHandle, point: SyncobjPoint| { + syncobj_handle_to_fd( + self.inner.drm.raw(), + handle.0, + DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE | flags, + point.0, + ) + .map(Rc::new) + .map(SyncFile) + }; + if self.supports_timeline_import() { + let res = export( + DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_TIMELINE, + self.get_handle(syncobj)?, + point, + ); + match res { + Ok(sf) => return Ok(sf), + Err(e) if e.0 == c::EINVAL => {} + Err(e) => return Err(DrmError::ExportSyncFile(e)), + } + } + let dummy = self.get_dummy()?; + let zero = SyncobjPoint(0); + self.transfer( + syncobj, + point, + &dummy, + zero, + DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT, + )?; + export(0, self.get_handle(&dummy)?, zero).map_err(DrmError::ExportSyncFile) } fn transfer( @@ -259,6 +304,7 @@ impl SyncobjCtx { src_point: SyncobjPoint, dst_syncobj: &Syncobj, dst_point: SyncobjPoint, + flags: u32, ) -> Result<(), DrmError> { let src_handle = self.get_handle(src_syncobj)?; let dst_handle = self.get_handle(dst_syncobj)?; @@ -268,7 +314,7 @@ impl SyncobjCtx { src_point.0, dst_handle.0, dst_point.0, - 0, + flags, ) .map_err(DrmError::TransferPoint) } @@ -283,6 +329,11 @@ impl SyncobjCtx { } } } + + pub fn query_last_signaled(&self, syncobj: &Syncobj) -> Result { + let handle = self.get_handle(syncobj)?; + syncobj_query(self.inner.drm.raw(), handle.0).map_err(DrmError::QuerySyncobj) + } } impl Drop for SyncobjCtx { diff --git a/src/video/drm/sys.rs b/src/video/drm/sys.rs index 0843d4fe..e0a55ccf 100644 --- a/src/video/drm/sys.rs +++ b/src/video/drm/sys.rs @@ -171,7 +171,10 @@ pub fn get_device_name_from_fd2(fd: c::c_int) -> Result { pub fn get_nodes(fd: c::c_int) -> Result, OsError> { let (_, maj, min) = drm_stat(fd)?; + get_drm_nodes_from_dev(maj, min) +} +pub fn get_drm_nodes_from_dev(maj: u64, min: u64) -> Result, OsError> { let dir = device_dir(maj, min); let mut dir = uapi::opendir(dir)?; @@ -1223,6 +1226,7 @@ pub const DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE: u32 = 1 << 0; pub const DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_TIMELINE: u32 = 1 << 1; pub const DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE: u32 = 1 << 0; +pub const DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_TIMELINE: u32 = 1 << 1; #[repr(C)] struct drm_syncobj_handle { @@ -1236,13 +1240,18 @@ struct drm_syncobj_handle { const DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD: u64 = drm_iowr::(0xC1); const DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE: u64 = drm_iowr::(0xC2); -pub fn syncobj_handle_to_fd(drm: c::c_int, handle: u32, flags: u32) -> Result { +pub fn syncobj_handle_to_fd( + drm: c::c_int, + handle: u32, + flags: u32, + point: u64, +) -> Result { let mut res = drm_syncobj_handle { handle, flags, fd: 0, pad: 0, - point: 0, + point, }; unsafe { ioctl(drm, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &mut res)?; @@ -1271,7 +1280,7 @@ pub fn syncobj_fd_to_handle( } // pub const DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL: u32 = 1 << 0; -// pub const DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT: u32 = 1 << 1; +pub const DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT: u32 = 1 << 1; pub const DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE: u32 = 1 << 2; #[repr(C)] @@ -1328,6 +1337,22 @@ pub fn syncobj_signal(drm: c::c_int, handle: u32, point: u64) -> Result<(), OsEr Ok(()) } +const DRM_IOCTL_SYNCOBJ_QUERY: u64 = drm_iowr::(0xCB); + +pub fn syncobj_query(drm: c::c_int, handle: u32) -> Result { + let mut point = 0u64; + let mut res = drm_syncobj_timeline_array { + handles: &handle as *const u32 as u64, + points: &mut point as *mut u64 as u64, + count_handles: 1, + flags: 0, + }; + unsafe { + ioctl(drm, DRM_IOCTL_SYNCOBJ_QUERY, &mut res)?; + } + Ok(point) +} + #[repr(C)] struct drm_syncobj_transfer { src_handle: u32, diff --git a/src/vulkan_core.rs b/src/vulkan_core.rs index b9261c3b..397a5356 100644 --- a/src/vulkan_core.rs +++ b/src/vulkan_core.rs @@ -1,4 +1,5 @@ use { + crate::{eventfd_cache::EventfdError, video::drm::DrmError}, ahash::{AHashMap, AHashSet}, ash::{ Entry, Instance, LoadingError, @@ -6,9 +7,13 @@ use { vk::{ self, API_VERSION_1_3, ApplicationInfo, Bool32, DebugUtilsMessageSeverityFlagsEXT, DebugUtilsMessageTypeFlagsEXT, DebugUtilsMessengerCallbackDataEXT, - DebugUtilsMessengerCreateInfoEXT, DebugUtilsMessengerEXT, ExtensionProperties, FALSE, - InstanceCreateInfo, LayerProperties, ValidationFeaturesEXT, api_version_major, - api_version_minor, api_version_patch, api_version_variant, + DebugUtilsMessengerCreateInfoEXT, DebugUtilsMessengerEXT, ExtensionProperties, + ExternalSemaphoreFeatureFlags, ExternalSemaphoreHandleTypeFlags, + ExternalSemaphoreProperties, FALSE, InstanceCreateInfo, LayerProperties, + PhysicalDevice, PhysicalDeviceExternalSemaphoreInfo, PhysicalDeviceFeatures, + PhysicalDeviceFeatures2, PhysicalDeviceTimelineSemaphoreFeatures, SemaphoreType, + SemaphoreTypeCreateInfo, ValidationFeaturesEXT, api_version_major, api_version_minor, + api_version_patch, api_version_variant, }, }, isnt::std_1::collections::IsntHashMapExt, @@ -25,6 +30,11 @@ use { uapi::{Ustr, ustr}, }; +pub mod device; +pub mod fence; +pub mod sync; +pub mod timeline_semaphore; + static VULKAN_ENTRY: Lazy>> = Lazy::new(|| unsafe { Entry::load() }.map_err(Arc::new)); @@ -45,6 +55,26 @@ pub enum VulkanCoreError { CreateInstance(#[source] vk::Result), #[error("Could not create a debug-utils messenger")] Messenger(#[source] vk::Result), + #[error("Could not create a fence")] + CreateFence(#[source] vk::Result), + #[error("Could not export a sync file from a semaphore")] + ExportSyncFile(#[source] vk::Result), + #[error("Could not create a semaphore")] + CreateSemaphore(#[source] vk::Result), + #[error("Device does not support timeline semaphore export")] + TimelineExportNotSupported, + #[error("Could not export an opaque fd from a semaphore")] + ExportTimelineSemaphore(#[source] vk::Result), + #[error("Could not signal the timeline semaphore")] + SignalSemaphore(#[source] vk::Result), + #[error("Could not query last signaled sync obj point")] + QueryLastSignaled(#[source] DrmError), + #[error("Mapping between syncobj points and timeline semaphore points is unexpected")] + UnsupportedPointMapping, + #[error("Could not acquire an eventfd")] + AcquireEventfd(#[source] EventfdError), + #[error("Could not create a sync obj eventfd wait")] + CreateSyncobjWait(#[source] DrmError), } pub struct VulkanCoreInstance { @@ -53,6 +83,13 @@ pub struct VulkanCoreInstance { debug_utils: debug_utils::Instance, messenger: DebugUtilsMessengerEXT, pub log_level: Level, + pub validation: bool, +} + +pub struct VulkanDeviceFeatures { + #[expect(dead_code)] + pub features: PhysicalDeviceFeatures, + pub semaphore_features: PhysicalDeviceTimelineSemaphoreFeatures<'static>, } impl VulkanCoreInstance { @@ -78,7 +115,8 @@ impl VulkanCoreInstance { let mut severity = DebugUtilsMessageSeverityFlagsEXT::empty() | DebugUtilsMessageSeverityFlagsEXT::ERROR | DebugUtilsMessageSeverityFlagsEXT::WARNING; - if *VULKAN_VALIDATION { + let validation = *VULKAN_VALIDATION; + if validation { severity |= DebugUtilsMessageSeverityFlagsEXT::INFO | DebugUtilsMessageSeverityFlagsEXT::VERBOSE; } @@ -102,7 +140,7 @@ impl VulkanCoreInstance { .application_info(&app_info) .push_next(&mut debug_info); let validation_layer_name = VALIDATION_LAYER.as_ptr(); - if *VULKAN_VALIDATION { + if validation { if get_available_layers(entry)?.contains(VALIDATION_LAYER) { create_info = create_info.enabled_layer_names(slice::from_ref(&validation_layer_name)); @@ -138,8 +176,49 @@ impl VulkanCoreInstance { debug_utils, messenger, log_level, + validation, }) } + + pub fn get_features(&self, phy_dev: PhysicalDevice) -> VulkanDeviceFeatures { + let mut semaphore_features = PhysicalDeviceTimelineSemaphoreFeatures::default(); + let mut features = PhysicalDeviceFeatures2::default().push_next(&mut semaphore_features); + unsafe { + self.instance + .get_physical_device_features2(phy_dev, &mut features); + } + VulkanDeviceFeatures { + features: features.features, + semaphore_features, + } + } + + pub fn supports_timeline_opaque_export( + &self, + phy_dev: PhysicalDevice, + features: &VulkanDeviceFeatures, + ) -> bool { + if features.semaphore_features.timeline_semaphore == vk::TRUE { + return self.supports_semaphore_opaque_export(phy_dev); + } + false + } + + fn supports_semaphore_opaque_export(&self, phy_dev: PhysicalDevice) -> bool { + let mut props = ExternalSemaphoreProperties::default(); + let mut type_info = + SemaphoreTypeCreateInfo::default().semaphore_type(SemaphoreType::TIMELINE); + let info = PhysicalDeviceExternalSemaphoreInfo::default() + .handle_type(ExternalSemaphoreHandleTypeFlags::OPAQUE_FD) + .push_next(&mut type_info); + unsafe { + self.instance + .get_physical_device_external_semaphore_properties(phy_dev, &info, &mut props); + } + props + .external_semaphore_features + .contains(ExternalSemaphoreFeatureFlags::EXPORTABLE) + } } impl Drop for VulkanCoreInstance { diff --git a/src/vulkan_core/device.rs b/src/vulkan_core/device.rs new file mode 100644 index 00000000..ab4ceb1a --- /dev/null +++ b/src/vulkan_core/device.rs @@ -0,0 +1,21 @@ +use { + crate::{ + eventfd_cache::EventfdCache, video::drm::syncobj::SyncobjCtx, + vulkan_core::VulkanCoreInstance, + }, + ash::{ + Device, + khr::{external_fence_fd, external_semaphore_fd}, + }, + std::rc::Rc, +}; + +pub trait VulkanDeviceInf: Sized { + fn instance(&self) -> &VulkanCoreInstance; + fn device(&self) -> &Device; + fn external_fence_fd(&self) -> &external_fence_fd::Device; + fn external_semaphore_fd(&self) -> &external_semaphore_fd::Device; + fn supports_timeline_opaque_export(&self) -> bool; + fn sync_ctx(&self) -> &Rc; + fn eventfd_cache(&self) -> &Rc; +} diff --git a/src/gfx_apis/vulkan/fence.rs b/src/vulkan_core/fence.rs similarity index 51% rename from src/gfx_apis/vulkan/fence.rs rename to src/vulkan_core/fence.rs index ad0fde1e..324a8f01 100644 --- a/src/gfx_apis/vulkan/fence.rs +++ b/src/vulkan_core/fence.rs @@ -1,7 +1,7 @@ use { crate::{ gfx_api::SyncFile, - gfx_apis::vulkan::{VulkanError, device::VulkanDevice}, + vulkan_core::{VulkanCoreError, device::VulkanDeviceInf}, }, ash::vk::{ ExportFenceCreateInfo, ExternalFenceHandleTypeFlags, Fence, FenceCreateInfo, @@ -11,27 +11,40 @@ use { uapi::OwnedFd, }; -pub struct VulkanFence { - pub(super) device: Rc, - pub(super) fence: Fence, +pub struct VulkanFence +where + D: VulkanDeviceInf, +{ + pub device: Rc, + pub fence: Fence, } -impl Drop for VulkanFence { +pub trait VulkanDeviceFenceExt: VulkanDeviceInf { + fn create_fence(self: &Rc) -> Result>, VulkanCoreError>; +} + +impl Drop for VulkanFence +where + D: VulkanDeviceInf, +{ fn drop(&mut self) { unsafe { - self.device.device.destroy_fence(self.fence, None); + self.device.device().destroy_fence(self.fence, None); } } } -impl VulkanDevice { - pub fn create_fence(self: &Rc) -> Result, VulkanError> { +impl VulkanDeviceFenceExt for D +where + D: VulkanDeviceInf, +{ + fn create_fence(self: &Rc) -> Result>, VulkanCoreError> { let fence = { let mut export_info = ExportFenceCreateInfo::default() .handle_types(ExternalFenceHandleTypeFlags::SYNC_FD); let create_info = FenceCreateInfo::default().push_next(&mut export_info); - let fence = unsafe { self.device.create_fence(&create_info, None) }; - fence.map_err(VulkanError::CreateFence)? + let fence = unsafe { self.device().create_fence(&create_info, None) }; + fence.map_err(VulkanCoreError::CreateFence)? }; Ok(Rc::new(VulkanFence { device: self.clone(), @@ -40,13 +53,16 @@ impl VulkanDevice { } } -impl VulkanFence { - pub fn export_sync_file(&self) -> Result, VulkanError> { +impl VulkanFence +where + D: VulkanDeviceInf, +{ + pub fn export_sync_file(&self) -> Result, VulkanCoreError> { let info = FenceGetFdInfoKHR::default() .fence(self.fence) .handle_type(ExternalFenceHandleTypeFlags::SYNC_FD); - let res = unsafe { self.device.external_fence_fd.get_fence_fd(&info) }; - let fd = res.map_err(VulkanError::ExportSyncFile)?; + let res = unsafe { self.device.external_fence_fd().get_fence_fd(&info) }; + let fd = res.map_err(VulkanCoreError::ExportSyncFile)?; if fd == -1 { Ok(None) } else { diff --git a/src/vulkan_core/sync.rs b/src/vulkan_core/sync.rs new file mode 100644 index 00000000..aaa7e317 --- /dev/null +++ b/src/vulkan_core/sync.rs @@ -0,0 +1,136 @@ +use { + crate::{ + gfx_api::{FdSync, ReservedSyncobjPoint}, + utils::errorfmt::ErrorFmt, + video::drm::syncobj::SyncobjPoint, + vulkan_core::{ + VulkanCoreError, + device::VulkanDeviceInf, + fence::{VulkanDeviceFenceExt, VulkanFence}, + timeline_semaphore::VulkanTimelineSemaphore, + }, + }, + ash::vk::{Fence, PipelineStageFlags2, SemaphoreSubmitInfo, SemaphoreWaitInfo, SubmitInfo2}, + std::{rc::Rc, slice}, +}; + +pub enum VulkanSync +where + D: VulkanDeviceInf, +{ + Fence(Rc>), + TimelineSemaphore { + tls: Rc>, + pending: Rc, + }, +} + +impl VulkanSync +where + D: VulkanDeviceInf, +{ + pub fn handle_validation(&self) { + if let VulkanSync::TimelineSemaphore { tls, pending } = self + && tls.device.instance().validation + { + let info = SemaphoreWaitInfo::default() + .semaphores(slice::from_ref(&tls.semaphore)) + .values(slice::from_ref(&pending.point.0)); + unsafe { + let _ = tls.device.device().wait_semaphores(&info, 0); + } + } + } + + pub fn fence(&self) -> Fence { + match self { + VulkanSync::Fence(f) => f.fence, + VulkanSync::TimelineSemaphore { .. } => Fence::null(), + } + } + + pub fn to_sync(&self, block: impl FnOnce()) -> Option { + match self { + VulkanSync::Fence(release_fence) => { + zone!("export_sync_file"); + let release_sync_file = match release_fence.export_sync_file() { + Ok(s) => s, + Err(e) => { + log::error!("Could not export sync file from fence: {}", ErrorFmt(e)); + block(); + None + } + }; + release_sync_file.map(FdSync::SyncFile) + } + VulkanSync::TimelineSemaphore { pending, .. } => Some(FdSync::Syncobj(pending.clone())), + } + } +} + +pub trait VulkanDeviceSyncExt: VulkanDeviceInf { + fn create_sync<'a>( + self: &Rc, + tls: Option<&Rc>>, + semaphore_submit_info: &'a mut SemaphoreSubmitInfo, + submit_info: &mut SubmitInfo2<'a>, + ) -> Result, VulkanCoreError>; +} + +impl VulkanDeviceSyncExt for D +where + D: VulkanDeviceInf, +{ + fn create_sync<'a>( + self: &Rc, + tls: Option<&Rc>>, + semaphore_submit_info: &'a mut SemaphoreSubmitInfo, + submit_info: &mut SubmitInfo2<'a>, + ) -> Result, VulkanCoreError> { + if let Some(tls) = tls { + match create_tls_sync(self, tls, semaphore_submit_info, submit_info) { + Ok(s) => return Ok(s), + Err(e) => { + log::warn!("Could not create sync obj sync: {}", ErrorFmt(e)); + } + } + } + self.create_fence().map(VulkanSync::Fence) + } +} + +fn create_tls_sync<'a, D>( + device: &Rc, + tls: &Rc>, + semaphore_submit_info: &'a mut SemaphoreSubmitInfo, + submit_info: &mut SubmitInfo2<'a>, +) -> Result, VulkanCoreError> +where + D: VulkanDeviceInf, +{ + let point = SyncobjPoint(tls.next_point.fetch_add(1)); + let eventfd = device + .eventfd_cache() + .acquire() + .map_err(VulkanCoreError::AcquireEventfd)?; + device + .sync_ctx() + .wait_for_point(&eventfd.fd, &tls.syncobj, point, true) + .map_err(VulkanCoreError::CreateSyncobjWait)?; + let pending = Rc::new(ReservedSyncobjPoint { + ctx: device.sync_ctx().clone(), + syncobj: tls.syncobj.clone(), + point, + sync_file: Default::default(), + signaled: eventfd, + }); + *semaphore_submit_info = SemaphoreSubmitInfo::default() + .semaphore(tls.semaphore) + .value(point.0) + .stage_mask(PipelineStageFlags2::ALL_COMMANDS); + *submit_info = submit_info.signal_semaphore_infos(slice::from_ref(semaphore_submit_info)); + Ok(VulkanSync::TimelineSemaphore { + tls: tls.clone(), + pending, + }) +} diff --git a/src/vulkan_core/timeline_semaphore.rs b/src/vulkan_core/timeline_semaphore.rs new file mode 100644 index 00000000..7f46c689 --- /dev/null +++ b/src/vulkan_core/timeline_semaphore.rs @@ -0,0 +1,116 @@ +use { + crate::{ + utils::{errorfmt::ErrorFmt, numcell::NumCell}, + video::drm::syncobj::Syncobj, + vulkan_core::{VulkanCoreError, device::VulkanDeviceInf}, + }, + ash::vk::{ + ExportSemaphoreCreateInfo, ExternalSemaphoreHandleTypeFlags, Semaphore, + SemaphoreCreateInfo, SemaphoreGetFdInfoKHR, SemaphoreSignalInfo, SemaphoreType, + SemaphoreTypeCreateInfo, + }, + run_on_drop::on_drop, + std::rc::Rc, + uapi::OwnedFd, +}; + +pub struct VulkanTimelineSemaphore +where + D: VulkanDeviceInf, +{ + pub(super) device: Rc, + pub(super) semaphore: Semaphore, + pub(super) syncobj: Rc, + pub(super) next_point: NumCell, +} + +impl Drop for VulkanTimelineSemaphore +where + D: VulkanDeviceInf, +{ + fn drop(&mut self) { + unsafe { + self.device.device().destroy_semaphore(self.semaphore, None); + } + } +} + +pub trait VulkanDeviceTimelineSemaphoreExt: VulkanDeviceInf { + fn create_timeline_semaphore( + self: &Rc, + ) -> Result>, VulkanCoreError>; + + fn create_timeline_semaphore_or_log( + self: &Rc, + ) -> Option>>; +} + +impl VulkanDeviceTimelineSemaphoreExt for D +where + D: VulkanDeviceInf, +{ + fn create_timeline_semaphore( + self: &Rc, + ) -> Result>, VulkanCoreError> { + if !self.supports_timeline_opaque_export() { + return Err(VulkanCoreError::TimelineExportNotSupported); + } + let sem = { + let mut export_info = ExportSemaphoreCreateInfo::default() + .handle_types(ExternalSemaphoreHandleTypeFlags::OPAQUE_FD); + let mut type_info = + SemaphoreTypeCreateInfo::default().semaphore_type(SemaphoreType::TIMELINE); + let info = SemaphoreCreateInfo::default() + .push_next(&mut export_info) + .push_next(&mut type_info); + let sem = unsafe { self.device().create_semaphore(&info, None) }; + sem.map_err(VulkanCoreError::CreateSemaphore)? + }; + let destroy_sem = on_drop(|| unsafe { self.device().destroy_semaphore(sem, None) }); + let syncobj = { + let info = SemaphoreGetFdInfoKHR::default() + .semaphore(sem) + .handle_type(ExternalSemaphoreHandleTypeFlags::OPAQUE_FD); + let res = unsafe { self.external_semaphore_fd().get_semaphore_fd(&info) }; + let fd = res.map_err(VulkanCoreError::ExportTimelineSemaphore)?; + Syncobj::new(&Rc::new(OwnedFd::new(fd))) + }; + let signal = |p| { + let info = SemaphoreSignalInfo::default().semaphore(sem).value(p); + unsafe { + self.device() + .signal_semaphore(&info) + .map_err(VulkanCoreError::SignalSemaphore) + } + }; + let next_point = NumCell::new(1); + for _ in 0..2 { + let n = next_point.fetch_add(1); + signal(n)?; + let signaled = self + .sync_ctx() + .query_last_signaled(&syncobj) + .map_err(VulkanCoreError::QueryLastSignaled)?; + if signaled != n { + return Err(VulkanCoreError::UnsupportedPointMapping); + } + } + destroy_sem.forget(); + Ok(Rc::new(VulkanTimelineSemaphore { + device: self.clone(), + semaphore: sem, + syncobj: Rc::new(syncobj), + next_point, + })) + } + + fn create_timeline_semaphore_or_log( + self: &Rc, + ) -> Option>> { + self.create_timeline_semaphore() + .inspect_err(|e| { + log::warn!("Could not create timeline semaphore: {}", ErrorFmt(e)); + }) + .ok() + } +}