From 0f556fc054bc65dc97b8083aa166e262fcd7d875 Mon Sep 17 00:00:00 2001 From: Julian Orth Date: Sun, 15 Feb 2026 14:13:30 +0100 Subject: [PATCH 1/5] gfx: return formats by reference --- src/drm_feedback.rs | 2 +- src/gfx_api.rs | 2 +- src/gfx_apis/gl/renderer/context.rs | 6 +++--- src/gfx_apis/vulkan.rs | 4 ++-- src/ifs/zwp_linux_buffer_params_v1.rs | 2 +- src/it/test_gfx_api.rs | 4 ++-- src/portal/ptl_display.rs | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/drm_feedback.rs b/src/drm_feedback.rs index e83a092d..7ac9b0ee 100644 --- a/src/drm_feedback.rs +++ b/src/drm_feedback.rs @@ -103,7 +103,7 @@ fn create_fd_data(ctx: &dyn GfxContext) -> (Vec, AHashMap<(u32, Modifier), u let mut vec = vec![]; let mut map = AHashMap::new(); let mut pos = 0; - for (format, info) in &*ctx.formats() { + for (format, info) in &**ctx.formats() { for modifier in &info.read_modifiers { vec.write_u32::(*format).unwrap(); vec.write_u32::(0).unwrap(); diff --git a/src/gfx_api.rs b/src/gfx_api.rs index 6fbddab2..7adf80a5 100644 --- a/src/gfx_api.rs +++ b/src/gfx_api.rs @@ -747,7 +747,7 @@ pub trait GfxContext: Debug { fn render_node(&self) -> Option>; - fn formats(&self) -> Rc>; + fn formats(&self) -> &Rc>; fn fast_ram_access(&self) -> bool; diff --git a/src/gfx_apis/gl/renderer/context.rs b/src/gfx_apis/gl/renderer/context.rs index 2be8defb..17441ea3 100644 --- a/src/gfx_apis/gl/renderer/context.rs +++ b/src/gfx_apis/gl/renderer/context.rs @@ -187,8 +187,8 @@ impl GlRenderContext { self.render_node.clone() } - pub fn formats(&self) -> Rc> { - self.ctx.formats.clone() + pub fn formats(&self) -> &Rc> { + &self.ctx.formats } fn dmabuf_fb(self: &Rc, buf: &DmaBuf) -> Result, RenderError> { @@ -253,7 +253,7 @@ impl GfxContext for GlRenderContext { Some(self.render_node()) } - fn formats(&self) -> Rc> { + fn formats(&self) -> &Rc> { self.formats() } diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs index 8ab6168f..ad0a5943 100644 --- a/src/gfx_apis/vulkan.rs +++ b/src/gfx_apis/vulkan.rs @@ -263,8 +263,8 @@ impl GfxContext for Context { Some(self.0.device.render_node.clone()) } - fn formats(&self) -> Rc> { - self.0.formats.clone() + fn formats(&self) -> &Rc> { + &self.0.formats } fn fast_ram_access(&self) -> bool { diff --git a/src/ifs/zwp_linux_buffer_params_v1.rs b/src/ifs/zwp_linux_buffer_params_v1.rs index 13b62538..eac91ff6 100644 --- a/src/ifs/zwp_linux_buffer_params_v1.rs +++ b/src/ifs/zwp_linux_buffer_params_v1.rs @@ -74,7 +74,7 @@ impl ZwpLinuxBufferParamsV1 { Some(ctx) => ctx, None => return Err(ZwpLinuxBufferParamsV1Error::NoRenderContext), }; - let formats = ctx.formats(); + let formats = ctx.formats().clone(); let format = match formats.get(&format) { Some(f) => f, None => return Err(ZwpLinuxBufferParamsV1Error::InvalidFormat(format)), diff --git a/src/it/test_gfx_api.rs b/src/it/test_gfx_api.rs index 596de1de..44993212 100644 --- a/src/it/test_gfx_api.rs +++ b/src/it/test_gfx_api.rs @@ -103,8 +103,8 @@ impl GfxContext for TestGfxCtx { None } - fn formats(&self) -> Rc> { - self.formats.clone() + fn formats(&self) -> &Rc> { + &self.formats } fn fast_ram_access(&self) -> bool { diff --git a/src/portal/ptl_display.rs b/src/portal/ptl_display.rs index 3b5c78fa..e52e2e0c 100644 --- a/src/portal/ptl_display.rs +++ b/src/portal/ptl_display.rs @@ -211,9 +211,9 @@ impl UsrJayRenderCtxOwner for PortalDisplay { if let Some(ctx) = render_ctx { let client_formats = ctx.ctx.formats(); let usable_formats = match &server_formats { - None => client_formats, + None => client_formats.clone(), Some(server_formats) => { - Rc::new(cross_intersect_formats(&client_formats, server_formats)) + Rc::new(cross_intersect_formats(client_formats, server_formats)) } }; self.render_ctx.set(Some(Rc::new(PortalServerRenderCtx { From f2a0221c9e8f816396f95ded11ef3bf3944ea72f Mon Sep 17 00:00:00 2001 From: Julian Orth Date: Fri, 13 Feb 2026 17:10:09 +0100 Subject: [PATCH 2/5] metal: refactor buffer allocation --- src/backends/metal.rs | 135 +---- src/backends/metal/allocator.rs | 938 ++++++++++++++++++++++++++++++ src/backends/metal/present.rs | 24 +- src/backends/metal/transaction.rs | 33 +- src/backends/metal/video.rs | 460 +-------------- 5 files changed, 1000 insertions(+), 590 deletions(-) create mode 100644 src/backends/metal/allocator.rs diff --git a/src/backends/metal.rs b/src/backends/metal.rs index 978e0b98..de329aa3 100644 --- a/src/backends/metal.rs +++ b/src/backends/metal.rs @@ -1,3 +1,4 @@ +mod allocator; mod input; mod monitor; mod present; @@ -12,13 +13,15 @@ use { InputDeviceClickMethod, InputDeviceGroupId, InputDeviceId, InputEvent, KeyState, Leds, TransformMatrix, transaction::BackendConnectorTransactionError, }, - backends::metal::video::{ - MetalDrmDeviceData, MetalLeaseData, MetalRenderContext, PendingDrmDevice, - PersistentDisplayData, + backends::metal::{ + allocator::{RenderBufferError, ScanoutBufferError, ScanoutBufferErrors}, + video::{ + MetalDrmDeviceData, MetalLeaseData, MetalRenderContext, PendingDrmDevice, + PersistentDisplayData, + }, }, dbus::{DbusError, SignalHandler}, drm_feedback::DrmFeedback, - format::Format, gfx_api::{GfxError, SyncFile}, ifs::{ wl_output::OutputId, @@ -49,15 +52,14 @@ use { smallmap::SmallMap, syncqueue::SyncQueue, }, - video::{Modifier, drm::DrmError, gbm::GbmError}, + video::{drm::DrmError, gbm::GbmError}, }, bstr::ByteSlice, - indexmap::IndexSet, std::{ cell::{Cell, RefCell}, error::Error, ffi::{CStr, CString}, - fmt::{Debug, Display, Formatter}, + fmt::{Debug, Formatter}, future::pending, rc::Rc, }, @@ -105,8 +107,6 @@ pub enum MetalError { DeviceResumeSignalHandler(#[source] DbusError), #[error("Could not render the frame")] RenderFrame(#[source] GfxError), - #[error("Could not copy frame to output device")] - CopyToOutput(#[source] GfxError), #[error("Could not perform atomic commit")] Commit(#[source] DrmError), #[error("The present configuration is out of date")] @@ -117,119 +117,14 @@ pub enum MetalError { CalculateDrmState(#[source] BackendConnectorTransactionError), #[error("Could not calculate DRM change set")] CalculateDrmChange(#[source] BackendConnectorTransactionError), - #[error("Could not create plane buffer")] + #[error("Could not create non-prime plane buffer")] AllocateScanoutBuffer(#[source] Box), -} - -#[derive(Debug, Error)] -pub enum ScanoutBufferErrorKind { - #[error("Scanout device: The format is not supported")] - SodUnsupportedFormat, - #[error( - "Scanout device: The intersection of the modifiers supported by the plane and modifiers writable by the gfx API is empty" - )] - SodNoWritableModifier, - #[error("Scanout device: Buffer allocation failed")] - SodBufferAllocation(#[source] GbmError), - #[error("Scanout device: addfb2 failed")] - SodAddfb2(#[source] DrmError), - #[error("Scanout device: Could not import SCANOUT buffer into the gfx API")] - SodImportSodImage(#[source] GfxError), - #[error("Scanout device: Could not turn imported SCANOUT buffer into gfx API FB")] - SodImportFb(#[source] GfxError), - #[error("Scanout device: Could not clear SCANOUT buffer")] - SodClear(#[source] GfxError), - #[error("Scanout device: Could not turn imported SCANOUT buffer into gfx API texture")] - SodImportSodTexture(#[source] GfxError), - #[error("Render device: The format is not supported")] + #[error("Could not create non-prime plane buffer")] + AllocateScanoutBufferPrime(#[source] ScanoutBufferErrors), + #[error("Could not copy frame to output device")] + CopyToDev(#[source] RenderBufferError), + #[error("The render device does not support the format")] RenderUnsupportedFormat, - #[error( - "Render device: The intersection of the modifiers readable by the scanout device and modifiers writable by the gfx API is empty" - )] - RenderNoWritableModifier, - #[error("Render device: Buffer allocation failed")] - RenderBufferAllocation(#[source] GbmError), - #[error("Render device: Could not import RENDER buffer into the gfx API")] - RenderImportImage(#[source] GfxError), - #[error("Render device: Could not turn imported RENDER buffer into gfx API FB")] - RenderImportFb(#[source] GfxError), - #[error("Render device: Could not clear RENDER buffer")] - RenderClear(#[source] GfxError), - #[error("Render device: Could not turn imported RENDER buffer into gfx API texture")] - RenderImportRenderTexture(#[source] GfxError), - #[error("Scanout device: Could not import RENDER buffer into the gfx API")] - SodImportRenderImage(#[source] GfxError), - #[error("Scanout device: Could not turn imported RENDER buffer into gfx API texture")] - SodImportRenderTexture(#[source] GfxError), -} - -#[derive(Debug)] -pub struct ScanoutBufferError { - dev: String, - format: &'static Format, - plane_modifiers: IndexSet, - width: i32, - height: i32, - cursor: bool, - dev_gfx_write_modifiers: Option>, - dev_gfx_read_modifiers: Option>, - dev_modifiers_possible: Option>, - dev_usage: Option, - dev_modifier: Option, - render_name: Option, - render_gfx_write_modifiers: Option>, - render_modifiers_possible: Option>, - render_usage: Option, - render_modifier: Option, - kind: ScanoutBufferErrorKind, -} - -impl Display for ScanoutBufferError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - writeln!(f)?; - writeln!(f, "scanout device: {}", self.dev)?; - writeln!(f, "format: {}", self.format.name)?; - writeln!(f, "plane modifiers: {:x?}", self.plane_modifiers)?; - writeln!(f, "size: {}x{}", self.width, self.height)?; - writeln!(f, "cursor: {}", self.cursor)?; - if let Some(v) = &self.dev_gfx_write_modifiers { - writeln!(f, "scanout gfx writable modifiers: {:x?}", v)?; - } - if let Some(v) = &self.dev_modifiers_possible { - writeln!(f, "scanout dev possible modifiers: {:x?}", v)?; - } - if let Some(v) = &self.dev_usage { - writeln!(f, "scanout dev gbm usage: {:x}", v)?; - } - if let Some(v) = &self.dev_modifier { - writeln!(f, "scanout dev modifier: {:x}", v)?; - } - if let Some(v) = &self.render_name { - writeln!(f, "render device: {}", v)?; - } - if let Some(v) = &self.render_gfx_write_modifiers { - writeln!(f, "render gfx writable modifiers: {:x?}", v)?; - } - if let Some(v) = &self.dev_gfx_read_modifiers { - writeln!(f, "scanout gfx readable modifiers: {:x?}", v)?; - } - if let Some(v) = &self.render_modifiers_possible { - writeln!(f, "render dev possible modifiers: {:x?}", v)?; - } - if let Some(v) = &self.render_usage { - writeln!(f, "render dev gbm usage: {:x}", v)?; - } - if let Some(v) = &self.render_modifier { - writeln!(f, "render dev modifier: {:x}", v)?; - } - Ok(()) - } -} - -impl Error for ScanoutBufferError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - Some(&self.kind) - } } pub struct MetalBackend { diff --git a/src/backends/metal/allocator.rs b/src/backends/metal/allocator.rs new file mode 100644 index 00000000..958299bb --- /dev/null +++ b/src/backends/metal/allocator.rs @@ -0,0 +1,938 @@ +use { + crate::{ + allocator::BufferObject, + backends::metal::{ + MetalBackend, MetalError, + video::{MetalDrmDevice, MetalRenderContext}, + }, + cmm::cmm_description::ColorDescription, + format::Format, + gfx_api::{ + AcquireSync, GfxBlendBuffer, GfxError, GfxFormat, GfxFramebuffer, GfxTexture, + GfxWriteModifier, ReleaseSync, SyncFile, needs_render_usage, + }, + rect::{DamageQueue, Rect}, + utils::{errorfmt::ErrorFmt, rc_eq::rc_eq}, + video::{ + Modifier, + dmabuf::DmaBuf, + drm::{DrmError, DrmFramebuffer}, + gbm::{GBM_BO_USE_LINEAR, GBM_BO_USE_RENDERING, GBM_BO_USE_SCANOUT, GbmBo, GbmError}, + }, + }, + arrayvec::ArrayVec, + bstr::ByteSlice, + indexmap::{IndexMap, IndexSet}, + isnt::std_1::primitive::IsntSliceExt, + linearize::{Linearize, LinearizeExt, StaticMap}, + log::Level, + run_on_drop::on_drop, + std::{ + cell::{Cell, RefCell}, + error::Error, + fmt::{self, Debug, Display, Formatter}, + rc::Rc, + sync::LazyLock, + }, + thiserror::Error, +}; + +#[derive(Debug)] +pub struct RenderBuffer { + pub width: i32, + pub height: i32, + pub locked: Cell, + pub format: &'static Format, + pub drm: Rc, + pub damage_queue: DamageQueue, + pub blend_buffer: Option>, + pub render: RenderBufferRender, + pub dev_ctx: Rc, + pub prime: RenderBufferPrime, +} + +#[derive(Debug)] +pub struct RenderBufferRender { + pub ctx: Rc, + pub bo: GbmBo, + pub tex: Rc, + pub fb: Rc, +} + +#[derive(Debug)] +pub enum RenderBufferPrime { + None, + Sampling { + dev_bo: GbmBo, + dev_fb: Rc, + // Import of the render dmabuf into the dev ctx. + dev_render_tex: Rc, + }, +} + +#[derive(Debug, Error)] +pub enum RenderBufferError { + #[error("Cannot copy between buffers of different size")] + NotSameSize, + #[error(transparent)] + GfxError(#[from] GfxError), + #[error("Could not copy frame to output device")] + CopyToOutput(#[source] GfxError), +} + +#[derive(Default)] +pub struct RenderBufferCopy { + pub render_block: Option, + pub present_block: Option, +} + +impl RenderBufferCopy { + pub fn for_both(sf: Option) -> Self { + Self { + render_block: sf.clone(), + present_block: sf, + } + } +} + +impl RenderBuffer { + pub fn copy_to_dev( + &self, + cd: &Rc, + sync_file: Option, + ) -> Result { + match &self.prime { + RenderBufferPrime::None => Ok(RenderBufferCopy { + render_block: None, + present_block: sync_file, + }), + RenderBufferPrime::Sampling { + dev_render_tex, + dev_fb, + .. + } => dev_fb + .copy_texture( + AcquireSync::Unnecessary, + ReleaseSync::Explicit, + cd, + dev_render_tex, + cd, + None, + AcquireSync::from_sync_file(sync_file), + ReleaseSync::None, + 0, + 0, + ) + .map_err(RenderBufferError::CopyToOutput) + .map(RenderBufferCopy::for_both), + } + } + + pub fn damage_full(&self) { + let rect = Rect::new_sized_saturating(0, 0, self.width, self.height); + self.damage_queue.clear_all(); + self.damage_queue.damage(&[rect]); + } + + pub fn clear(&self, cd: &Rc) -> Result, RenderBufferError> { + match &self.prime { + RenderBufferPrime::None => self + .render + .fb + .clear(AcquireSync::Unnecessary, ReleaseSync::Explicit, cd) + .map_err(Into::into), + RenderBufferPrime::Sampling { dev_fb, .. } => dev_fb + .clear(AcquireSync::Unnecessary, ReleaseSync::Explicit, cd) + .map_err(Into::into), + } + } + + pub fn copy_to_new( + &self, + new: &Self, + cd: &Rc, + ) -> Result, RenderBufferError> { + let old = self; + + if (old.width, old.height) != (new.width, new.height) { + return Err(RenderBufferError::NotSameSize); + } + + let copy_texture_impl = |fb: &Rc, tex: &Rc| { + fb.copy_texture( + AcquireSync::Unnecessary, + ReleaseSync::Explicit, + cd, + tex, + cd, + None, + AcquireSync::Unnecessary, + ReleaseSync::Explicit, + 0, + 0, + ) + }; + let copy_texture = |new_ctx: &Rc, + fb: &Rc, + old_ctx: &Rc, + tex: &Rc, + dma_buf: &DmaBuf| { + if rc_eq(&new_ctx.gfx, &old_ctx.gfx) { + return copy_texture_impl(fb, tex); + } + let tex = new_ctx.gfx.clone().dmabuf_img(dma_buf)?.to_texture()?; + copy_texture_impl(fb, &tex) + }; + + let sf = match &old.prime { + RenderBufferPrime::None => match &new.prime { + RenderBufferPrime::None => copy_texture( + &new.render.ctx, + &new.render.fb, + &old.render.ctx, + &old.render.tex, + old.render.bo.dmabuf(), + )?, + RenderBufferPrime::Sampling { + dev_fb: new_dev_fb, .. + } => copy_texture( + &new.dev_ctx, + new_dev_fb, + &old.render.ctx, + &old.render.tex, + old.render.bo.dmabuf(), + )?, + }, + RenderBufferPrime::Sampling { + dev_render_tex: old_dev_render_tex, + dev_bo: old_dev_bo, + .. + } => match &new.prime { + RenderBufferPrime::None => copy_texture( + &new.render.ctx, + &new.render.fb, + &old.dev_ctx, + old_dev_render_tex, + old_dev_bo.dmabuf(), + )?, + RenderBufferPrime::Sampling { + dev_fb: new_dev_fb, .. + } => copy_texture( + &new.dev_ctx, + &new_dev_fb, + &old.dev_ctx, + old_dev_render_tex, + old_dev_bo.dmabuf(), + )?, + }, + }; + Ok(sf) + } + + pub fn dev_bo(&self) -> &GbmBo { + match &self.prime { + RenderBufferPrime::None => &self.render.bo, + RenderBufferPrime::Sampling { dev_bo, .. } => dev_bo, + } + } +} + +struct Builder<'a> { + slf: &'a MetalBackend, + dev: &'a Rc, + dev_ctx: &'a Rc, + format: &'static Format, + render_fmt: &'a GfxFormat, + plane_modifiers: &'a IndexSet, + width: i32, + height: i32, + render_ctx: &'a Rc, + cursor: bool, + blend_buffer: Option<&'a Rc>, +} + +struct BoAllocationSettings { + modifiers: Vec, + usage: u32, +} + +struct NoPrime { + allocation_settings: BoAllocationSettings, +} + +struct PrimeSampling { + render_allocation_settings: BoAllocationSettings, + dev_allocation_settings: BoAllocationSettings, +} + +impl MetalBackend { + pub fn create_scanout_buffers( + &self, + dev: &Rc, + format: &'static Format, + plane_modifiers: &IndexSet, + width: i32, + height: i32, + render_ctx: &Rc, + cursor: bool, + ) -> Result<[RenderBuffer; N], MetalError> { + let Some(render_fmt) = render_ctx.gfx.formats().get(&format.drm) else { + return Err(MetalError::RenderUnsupportedFormat); + }; + let mut blend_buffer = None; + if !cursor { + match render_ctx.gfx.acquire_blend_buffer(width, height) { + Ok(bb) => blend_buffer = Some(bb), + Err(e) => { + log::warn!("Could not create blend buffer: {}", ErrorFmt(e)); + } + } + } + let builder = Builder { + slf: self, + dev, + dev_ctx: &dev.ctx.get(), + format, + render_fmt, + plane_modifiers, + width, + height, + render_ctx, + cursor, + blend_buffer: blend_buffer.as_ref(), + }; + if render_ctx.dev_id == dev.id { + return wrap_error(&builder, None, |dbg| { + let prepared = &builder.prepare_prime_none(dbg)?; + self.create_scanout_buffers_(|damage| { + builder.create_prime_none(prepared, damage, dbg) + }) + }) + .map_err(MetalError::AllocateScanoutBuffer); + } + let mut errors = ScanoutBufferErrors::default(); + for &method in &*PRIME_METHODS { + let res = wrap_error(&builder, Some(method), |dbg| { + macro_rules! x { + ($prepare:ident, $create:ident $(,)?) => {{ + let prepared = &builder.$prepare(dbg)?; + self.create_scanout_buffers_(|damage| { + builder.$create(prepared, damage, dbg) + }) + }}; + } + match method { + PrimeMethod::Sampling => { + x!(prepare_prime_sampling, create_prime_sampling) + } + } + }); + match res { + Err(e) => errors.errors.push(e), + Ok(b) => { + if errors.errors.is_not_empty() { + log::warn!("Preferred prime methods failed"); + let debug = log::log_enabled!(Level::Debug); + for error in &errors.errors { + let Some(method) = error.prime else { + continue; + }; + if debug { + log::warn!("- {method}: {}", ErrorFmt(error)); + } else { + log::warn!("- {method}: {}", ErrorFmt(&error.kind)); + } + } + } + return Ok(b); + } + } + } + Err(MetalError::AllocateScanoutBufferPrime(errors)) + } + + fn create_scanout_buffers_( + &self, + allocate: impl Fn(DamageQueue) -> Result, + ) -> Result<[RenderBuffer; N], ScanoutBufferErrorKind> { + let mut damage_queue = ArrayVec::from(DamageQueue::new::()); + let mut array = ArrayVec::<_, N>::new(); + for _ in 0..N { + let damage_queue = damage_queue.pop().unwrap(); + array.push(allocate(damage_queue)?); + } + if let Some(buffer) = array.first() { + buffer.damage_full(); + } + Ok(array.into_inner().unwrap()) + } +} + +#[derive(Debug, Error)] +pub enum ScanoutBufferErrorKind { + #[error("Scanout device: The format is not supported")] + SodUnsupportedFormat, + #[error("Scanout device: Buffer allocation failed")] + SodBufferAllocation(#[source] GbmError), + #[error("Scanout device: addfb2 failed")] + SodAddfb2(#[source] DrmError), + #[error("Scanout device: Could not import SCANOUT buffer into the gfx API")] + SodImportSodImage(#[source] GfxError), + #[error("Scanout device: Could not turn imported SCANOUT buffer into gfx API FB")] + SodImportFb(#[source] GfxError), + #[error("Render device: The intersection of render/sample/sod_sample modifiers is empty")] + RenderWriteReadSodReadIntersection, + #[error("Scanout device: The intersection of render/sample/plane modifiers is empty")] + SodWriteReadPlaneIntersection, + #[error("Scanout device: The intersection of render/plane modifiers is empty")] + SodWritePlaneIntersection, + #[error("Render device: Buffer allocation failed")] + RenderBufferAllocation(#[source] GbmError), + #[error("Render device: Could not import RENDER buffer into the gfx API")] + RenderImportImage(#[source] GfxError), + #[error("Render device: Could not turn imported RENDER buffer into gfx API FB")] + RenderImportFb(#[source] GfxError), + #[error("Render device: Could not clear RENDER buffer")] + RenderClear(#[source] GfxError), + #[error("Render device: Could not turn imported RENDER buffer into gfx API texture")] + RenderImportRenderTexture(#[source] GfxError), + #[error("Scanout device: Could not import RENDER buffer into the gfx API")] + SodImportRenderImage(#[source] GfxError), + #[error("Scanout device: Could not turn imported RENDER buffer into gfx API texture")] + SodImportRenderTexture(#[source] GfxError), +} + +#[derive(Default, Debug)] +pub struct ScanoutBufferErrors { + #[expect(clippy::vec_box)] + errors: Vec>, +} + +#[derive(Debug)] +pub struct ScanoutBufferError { + dev: String, + render_name: Option, + format: &'static Format, + plane_modifiers: IndexSet, + width: i32, + height: i32, + cursor: bool, + dbg: RenderBufferAllocationDebug, + kind: ScanoutBufferErrorKind, + prime: Option, +} + +#[derive(Copy, Clone, Linearize)] +pub enum PrimeMethod { + Sampling, +} + +impl PrimeMethod { + pub fn name(self) -> &'static str { + match self { + PrimeMethod::Sampling => "direct-sampling", + } + } +} + +impl Display for PrimeMethod { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str(self.name()) + } +} + +impl Debug for PrimeMethod { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str(self.name()) + } +} + +impl RenderBufferPrime { + pub fn method(&self) -> Option { + let method = match self { + RenderBufferPrime::None => return None, + RenderBufferPrime::Sampling { .. } => PrimeMethod::Sampling, + }; + Some(method) + } +} + +#[derive(Default, Debug)] +struct RenderBufferAllocationDebug { + dev_gfx_write_modifiers: Option>, + dev_gfx_read_modifiers: Option>, + dev_modifiers_possible: Option>, + dev_usage: Option, + dev_modifier: Option, + render_gfx_write_modifiers: Option>, + render_gfx_read_modifiers: Option>, + render_modifiers_possible: Option>, + render_usage: Option, + render_modifier: Option, +} + +impl Display for ScanoutBufferErrors { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + for (idx, error) in self.errors.iter().enumerate() { + if idx > 0 { + writeln!(f, "\n------")?; + } + write!(f, "{}", ErrorFmt(error))?; + } + Ok(()) + } +} + +impl Error for ScanoutBufferErrors {} + +impl Display for ScanoutBufferError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + writeln!(f)?; + if let Some(v) = self.prime { + writeln!(f, "prime type: {}", v)?; + } + writeln!(f, "scanout device: {}", self.dev)?; + writeln!(f, "format: {}", self.format.name)?; + writeln!(f, "plane modifiers: {:x?}", self.plane_modifiers)?; + writeln!(f, "size: {}x{}", self.width, self.height)?; + writeln!(f, "cursor: {}", self.cursor)?; + if let Some(v) = &self.dbg.dev_gfx_write_modifiers { + writeln!(f, "scanout gfx writable modifiers: {:x?}", v)?; + } + if let Some(v) = &self.dbg.dev_modifiers_possible { + writeln!(f, "scanout dev possible modifiers: {:x?}", v)?; + } + if let Some(v) = &self.dbg.dev_usage { + writeln!(f, "scanout dev gbm usage: {:x}", v)?; + } + if let Some(v) = &self.dbg.dev_modifier { + writeln!(f, "scanout dev modifier: {:x}", v)?; + } + if let Some(v) = &self.render_name { + writeln!(f, "render device: {}", v)?; + } + if let Some(v) = &self.dbg.render_gfx_write_modifiers { + writeln!(f, "render gfx writable modifiers: {:x?}", v)?; + } + if let Some(v) = &self.dbg.render_gfx_read_modifiers { + writeln!(f, "render gfx readable modifiers: {:x?}", v)?; + } + if let Some(v) = &self.dbg.dev_gfx_read_modifiers { + writeln!(f, "scanout gfx readable modifiers: {:x?}", v)?; + } + if let Some(v) = &self.dbg.render_modifiers_possible { + writeln!(f, "render dev possible modifiers: {:x?}", v)?; + } + if let Some(v) = &self.dbg.render_usage { + writeln!(f, "render dev gbm usage: {:x}", v)?; + } + if let Some(v) = &self.dbg.render_modifier { + writeln!(f, "render dev modifier: {:x}", v)?; + } + Ok(()) + } +} + +impl Error for ScanoutBufferError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + Some(&self.kind) + } +} + +fn wrap_error( + common: &Builder<'_>, + prime: Option, + f: impl FnOnce(&RefCell) -> Result, +) -> Result> { + let dbg = Default::default(); + f(&dbg) + .map_err(|kind| ScanoutBufferError { + dev: common.dev.devnode.as_bytes().as_bstr().to_string(), + render_name: (common.dev.id != common.render_ctx.dev_id) + .then(|| common.render_ctx.devnode.as_bytes().as_bstr().to_string()), + format: common.format, + plane_modifiers: common.plane_modifiers.clone(), + width: common.width, + height: common.height, + cursor: common.cursor, + dbg: dbg.into_inner(), + kind, + prime, + }) + .map_err(Box::new) +} + +impl BoAllocationSettings { + fn new1( + common: &Builder<'_>, + modifiers: &IndexMap, + scanout: bool, + rendering: bool, + usage_out: &mut Option, + ) -> Self { + let needs_render_usage = rendering && needs_render_usage(modifiers.values().copied()); + Self::new3( + common, + modifiers.keys(), + scanout, + needs_render_usage, + usage_out, + ) + } + + #[expect(dead_code)] + fn new2<'a>( + common: &Builder<'_>, + modifiers: impl IntoIterator + Clone, + fmt: &GfxFormat, + scanout: bool, + rendering: bool, + usage_out: &mut Option, + ) -> Self { + let needs_render_usage = rendering + && needs_render_usage( + modifiers + .clone() + .into_iter() + .filter_map(|m| fmt.write_modifiers.get(m)), + ); + Self::new3(common, modifiers, scanout, needs_render_usage, usage_out) + } + + fn new3<'a>( + common: &Builder<'_>, + modifiers: impl IntoIterator, + scanout: bool, + needs_render_usage: bool, + usage_out: &mut Option, + ) -> Self { + let mut usage = 0; + if scanout { + usage |= GBM_BO_USE_SCANOUT; + if common.cursor { + usage |= GBM_BO_USE_LINEAR; + } + } + if needs_render_usage { + usage |= GBM_BO_USE_RENDERING; + } + *usage_out = Some(usage); + Self { + modifiers: modifiers.into_iter().copied().collect(), + usage, + } + } +} + +impl Builder<'_> { + fn create( + &self, + drm: Rc, + damage_queue: DamageQueue, + render: RenderBufferRender, + prime: RenderBufferPrime, + ) -> Result { + let Self { + dev_ctx, + format, + width, + height, + blend_buffer, + .. + } = *self; + Ok(RenderBuffer { + width, + height, + locked: Cell::new(true), + format, + drm, + damage_queue, + blend_buffer: blend_buffer.cloned(), + render, + dev_ctx: dev_ctx.clone(), + prime, + }) + } + + fn create_bo( + &self, + ctx: &MetalRenderContext, + settings: &BoAllocationSettings, + ) -> Result { + ctx.gbm.create_bo( + &self.slf.state.dma_buf_ids, + self.width, + self.height, + self.format, + &settings.modifiers, + settings.usage, + ) + } + + fn create_dev_bo( + &self, + settings: &BoAllocationSettings, + dbg: &RefCell, + ) -> Result<(GbmBo, Rc), ScanoutBufferErrorKind> { + let bo = self + .create_bo(self.dev_ctx, settings) + .map_err(ScanoutBufferErrorKind::SodBufferAllocation)?; + let send_dev_modifier = on_drop(|| { + dbg.borrow_mut().dev_modifier = Some(bo.dmabuf().modifier); + }); + let drm = self + .dev + .master + .add_fb(bo.dmabuf(), None) + .map(Rc::new) + .map_err(ScanoutBufferErrorKind::SodAddfb2)?; + send_dev_modifier.forget(); + Ok((bo, drm)) + } + + fn create_render_buffer_render( + &self, + settings: &BoAllocationSettings, + dbg: &RefCell, + ) -> Result { + let Self { render_ctx, .. } = *self; + let bo = self + .create_bo(render_ctx, settings) + .map_err(ScanoutBufferErrorKind::RenderBufferAllocation)?; + let send_render_modifier = on_drop(|| { + dbg.borrow_mut().render_modifier = Some(bo.dmabuf().modifier); + }); + let img = render_ctx + .gfx + .clone() + .dmabuf_img(bo.dmabuf()) + .map_err(ScanoutBufferErrorKind::RenderImportImage)?; + let tex = img + .clone() + .to_texture() + .map_err(ScanoutBufferErrorKind::RenderImportRenderTexture)?; + let fb = img + .to_framebuffer() + .map_err(ScanoutBufferErrorKind::RenderImportFb)?; + fb.clear( + AcquireSync::Unnecessary, + ReleaseSync::None, + self.slf.state.color_manager.srgb_gamma22(), + ) + .map_err(ScanoutBufferErrorKind::RenderClear)?; + send_render_modifier.forget(); + Ok(RenderBufferRender { + ctx: render_ctx.clone(), + bo, + tex, + fb, + }) + } + + fn prepare_prime_none( + &self, + dbg: &RefCell, + ) -> Result { + let dbg = &mut *dbg.borrow_mut(); + let Self { + render_fmt, + plane_modifiers, + .. + } = *self; + let modifiers: IndexMap<_, _> = render_fmt + .write_modifiers + .iter() + .map(|(m, v)| (*m, v)) + .filter(|(m, _)| plane_modifiers.contains(m)) + .filter(|(m, _)| render_fmt.read_modifiers.contains(m)) + .collect(); + dbg.dev_gfx_write_modifiers = Some(render_modifiers(render_fmt)); + dbg.dev_gfx_read_modifiers = Some(sample_modifiers(render_fmt)); + dbg.dev_modifiers_possible = Some(modifiers.keys().copied().collect()); + if modifiers.is_empty() { + return Err(ScanoutBufferErrorKind::SodWriteReadPlaneIntersection); + } + let allocation_settings = + BoAllocationSettings::new1(self, &modifiers, true, true, &mut dbg.render_usage); + Ok(NoPrime { + allocation_settings, + }) + } + + fn create_prime_none( + &self, + prepared: &NoPrime, + damage_queue: DamageQueue, + dbg: &RefCell, + ) -> Result { + let NoPrime { + allocation_settings, + } = prepared; + let Self { dev, .. } = *self; + let render = self.create_render_buffer_render(allocation_settings, dbg)?; + let send_dev_modifier = on_drop(|| { + dbg.borrow_mut().dev_modifier = Some(render.bo.dmabuf().modifier); + }); + let drm = dev + .master + .add_fb(render.bo.dmabuf(), None) + .map(Rc::new) + .map_err(ScanoutBufferErrorKind::SodAddfb2)?; + send_dev_modifier.forget(); + let prime = RenderBufferPrime::None; + self.create(drm, damage_queue, render, prime) + } + + fn prepare_prime_sampling( + &self, + dbg: &RefCell, + ) -> Result { + let dbg = &mut *dbg.borrow_mut(); + let Self { + dev_ctx, + format, + render_fmt, + plane_modifiers, + .. + } = *self; + let Some(dev_fmt) = dev_ctx.gfx.formats().get(&format.drm) else { + return Err(ScanoutBufferErrorKind::SodUnsupportedFormat); + }; + let render_modifiers_possible: IndexMap<_, _> = render_fmt + .write_modifiers + .iter() + .filter(|(m, _)| render_fmt.read_modifiers.contains(*m)) + .filter(|(m, _)| dev_fmt.read_modifiers.contains(*m)) + .map(|(m, v)| (*m, v)) + .collect(); + dbg.dev_gfx_read_modifiers = Some(sample_modifiers(dev_fmt)); + dbg.render_gfx_write_modifiers = Some(render_modifiers(render_fmt)); + dbg.render_gfx_read_modifiers = Some(sample_modifiers(render_fmt)); + dbg.render_modifiers_possible = Some(render_modifiers_possible.keys().copied().collect()); + if render_modifiers_possible.is_empty() { + return Err(ScanoutBufferErrorKind::RenderWriteReadSodReadIntersection); + } + let dev_modifiers_possible: IndexMap<_, _> = dev_fmt + .write_modifiers + .iter() + .filter(|(m, _)| plane_modifiers.contains(*m)) + .map(|(m, v)| (*m, v)) + .collect(); + dbg.dev_gfx_write_modifiers = Some(render_modifiers(dev_fmt)); + dbg.dev_modifiers_possible = Some(dev_modifiers_possible.keys().copied().collect()); + if dev_modifiers_possible.is_empty() { + return Err(ScanoutBufferErrorKind::SodWritePlaneIntersection); + } + let render_allocation_settings = BoAllocationSettings::new1( + self, + &render_modifiers_possible, + false, + true, + &mut dbg.render_usage, + ); + let dev_allocation_settings = BoAllocationSettings::new1( + self, + &dev_modifiers_possible, + true, + true, + &mut dbg.dev_usage, + ); + Ok(PrimeSampling { + render_allocation_settings, + dev_allocation_settings, + }) + } + + fn create_prime_sampling( + &self, + prepared: &PrimeSampling, + damage_queue: DamageQueue, + dbg: &RefCell, + ) -> Result { + let PrimeSampling { + render_allocation_settings, + dev_allocation_settings, + } = prepared; + let Self { dev_ctx, .. } = *self; + let render = self.create_render_buffer_render(render_allocation_settings, dbg)?; + let send_render_modifier = on_drop(|| { + dbg.borrow_mut().render_modifier = Some(render.bo.dmabuf().modifier); + }); + let dev_render_tex = dev_ctx + .gfx + .clone() + .dmabuf_img(render.bo.dmabuf()) + .map_err(ScanoutBufferErrorKind::SodImportRenderImage)? + .to_texture() + .map_err(ScanoutBufferErrorKind::SodImportRenderTexture)?; + let (dev_bo, drm) = self.create_dev_bo(dev_allocation_settings, dbg)?; + let send_dev_modifier = on_drop(|| { + dbg.borrow_mut().dev_modifier = Some(dev_bo.dmabuf().modifier); + }); + let dev_fb = dev_ctx + .gfx + .clone() + .dmabuf_img(dev_bo.dmabuf()) + .map_err(ScanoutBufferErrorKind::SodImportSodImage)? + .to_framebuffer() + .map_err(ScanoutBufferErrorKind::SodImportFb)?; + send_dev_modifier.forget(); + send_render_modifier.forget(); + let prime = RenderBufferPrime::Sampling { + dev_bo, + dev_fb, + dev_render_tex, + }; + self.create(drm, damage_queue, render, prime) + } +} + +const JAY_PRIME_METHODS: &str = "JAY_PRIME_METHODS"; + +type PrimeMethods = ArrayVec; + +static PRIME_METHODS: LazyLock = LazyLock::new(prime_methods); + +fn prime_methods() -> PrimeMethods { + let mut res = PrimeMethods::new(); + let mut seen = StaticMap::<_, bool>::default(); + let mut apply = |method: PrimeMethod, allow: bool| { + if !seen[method] { + seen[method] = true; + if allow { + res.push(method); + } + } + }; + if let Ok(var) = std::env::var(JAY_PRIME_METHODS) { + for mut name in var.split(",") { + name = name.trim(); + if name.is_empty() { + continue; + } + let mut allow = true; + if let Some(m) = name.strip_prefix("-") { + name = m; + allow = false; + } + let Some(method) = PrimeMethod::variants().find(|m| m.name() == name) else { + log::warn!("Unknown prime method {}", name); + continue; + }; + apply(method, allow); + } + } + for method in PrimeMethod::variants() { + apply(method, true); + } + log::info!("Prime methods: {:?}", res); + res +} + +fn sample_modifiers(fmt: &GfxFormat) -> Vec { + fmt.read_modifiers.iter().copied().collect() +} + +fn render_modifiers(fmt: &GfxFormat) -> Vec { + fmt.write_modifiers.keys().copied().collect() +} diff --git a/src/backends/metal/present.rs b/src/backends/metal/present.rs index 7bc70427..b1493ea8 100644 --- a/src/backends/metal/present.rs +++ b/src/backends/metal/present.rs @@ -3,11 +3,9 @@ use { backend::Connector, backends::metal::{ MetalError, + allocator::{RenderBuffer, RenderBufferCopy}, transaction::{DrmConnectorState, DrmPlaneState}, - video::{ - MetalConnector, MetalCrtc, MetalHardwareCursorChange, MetalPlane, RenderBuffer, - RenderBufferCopy, - }, + video::{MetalConnector, MetalCrtc, MetalHardwareCursorChange, MetalPlane}, }, cmm::cmm_description::ColorDescription, gfx_api::{ @@ -557,7 +555,11 @@ impl MetalConnector { let swap_buffers = c.cursor_swap_buffer.is_some(); self.cursor_swap_buffer.set(swap_buffers); if let Some(sf) = c.cursor_swap_buffer.take() { - let sf = c.cursor_buffer.copy_to_dev(cd, sf)?.present_block; + let sf = c + .cursor_buffer + .copy_to_dev(cd, sf) + .map_err(MetalError::CopyToDev)? + .present_block; self.cursor_sync_file.set(sf); } let mut cursor_changed = false; @@ -588,13 +590,12 @@ impl MetalConnector { } let buffer_idx = (front_buffer % buffers.len() as u64) as usize; let buffer = &buffers[buffer_idx]; - let (width, height) = buffer.dev_fb.physical_size(); CursorProgrammingType::Enable { fb: buffer.drm.clone(), x: self.cursor_x.get(), y: self.cursor_y.get(), - width, - height, + width: buffer.width, + height: buffer.height, swap, } } else { @@ -862,7 +863,8 @@ impl MetalConnector { match &direct_scanout_data { None => { let sf = buffer - .render_fb() + .render + .fb .perform_render_pass( AcquireSync::Unnecessary, ReleaseSync::Explicit, @@ -873,9 +875,9 @@ impl MetalConnector { blend_cd, ) .map_err(MetalError::RenderFrame)?; - copy = buffer.copy_to_dev(cd, sf)?; + copy = buffer.copy_to_dev(cd, sf).map_err(MetalError::CopyToDev)?; fb = buffer.drm.clone(); - tex = buffer.render_tex.clone(); + tex = buffer.render.tex.clone(); } Some(dsd) => { let sf = match &dsd.acquire_sync { diff --git a/src/backends/metal/transaction.rs b/src/backends/metal/transaction.rs index 417fdd24..78fb7237 100644 --- a/src/backends/metal/transaction.rs +++ b/src/backends/metal/transaction.rs @@ -8,9 +8,11 @@ use { BackendConnectorTransactionError, BackendPreparedConnectorTransaction, }, }, - backends::metal::video::{ - FrontState, MetalConnector, MetalCrtc, MetalDrmDeviceData, MetalPlane, PlaneType, - RenderBuffer, + backends::metal::{ + allocator::RenderBuffer, + video::{ + FrontState, MetalConnector, MetalCrtc, MetalDrmDeviceData, MetalPlane, PlaneType, + }, }, format::{ARGB8888, Format}, gfx_api::SyncFile, @@ -464,7 +466,7 @@ impl MetalDeviceTransaction { if b[0].width != width || b[0].height != height || b[0].format != format { discard!(); } - if !rc_eq(render_ctx, &b[0].render_ctx) { + if !rc_eq(render_ctx, &b[0].render.ctx) { discard!(); } if !rc_eq(&dev_ctx, &b[0].dev_ctx) { @@ -472,7 +474,7 @@ impl MetalDeviceTransaction { } let modifiers = &plane.obj.formats.get(&format.drm).unwrap().modifiers; for b in &**b { - if !modifiers.contains(&b.dev_bo.dmabuf().modifier) { + if !modifiers.contains(&b.dev_bo().dmabuf().modifier) { discard!(); } } @@ -517,7 +519,20 @@ impl MetalDeviceTransaction { *plane_id = DrmPlane::NONE; continue; } - let buffers = Rc::new(res?); + let res = res?; + if let Some(method) = res[0].prime.method() { + let plane = match plane.obj.ty { + PlaneType::Overlay => "overlay", + PlaneType::Primary => "primary", + PlaneType::Cursor => "cursor", + }; + let dev = slf.dev.dev.devnode.as_bytes().as_bstr(); + let connector = connector.obj.kernel_id(); + log::info!( + "Using prime method {method} for {dev} {connector} ({plane})", + ); + } + let buffers = Rc::new(res); plane.new.buffers = Some(buffers.clone()); new_buffers = Some(buffers.clone()); buffers @@ -572,7 +587,7 @@ impl MetalDeviceTransaction { match prev.copy_to_new(new_buffer, &cd) { Ok(sf) => Ok(sf), Err(e) => { - log::warn!("Could not copy from old buffer: {}", ErrorFmt(&*e)); + log::warn!("Could not copy from old buffer: {}", ErrorFmt(e)); new_buffer.clear(&cd) } } @@ -582,7 +597,7 @@ impl MetalDeviceTransaction { match res { Ok(sf) => sync_files.extend(sf), Err(e) => { - log::warn!("Could not clear new buffer: {}", ErrorFmt(&*e)); + log::warn!("Could not clear new buffer: {}", ErrorFmt(e)); } } } else { @@ -623,7 +638,7 @@ impl MetalDeviceTransaction { Err(e) => { log::error!( "Could not black out old buffer: {}", - ErrorFmt(&*e), + ErrorFmt(e), ); } } diff --git a/src/backends/metal/video.rs b/src/backends/metal/video.rs index 0faa76bc..89fd53f8 100644 --- a/src/backends/metal/video.rs +++ b/src/backends/metal/video.rs @@ -1,6 +1,5 @@ use { crate::{ - allocator::BufferObject, async_engine::{Phase, SpawnedFuture}, backend::{ BackendColorSpace, BackendConnectorState, BackendDrmDevice, BackendDrmLease, @@ -14,7 +13,8 @@ use { }, }, backends::metal::{ - MetalBackend, MetalError, ScanoutBufferError, ScanoutBufferErrorKind, + MetalBackend, MetalError, + allocator::RenderBuffer, present::{ DEFAULT_POST_COMMIT_MARGIN, DEFAULT_PRE_COMMIT_MARGIN, DirectScanoutCache, POST_COMMIT_MARGIN_DELTA, PresentFb, @@ -25,15 +25,11 @@ use { drm_feedback::DrmFeedback, edid::{CtaDataBlock, Descriptor, EdidExtension}, format::{Format, XRGB8888}, - gfx_api::{ - AcquireSync, GfxBlendBuffer, GfxContext, GfxFramebuffer, GfxTexture, ReleaseSync, - SyncFile, needs_render_usage, - }, + gfx_api::{GfxContext, GfxFramebuffer, SyncFile}, ifs::{ wl_output::OutputId, wp_presentation_feedback::{KIND_HW_COMPLETION, KIND_VSYNC, KIND_ZERO_COPY}, }, - rect::{DamageQueue, Rect}, state::State, tree::OutputNode, udev::UdevDevice, @@ -41,32 +37,29 @@ use { asyncevent::AsyncEvent, binary_search_map::BinarySearchMap, bitflags::BitflagsExt, cell_ext::CellExt, clonecell::CloneCell, copyhashmap::CopyHashMap, errorfmt::ErrorFmt, geometric_decay::GeometricDecay, numcell::NumCell, on_change::OnChange, - opaque_cell::OpaqueCell, ordered_float::F64, oserror::OsError, rc_eq::rc_eq, + opaque_cell::OpaqueCell, ordered_float::F64, oserror::OsError, }, video::{ INVALID_MODIFIER, Modifier, dmabuf::DmaBufId, drm::{ ConnectorStatus, ConnectorType, DRM_CLIENT_CAP_ATOMIC, DrmBlob, DrmConnector, - DrmCrtc, DrmEncoder, DrmError, DrmEvent, DrmFb, DrmFramebuffer, DrmLease, - DrmMaster, DrmModeInfo, DrmObject, DrmPlane, DrmProperty, DrmPropertyDefinition, - DrmPropertyType, DrmVersion, HDMI_EOTF_TRADITIONAL_GAMMA_SDR, drm_mode_modeinfo, + DrmCrtc, DrmEncoder, DrmError, DrmEvent, DrmFb, DrmLease, DrmMaster, DrmModeInfo, + DrmObject, DrmPlane, DrmProperty, DrmPropertyDefinition, DrmPropertyType, + DrmVersion, HDMI_EOTF_TRADITIONAL_GAMMA_SDR, drm_mode_modeinfo, hdr_output_metadata, }, - gbm::{GBM_BO_USE_LINEAR, GBM_BO_USE_RENDERING, GBM_BO_USE_SCANOUT, GbmBo, GbmDevice}, + gbm::GbmDevice, }, }, ahash::{AHashMap, AHashSet}, - arrayvec::ArrayVec, bstr::{BString, ByteSlice}, - indexmap::{IndexMap, IndexSet, indexset}, + indexmap::{IndexSet, indexset}, isnt::std_1::collections::IsntHashMapExt, jay_config::video::GfxApi, - run_on_drop::on_drop, std::{ cell::{Cell, RefCell}, collections::hash_map::Entry, - error::Error, ffi::CString, fmt::{Debug, Formatter}, mem, @@ -588,7 +581,7 @@ impl HardwareCursorUpdate for MetalHardwareCursorChange<'_> { } fn get_buffer(&self) -> Rc { - self.cursor_buffer.render_fb() + self.cursor_buffer.render.fb.clone() } fn set_position(&mut self, x: i32, y: i32) { @@ -2603,315 +2596,6 @@ impl MetalBackend { Ok(()) } - pub fn create_scanout_buffers( - &self, - dev: &Rc, - format: &'static Format, - plane_modifiers: &IndexSet, - width: i32, - height: i32, - render_ctx: &Rc, - cursor: bool, - ) -> Result<[RenderBuffer; N], MetalError> { - let mut blend_buffer = None; - if !cursor { - match render_ctx.gfx.acquire_blend_buffer(width, height) { - Ok(bb) => blend_buffer = Some(bb), - Err(e) => { - log::warn!("Could not create blend buffer: {}", ErrorFmt(e)); - } - } - } - let mut damage_queue = ArrayVec::from(DamageQueue::new::()); - let mut create = || { - self.create_scanout_buffer( - dev, - format, - plane_modifiers, - width, - height, - render_ctx, - cursor, - damage_queue.pop().unwrap(), - blend_buffer.clone(), - ) - }; - let mut array = ArrayVec::<_, N>::new(); - for _ in 0..N { - array.push(create()?); - } - if let Some(buffer) = array.first() { - buffer.damage_full(); - } - Ok(array.into_inner().unwrap()) - } - - fn create_scanout_buffer( - &self, - dev: &Rc, - format: &'static Format, - plane_modifiers: &IndexSet, - width: i32, - height: i32, - render_ctx: &Rc, - cursor: bool, - damage_queue: DamageQueue, - blend_buffer: Option>, - ) -> Result { - let mut dev_gfx_write_modifiers = None; - let mut dev_gfx_read_modifiers = None; - let mut dev_modifiers_possible = None; - let mut dev_usage = None; - let mut dev_modifier = None; - let mut render_name = None; - let mut render_gfx_write_modifiers = None; - let mut render_modifiers_possible = None; - let mut render_usage = None; - let mut render_modifier = None; - self.create_scanout_buffer_( - dev, - format, - plane_modifiers, - width, - height, - render_ctx, - cursor, - damage_queue, - blend_buffer, - &mut dev_gfx_write_modifiers, - &mut dev_gfx_read_modifiers, - &mut dev_modifiers_possible, - &mut dev_usage, - &mut dev_modifier, - &mut render_name, - &mut render_gfx_write_modifiers, - &mut render_modifiers_possible, - &mut render_usage, - &mut render_modifier, - ) - .map_err(|kind| ScanoutBufferError { - dev: dev.devnode.as_bytes().as_bstr().to_string(), - format, - plane_modifiers: plane_modifiers.clone(), - width, - height, - cursor, - dev_gfx_write_modifiers, - dev_gfx_read_modifiers, - dev_modifiers_possible, - dev_usage, - dev_modifier, - render_name, - render_gfx_write_modifiers, - render_modifiers_possible, - render_usage, - render_modifier, - kind, - }) - .map_err(Box::new) - .map_err(MetalError::AllocateScanoutBuffer) - } - - fn create_scanout_buffer_( - &self, - dev: &Rc, - format: &'static Format, - plane_modifiers: &IndexSet, - width: i32, - height: i32, - render_ctx: &Rc, - cursor: bool, - damage_queue: DamageQueue, - blend_buffer: Option>, - dbg_dev_gfx_write_modifiers: &mut Option>, - dbg_dev_gfx_read_modifiers: &mut Option>, - dbg_dev_modifiers_possible: &mut Option>, - dbg_dev_usage: &mut Option, - dbg_dev_modifier: &mut Option, - dbg_render_name: &mut Option, - dbg_render_gfx_write_modifiers: &mut Option>, - dbg_render_modifiers_possible: &mut Option>, - dbg_render_usage: &mut Option, - dbg_render_modifier: &mut Option, - ) -> Result { - let dev_ctx = dev.ctx.get(); - let dev_gfx_formats = dev_ctx.gfx.formats(); - let Some(dev_gfx_format) = dev_gfx_formats.get(&format.drm) else { - return Err(ScanoutBufferErrorKind::SodUnsupportedFormat); - }; - let send_dev_gfx_write_modifiers = on_drop(|| { - *dbg_dev_gfx_write_modifiers = - Some(dev_gfx_format.write_modifiers.keys().copied().collect()) - }); - let possible_modifiers: IndexMap<_, _> = dev_gfx_format - .write_modifiers - .iter() - .filter(|(m, _)| plane_modifiers.contains(*m)) - .map(|(m, v)| (*m, v)) - .collect(); - let send_dev_modifiers_possible = on_drop(|| { - *dbg_dev_modifiers_possible = Some(possible_modifiers.keys().copied().collect()) - }); - if possible_modifiers.is_empty() { - return Err(ScanoutBufferErrorKind::SodNoWritableModifier); - } - let mut usage = GBM_BO_USE_RENDERING | GBM_BO_USE_SCANOUT; - if !needs_render_usage(possible_modifiers.values().copied()) { - usage &= !GBM_BO_USE_RENDERING; - } - if cursor { - usage |= GBM_BO_USE_LINEAR; - }; - *dbg_dev_usage = Some(usage); - let dev_bo = dev.gbm.create_bo( - &self.state.dma_buf_ids, - width, - height, - format, - possible_modifiers.keys(), - usage, - ); - let dev_bo = match dev_bo { - Ok(b) => b, - Err(e) => return Err(ScanoutBufferErrorKind::SodBufferAllocation(e)), - }; - *dbg_dev_modifier = Some(dev_bo.dmabuf().modifier); - let drm_fb = match dev.master.add_fb(dev_bo.dmabuf(), None) { - Ok(fb) => Rc::new(fb), - Err(e) => return Err(ScanoutBufferErrorKind::SodAddfb2(e)), - }; - let dev_img = match dev_ctx.gfx.clone().dmabuf_img(dev_bo.dmabuf()) { - Ok(img) => img, - Err(e) => return Err(ScanoutBufferErrorKind::SodImportSodImage(e)), - }; - let dev_fb = match dev_img.clone().to_framebuffer() { - Ok(fb) => fb, - Err(e) => return Err(ScanoutBufferErrorKind::SodImportFb(e)), - }; - dev_fb - .clear( - AcquireSync::Unnecessary, - ReleaseSync::None, - self.state.color_manager.srgb_gamma22(), - ) - .map_err(ScanoutBufferErrorKind::SodClear)?; - let render_gfx_formats; - let render_possible_modifiers: IndexMap<_, _>; - let mut send_render_dev_name = None; - let mut send_render_gfx_write_modifiers = None; - let mut send_dev_gfx_read_modifiers = None; - let mut send_render_possible_modifiers = None; - let (dev_tex, render_tex, render_fb, render_bo) = if dev.id == render_ctx.dev_id { - let render_tex = match dev_img.to_texture() { - Ok(fb) => fb, - Err(e) => return Err(ScanoutBufferErrorKind::SodImportSodTexture(e)), - }; - (None, render_tex, None, None) - } else { - send_render_dev_name = Some(on_drop(|| { - *dbg_render_name = Some(render_ctx.devnode.as_bytes().as_bstr().to_string()); - })); - // Create a _bridge_ BO in the render device - render_gfx_formats = render_ctx.gfx.formats(); - let render_gfx_format = match render_gfx_formats.get(&format.drm) { - None => return Err(ScanoutBufferErrorKind::RenderUnsupportedFormat), - Some(f) => f, - }; - send_render_gfx_write_modifiers = Some(on_drop(|| { - *dbg_render_gfx_write_modifiers = - Some(render_gfx_format.write_modifiers.keys().copied().collect()) - })); - send_dev_gfx_read_modifiers = Some(on_drop(|| { - *dbg_dev_gfx_read_modifiers = Some(dev_gfx_format.read_modifiers.clone()); - })); - render_possible_modifiers = render_gfx_format - .write_modifiers - .iter() - .filter(|(m, _)| dev_gfx_format.read_modifiers.contains(*m)) - .map(|(m, v)| (*m, v)) - .collect(); - send_render_possible_modifiers = Some(on_drop(|| { - *dbg_render_modifiers_possible = - Some(render_possible_modifiers.keys().copied().collect()) - })); - if render_possible_modifiers.is_empty() { - return Err(ScanoutBufferErrorKind::RenderNoWritableModifier); - } - usage = GBM_BO_USE_RENDERING | GBM_BO_USE_LINEAR; - if !needs_render_usage(render_possible_modifiers.values().copied()) { - usage &= !GBM_BO_USE_RENDERING; - } - *dbg_render_usage = Some(usage); - let render_bo = render_ctx.gbm.create_bo( - &self.state.dma_buf_ids, - width, - height, - format, - render_possible_modifiers.keys(), - usage, - ); - let render_bo = match render_bo { - Ok(b) => b, - Err(e) => return Err(ScanoutBufferErrorKind::RenderBufferAllocation(e)), - }; - *dbg_render_modifier = Some(render_bo.dmabuf().modifier); - let render_img = match render_ctx.gfx.clone().dmabuf_img(render_bo.dmabuf()) { - Ok(img) => img, - Err(e) => return Err(ScanoutBufferErrorKind::RenderImportImage(e)), - }; - let render_fb = match render_img.clone().to_framebuffer() { - Ok(fb) => fb, - Err(e) => return Err(ScanoutBufferErrorKind::RenderImportFb(e)), - }; - render_fb - .clear( - AcquireSync::Unnecessary, - ReleaseSync::None, - self.state.color_manager.srgb_gamma22(), - ) - .map_err(ScanoutBufferErrorKind::RenderClear)?; - let render_tex = match render_img.to_texture() { - Ok(fb) => fb, - Err(e) => return Err(ScanoutBufferErrorKind::RenderImportRenderTexture(e)), - }; - - // Import the bridge BO into the current device - let dev_img = match dev_ctx.gfx.clone().dmabuf_img(render_bo.dmabuf()) { - Ok(img) => img, - Err(e) => return Err(ScanoutBufferErrorKind::SodImportRenderImage(e)), - }; - let dev_tex = match dev_img.to_texture() { - Ok(fb) => fb, - Err(e) => return Err(ScanoutBufferErrorKind::SodImportRenderTexture(e)), - }; - - (Some(dev_tex), render_tex, Some(render_fb), Some(render_bo)) - }; - send_dev_gfx_write_modifiers.forget(); - send_dev_modifiers_possible.forget(); - send_render_dev_name.map(|o| o.forget()); - send_render_gfx_write_modifiers.map(|o| o.forget()); - send_dev_gfx_read_modifiers.map(|o| o.forget()); - send_render_possible_modifiers.map(|o| o.forget()); - Ok(RenderBuffer { - width, - height, - locked: Cell::new(true), - format, - dev_ctx, - render_ctx: render_ctx.clone(), - drm: drm_fb, - damage_queue, - dev_bo, - _render_bo: render_bo, - blend_buffer, - dev_fb, - dev_tex, - render_tex, - render_fb, - }) - } - fn start_connector(&self, connector: &Rc, log_mode: bool) { let dd = &*connector.display.borrow(); self.send_connected(connector, dd); @@ -2938,127 +2622,3 @@ impl MetalBackend { connector.schedule_present(); } } - -#[derive(Debug)] -pub struct RenderBuffer { - pub width: i32, - pub height: i32, - pub locked: Cell, - pub format: &'static Format, - pub dev_ctx: Rc, - pub render_ctx: Rc, - pub drm: Rc, - pub damage_queue: DamageQueue, - pub dev_bo: GbmBo, - pub _render_bo: Option, - pub blend_buffer: Option>, - // ctx = dev - // buffer location = dev - pub dev_fb: Rc, - // ctx = dev - // buffer location = render - pub dev_tex: Option>, - // ctx = render - // buffer location = render - pub render_tex: Rc, - // ctx = render - // buffer location = render - pub render_fb: Option>, -} - -#[derive(Default)] -pub struct RenderBufferCopy { - pub render_block: Option, - pub present_block: Option, -} - -impl RenderBufferCopy { - pub fn for_both(sf: Option) -> Self { - Self { - render_block: sf.clone(), - present_block: sf, - } - } -} - -impl RenderBuffer { - pub fn render_fb(&self) -> Rc { - self.render_fb - .clone() - .unwrap_or_else(|| self.dev_fb.clone()) - } - - pub fn copy_to_dev( - &self, - cd: &Rc, - sync_file: Option, - ) -> Result { - let Some(tex) = &self.dev_tex else { - return Ok(RenderBufferCopy { - render_block: None, - present_block: sync_file, - }); - }; - self.dev_fb - .copy_texture( - AcquireSync::Unnecessary, - ReleaseSync::Explicit, - cd, - tex, - cd, - None, - AcquireSync::from_sync_file(sync_file), - ReleaseSync::None, - 0, - 0, - ) - .map_err(MetalError::CopyToOutput) - .map(RenderBufferCopy::for_both) - } - - pub fn damage_full(&self) { - let dmabuf = self.dev_bo.dmabuf(); - let rect = Rect::new_sized_saturating(0, 0, dmabuf.width, dmabuf.height); - self.damage_queue.clear_all(); - self.damage_queue.damage(&[rect]); - } - - pub fn clear(&self, cd: &Rc) -> Result, Box> { - self.dev_fb - .clear(AcquireSync::Unnecessary, ReleaseSync::Explicit, cd) - .map_err(Into::into) - } - - pub fn copy_to_new( - &self, - new: &Self, - cd: &Rc, - ) -> Result, Box> { - let old = self; - let copy_texture = |new: &Rc, old: &Rc| { - new.copy_texture( - AcquireSync::Unnecessary, - ReleaseSync::Explicit, - cd, - old, - cd, - None, - AcquireSync::Unnecessary, - ReleaseSync::Explicit, - 0, - 0, - ) - .map_err(Into::into) - }; - if rc_eq(&old.dev_ctx, &new.dev_ctx) { - return copy_texture(&new.dev_fb, old.dev_tex.as_ref().unwrap_or(&old.render_tex)); - } - let tex = new - .dev_ctx - .gfx - .clone() - .dmabuf_img(old.dev_bo.dmabuf())? - .to_texture()?; - copy_texture(&new.dev_fb, &tex) - } -} From fa897f0f763a26e8a38b09c1e3088a1d16f2dc3a Mon Sep 17 00:00:00 2001 From: Julian Orth Date: Sat, 14 Feb 2026 23:03:09 +0100 Subject: [PATCH 3/5] copy_device: add new utility --- src/compositor.rs | 2 + src/copy_device.rs | 2048 +++++++++++++++++++++++++++++++++ src/gfx_api.rs | 2 +- src/gfx_apis/vulkan/device.rs | 10 +- src/main.rs | 1 + src/state.rs | 3 + src/udmabuf.rs | 5 +- src/video.rs | 3 + src/vulkan_core.rs | 4 +- 9 files changed, 2068 insertions(+), 10 deletions(-) create mode 100644 src/copy_device.rs diff --git a/src/compositor.rs b/src/compositor.rs index eedb883f..48191a77 100644 --- a/src/compositor.rs +++ b/src/compositor.rs @@ -14,6 +14,7 @@ use { clientmem::{self, ClientMemError}, cmm::{cmm_manager::ColorManager, cmm_primaries::Primaries}, config::ConfigProxy, + copy_device::CopyDeviceRegistry, cpu_worker::{CpuWorker, CpuWorkerError}, criteria::{ CritMatcherIds, @@ -363,6 +364,7 @@ fn start_compositor2( outputs_without_hc: Default::default(), udmabuf: Default::default(), gfx_ctx_changed: Default::default(), + copy_device_registry: Rc::new(CopyDeviceRegistry::new(&ring, &engine)), }); state.tracker.register(ClientId::from_raw(0)); create_dummy_output(&state); diff --git a/src/copy_device.rs b/src/copy_device.rs new file mode 100644 index 00000000..512a297a --- /dev/null +++ b/src/copy_device.rs @@ -0,0 +1,2048 @@ +use { + crate::{ + async_engine::{AsyncEngine, SpawnedFuture}, + format::{FORMATS, Format}, + gfx_api::SyncFile, + io_uring::IoUring, + rect::{Rect, Region}, + utils::{ + clonecell::CloneCell, copyhashmap::CopyHashMap, errorfmt::ErrorFmt, queue::AsyncQueue, + stack::Stack, + }, + video::{ + LINEAR_MODIFIER, LINEAR_STRIDE_ALIGN, Modifier, + dmabuf::{DmaBuf, DmaBufIds, DmaBufPlane, PlaneVec}, + }, + vulkan_core::{ + VULKAN_API_VERSION, VulkanCoreError, VulkanCoreInstance, map_extension_properties, + }, + }, + ahash::{AHashMap, AHashSet}, + arrayvec::ArrayVec, + ash::{ + Device, + ext::{ + external_memory_dma_buf, image_drm_format_modifier, physical_device_drm, + queue_family_foreign, + }, + khr::{external_fence_fd, external_memory_fd, external_semaphore_fd}, + vk::{ + self, AccessFlags2, BindImageMemoryInfo, BindImagePlaneMemoryInfo, BlitImageInfo2, + BufferCopy2, BufferCreateInfo, BufferImageCopy2, BufferMemoryBarrier2, + BufferUsageFlags, CommandBuffer, CommandBufferAllocateInfo, CommandBufferBeginInfo, + CommandBufferSubmitInfo, CommandBufferUsageFlags, CommandPoolCreateFlags, + CommandPoolCreateInfo, CopyBufferInfo2, CopyBufferToImageInfo2, CopyImageInfo2, + CopyImageToBufferInfo2, DependencyInfo, DeviceCreateInfo, DeviceMemory, + DeviceQueueCreateInfo, DrmFormatModifierPropertiesEXT, + DrmFormatModifierPropertiesListEXT, ExportFenceCreateInfo, ExportMemoryAllocateInfo, + Extent3D, ExternalBufferProperties, ExternalFenceFeatureFlags, + ExternalFenceHandleTypeFlags, ExternalFenceProperties, + ExternalImageFormatPropertiesKHR, ExternalMemoryBufferCreateInfo, + ExternalMemoryBufferCreateInfoKHR, ExternalMemoryFeatureFlags, + ExternalMemoryHandleTypeFlags, ExternalMemoryImageCreateInfo, + ExternalSemaphoreFeatureFlags, ExternalSemaphoreHandleTypeFlags, + ExternalSemaphoreProperties, Fence, FenceCreateInfo, FenceGetFdInfoKHR, Filter, + FormatFeatureFlags, FormatProperties2, ImageAspectFlags, ImageBlit2, ImageCopy2, + ImageCreateFlags, ImageCreateInfo, ImageDrmFormatModifierExplicitCreateInfoEXT, + ImageFormatProperties2, ImageLayout, ImageMemoryBarrier2, ImageMemoryRequirementsInfo2, + ImagePlaneMemoryRequirementsInfo, ImageSubresourceLayers, ImageSubresourceRange, + ImageTiling, ImageType, ImageUsageFlags, ImportMemoryFdInfoKHR, + ImportSemaphoreFdInfoKHR, MemoryAllocateInfo, MemoryDedicatedAllocateInfo, + MemoryFdPropertiesKHR, MemoryGetFdInfoKHR, MemoryPropertyFlags, MemoryRequirements2, + MemoryType, Offset3D, PhysicalDevice, PhysicalDeviceDrmPropertiesEXT, + PhysicalDeviceExternalBufferInfo, PhysicalDeviceExternalFenceInfo, + PhysicalDeviceExternalImageFormatInfoKHR, PhysicalDeviceExternalSemaphoreInfo, + PhysicalDeviceFeatures2, PhysicalDeviceImageDrmFormatModifierInfoEXT, + PhysicalDeviceImageFormatInfo2, PhysicalDeviceProperties2, + PhysicalDeviceSynchronization2Features, PipelineStageFlags2, QUEUE_FAMILY_FOREIGN_EXT, + Queue, QueueFlags, SampleCountFlags, SemaphoreCreateInfo, SemaphoreImportFlags, + SemaphoreSubmitInfo, SharingMode, SubmitInfo2, SubresourceLayout, WHOLE_SIZE, + }, + }, + bstr::ByteSlice, + isnt::std_1::collections::IsntHashMapExt, + linearize::{Linearize, LinearizeExt, StaticCopyMap, StaticMap, static_copy_map, static_map}, + log::Level, + run_on_drop::on_drop, + std::{ + cell::{Cell, RefCell}, + ffi::CStr, + fmt::{Debug, Formatter}, + io, + ops::Deref, + rc::Rc, + slice, + }, + thiserror::Error, + uapi::{AsUstr, OwnedFd, c}, + vk::{Buffer, CommandPool, Image, Semaphore}, +}; + +#[derive(Debug, Error)] +pub enum CopyDeviceError { + #[error(transparent)] + Core(#[from] VulkanCoreError), + #[error("Could not create a semaphore")] + CreateSemaphore(#[source] vk::Result), + #[error("Could not create a fence")] + CreateFence(#[source] vk::Result), + #[error("Could not dup a sync file")] + DupSyncFile(#[source] io::Error), + #[error("Could not dup a dma buf")] + DupDmaBuf(#[source] io::Error), + #[error("Could not import a sync file")] + ImportSyncFile(#[source] vk::Result), + #[error("Could not export a sync file")] + ExportSyncFile(#[source] vk::Result), + #[error("Could not submit the copy")] + SubmitCopy(#[source] vk::Result), + #[error("Could not enumerate the physical devices")] + EnumeratePhysicalDevice(#[source] vk::Result), + #[error("Could not find a corresponding vulkan device")] + NoVulkanDevice, + #[error("Device does not support vulkan 1.3")] + NoVulkan13, + #[error("Device does not support the synchronization2 feature")] + NoSynchronization2, + #[error("Device does not support the device extension {}", .0.as_ustr().as_bytes().as_bstr())] + MissingDeviceExtensions(&'static CStr), + #[error("Device does not support importing sync files")] + NoSyncFileImport, + #[error("Device does not support exporting sync files")] + NoSyncFileExport, + #[error("Device does not support importing dma bufs as buffers")] + NoDmaBufBufferImport, + #[error("Device does not have a graphics queue family")] + NoGfxQueueFamily, + #[error("Could not create the device")] + CreateDevice(#[source] vk::Result), + #[error("Could not create a command pool")] + CreateCommandPool(#[source] vk::Result), + #[error("Could not create a command buffer")] + CreateCommandBuffer(#[source] vk::Result), + #[error("Copy source and destination must have the same size")] + NotSameSize, + #[error("Copy source has a non-positive size")] + NonPositiveSize, + #[error("The size calculation overflowed")] + SizeOverflow, + #[error("The format and/or modifier is not supported")] + UnsupportedFormat, + #[error("the image is too large")] + TooLarge, + #[error("Copy source has an incorrect number of planes")] + WrongNumberOfPlanes, + #[error("Could not create a buffer")] + CreateBuffer(#[source] vk::Result), + #[error("Device returned an unexpected required buffer size")] + UnexpectedBufferSize, + #[error("Could not query memory fd properties")] + GetMemoryFdProperties(#[source] vk::Result), + #[error("Could not find a memory type for import")] + NoMemoryTypeForImport, + #[error("Could not import memory")] + ImportMemory(#[source] vk::Result), + #[error("Could not bind buffer memory")] + BindBufferMemory(#[source] vk::Result), + #[error("Could not bind image memory")] + BindImageMemory(#[source] vk::Result), + #[error("Could not create an image")] + CreateImage(#[source] vk::Result), + #[error("Could not begin a command buffer")] + BeginCommandBuffer(#[source] vk::Result), + #[error("Could not end a command buffer")] + EndCommandBuffer(#[source] vk::Result), + #[error("The previous copy is still executing")] + Busy, + #[error("The device does not support dmabuf export")] + NoDmabufExport, + #[error("Could not find a memory type for import")] + NoMemoryTypeForAllocation, + #[error("Could not allocate memory")] + AllocateMemory(#[source] vk::Result), + #[error("Could not export a dmabuf")] + ExportDmabuf(#[source] vk::Result), + #[error("Both buffers are off device")] + BothOffDevice, + #[error("Cannot blit between these formats")] + BlitNotSupported, +} + +type Keyed = StaticMap; +type KeyedCopy = StaticCopyMap; + +pub struct PhysicalCopyDevice { + ring: Rc, + eng: Rc, + instance: VulkanCoreInstance, + physical_device: PhysicalDevice, + support: AHashMap>>, + queues_to_allocate: Vec, + queues: KeyedCopy, + supports_dmabuf_export: bool, + memory_types: Vec, + rects: RefCell>, + buffer_copy_2: RefCell>>, + buffer_image_copy_2: RefCell>>, + image_copy_2: RefCell>>, + image_blit_2: RefCell>>, +} + +#[derive(Debug)] +struct QueueToAllocate { + family: u32, + num: usize, +} + +#[derive(Copy, Clone, Default, Debug)] +struct QueueIndex { + allocate_idx: usize, + family: u32, + idx_within_family: u32, + transfer_granularity_mask: (u32, u32), +} + +pub struct CopyDevice { + _tasks: Vec>, + dev: Rc, +} + +struct CopyDeviceInner { + phy: Rc, + dev: Device, + unique_pools: Vec, + pools: Keyed, + queues: KeyedCopy, + external_semaphore_fd: external_semaphore_fd::Device, + external_fence_fd: external_fence_fd::Device, + external_memory_fd: external_memory_fd::Device, + semaphores: Stack, + fences: Stack, + submissions: Keyed>, +} + +#[derive(Default)] +struct PendingSubmissions { + task_has_pending: Cell, + pending: AsyncQueue, +} + +pub struct CopyDeviceCopy { + inner: Rc, + _dev: Rc, +} + +struct CopyDeviceCopyInner { + dev: Rc, + busy: CloneCell>, + width: u32, + height: u32, + command_buffer: CommandBuffer, + tt: TransferType, + ty: CopyDeviceCopyType, +} + +enum CopyDeviceCopyType { + BufferToBuffer { + src: VulkanBuffer, + dst: VulkanBuffer, + stride: u32, + bpp: u32, + }, + BufferToImage { + buf: VulkanBuffer, + buf_format: &'static Format, + buf_stride: u32, + img: VulkanImage, + }, + ImageToBuffer { + img: VulkanImage, + buf: VulkanBuffer, + buf_format: &'static Format, + buf_stride: u32, + }, + ImageToImage { + src: VulkanImage, + dst: VulkanImage, + }, + Blit { + src: VulkanImage, + dst: VulkanImage, + }, +} + +struct Pending { + dev: Rc, + sync_file: Option, + copy: Rc, + semaphore: Option, + fence: Option, +} + +struct VulkanSemaphore { + dev: Rc, + semaphore: Semaphore, +} + +struct VulkanFence { + dev: Rc, + fence: Fence, +} + +struct VulkanBuffer { + dev: Rc, + buf: Buffer, + mem: DeviceMemory, +} + +struct VulkanImage { + dev: Rc, + img: Image, + mem: PlaneVec, +} + +#[derive(Copy, Clone)] +pub struct CopyDeviceSupport { + pub modifier: Modifier, + pub planes: usize, + pub max_width: u32, + pub max_height: u32, + pub blit: bool, +} + +pub struct CopyDeviceBuffer { + device: Rc, + memory: DeviceMemory, + dmabuf: DmaBuf, +} + +#[derive(Copy, Clone, Debug, Linearize)] +enum TransferType { + Blit, + Intra, + Download, + Upload, +} + +#[derive(Copy, Clone, Debug, Linearize)] +enum Dir { + Src, + Dst, +} + +struct ClassifiedDmabuf<'a> { + fd_props: PlaneVec>, + on_device: bool, + buffer_possible: bool, + format: &'a CopyDeviceSupport, +} + +pub struct CopyDeviceRegistry { + ring: Rc, + eng: Rc, + devs: CopyHashMap>>, +} + +const DEVICE_EXTENSIONS: [&CStr; 6] = [ + external_semaphore_fd::NAME, + external_fence_fd::NAME, + external_memory_fd::NAME, + external_memory_dma_buf::NAME, + image_drm_format_modifier::NAME, + queue_family_foreign::NAME, +]; + +impl PhysicalCopyDevice { + fn new( + ring: &Rc, + eng: &Rc, + dev: c::dev_t, + ) -> Result, CopyDeviceError> { + let core_instance = VulkanCoreInstance::new(Level::Debug)?; + let instance = &core_instance.instance; + let physical_device; + let device_extensions; + let device_properties; + let supports_dmabuf_export; + 'find_device: { + let devices = unsafe { + instance + .enumerate_physical_devices() + .map_err(CopyDeviceError::EnumeratePhysicalDevice)? + }; + 'outer: for phy in devices { + let res = unsafe { instance.enumerate_device_extension_properties(phy) }; + let exts = match res { + Ok(res) => map_extension_properties(res), + Err(e) => { + log::error!( + "Could not enumerate extensions of physical device: {}", + ErrorFmt(e), + ); + continue; + } + }; + if exts.not_contains_key(physical_device_drm::NAME) { + continue 'outer; + } + let mut drm_props = PhysicalDeviceDrmPropertiesEXT::default(); + let mut props = PhysicalDeviceProperties2::default().push_next(&mut drm_props); + unsafe { + instance.get_physical_device_properties2(phy, &mut props); + } + let props = props.properties; + let major = uapi::major(dev) as i64; + let minor = uapi::minor(dev) as i64; + let matches = (drm_props.has_primary == vk::TRUE + && drm_props.primary_major == major + && drm_props.primary_minor == minor) + || (drm_props.has_render == vk::TRUE + && drm_props.render_major == major + && drm_props.render_minor == minor); + if matches { + physical_device = phy; + device_extensions = exts; + device_properties = props; + break 'find_device; + } + } + return Err(CopyDeviceError::NoVulkanDevice); + } + if device_properties.api_version < VULKAN_API_VERSION { + return Err(CopyDeviceError::NoVulkan13); + } + for ext in DEVICE_EXTENSIONS { + if device_extensions.not_contains_key(ext) { + return Err(CopyDeviceError::MissingDeviceExtensions(ext)); + } + } + { + let mut synchronization2_features = PhysicalDeviceSynchronization2Features::default(); + let mut physical_device_features = + PhysicalDeviceFeatures2::default().push_next(&mut synchronization2_features); + unsafe { + instance + .get_physical_device_features2(physical_device, &mut physical_device_features); + } + if synchronization2_features.synchronization2 != vk::TRUE { + return Err(CopyDeviceError::NoSynchronization2); + } + } + { + let info = PhysicalDeviceExternalSemaphoreInfo::default() + .handle_type(ExternalSemaphoreHandleTypeFlags::SYNC_FD); + let mut props = ExternalSemaphoreProperties::default(); + unsafe { + instance.get_physical_device_external_semaphore_properties( + physical_device, + &info, + &mut props, + ); + } + let supported = props + .external_semaphore_features + .contains(ExternalSemaphoreFeatureFlags::IMPORTABLE); + if !supported { + return Err(CopyDeviceError::NoSyncFileImport); + } + } + { + let info = PhysicalDeviceExternalFenceInfo::default() + .handle_type(ExternalFenceHandleTypeFlags::SYNC_FD); + let mut props = ExternalFenceProperties::default(); + unsafe { + instance.get_physical_device_external_fence_properties( + physical_device, + &info, + &mut props, + ); + } + let supported = props + .external_fence_features + .contains(ExternalFenceFeatureFlags::EXPORTABLE); + if !supported { + return Err(CopyDeviceError::NoSyncFileExport); + } + } + { + let info = PhysicalDeviceExternalBufferInfo::default() + .handle_type(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT) + .usage(BufferUsageFlags::TRANSFER_SRC | BufferUsageFlags::TRANSFER_DST); + let mut props = ExternalBufferProperties::default(); + unsafe { + instance.get_physical_device_external_buffer_properties( + physical_device, + &info, + &mut props, + ); + } + let features = props.external_memory_properties.external_memory_features; + let supported = features.contains(ExternalMemoryFeatureFlags::IMPORTABLE); + if !supported { + return Err(CopyDeviceError::NoDmaBufBufferImport); + } + supports_dmabuf_export = features.contains(ExternalMemoryFeatureFlags::EXPORTABLE); + } + let (queues_to_allocate, queue_indices) = { + let families = + unsafe { instance.get_physical_device_queue_family_properties(physical_device) }; + let mut transfer_only = None; + let mut compute_only = None; + let mut gfx = None; + for (idx, family) in families.iter().enumerate() { + let idx = idx as u32; + let g = family.min_image_transfer_granularity; + let g = (g.width.wrapping_sub(1), g.height.wrapping_sub(1)); + if g.0 == u32::MAX || g.1 == u32::MAX { + continue; + } + let count = family.queue_count; + if count == 0 { + continue; + } + let v = (idx, g, count); + let flags = family.queue_flags; + if flags.contains(QueueFlags::GRAPHICS) { + if gfx.is_none() { + gfx = Some(v); + } + } else if flags.contains(QueueFlags::COMPUTE) { + if compute_only.is_none() { + compute_only = Some(v); + } + } else if flags.contains(QueueFlags::TRANSFER) { + if transfer_only.is_none() { + transfer_only = Some(v); + } + } + } + let gfx = gfx.ok_or(CopyDeviceError::NoGfxQueueFamily)?; + allocate_queues(gfx, compute_only, transfer_only) + }; + let mut support = AHashMap::default(); + for format in FORMATS { + let mut list = vec![]; + for attach in [false, true] { + let mut modifiers = DrmFormatModifierPropertiesListEXT::default(); + if attach { + modifiers = modifiers.drm_format_modifier_properties(&mut list); + } + let mut out = FormatProperties2::default().push_next(&mut modifiers); + unsafe { + instance.get_physical_device_format_properties2( + physical_device, + format.vk_format, + &mut out, + ); + } + if !attach { + list = vec![ + DrmFormatModifierPropertiesEXT::default(); + modifiers.drm_format_modifier_count as usize + ]; + } + } + let mut format_support = StaticMap::<_, Vec<_>>::default(); + for modifier in list { + for dir in Dir::variants() { + let format_feature_flags = match dir { + Dir::Src => FormatFeatureFlags::TRANSFER_SRC, + Dir::Dst => FormatFeatureFlags::TRANSFER_DST, + }; + let blit_feature_flags = match dir { + Dir::Src => FormatFeatureFlags::BLIT_SRC, + Dir::Dst => FormatFeatureFlags::BLIT_DST, + }; + let image_usage_flags = match dir { + Dir::Src => ImageUsageFlags::TRANSFER_SRC, + Dir::Dst => ImageUsageFlags::TRANSFER_DST, + }; + let image_features = modifier.drm_format_modifier_tiling_features; + if !image_features.contains(format_feature_flags) { + continue; + } + let supports_blit = image_features.contains(blit_feature_flags); + let mut modifier_info = PhysicalDeviceImageDrmFormatModifierInfoEXT::default() + .drm_format_modifier(modifier.drm_format_modifier); + let mut external_memory_info = + PhysicalDeviceExternalImageFormatInfoKHR::default() + .handle_type(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let info = PhysicalDeviceImageFormatInfo2::default() + .format(format.vk_format) + .ty(ImageType::TYPE_2D) + .tiling(ImageTiling::DRM_FORMAT_MODIFIER_EXT) + .usage(image_usage_flags) + .push_next(&mut external_memory_info) + .push_next(&mut modifier_info); + let mut external_memory_prop = ExternalImageFormatPropertiesKHR::default(); + let mut prop = + ImageFormatProperties2::default().push_next(&mut external_memory_prop); + let res = unsafe { + instance.get_physical_device_image_format_properties2( + physical_device, + &info, + &mut prop, + ) + }; + if res.is_err() { + continue; + } + let prop = prop.image_format_properties; + let memory_features = external_memory_prop + .external_memory_properties + .external_memory_features; + if !memory_features.contains(ExternalMemoryFeatureFlags::IMPORTABLE) { + continue; + } + let me = prop.max_extent; + if me.width > 0 && me.height > 0 && me.depth > 0 { + format_support[dir].push(CopyDeviceSupport { + modifier: modifier.drm_format_modifier, + planes: modifier.drm_format_modifier_plane_count as usize, + max_width: me.width, + max_height: me.height, + blit: supports_blit, + }); + } + } + } + support.insert(format.drm, format_support); + } + let memory_info = + unsafe { instance.get_physical_device_memory_properties(physical_device) }; + let dev = Rc::new(PhysicalCopyDevice { + ring: ring.clone(), + eng: eng.clone(), + instance: core_instance, + physical_device, + support, + queues_to_allocate, + queues: queue_indices, + supports_dmabuf_export, + memory_types: memory_info.memory_types_as_slice().to_vec(), + rects: Default::default(), + buffer_copy_2: Default::default(), + image_blit_2: Default::default(), + image_copy_2: Default::default(), + buffer_image_copy_2: Default::default(), + }); + Ok(dev) + } + + #[expect(dead_code)] + pub fn src_support(&self, format: &Format) -> &[CopyDeviceSupport] { + self.support(format, Dir::Src) + } + + #[expect(dead_code)] + pub fn dst_support(&self, format: &Format) -> &[CopyDeviceSupport] { + self.support(format, Dir::Dst) + } + + fn support(&self, format: &Format, dir: Dir) -> &[CopyDeviceSupport] { + self.support + .get(&format.drm) + .map(|s| s[dir].as_slice()) + .unwrap_or_default() + } + + #[expect(dead_code)] + pub fn create_device(self: &Rc) -> Result, CopyDeviceError> { + let instance = &self.instance.instance; + let device = { + let priorities = [1.0; TransferType::LENGTH]; + let queue_create_info: Vec<_> = self + .queues_to_allocate + .iter() + .map(|q| { + DeviceQueueCreateInfo::default() + .queue_family_index(q.family) + .queue_priorities(&priorities[..q.num]) + }) + .collect(); + let extensions = DEVICE_EXTENSIONS.map(|e| e.as_ptr()); + let mut synchronization2_features = + PhysicalDeviceSynchronization2Features::default().synchronization2(true); + let info = DeviceCreateInfo::default() + .queue_create_infos(&queue_create_info) + .enabled_extension_names(&extensions) + .push_next(&mut synchronization2_features); + unsafe { + instance + .create_device(self.physical_device, &info, None) + .map_err(CopyDeviceError::CreateDevice)? + } + }; + let destroy_device = on_drop(|| unsafe { device.destroy_device(None) }); + let external_semaphore_fd = external_semaphore_fd::Device::new(instance, &device); + let external_fence_fd = external_fence_fd::Device::new(instance, &device); + let external_memory_fd = external_memory_fd::Device::new(instance, &device); + let queues = self.queues.map_values(|idx| unsafe { + device.get_device_queue(idx.family, idx.idx_within_family) + }); + let mut unique_pools = vec![]; + let mut destroy_pools = vec![]; + for q in &self.queues_to_allocate { + let info = CommandPoolCreateInfo::default() + .queue_family_index(q.family) + .flags(CommandPoolCreateFlags::RESET_COMMAND_BUFFER); + let pool = unsafe { + device + .create_command_pool(&info, None) + .map_err(CopyDeviceError::CreateCommandPool)? + }; + unique_pools.push(pool); + let device = &device; + let destroy_pool = on_drop(move || unsafe { device.destroy_command_pool(pool, None) }); + destroy_pools.push(destroy_pool); + } + let pools: StaticMap = static_map! { + tt => unique_pools[self.queues[tt].allocate_idx] + }; + let submissions_list: Vec>> = self + .queues_to_allocate + .iter() + .map(|q| vec![Default::default(); q.num]) + .collect(); + let submissions = self + .queues + .into_static_map() + .map_values(|q| submissions_list[q.allocate_idx][q.idx_within_family as usize].clone()); + destroy_pools.into_iter().for_each(|v| v.forget()); + destroy_device.forget(); + let dev = Rc::new(CopyDeviceInner { + phy: self.clone(), + dev: device, + unique_pools, + pools, + queues, + external_semaphore_fd, + external_fence_fd, + external_memory_fd, + semaphores: Default::default(), + fences: Default::default(), + submissions, + }); + let mut tasks = vec![]; + for submissions in submissions_list.iter().flatten().cloned() { + let future = wait_for_submissions(submissions, dev.clone(), self.ring.clone()); + let task = self.eng.spawn("copy-device-await-pending", future); + tasks.push(task); + } + let queue = Rc::new(CopyDevice { dev, _tasks: tasks }); + Ok(queue) + } +} + +async fn wait_for_submissions( + submissions: Rc, + dev: Rc, + ring: Rc, +) { + loop { + submissions.task_has_pending.set(false); + let pending = submissions.pending.pop().await; + submissions.task_has_pending.set(true); + if let Some(sync_file) = &pending.sync_file + && let Err(e) = ring.readable(sync_file).await + { + log::warn!( + "Could not wait for sync file to become readable: {}", + ErrorFmt(e), + ); + dev.wait_idle(); + } + } +} + +impl CopyDevice { + fn classify_dmabuf( + &self, + buf: &DmaBuf, + dir: Dir, + ) -> Result, CopyDeviceError> { + if buf.width <= 0 || buf.height <= 0 { + return Err(CopyDeviceError::NonPositiveSize); + } + let width = buf.width as u32; + let height = buf.height as u32; + let Some(format) = self + .dev + .phy + .support(buf.format, dir) + .iter() + .find(|s| s.modifier == buf.modifier) + else { + return Err(CopyDeviceError::UnsupportedFormat); + }; + if width > format.max_width || height > format.max_height { + return Err(CopyDeviceError::TooLarge); + } + if buf.planes.len() != format.planes { + return Err(CopyDeviceError::WrongNumberOfPlanes); + } + let mut fd_props = PlaneVec::new(); + for plane in &buf.planes { + let mut props = MemoryFdPropertiesKHR::default(); + unsafe { + self.dev + .external_memory_fd + .get_memory_fd_properties( + ExternalMemoryHandleTypeFlags::DMA_BUF_EXT, + plane.fd.raw(), + &mut props, + ) + .map_err(CopyDeviceError::GetMemoryFdProperties)?; + } + fd_props.push(props); + if buf.is_one_file() { + break; + } + } + let mut on_device = true; + for prop in &fd_props { + let mut plane_on_device = false; + for (idx, ty) in self.dev.phy.memory_types.iter().enumerate() { + if prop.memory_type_bits & (1 << idx) != 0 + && ty + .property_flags + .contains(MemoryPropertyFlags::DEVICE_LOCAL) + { + plane_on_device = true; + break; + } + } + if !plane_on_device { + on_device = false; + break; + } + } + let buffer_possible = buf.modifier == LINEAR_MODIFIER + && buf.planes.len() == 1 + && buf.planes[0].stride % buf.format.bpp == 0 + && width <= buf.planes[0].stride / buf.format.bpp; + Ok(ClassifiedDmabuf { + fd_props, + on_device, + buffer_possible, + format, + }) + } + + fn import_buffer( + &self, + tt: TransferType, + class: &ClassifiedDmabuf, + buf: &DmaBuf, + dir: Dir, + ) -> Result { + assert!(class.buffer_possible); + let height = buf.height as u32; + let plane = &buf.planes[0]; + let queue_family = self.dev.phy.queues[tt].family; + let buffer_size = plane.stride as u64 * height as u64; + let buffer = { + let buffer_usage_flags = match dir { + Dir::Src => BufferUsageFlags::TRANSFER_SRC, + Dir::Dst => BufferUsageFlags::TRANSFER_DST, + }; + let mut external_info = ExternalMemoryBufferCreateInfoKHR::default() + .handle_types(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let info = BufferCreateInfo::default() + .size(buffer_size) + .usage(buffer_usage_flags) + .queue_family_indices(slice::from_ref(&queue_family)) + .push_next(&mut external_info); + unsafe { + self.dev + .dev + .create_buffer(&info, None) + .map_err(CopyDeviceError::CreateBuffer)? + } + }; + let destroy_buffer = on_drop(|| unsafe { self.dev.dev.destroy_buffer(buffer, None) }); + let memory = { + let out = unsafe { self.dev.dev.get_buffer_memory_requirements(buffer) }; + if out.size > buffer_size { + return Err(CopyDeviceError::UnexpectedBufferSize); + } + let memory_type_bits = class.fd_props[0].memory_type_bits & out.memory_type_bits; + if memory_type_bits == 0 { + return Err(CopyDeviceError::NoMemoryTypeForImport); + } + let fd = uapi::fcntl_dupfd_cloexec(plane.fd.raw(), 0) + .map_err(Into::into) + .map_err(CopyDeviceError::DupDmaBuf)?; + let mut dedicated_allocation = MemoryDedicatedAllocateInfo::default().buffer(buffer); + let mut external_memory = ImportMemoryFdInfoKHR::default() + .handle_type(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT) + .fd(fd.raw()); + let allocate_info = MemoryAllocateInfo::default() + .allocation_size(out.size) + .memory_type_index(memory_type_bits.trailing_zeros() as _) + .push_next(&mut external_memory) + .push_next(&mut dedicated_allocation); + let memory = unsafe { + self.dev + .dev + .allocate_memory(&allocate_info, None) + .map_err(CopyDeviceError::ImportMemory)? + }; + let _ = fd.unwrap(); + memory + }; + let free_memory = on_drop(|| unsafe { self.dev.dev.free_memory(memory, None) }); + unsafe { + self.dev + .dev + .bind_buffer_memory(buffer, memory, 0) + .map_err(CopyDeviceError::BindBufferMemory)?; + } + free_memory.forget(); + destroy_buffer.forget(); + Ok(VulkanBuffer { + dev: self.dev.clone(), + buf: buffer, + mem: memory, + }) + } + + fn import_image( + &self, + tt: TransferType, + class: &ClassifiedDmabuf, + buf: &DmaBuf, + dir: Dir, + ) -> Result { + let dev = &self.dev.dev; + let disjoint = buf.is_disjoint(); + let queue_family = self.dev.phy.queues[tt].family; + let image = { + let image_create_flags = match disjoint { + true => ImageCreateFlags::DISJOINT, + false => ImageCreateFlags::empty(), + }; + let image_usage_flags = match dir { + Dir::Src => ImageUsageFlags::TRANSFER_SRC, + Dir::Dst => ImageUsageFlags::TRANSFER_DST, + }; + let plane_layouts: PlaneVec<_> = buf + .planes + .iter() + .map(|p| SubresourceLayout { + offset: p.offset as _, + row_pitch: p.stride as _, + size: 0, + array_pitch: 0, + depth_pitch: 0, + }) + .collect(); + let mut mod_info = ImageDrmFormatModifierExplicitCreateInfoEXT::default() + .drm_format_modifier(buf.modifier) + .plane_layouts(&plane_layouts); + let mut memory_image_create_info = ExternalMemoryImageCreateInfo::default() + .handle_types(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let info = ImageCreateInfo::default() + .flags(image_create_flags) + .image_type(ImageType::TYPE_2D) + .format(buf.format.vk_format) + .extent(Extent3D { + width: buf.width as _, + height: buf.height as _, + depth: 1, + }) + .mip_levels(1) + .array_layers(1) + .samples(SampleCountFlags::TYPE_1) + .tiling(ImageTiling::DRM_FORMAT_MODIFIER_EXT) + .usage(image_usage_flags) + .sharing_mode(SharingMode::EXCLUSIVE) + .queue_family_indices(slice::from_ref(&queue_family)) + .initial_layout(ImageLayout::UNDEFINED) + .push_next(&mut mod_info) + .push_next(&mut memory_image_create_info); + unsafe { + dev.create_image(&info, None) + .map_err(CopyDeviceError::CreateImage)? + } + }; + let destroy_image = on_drop(|| unsafe { dev.destroy_image(image, None) }); + let mut memories = PlaneVec::new(); + let mut free_memories = PlaneVec::new(); + { + let num_device_memories = match disjoint { + true => buf.planes.len(), + false => 1, + }; + let mut bind_image_plane_memory_infos = PlaneVec::new(); + for plane_idx in 0..num_device_memories { + let dma_buf_plane = &buf.planes[plane_idx]; + let mut image_memory_requirements_info = + ImageMemoryRequirementsInfo2::default().image(image); + let mut image_plane_memory_requirements_info; + if disjoint { + let plane_aspect = match plane_idx { + 0 => ImageAspectFlags::MEMORY_PLANE_0_EXT, + 1 => ImageAspectFlags::MEMORY_PLANE_1_EXT, + 2 => ImageAspectFlags::MEMORY_PLANE_2_EXT, + 3 => ImageAspectFlags::MEMORY_PLANE_3_EXT, + _ => unreachable!(), + }; + image_plane_memory_requirements_info = + ImagePlaneMemoryRequirementsInfo::default().plane_aspect(plane_aspect); + image_memory_requirements_info = image_memory_requirements_info + .push_next(&mut image_plane_memory_requirements_info); + bind_image_plane_memory_infos + .push(BindImagePlaneMemoryInfo::default().plane_aspect(plane_aspect)); + } + let mut memory_requirements = MemoryRequirements2::default(); + unsafe { + dev.get_image_memory_requirements2( + &image_memory_requirements_info, + &mut memory_requirements, + ); + } + let memory_type_bits = memory_requirements.memory_requirements.memory_type_bits + & class.fd_props[plane_idx].memory_type_bits; + if memory_type_bits == 0 { + return Err(CopyDeviceError::NoMemoryTypeForImport); + } + let fd = uapi::fcntl_dupfd_cloexec(dma_buf_plane.fd.raw(), 0) + .map_err(Into::into) + .map_err(CopyDeviceError::DupDmaBuf)?; + let mut memory_dedicated_allocate_info = + MemoryDedicatedAllocateInfo::default().image(image); + let mut import_memory_fd_info = ImportMemoryFdInfoKHR::default() + .fd(fd.raw()) + .handle_type(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let memory_allocate_info = MemoryAllocateInfo::default() + .allocation_size(memory_requirements.memory_requirements.size) + .memory_type_index(memory_type_bits.trailing_zeros() as _) + .push_next(&mut import_memory_fd_info) + .push_next(&mut memory_dedicated_allocate_info); + let device_memory = unsafe { + dev.allocate_memory(&memory_allocate_info, None) + .map_err(CopyDeviceError::ImportMemory)? + }; + let _ = fd.unwrap(); + memories.push(device_memory); + free_memories.push(on_drop(move || unsafe { + dev.free_memory(device_memory, None) + })); + } + let mut bind_image_memory_infos = PlaneVec::new(); + let mut bind_image_plane_memory_infos = bind_image_plane_memory_infos.iter_mut(); + for mem in memories.iter().copied() { + let mut info = BindImageMemoryInfo::default().image(image).memory(mem); + if disjoint { + info = info.push_next(bind_image_plane_memory_infos.next().unwrap()); + } + bind_image_memory_infos.push(info); + } + unsafe { + dev.bind_image_memory2(&bind_image_memory_infos) + .map_err(CopyDeviceError::BindImageMemory)?; + } + } + free_memories.into_iter().for_each(|f| f.forget()); + destroy_image.forget(); + Ok(VulkanImage { + dev: self.dev.clone(), + img: image, + mem: memories, + }) + } + + #[expect(dead_code)] + pub fn create_copy( + self: &Rc, + src: &DmaBuf, + dst: &DmaBuf, + ) -> Result { + if (dst.width, dst.height) != (src.width, src.height) { + return Err(CopyDeviceError::NotSameSize); + } + let src_class = self.classify_dmabuf(src, Dir::Src)?; + let dst_class = self.classify_dmabuf(dst, Dir::Dst)?; + let blit = src.format != dst.format; + if blit && (!src_class.format.blit || !dst_class.format.blit) { + return Err(CopyDeviceError::BlitNotSupported); + } + let tt = match (src_class.on_device, dst_class.on_device) { + (false, false) => return Err(CopyDeviceError::BothOffDevice), + _ if blit => TransferType::Blit, + (false, true) => TransferType::Upload, + (true, false) => TransferType::Download, + (true, true) => TransferType::Intra, + }; + let dev = &self.dev.dev; + let command_buffer = { + let info = CommandBufferAllocateInfo::default() + .command_pool(self.dev.pools[tt]) + .command_buffer_count(1); + let mut buf = unsafe { + dev.allocate_command_buffers(&info) + .map_err(CopyDeviceError::CreateCommandBuffer)? + }; + assert_eq!(buf.len(), 1); + buf.pop().unwrap() + }; + let free_command_buffer = + on_drop(|| unsafe { dev.free_command_buffers(self.dev.pools[tt], &[command_buffer]) }); + let ty = if blit { + CopyDeviceCopyType::Blit { + src: self.import_image(tt, &src_class, src, Dir::Src)?, + dst: self.import_image(tt, &dst_class, dst, Dir::Dst)?, + } + } else if !src_class.buffer_possible && !dst_class.buffer_possible { + CopyDeviceCopyType::ImageToImage { + src: self.import_image(tt, &src_class, src, Dir::Src)?, + dst: self.import_image(tt, &dst_class, dst, Dir::Dst)?, + } + } else if src_class.buffer_possible + && dst_class.buffer_possible + && src.planes[0].stride == dst.planes[0].stride + { + CopyDeviceCopyType::BufferToBuffer { + src: self.import_buffer(tt, &src_class, src, Dir::Src)?, + dst: self.import_buffer(tt, &dst_class, dst, Dir::Dst)?, + stride: src.planes[0].stride, + bpp: src.format.bpp, + } + } else if src_class.buffer_possible { + CopyDeviceCopyType::BufferToImage { + buf: self.import_buffer(tt, &src_class, src, Dir::Src)?, + buf_format: src.format, + buf_stride: src.planes[0].stride, + img: self.import_image(tt, &dst_class, dst, Dir::Dst)?, + } + } else { + CopyDeviceCopyType::ImageToBuffer { + img: self.import_image(tt, &src_class, src, Dir::Src)?, + buf: self.import_buffer(tt, &dst_class, dst, Dir::Dst)?, + buf_format: dst.format, + buf_stride: dst.planes[0].stride, + } + }; + free_command_buffer.forget(); + Ok(CopyDeviceCopy { + inner: Rc::new(CopyDeviceCopyInner { + dev: self.dev.clone(), + busy: Default::default(), + width: src.width as _, + height: src.height as _, + command_buffer, + tt, + ty, + }), + _dev: self.clone(), + }) + } + + #[expect(dead_code)] + pub fn create_buffer( + &self, + dma_buf_ids: &DmaBufIds, + width: i32, + height: i32, + format: &'static Format, + ) -> Result { + if !self.dev.phy.supports_dmabuf_export { + return Err(CopyDeviceError::NoDmabufExport); + } + if width <= 0 || height <= 0 { + return Err(CopyDeviceError::NonPositiveSize); + } + let stride = width as u32 * format.bpp as u32; + let Some(stride) = stride.checked_next_multiple_of(LINEAR_STRIDE_ALIGN as u32) else { + return Err(CopyDeviceError::SizeOverflow); + }; + let Some(size) = (stride as u64).checked_mul(height as u64) else { + return Err(CopyDeviceError::SizeOverflow); + }; + let dev = &self.dev.dev; + let buffer = { + let mut external_info = ExternalMemoryBufferCreateInfo::default() + .handle_types(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let info = BufferCreateInfo::default() + .size(size) + .usage(BufferUsageFlags::TRANSFER_SRC | BufferUsageFlags::TRANSFER_DST) + .sharing_mode(SharingMode::EXCLUSIVE) + .push_next(&mut external_info); + unsafe { + dev.create_buffer(&info, None) + .map_err(CopyDeviceError::CreateBuffer)? + } + }; + let _destroy_buffer = on_drop(|| unsafe { dev.destroy_buffer(buffer, None) }); + let memory = { + let memory_requirements = unsafe { dev.get_buffer_memory_requirements(buffer) }; + let required_flags = + MemoryPropertyFlags::DEVICE_LOCAL | MemoryPropertyFlags::HOST_VISIBLE; + let index = 'index: { + for (idx, ty) in self.dev.phy.memory_types.iter().enumerate() { + if memory_requirements.memory_type_bits & (1 << idx) != 0 + && ty.property_flags.contains(required_flags) + { + break 'index idx; + } + } + return Err(CopyDeviceError::NoMemoryTypeForAllocation); + }; + let mut dedicated_allocation = MemoryDedicatedAllocateInfo::default().buffer(buffer); + let mut external_memory = ExportMemoryAllocateInfo::default() + .handle_types(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let info = MemoryAllocateInfo::default() + .allocation_size(memory_requirements.size) + .memory_type_index(index as _) + .push_next(&mut external_memory) + .push_next(&mut dedicated_allocation); + unsafe { + dev.allocate_memory(&info, None) + .map_err(CopyDeviceError::AllocateMemory)? + } + }; + let free_memory = on_drop(|| unsafe { dev.free_memory(memory, None) }); + let fd = { + let info = MemoryGetFdInfoKHR::default() + .memory(memory) + .handle_type(ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + unsafe { + self.dev + .external_memory_fd + .get_memory_fd(&info) + .map_err(CopyDeviceError::ExportDmabuf)? + } + }; + let fd = Rc::new(OwnedFd::new(fd)); + let mut dmabuf = DmaBuf { + id: dma_buf_ids.next(), + width, + height, + format, + modifier: LINEAR_MODIFIER, + planes: Default::default(), + is_disjoint: Default::default(), + }; + dmabuf.planes.push(DmaBufPlane { + offset: 0, + stride, + fd, + }); + free_memory.forget(); + Ok(CopyDeviceBuffer { + device: self.dev.clone(), + memory, + dmabuf, + }) + } +} + +impl CopyDeviceInner { + fn wait_idle(&self) { + log::warn!("Blocking"); + let res = unsafe { self.dev.device_wait_idle() }; + if let Err(e) = res { + log::error!("Could not wait for device idle: {}", ErrorFmt(e)); + log::error!("This is unsound."); + } + for submissions in self.submissions.values() { + submissions.pending.clear(); + } + } + + fn create_semaphore(self: &Rc) -> Result { + let create_info = SemaphoreCreateInfo::default(); + let semaphore = unsafe { + self.dev + .create_semaphore(&create_info, None) + .map_err(CopyDeviceError::CreateSemaphore)? + }; + Ok(VulkanSemaphore { + dev: self.clone(), + semaphore, + }) + } + + fn create_fence(self: &Rc) -> Result { + let mut export_info = + ExportFenceCreateInfo::default().handle_types(ExternalFenceHandleTypeFlags::SYNC_FD); + let create_info = FenceCreateInfo::default().push_next(&mut export_info); + let fence = unsafe { + self.dev + .create_fence(&create_info, None) + .map_err(CopyDeviceError::CreateFence)? + }; + Ok(VulkanFence { + dev: self.clone(), + fence, + }) + } +} + +impl CopyDeviceCopy { + fn ensure_not_busy(&self) -> Result<(), CopyDeviceError> { + let slf = &*self.inner; + let Some(busy) = slf.busy.get() else { + return Ok(()); + }; + let mut pollfd = c::pollfd { + fd: busy.raw(), + events: c::POLLIN, + revents: 0, + }; + let res = uapi::poll(slice::from_mut(&mut pollfd), 0); + if res != Ok(1) { + return Err(CopyDeviceError::Busy); + } + slf.busy.take(); + Ok(()) + } + + #[expect(dead_code)] + pub fn execute( + &self, + sync_file: Option<&SyncFile>, + region: Option<&Region>, + ) -> Result, CopyDeviceError> { + self.ensure_not_busy()?; + let slf = &*self.inner; + let tt = slf.tt; + let dev = &slf.dev.dev; + let cmd = slf.command_buffer; + let queue_family = slf.dev.phy.queues[tt].family; + let region_buf; + let width = slf.width; + let height = slf.height; + let region = match region { + Some(r) => r, + _ => { + region_buf = Region::new(Rect::new_saturating(0, 0, width as i32, height as i32)); + ®ion_buf + } + }; + let (x_mask, y_mask) = slf.dev.phy.queues[tt].transfer_granularity_mask; + let rects = &mut *slf.dev.phy.rects.borrow_mut(); + rects.clear(); + for rect in region.iter() { + let x1 = (rect.x1().max(0) as u32 & !x_mask).min(width); + let y1 = (rect.y1().max(0) as u32 & !y_mask).min(height); + let x2 = ((rect.x2().max(0) as u32 + x_mask) & !x_mask).min(width); + let y2 = ((rect.y2().max(0) as u32 + y_mask) & !y_mask).min(height); + let width = x2 - x1; + let height = y2 - y1; + if width == 0 || height == 0 { + continue; + } + rects.push((x1 as i32, y1 as i32, width, height)); + } + if rects.is_empty() { + return Ok(None); + } + let begin_info = + CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT); + unsafe { + dev.begin_command_buffer(cmd, &begin_info) + .map_err(CopyDeviceError::BeginCommandBuffer)?; + } + macro_rules! initial_buffer_barriers { + ($($buf:expr, $access:expr;)*) => { + [$( + BufferMemoryBarrier2::default() + .dst_stage_mask(PipelineStageFlags2::TRANSFER) + .dst_access_mask($access) + .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) + .dst_queue_family_index(queue_family) + .buffer($buf.buf) + .size(WHOLE_SIZE), + )*] + }; + } + macro_rules! final_buffer_barriers { + ($($buf:expr, $access:expr;)*) => { + [$( + BufferMemoryBarrier2::default() + .src_stage_mask(PipelineStageFlags2::TRANSFER) + .src_access_mask($access) + .src_queue_family_index(queue_family) + .dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) + .buffer($buf.buf) + .size(WHOLE_SIZE), + )*] + }; + } + let image_subresource_range = ImageSubresourceRange { + aspect_mask: ImageAspectFlags::COLOR, + base_mip_level: 0, + level_count: 1, + base_array_layer: 0, + layer_count: 1, + }; + let image_subresource = ImageSubresourceLayers { + aspect_mask: ImageAspectFlags::COLOR, + mip_level: 0, + base_array_layer: 0, + layer_count: 1, + }; + macro_rules! initial_image_barriers { + ($($img:expr, $layout:expr, $access:expr;)*) => { + [$( + ImageMemoryBarrier2::default() + .dst_stage_mask(PipelineStageFlags2::TRANSFER) + .dst_access_mask($access) + .old_layout(ImageLayout::GENERAL) + .new_layout(ImageLayout::GENERAL) + .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) + .dst_queue_family_index(queue_family) + .image($img.img) + .subresource_range(image_subresource_range), + ImageMemoryBarrier2::default() + .src_stage_mask(PipelineStageFlags2::TRANSFER) + .src_access_mask($access) + .dst_stage_mask(PipelineStageFlags2::TRANSFER) + .dst_access_mask($access) + .old_layout(ImageLayout::GENERAL) + .new_layout($layout) + .src_queue_family_index(queue_family) + .dst_queue_family_index(queue_family) + .image($img.img) + .subresource_range(image_subresource_range), + )*] + }; + } + macro_rules! final_image_barriers { + ($($img:expr, $layout:expr, $access:expr;)*) => { + [$( + ImageMemoryBarrier2::default() + .src_stage_mask(PipelineStageFlags2::TRANSFER) + .src_access_mask($access) + .dst_stage_mask(PipelineStageFlags2::TRANSFER) + .dst_access_mask($access) + .old_layout($layout) + .new_layout(ImageLayout::GENERAL) + .src_queue_family_index(queue_family) + .dst_queue_family_index(queue_family) + .image($img.img) + .subresource_range(image_subresource_range), + ImageMemoryBarrier2::default() + .src_stage_mask(PipelineStageFlags2::TRANSFER) + .src_access_mask($access) + .old_layout(ImageLayout::GENERAL) + .new_layout(ImageLayout::GENERAL) + .src_queue_family_index(queue_family) + .dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) + .image($img.img) + .subresource_range(image_subresource_range), + )*] + }; + } + match &slf.ty { + CopyDeviceCopyType::BufferToBuffer { + src, + dst, + stride, + bpp, + } => { + let regions = &mut *slf.dev.phy.buffer_copy_2.borrow_mut(); + regions.clear(); + let stride = *stride as u64; + let bpp = *bpp as u64; + for &mut (x, y, width, height) in rects { + let lo = y as u64 * stride + x as u64 * bpp; + let size = (height as u64 - 1) * stride + width as u64 * bpp; + let region = BufferCopy2::default() + .src_offset(lo) + .dst_offset(lo) + .size(size); + regions.push(region); + } + use AccessFlags2 as A; + let initial_barriers = initial_buffer_barriers![ + src, A::TRANSFER_READ; + dst, A::TRANSFER_WRITE; + ]; + let final_barriers = final_buffer_barriers![ + src, A::TRANSFER_READ; + dst, A::TRANSFER_WRITE; + ]; + let initial_dependency_info = + DependencyInfo::default().buffer_memory_barriers(&initial_barriers); + let final_dependency_info = + DependencyInfo::default().buffer_memory_barriers(&final_barriers); + let copy_buffer_info = CopyBufferInfo2::default() + .src_buffer(src.buf) + .dst_buffer(dst.buf) + .regions(regions); + unsafe { + dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info); + dev.cmd_copy_buffer2(cmd, ©_buffer_info); + dev.cmd_pipeline_barrier2(cmd, &final_dependency_info); + } + } + CopyDeviceCopyType::BufferToImage { + buf, + buf_format, + buf_stride, + img, + } + | CopyDeviceCopyType::ImageToBuffer { + img, + buf, + buf_format, + buf_stride, + } => { + let regions = &mut *slf.dev.phy.buffer_image_copy_2.borrow_mut(); + regions.clear(); + for &mut (x, y, width, height) in rects { + let offset = y as u64 * *buf_stride as u64 + x as u64 * buf_format.bpp as u64; + let region = BufferImageCopy2::default() + .buffer_offset(offset) + .buffer_row_length(*buf_stride / buf_format.bpp) + .buffer_image_height(slf.height) + .image_subresource(image_subresource) + .image_offset(Offset3D { x, y, z: 0 }) + .image_extent(Extent3D { + width, + height, + depth: 1, + }); + regions.push(region); + } + let buffer_to_image = match &slf.ty { + CopyDeviceCopyType::BufferToImage { .. } => true, + CopyDeviceCopyType::ImageToBuffer { .. } => false, + _ => unreachable!(), + }; + let image_access_mask; + let image_layout; + let buffer_access_mask; + match buffer_to_image { + true => { + image_access_mask = AccessFlags2::TRANSFER_WRITE; + image_layout = ImageLayout::TRANSFER_DST_OPTIMAL; + buffer_access_mask = AccessFlags2::TRANSFER_READ; + } + false => { + image_access_mask = AccessFlags2::TRANSFER_READ; + image_layout = ImageLayout::TRANSFER_SRC_OPTIMAL; + buffer_access_mask = AccessFlags2::TRANSFER_WRITE; + } + } + let initial_image_barriers = initial_image_barriers![ + img, image_layout, image_access_mask; + ]; + let final_image_barriers = final_image_barriers![ + img, image_layout, image_access_mask; + ]; + let initial_buffer_barriers = initial_buffer_barriers![ + buf, buffer_access_mask; + ]; + let final_buffer_barriers = final_buffer_barriers![ + buf, buffer_access_mask; + ]; + let initial_dependency_info = DependencyInfo::default() + .buffer_memory_barriers(&initial_buffer_barriers) + .image_memory_barriers(&initial_image_barriers); + let final_dependency_info = DependencyInfo::default() + .buffer_memory_barriers(&final_buffer_barriers) + .image_memory_barriers(&final_image_barriers); + unsafe { + dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info); + match buffer_to_image { + true => { + let copy = CopyBufferToImageInfo2::default() + .src_buffer(buf.buf) + .dst_image(img.img) + .dst_image_layout(image_layout) + .regions(®ions); + dev.cmd_copy_buffer_to_image2(cmd, ©); + } + false => { + let copy = CopyImageToBufferInfo2::default() + .src_image(img.img) + .src_image_layout(image_layout) + .dst_buffer(buf.buf) + .regions(®ions); + dev.cmd_copy_image_to_buffer2(cmd, ©); + } + } + dev.cmd_pipeline_barrier2(cmd, &final_dependency_info); + } + } + CopyDeviceCopyType::ImageToImage { src, dst } => { + let regions = &mut *slf.dev.phy.image_copy_2.borrow_mut(); + regions.clear(); + for &mut (x, y, width, height) in rects { + let region = ImageCopy2::default() + .src_subresource(image_subresource) + .src_offset(Offset3D { x, y, z: 0 }) + .dst_subresource(image_subresource) + .dst_offset(Offset3D { x, y, z: 0 }) + .extent(Extent3D { + width, + height, + depth: 1, + }); + regions.push(region); + } + use {AccessFlags2 as A, ImageLayout as L}; + let initial_barriers = initial_image_barriers![ + src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ; + dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE; + ]; + let final_barriers = final_image_barriers![ + src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ; + dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE; + ]; + let initial_dependency_info = + DependencyInfo::default().image_memory_barriers(&initial_barriers); + let final_dependency_info = + DependencyInfo::default().image_memory_barriers(&final_barriers); + let copy_image_info = CopyImageInfo2::default() + .src_image(src.img) + .src_image_layout(L::TRANSFER_SRC_OPTIMAL) + .dst_image(dst.img) + .dst_image_layout(L::TRANSFER_DST_OPTIMAL) + .regions(regions); + unsafe { + dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info); + dev.cmd_copy_image2(cmd, ©_image_info); + dev.cmd_pipeline_barrier2(cmd, &final_dependency_info); + } + } + CopyDeviceCopyType::Blit { src, dst } => { + let regions = &mut *slf.dev.phy.image_blit_2.borrow_mut(); + regions.clear(); + for &mut (x, y, width, height) in rects { + let x1 = x; + let y1 = y; + let x2 = x1 + width as i32; + let y2 = y1 + height as i32; + let offsets = [ + Offset3D { x: x1, y: y1, z: 0 }, + Offset3D { x: x2, y: y2, z: 1 }, + ]; + let region = ImageBlit2::default() + .src_subresource(image_subresource) + .src_offsets(offsets) + .dst_subresource(image_subresource) + .dst_offsets(offsets); + regions.push(region); + } + use {AccessFlags2 as A, ImageLayout as L}; + let initial_barriers = initial_image_barriers![ + src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ; + dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE; + ]; + let final_barriers = final_image_barriers![ + src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ; + dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE; + ]; + let initial_dependency_info = + DependencyInfo::default().image_memory_barriers(&initial_barriers); + let final_dependency_info = + DependencyInfo::default().image_memory_barriers(&final_barriers); + let blit_image_info = BlitImageInfo2::default() + .src_image(src.img) + .src_image_layout(L::TRANSFER_SRC_OPTIMAL) + .dst_image(dst.img) + .dst_image_layout(L::TRANSFER_DST_OPTIMAL) + .regions(regions) + .filter(Filter::NEAREST); + unsafe { + dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info); + dev.cmd_blit_image2(cmd, &blit_image_info); + dev.cmd_pipeline_barrier2(cmd, &final_dependency_info); + } + } + }; + unsafe { + dev.end_command_buffer(cmd) + .map_err(CopyDeviceError::EndCommandBuffer)?; + } + let mut wait_semaphore = None; + let mut wait_semaphores = ArrayVec::<_, 1>::new(); + if let Some(sync_file) = sync_file { + let semaphore = match slf.dev.semaphores.pop() { + Some(s) => s, + _ => slf.dev.create_semaphore()?, + }; + semaphore.import(sync_file)?; + let info = SemaphoreSubmitInfo::default() + .semaphore(semaphore.semaphore) + .stage_mask(PipelineStageFlags2::TRANSFER); + wait_semaphores.push(info); + wait_semaphore = Some(semaphore); + } + let signal_fence = match slf.dev.fences.pop() { + Some(s) => s, + _ => slf.dev.create_fence()?, + }; + let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd); + let submit_info = SubmitInfo2::default() + .command_buffer_infos(slice::from_ref(&command_buffer_info)) + .wait_semaphore_infos(&wait_semaphores); + unsafe { + slf.dev + .dev + .queue_submit2( + slf.dev.queues[tt], + slice::from_ref(&submit_info), + signal_fence.fence, + ) + .map_err(CopyDeviceError::SubmitCopy)?; + } + let sync_file = match signal_fence.export() { + Ok(f) => f, + Err(e) => { + log::error!("Could not export signal fence: {}", ErrorFmt(e)); + slf.dev.wait_idle(); + None + } + }; + slf.busy.set(sync_file.clone()); + let pending = Pending { + dev: slf.dev.clone(), + sync_file: sync_file.clone(), + copy: self.inner.clone(), + semaphore: wait_semaphore, + fence: Some(signal_fence), + }; + slf.dev.submissions[tt].pending.push(pending); + Ok(sync_file) + } +} + +impl VulkanSemaphore { + fn import(&self, sync_file: &OwnedFd) -> Result<(), CopyDeviceError> { + let fd = uapi::fcntl_dupfd_cloexec(sync_file.raw(), 0) + .map_err(Into::into) + .map_err(CopyDeviceError::DupSyncFile)?; + let info = ImportSemaphoreFdInfoKHR::default() + .flags(SemaphoreImportFlags::TEMPORARY) + .semaphore(self.semaphore) + .handle_type(ExternalSemaphoreHandleTypeFlags::SYNC_FD) + .fd(fd.raw()); + unsafe { + self.dev + .external_semaphore_fd + .import_semaphore_fd(&info) + .map_err(CopyDeviceError::ImportSyncFile)?; + } + let _ = fd.unwrap(); + Ok(()) + } +} + +impl VulkanFence { + fn export(&self) -> Result, CopyDeviceError> { + let info = FenceGetFdInfoKHR::default() + .fence(self.fence) + .handle_type(ExternalFenceHandleTypeFlags::SYNC_FD); + let fd = unsafe { + self.dev + .external_fence_fd + .get_fence_fd(&info) + .map_err(CopyDeviceError::ExportSyncFile)? + }; + let fd = if fd == -1 { + None + } else { + Some(SyncFile(Rc::new(OwnedFd::new(fd)))) + }; + Ok(fd) + } +} + +impl CopyDeviceRegistry { + pub fn new(ring: &Rc, eng: &Rc) -> Self { + Self { + ring: ring.clone(), + eng: eng.clone(), + devs: Default::default(), + } + } + + #[expect(dead_code)] + pub fn remove(&self, dev: c::dev_t) { + self.devs.remove(&dev); + } + + #[expect(dead_code)] + pub fn get(&self, dev: c::dev_t) -> Option> { + if let Some(dev) = self.devs.get(&dev) { + return dev; + } + match PhysicalCopyDevice::new(&self.ring, &self.eng, dev).map(Some) { + Ok(cd) => { + self.devs.set(dev, cd.clone()); + cd + } + Err(e) => { + let maj = uapi::major(dev); + let min = uapi::minor(dev); + log::warn!( + "Could not create physical copy device for {maj}:{min}: {}", + ErrorFmt(e), + ); + self.devs.set(dev, None); + None + } + } + } +} + +impl Drop for VulkanSemaphore { + fn drop(&mut self) { + unsafe { + self.dev.dev.destroy_semaphore(self.semaphore, None); + } + } +} + +impl Drop for VulkanFence { + fn drop(&mut self) { + unsafe { + self.dev.dev.destroy_fence(self.fence, None); + } + } +} + +impl Drop for CopyDeviceCopyInner { + fn drop(&mut self) { + unsafe { + self.dev.dev.free_command_buffers( + self.dev.pools[self.tt], + slice::from_ref(&self.command_buffer), + ); + } + } +} + +impl Drop for CopyDeviceInner { + fn drop(&mut self) { + unsafe { + for &pool in &self.unique_pools { + self.dev.destroy_command_pool(pool, None); + } + self.dev.destroy_device(None); + } + } +} + +impl Drop for CopyDevice { + fn drop(&mut self) { + let dev = &self.dev; + let has_pending = dev + .submissions + .values() + .any(|s| s.task_has_pending.get() || s.pending.is_not_empty()); + if has_pending { + dev.wait_idle(); + } + dev.semaphores.take(); + dev.fences.take(); + } +} + +impl Drop for Pending { + fn drop(&mut self) { + if let Some(v) = self.semaphore.take() { + self.dev.semaphores.push(v); + } + if let Some(v) = self.fence.take() { + self.dev.fences.push(v); + } + if self.copy.busy.get() == self.sync_file { + self.copy.busy.take(); + } + } +} + +impl CopyDeviceBuffer { + #[expect(dead_code)] + pub fn dmabuf(&self) -> &DmaBuf { + &self.dmabuf + } +} + +impl Drop for CopyDeviceBuffer { + fn drop(&mut self) { + unsafe { + self.device.dev.free_memory(self.memory, None); + } + } +} + +impl Debug for CopyDeviceBuffer { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CopyDeviceBuffer").finish_non_exhaustive() + } +} + +impl Debug for PhysicalCopyDevice { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PhysicalCopyDevice").finish_non_exhaustive() + } +} + +impl Debug for CopyDevice { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CopyDevice").finish_non_exhaustive() + } +} + +impl Debug for CopyDeviceCopy { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CopyDeviceCopy").finish_non_exhaustive() + } +} + +impl Drop for VulkanBuffer { + fn drop(&mut self) { + let dev = &self.dev.dev; + unsafe { + dev.destroy_buffer(self.buf, None); + dev.free_memory(self.mem, None); + } + } +} + +impl Drop for VulkanImage { + fn drop(&mut self) { + let dev = &self.dev.dev; + unsafe { + dev.destroy_image(self.img, None); + for &mem in &self.mem { + dev.free_memory(mem, None); + } + } + } +} + +impl Deref for CopyDevice { + type Target = Rc; + + fn deref(&self) -> &Self::Target { + &self.dev.phy + } +} + +type QueueInfo = (u32, (u32, u32), u32); + +fn allocate_queues( + gfx: QueueInfo, + compute_only: Option, + transfer_only: Option, +) -> (Vec, KeyedCopy) { + let intra = compute_only.unwrap_or(gfx); + let cross = transfer_only.unwrap_or(intra); + let mut distinct_families = AHashSet::default(); + distinct_families.insert(cross); + distinct_families.insert(intra); + distinct_families.insert(gfx); + let mut queues_to_allocate = vec![]; + macro_rules! index { + ($qi:expr, $within:expr) => { + QueueIndex { + allocate_idx: queues_to_allocate.len(), + family: $qi.0, + idx_within_family: $within as u32, + transfer_granularity_mask: $qi.1, + } + }; + } + macro_rules! alloc { + ($qi:expr, $num:expr) => { + QueueToAllocate { + family: $qi.0, + num: $num as usize, + } + }; + } + let (blit, intra_idx, download, upload); + if distinct_families.len() == 3 { + let num_cross = cross.2.min(2) as usize; + blit = index!(gfx, 0); + queues_to_allocate.push(alloc!(gfx, 1)); + intra_idx = index!(intra, 0); + queues_to_allocate.push(alloc!(intra, 1)); + download = index!(cross, 0); + upload = index!(cross, num_cross - 1); + queues_to_allocate.push(alloc!(cross, num_cross)); + } else if distinct_families.len() == 1 { + let qi = cross; + let num = qi.2.min(4); + match num { + 1 => { + blit = index!(qi, 0); + intra_idx = index!(qi, 0); + download = index!(qi, 0); + upload = index!(qi, 0); + } + 2 => { + blit = index!(qi, 0); + intra_idx = index!(qi, 0); + download = index!(qi, 0); + upload = index!(qi, 1); + } + 3 => { + blit = index!(qi, 0); + intra_idx = index!(qi, 0); + download = index!(qi, 1); + upload = index!(qi, 2); + } + 4 => { + blit = index!(qi, 0); + intra_idx = index!(qi, 1); + download = index!(qi, 2); + upload = index!(qi, 3); + } + _ => unreachable!(), + } + queues_to_allocate.push(alloc!(qi, num)); + } else { + if gfx == intra { + let num_gfx = gfx.2.min(2); + blit = index!(gfx, 0); + intra_idx = index!(gfx, num_gfx - 1); + queues_to_allocate.push(alloc!(gfx, num_gfx)); + let num_cross = cross.2.min(2); + download = index!(cross, 0); + upload = index!(cross, num_cross - 1); + queues_to_allocate.push(alloc!(cross, num_cross)); + } else { + // if cross == gfx then intra == gfx + assert_eq!(intra, cross); + blit = index!(gfx, 0); + queues_to_allocate.push(alloc!(gfx, 1)); + let num_intra = intra.2.min(3); + match num_intra { + 1 => { + intra_idx = index!(intra, 0); + download = index!(intra, 0); + upload = index!(intra, 0); + } + 2 => { + intra_idx = index!(intra, 0); + download = index!(intra, 0); + upload = index!(intra, 1); + } + 3 => { + intra_idx = index!(intra, 0); + download = index!(intra, 1); + upload = index!(intra, 2); + } + _ => unreachable!(), + } + queues_to_allocate.push(alloc!(intra, num_intra)); + } + } + let queue_indices = static_copy_map! { + TransferType::Blit => blit, + TransferType::Intra => intra_idx, + TransferType::Download => download, + TransferType::Upload => upload, + }; + (queues_to_allocate, queue_indices) +} diff --git a/src/gfx_api.rs b/src/gfx_api.rs index 7adf80a5..44704c20 100644 --- a/src/gfx_api.rs +++ b/src/gfx_api.rs @@ -221,7 +221,7 @@ pub struct CopyTexture { pub cd: Rc, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] pub struct SyncFile(pub Rc); impl Deref for SyncFile { diff --git a/src/gfx_apis/vulkan/device.rs b/src/gfx_apis/vulkan/device.rs index 76aa4968..90106f7b 100644 --- a/src/gfx_apis/vulkan/device.rs +++ b/src/gfx_apis/vulkan/device.rs @@ -13,7 +13,9 @@ use { drm::{Drm, sync_obj::SyncObjCtx}, gbm::{GBM_BO_USE_RENDERING, GbmDevice}, }, - vulkan_core::{API_VERSION, ApiVersionDisplay, Extensions, map_extension_properties}, + vulkan_core::{ + ApiVersionDisplay, Extensions, VULKAN_API_VERSION, map_extension_properties, + }, }, ahash::AHashMap, arrayvec::ArrayVec, @@ -145,7 +147,7 @@ impl VulkanInstance { let mut devices = vec![]; for phy_dev in phy_devs { let props = unsafe { self.instance.get_physical_device_properties(phy_dev) }; - if props.api_version < API_VERSION { + if props.api_version < VULKAN_API_VERSION { devices.push((props, None, None)); continue; } @@ -218,7 +220,7 @@ impl VulkanInstance { }; for phy_dev in phy_devs { let props = unsafe { self.instance.get_physical_device_properties(phy_dev) }; - if props.api_version < API_VERSION { + if props.api_version < VULKAN_API_VERSION { continue; } if props.device_type == PhysicalDeviceType::CPU { @@ -618,7 +620,7 @@ fn log_device( Ustr::from_ptr(props.device_name.as_ptr()).display() ); } - if props.api_version < API_VERSION { + if props.api_version < VULKAN_API_VERSION { log::warn!(" device does not support vulkan 1.3"); } if let Some(extensions) = extensions { diff --git a/src/main.rs b/src/main.rs index 4d3c060c..138ab397 100644 --- a/src/main.rs +++ b/src/main.rs @@ -58,6 +58,7 @@ mod clientmem; mod cmm; mod compositor; mod config; +mod copy_device; mod cpu_worker; mod criteria; mod cursor; diff --git a/src/state.rs b/src/state.rs index c9556c15..01ebf3b4 100644 --- a/src/state.rs +++ b/src/state.rs @@ -15,6 +15,7 @@ use { cmm::{cmm_description::ColorDescription, cmm_manager::ColorManager}, compositor::LIBEI_SOCKET, config::ConfigProxy, + copy_device::CopyDeviceRegistry, cpu_worker::CpuWorker, criteria::{clm::ClMatcherManager, tlm::TlMatcherManager}, cursor::{Cursor, ServerCursors}, @@ -292,6 +293,8 @@ pub struct State { pub outputs_without_hc: NumCell, pub udmabuf: Rc, pub gfx_ctx_changed: EventSource, + #[expect(dead_code)] + pub copy_device_registry: Rc, } // impl Drop for State { diff --git a/src/udmabuf.rs b/src/udmabuf.rs index 4bff5667..bb35c463 100644 --- a/src/udmabuf.rs +++ b/src/udmabuf.rs @@ -7,7 +7,7 @@ use { oserror::OsError, page_size::page_size, }, video::{ - LINEAR_MODIFIER, Modifier, + LINEAR_MODIFIER, LINEAR_STRIDE_ALIGN, Modifier, dmabuf::{DmaBuf, DmaBufIds, DmaBufPlane, PlaneVec}, drm::Drm, }, @@ -139,8 +139,7 @@ impl Allocator for Udmabuf { if height > 1 << 16 || width > 1 << 16 { return Err(UdmabufError::Overflow.into()); } - let stride_mask = 255; - let stride = (width * format.bpp as u64 + stride_mask) & !stride_mask; + let stride = (width * format.bpp as u64).next_multiple_of(LINEAR_STRIDE_ALIGN); let size_mask = page_size() as u64 - 1; let size = (height * stride + size_mask) & !size_mask; let memfd = match uapi::memfd_create("udmabuf", MFD_ALLOW_SEALING) { diff --git a/src/video.rs b/src/video.rs index 572190a1..9f8906fe 100644 --- a/src/video.rs +++ b/src/video.rs @@ -6,3 +6,6 @@ pub type Modifier = u64; pub const INVALID_MODIFIER: Modifier = 0x00ff_ffff_ffff_ffff; pub const LINEAR_MODIFIER: Modifier = 0; + +// This is required by AMD and therefore everyone else uses this too. +pub const LINEAR_STRIDE_ALIGN: u64 = 256; diff --git a/src/vulkan_core.rs b/src/vulkan_core.rs index f269c3d0..b9261c3b 100644 --- a/src/vulkan_core.rs +++ b/src/vulkan_core.rs @@ -72,7 +72,7 @@ impl VulkanCoreInstance { .map(|c| c.as_ptr()) .collect(); let app_info = ApplicationInfo::default() - .api_version(API_VERSION) + .api_version(VULKAN_API_VERSION) .application_name(c"jay") .application_version(1); let mut severity = DebugUtilsMessageSeverityFlagsEXT::empty() @@ -240,4 +240,4 @@ impl Display for ApiVersionDisplay { } } -pub const API_VERSION: u32 = API_VERSION_1_3; +pub const VULKAN_API_VERSION: u32 = API_VERSION_1_3; From a77929741a8c0c20a58f2e3be9969c6ab0987548 Mon Sep 17 00:00:00 2001 From: Julian Orth Date: Sun, 15 Feb 2026 19:04:25 +0100 Subject: [PATCH 4/5] metal: add support for copy-device based prime methods --- src/backends/metal/allocator.rs | 110 +++++++++++++++++++++++++++++++- src/backends/metal/monitor.rs | 1 + src/backends/metal/present.rs | 6 +- src/backends/metal/video.rs | 48 +++++++++++++- src/copy_device.rs | 7 -- src/state.rs | 1 - src/udmabuf.rs | 100 ++++++++++++++++++----------- 7 files changed, 223 insertions(+), 50 deletions(-) diff --git a/src/backends/metal/allocator.rs b/src/backends/metal/allocator.rs index 958299bb..3f5bc257 100644 --- a/src/backends/metal/allocator.rs +++ b/src/backends/metal/allocator.rs @@ -6,20 +6,22 @@ use { video::{MetalDrmDevice, MetalRenderContext}, }, cmm::cmm_description::ColorDescription, + copy_device::{CopyDevice, CopyDeviceError, CopyDeviceSupport}, format::Format, gfx_api::{ AcquireSync, GfxBlendBuffer, GfxError, GfxFormat, GfxFramebuffer, GfxTexture, GfxWriteModifier, ReleaseSync, SyncFile, needs_render_usage, }, - rect::{DamageQueue, Rect}, + rect::{DamageQueue, Rect, Region}, utils::{errorfmt::ErrorFmt, rc_eq::rc_eq}, video::{ - Modifier, + LINEAR_MODIFIER, Modifier, dmabuf::DmaBuf, drm::{DrmError, DrmFramebuffer}, gbm::{GBM_BO_USE_LINEAR, GBM_BO_USE_RENDERING, GBM_BO_USE_SCANOUT, GbmBo, GbmError}, }, }, + ahash::HashSet, arrayvec::ArrayVec, bstr::ByteSlice, indexmap::{IndexMap, IndexSet}, @@ -78,6 +80,10 @@ pub enum RenderBufferError { GfxError(#[from] GfxError), #[error("Could not copy frame to output device")] CopyToOutput(#[source] GfxError), + #[error("Could not create a copy device copy")] + CreateCopyDeviceCopy(#[source] CopyDeviceError), + #[error("Could not execute a copy device copy")] + ExecuteCopyDeviceCopy(#[source] CopyDeviceError), } #[derive(Default)] @@ -99,6 +105,7 @@ impl RenderBuffer { pub fn copy_to_dev( &self, cd: &Rc, + _region: Option<&Region>, sync_file: Option, ) -> Result { match &self.prime { @@ -158,6 +165,14 @@ impl RenderBuffer { return Err(RenderBufferError::NotSameSize); } + if let Some(dev) = new.dev_copy_device().or(old.dev_copy_device()) { + return dev + .create_copy(old.dev_bo().dmabuf(), new.dev_bo().dmabuf()) + .map_err(RenderBufferError::CreateCopyDeviceCopy)? + .execute(None, None) + .map_err(RenderBufferError::ExecuteCopyDeviceCopy); + } + let copy_texture_impl = |fb: &Rc, tex: &Rc| { fb.copy_texture( AcquireSync::Unnecessary, @@ -235,6 +250,13 @@ impl RenderBuffer { RenderBufferPrime::Sampling { dev_bo, .. } => dev_bo, } } + + pub fn dev_copy_device(&self) -> Option<&Rc> { + match &self.prime { + RenderBufferPrime::None => None, + RenderBufferPrime::Sampling { .. } => None, + } + } } struct Builder<'a> { @@ -459,11 +481,15 @@ impl RenderBufferPrime { #[derive(Default, Debug)] struct RenderBufferAllocationDebug { + dev_copy_src_modifiers: Option>, + dev_copy_dst_modifiers: Option>, dev_gfx_write_modifiers: Option>, dev_gfx_read_modifiers: Option>, dev_modifiers_possible: Option>, dev_usage: Option, dev_modifier: Option, + render_copy_src_modifiers: Option>, + render_copy_dst_modifiers: Option>, render_gfx_write_modifiers: Option>, render_gfx_read_modifiers: Option>, render_modifiers_possible: Option>, @@ -496,6 +522,12 @@ impl Display for ScanoutBufferError { writeln!(f, "plane modifiers: {:x?}", self.plane_modifiers)?; writeln!(f, "size: {}x{}", self.width, self.height)?; writeln!(f, "cursor: {}", self.cursor)?; + if let Some(v) = &self.dbg.dev_copy_src_modifiers { + writeln!(f, "scanout copy src modifiers: {:x?}", v)?; + } + if let Some(v) = &self.dbg.dev_copy_dst_modifiers { + writeln!(f, "scanout copy dst modifiers: {:x?}", v)?; + } if let Some(v) = &self.dbg.dev_gfx_write_modifiers { writeln!(f, "scanout gfx writable modifiers: {:x?}", v)?; } @@ -511,6 +543,12 @@ impl Display for ScanoutBufferError { if let Some(v) = &self.render_name { writeln!(f, "render device: {}", v)?; } + if let Some(v) = &self.dbg.render_copy_src_modifiers { + writeln!(f, "render copy src modifiers: {:x?}", v)?; + } + if let Some(v) = &self.dbg.render_copy_dst_modifiers { + writeln!(f, "render copy dst modifiers: {:x?}", v)?; + } if let Some(v) = &self.dbg.render_gfx_write_modifiers { writeln!(f, "render gfx writable modifiers: {:x?}", v)?; } @@ -729,6 +767,43 @@ impl Builder<'_> { }) } + fn copy_modifiers_iter(&self, support: &[CopyDeviceSupport]) -> impl Iterator { + let Builder { width, height, .. } = *self; + support + .iter() + .filter(move |s| s.max_width >= width as _ && s.max_height >= height as _) + .map(move |s| s.modifier) + } + + fn copy_modifiers(&self, support: &[CopyDeviceSupport]) -> Vec { + self.copy_modifiers_iter(support).collect() + } + + #[expect(dead_code)] + fn copy_src_modifiers(&self, dev: &CopyDevice) -> Vec { + self.copy_modifiers(dev.src_support(self.format)) + } + + #[expect(dead_code)] + fn copy_dst_modifiers(&self, dev: &CopyDevice) -> Vec { + self.copy_modifiers(dev.dst_support(self.format)) + } + + fn copy_supports_linear(&self, support: &[CopyDeviceSupport]) -> bool { + self.copy_modifiers_iter(support) + .any(|m| m == LINEAR_MODIFIER) + } + + #[expect(dead_code)] + fn copy_src_supports_linear(&self, dev: &CopyDevice) -> bool { + self.copy_supports_linear(dev.src_support(self.format)) + } + + #[expect(dead_code)] + fn copy_dst_supports_linear(&self, dev: &CopyDevice) -> bool { + self.copy_supports_linear(dev.dst_support(self.format)) + } + fn prepare_prime_none( &self, dbg: &RefCell, @@ -936,3 +1011,34 @@ fn sample_modifiers(fmt: &GfxFormat) -> Vec { fn render_modifiers(fmt: &GfxFormat) -> Vec { fmt.write_modifiers.keys().copied().collect() } + +fn intersect_modifiers<'a>( + left: impl IntoIterator, + right: impl IntoIterator, +) -> Vec { + let right: HashSet<_> = right.into_iter().copied().collect(); + left.into_iter() + .copied() + .filter(|m| right.contains(m)) + .collect() +} + +#[expect(dead_code)] +fn intersect_render_modifiers<'a>( + left: &'a GfxFormat, + right: impl IntoIterator, +) -> Vec { + intersect_modifiers( + left.write_modifiers + .keys() + .filter(|m| left.read_modifiers.contains(*m)), + right, + ) +} + +#[expect(dead_code)] +fn make_linear_only(modifiers: &mut Vec) { + if modifiers.contains(&LINEAR_MODIFIER) { + *modifiers = vec![LINEAR_MODIFIER]; + } +} diff --git a/src/backends/metal/monitor.rs b/src/backends/metal/monitor.rs index e5ca6daf..db9a85a2 100644 --- a/src/backends/metal/monitor.rs +++ b/src/backends/metal/monitor.rs @@ -131,6 +131,7 @@ impl MetalBackend { } fn handle_drm_device_removed(self: &Rc, dev: &Rc) { + self.state.copy_device_registry.remove(dev.dev.devnum); log::info!("Device removed: {}", dev.dev.devnode.to_bytes().as_bstr()); } diff --git a/src/backends/metal/present.rs b/src/backends/metal/present.rs index b1493ea8..a4edf006 100644 --- a/src/backends/metal/present.rs +++ b/src/backends/metal/present.rs @@ -557,7 +557,7 @@ impl MetalConnector { if let Some(sf) = c.cursor_swap_buffer.take() { let sf = c .cursor_buffer - .copy_to_dev(cd, sf) + .copy_to_dev(cd, None, sf) .map_err(MetalError::CopyToDev)? .present_block; self.cursor_sync_file.set(sf); @@ -875,7 +875,9 @@ impl MetalConnector { blend_cd, ) .map_err(MetalError::RenderFrame)?; - copy = buffer.copy_to_dev(cd, sf).map_err(MetalError::CopyToDev)?; + copy = buffer + .copy_to_dev(cd, Some(&latched.damage), sf) + .map_err(MetalError::CopyToDev)?; fb = buffer.drm.clone(); tex = buffer.render.tex.clone(); } diff --git a/src/backends/metal/video.rs b/src/backends/metal/video.rs index 89fd53f8..1598f478 100644 --- a/src/backends/metal/video.rs +++ b/src/backends/metal/video.rs @@ -22,6 +22,7 @@ use { transaction::{DrmConnectorState, DrmCrtcState, DrmPlaneState, MetalDeviceTransaction}, }, cmm::{cmm_description::ColorDescription, cmm_primaries::Primaries}, + copy_device::{CopyDevice, CopyDeviceRegistry}, drm_feedback::DrmFeedback, edid::{CtaDataBlock, Descriptor, EdidExtension}, format::{Format, XRGB8888}, @@ -58,7 +59,7 @@ use { isnt::std_1::collections::IsntHashMapExt, jay_config::video::GfxApi, std::{ - cell::{Cell, RefCell}, + cell::{Cell, OnceCell, RefCell}, collections::hash_map::Entry, ffi::CString, fmt::{Debug, Formatter}, @@ -84,6 +85,19 @@ pub struct MetalRenderContext { pub gfx: Rc, pub gbm: Rc, pub devnode: CString, + pub copy_device: Rc, +} + +pub struct CopyDeviceHolder { + pub registry: Rc, + pub devnum: dev_t, + pub dev: OnceCell>>, +} + +impl Debug for CopyDeviceHolder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CopyDeviceHolder").finish_non_exhaustive() + } } pub struct MetalDrmDevice { @@ -105,6 +119,8 @@ pub struct MetalDrmDevice { pub gbm: Rc, pub handle_events: HandleEvents, pub ctx: CloneCell>, + #[expect(dead_code)] + pub copy_device: Rc, pub on_change: OnChange, pub direct_scanout_enabled: Cell>, pub is_nvidia: bool, @@ -1987,6 +2003,12 @@ impl MetalBackend { Err(e) => return Err(MetalError::GbmDevice(e)), }; + let copy_device = Rc::new(CopyDeviceHolder { + registry: self.state.copy_device_registry.clone(), + devnum: pending.devnum, + dev: Default::default(), + }); + let gfx = match self.state.create_gfx_context(master, None) { Ok(r) => r, Err(e) => return Err(MetalError::CreateRenderContex(e)), @@ -1996,6 +2018,7 @@ impl MetalBackend { gfx, gbm: gbm.clone(), devnode: pending.devnode.clone(), + copy_device: copy_device.clone(), }); let mut is_nvidia = false; @@ -2037,6 +2060,7 @@ impl MetalBackend { handle_events: Cell::new(None), }, ctx: CloneCell::new(ctx), + copy_device, on_change: Default::default(), direct_scanout_enabled: Default::default(), is_nvidia, @@ -2462,6 +2486,7 @@ impl MetalBackend { gfx, gbm: old_ctx.gbm.clone(), devnode: old_ctx.devnode.clone(), + copy_device: old_ctx.copy_device.clone(), })); if dev.is_render_device() { self.make_render_device(dev, true); @@ -2622,3 +2647,24 @@ impl MetalBackend { connector.schedule_present(); } } + +impl CopyDeviceHolder { + #[expect(dead_code)] + pub fn get(&self) -> Option> { + self.dev + .get_or_init( + || match self.registry.get(self.devnum)?.create_device().map(Some) { + Ok(d) => d, + Err(e) => { + log::error!( + "Could not get copy device for {}: {}", + self.devnum, + ErrorFmt(e), + ); + None + } + }, + ) + .clone() + } +} diff --git a/src/copy_device.rs b/src/copy_device.rs index 512a297a..4acd68a8 100644 --- a/src/copy_device.rs +++ b/src/copy_device.rs @@ -629,12 +629,10 @@ impl PhysicalCopyDevice { Ok(dev) } - #[expect(dead_code)] pub fn src_support(&self, format: &Format) -> &[CopyDeviceSupport] { self.support(format, Dir::Src) } - #[expect(dead_code)] pub fn dst_support(&self, format: &Format) -> &[CopyDeviceSupport] { self.support(format, Dir::Dst) } @@ -646,7 +644,6 @@ impl PhysicalCopyDevice { .unwrap_or_default() } - #[expect(dead_code)] pub fn create_device(self: &Rc) -> Result, CopyDeviceError> { let instance = &self.instance.instance; let device = { @@ -1053,7 +1050,6 @@ impl CopyDevice { }) } - #[expect(dead_code)] pub fn create_copy( self: &Rc, src: &DmaBuf, @@ -1299,7 +1295,6 @@ impl CopyDeviceCopy { Ok(()) } - #[expect(dead_code)] pub fn execute( &self, sync_file: Option<&SyncFile>, @@ -1763,12 +1758,10 @@ impl CopyDeviceRegistry { } } - #[expect(dead_code)] pub fn remove(&self, dev: c::dev_t) { self.devs.remove(&dev); } - #[expect(dead_code)] pub fn get(&self, dev: c::dev_t) -> Option> { if let Some(dev) = self.devs.get(&dev) { return dev; diff --git a/src/state.rs b/src/state.rs index 01ebf3b4..790f9a9a 100644 --- a/src/state.rs +++ b/src/state.rs @@ -293,7 +293,6 @@ pub struct State { pub outputs_without_hc: NumCell, pub udmabuf: Rc, pub gfx_ctx_changed: EventSource, - #[expect(dead_code)] pub copy_device_registry: Rc, } diff --git a/src/udmabuf.rs b/src/udmabuf.rs index bb35c463..e398d692 100644 --- a/src/udmabuf.rs +++ b/src/udmabuf.rs @@ -115,6 +115,64 @@ impl Udmabuf { }; Ok(dmabuf) } + + #[expect(dead_code)] + pub fn create_dmabuf( + &self, + dma_buf_ids: &DmaBufIds, + width: i32, + height: i32, + format: &'static Format, + ) -> Result { + Ok(self.create_bo(dma_buf_ids, width, height, format)?.buf) + } + + fn create_bo( + &self, + dma_buf_ids: &DmaBufIds, + width: i32, + height: i32, + format: &'static Format, + ) -> Result { + let height = height as u64; + let width = width as u64; + if height > 1 << 16 || width > 1 << 16 { + return Err(UdmabufError::Overflow); + } + let stride = (width * format.bpp as u64).next_multiple_of(LINEAR_STRIDE_ALIGN); + let size_mask = page_size() as u64 - 1; + let size = (height * stride + size_mask) & !size_mask; + let memfd = match uapi::memfd_create("udmabuf", MFD_ALLOW_SEALING) { + Ok(f) => f, + Err(e) => return Err(UdmabufError::Memfd(e.into())), + }; + if let Err(e) = uapi::ftruncate(memfd.raw(), size as _) { + return Err(UdmabufError::Truncate(e.into())); + } + if let Err(e) = uapi::fcntl_add_seals(memfd.raw(), F_SEAL_SHRINK) { + return Err(UdmabufError::Seal(e.into())); + } + let dmabuf = self.create_dmabuf_from_memfd(&memfd, 0, size as _)?; + let mut planes = PlaneVec::new(); + planes.push(DmaBufPlane { + offset: 0, + stride: stride as _, + fd: Rc::new(dmabuf), + }); + let dmabuf = DmaBuf { + id: dma_buf_ids.next(), + width: width as _, + height: height as _, + format, + modifier: LINEAR_MODIFIER, + planes, + is_disjoint: Default::default(), + }; + Ok(UdmabufBo { + buf: dmabuf, + size: size as _, + }) + } } impl Allocator for Udmabuf { @@ -134,44 +192,12 @@ impl Allocator for Udmabuf { if !modifiers.contains(&LINEAR_MODIFIER) { return Err(UdmabufError::Modifier.into()); } - let height = height as u64; - let width = width as u64; - if height > 1 << 16 || width > 1 << 16 { - return Err(UdmabufError::Overflow.into()); - } - let stride = (width * format.bpp as u64).next_multiple_of(LINEAR_STRIDE_ALIGN); - let size_mask = page_size() as u64 - 1; - let size = (height * stride + size_mask) & !size_mask; - let memfd = match uapi::memfd_create("udmabuf", MFD_ALLOW_SEALING) { - Ok(f) => f, - Err(e) => return Err(UdmabufError::Memfd(e.into()).into()), - }; - if let Err(e) = uapi::ftruncate(memfd.raw(), size as _) { - return Err(UdmabufError::Truncate(e.into()).into()); - } - if let Err(e) = uapi::fcntl_add_seals(memfd.raw(), F_SEAL_SHRINK) { - return Err(UdmabufError::Seal(e.into()).into()); - } - let dmabuf = self.create_dmabuf_from_memfd(&memfd, 0, size as _)?; - let mut planes = PlaneVec::new(); - planes.push(DmaBufPlane { - offset: 0, - stride: stride as _, - fd: Rc::new(dmabuf), - }); - let dmabuf = DmaBuf { - id: dma_buf_ids.next(), - width: width as _, - height: height as _, + Ok(Rc::new(self.create_bo( + dma_buf_ids, + width, + height, format, - modifier: LINEAR_MODIFIER, - planes, - is_disjoint: Default::default(), - }; - Ok(Rc::new(UdmabufBo { - buf: dmabuf, - size: size as _, - })) + )?)) } fn import_dmabuf( From 897944b5803af4325d3e1127aea93782599a55c6 Mon Sep 17 00:00:00 2001 From: Julian Orth Date: Sat, 14 Feb 2026 23:47:45 +0100 Subject: [PATCH 5/5] metal: add copy-device based prime methods --- src/backends/metal/allocator.rs | 591 ++++++++++++++++++++++++++++++-- src/backends/metal/video.rs | 2 - src/copy_device.rs | 2 - src/udmabuf.rs | 1 - 4 files changed, 571 insertions(+), 25 deletions(-) diff --git a/src/backends/metal/allocator.rs b/src/backends/metal/allocator.rs index 3f5bc257..63e24ad7 100644 --- a/src/backends/metal/allocator.rs +++ b/src/backends/metal/allocator.rs @@ -6,13 +6,16 @@ use { video::{MetalDrmDevice, MetalRenderContext}, }, cmm::cmm_description::ColorDescription, - copy_device::{CopyDevice, CopyDeviceError, CopyDeviceSupport}, + copy_device::{ + CopyDevice, CopyDeviceBuffer, CopyDeviceCopy, CopyDeviceError, CopyDeviceSupport, + }, format::Format, gfx_api::{ AcquireSync, GfxBlendBuffer, GfxError, GfxFormat, GfxFramebuffer, GfxTexture, GfxWriteModifier, ReleaseSync, SyncFile, needs_render_usage, }, rect::{DamageQueue, Rect, Region}, + udmabuf::{Udmabuf, UdmabufError}, utils::{errorfmt::ErrorFmt, rc_eq::rc_eq}, video::{ LINEAR_MODIFIER, Modifier, @@ -70,6 +73,29 @@ pub enum RenderBufferPrime { // Import of the render dmabuf into the dev ctx. dev_render_tex: Rc, }, + CopyUdmabuf { + render_copy: CopyDeviceCopy, + dev_copy_dev: Rc, + dev_copy: CopyDeviceCopy, + dev_bo: GbmBo, + }, + CopyDirectPull { + dev_copy_dev: Rc, + dev_copy: CopyDeviceCopy, + dev_bo: GbmBo, + }, + CopyIndirectPull { + render_copy: CopyDeviceCopy, + _render_secondary_bo: CopyDeviceBuffer, + dev_copy_dev: Rc, + dev_copy: CopyDeviceCopy, + dev_bo: GbmBo, + }, + CopyDirectPush { + render_copy: CopyDeviceCopy, + dev_copy_dev: Rc, + dev_bo: GbmBo, + }, } #[derive(Debug, Error)] @@ -80,10 +106,16 @@ pub enum RenderBufferError { GfxError(#[from] GfxError), #[error("Could not copy frame to output device")] CopyToOutput(#[source] GfxError), + #[error("Could not copy render bo to udmabuf")] + CopyRenderToUdmabuf(#[source] CopyDeviceError), + #[error("Could not copy udmabuf to dev bo")] + CopyUdmabufToDev(#[source] CopyDeviceError), #[error("Could not create a copy device copy")] CreateCopyDeviceCopy(#[source] CopyDeviceError), #[error("Could not execute a copy device copy")] ExecuteCopyDeviceCopy(#[source] CopyDeviceError), + #[error("Could not copy render bo to dev bo")] + CopyRenderToDev(#[source] CopyDeviceError), } #[derive(Default)] @@ -105,7 +137,7 @@ impl RenderBuffer { pub fn copy_to_dev( &self, cd: &Rc, - _region: Option<&Region>, + region: Option<&Region>, sync_file: Option, ) -> Result { match &self.prime { @@ -132,6 +164,34 @@ impl RenderBuffer { ) .map_err(RenderBufferError::CopyToOutput) .map(RenderBufferCopy::for_both), + RenderBufferPrime::CopyUdmabuf { + render_copy, + dev_copy, + .. + } + | RenderBufferPrime::CopyIndirectPull { + render_copy, + dev_copy, + .. + } => { + let render_block = render_copy + .execute(sync_file.as_ref(), region) + .map_err(RenderBufferError::CopyRenderToUdmabuf)?; + let present_block = dev_copy + .execute(render_block.as_ref(), region) + .map_err(RenderBufferError::CopyUdmabufToDev)?; + Ok(RenderBufferCopy { + render_block, + present_block, + }) + } + RenderBufferPrime::CopyDirectPull { dev_copy: copy, .. } + | RenderBufferPrime::CopyDirectPush { + render_copy: copy, .. + } => copy + .execute(sync_file.as_ref(), region) + .map_err(RenderBufferError::CopyRenderToDev) + .map(RenderBufferCopy::for_both), } } @@ -142,16 +202,27 @@ impl RenderBuffer { } pub fn clear(&self, cd: &Rc) -> Result, RenderBufferError> { - match &self.prime { - RenderBufferPrime::None => self - .render - .fb - .clear(AcquireSync::Unnecessary, ReleaseSync::Explicit, cd) - .map_err(Into::into), - RenderBufferPrime::Sampling { dev_fb, .. } => dev_fb - .clear(AcquireSync::Unnecessary, ReleaseSync::Explicit, cd) - .map_err(Into::into), - } + let sync_file = match &self.prime { + RenderBufferPrime::None => { + self.render + .fb + .clear(AcquireSync::Unnecessary, ReleaseSync::Explicit, cd)? + } + RenderBufferPrime::Sampling { dev_fb, .. } => { + dev_fb.clear(AcquireSync::Unnecessary, ReleaseSync::Explicit, cd)? + } + RenderBufferPrime::CopyUdmabuf { .. } + | RenderBufferPrime::CopyDirectPull { .. } + | RenderBufferPrime::CopyIndirectPull { .. } + | RenderBufferPrime::CopyDirectPush { .. } => { + let sf = + self.render + .fb + .clear(AcquireSync::Unnecessary, ReleaseSync::Explicit, cd)?; + self.copy_to_dev(cd, None, sf)?.present_block + } + }; + Ok(sync_file) } pub fn copy_to_new( @@ -217,6 +288,7 @@ impl RenderBuffer { &old.render.tex, old.render.bo.dmabuf(), )?, + _ => unreachable!(), }, RenderBufferPrime::Sampling { dev_render_tex: old_dev_render_tex, @@ -239,7 +311,9 @@ impl RenderBuffer { old_dev_render_tex, old_dev_bo.dmabuf(), )?, + _ => unreachable!(), }, + _ => unreachable!(), }; Ok(sf) } @@ -248,6 +322,10 @@ impl RenderBuffer { match &self.prime { RenderBufferPrime::None => &self.render.bo, RenderBufferPrime::Sampling { dev_bo, .. } => dev_bo, + RenderBufferPrime::CopyUdmabuf { dev_bo, .. } => dev_bo, + RenderBufferPrime::CopyDirectPull { dev_bo, .. } => dev_bo, + RenderBufferPrime::CopyDirectPush { dev_bo, .. } => dev_bo, + RenderBufferPrime::CopyIndirectPull { dev_bo, .. } => dev_bo, } } @@ -255,6 +333,10 @@ impl RenderBuffer { match &self.prime { RenderBufferPrime::None => None, RenderBufferPrime::Sampling { .. } => None, + RenderBufferPrime::CopyUdmabuf { dev_copy_dev, .. } + | RenderBufferPrime::CopyDirectPull { dev_copy_dev, .. } + | RenderBufferPrime::CopyIndirectPull { dev_copy_dev, .. } + | RenderBufferPrime::CopyDirectPush { dev_copy_dev, .. } => Some(dev_copy_dev), } } } @@ -287,6 +369,34 @@ struct PrimeSampling { dev_allocation_settings: BoAllocationSettings, } +struct DirectCopyPull { + dev_copy_dev: Rc, + render_allocation_settings: BoAllocationSettings, + dev_allocation_settings: BoAllocationSettings, +} + +struct DirectCopyPush { + render_copy_dev: Rc, + dev_copy_dev: Rc, + render_allocation_settings: BoAllocationSettings, + dev_allocation_settings: BoAllocationSettings, +} + +struct CopyUdmabuf { + udmabuf: Rc, + render_allocation_settings: BoAllocationSettings, + render_copy_dev: Rc, + dev_copy_dev: Rc, + dev_allocation_settings: BoAllocationSettings, +} + +struct IndirectCopyPull { + render_allocation_settings: BoAllocationSettings, + render_copy_dev: Rc, + dev_copy_dev: Rc, + dev_allocation_settings: BoAllocationSettings, +} + impl MetalBackend { pub fn create_scanout_buffers( &self, @@ -344,9 +454,21 @@ impl MetalBackend { }}; } match method { + PrimeMethod::DirectPull => { + x!(prepare_direct_copy_pull, create_direct_copy_pull) + } + PrimeMethod::DirectPush => { + x!(prepare_direct_copy_push, create_direct_copy_push) + } + PrimeMethod::Udmabuf => { + x!(prepare_copy_udmabuf, create_copy_udmabuf) + } PrimeMethod::Sampling => { x!(prepare_prime_sampling, create_prime_sampling) } + PrimeMethod::IndirectPull => { + x!(prepare_indirect_copy_pull, create_indirect_copy_pull) + } } }); match res { @@ -408,6 +530,14 @@ pub enum ScanoutBufferErrorKind { SodWriteReadPlaneIntersection, #[error("Scanout device: The intersection of render/plane modifiers is empty")] SodWritePlaneIntersection, + #[error("Render device: The intersection of render/sample/copy_src modifiers is empty")] + RenderWriteReadCopySrcIntersection, + #[error("Scanout device: The intersection of plane/render_copy_dst modifiers is empty")] + SodPlaneRenderCopyDstIntersection, + #[error("Render device: The intersection of render/sample/sod_copy_src modifiers is empty")] + RenderWriteReadSodCopySrcIntersection, + #[error("Scanout device: The intersection of plane/copy_dst modifiers is empty")] + SodPlaneCopyDstIntersection, #[error("Render device: Buffer allocation failed")] RenderBufferAllocation(#[source] GbmError), #[error("Render device: Could not import RENDER buffer into the gfx API")] @@ -422,6 +552,32 @@ pub enum ScanoutBufferErrorKind { SodImportRenderImage(#[source] GfxError), #[error("Scanout device: Could not turn imported RENDER buffer into gfx API texture")] SodImportRenderTexture(#[source] GfxError), + #[error("Udmabuf is not available")] + UdmabufNotAvailable, + #[error("Render device: Could not create a copy device")] + RenderNoCopyDevice, + #[error("Scanout device: Could not create a copy device")] + SodNoCopyDevice, + #[error("Render device: Cannot copy to linear")] + RenderNoCopyToLinear, + #[error("Scanout device: Cannot copy from linear")] + SodNoCopyFromLinear, + #[error("Could not create an udmabuf")] + CreateUdmabuf(#[source] UdmabufError), + #[error("Render device: Could not create a copy to udmabuf")] + RenderCreateCopyToUdmabuf(#[source] CopyDeviceError), + #[error("Render device: Could not create a copy to secondary")] + RenderCreateCopyToSecondary(#[source] CopyDeviceError), + #[error("Scanout device: Could not create a copy from udmabuf")] + SodCreateCopyFromUdmabuf(#[source] CopyDeviceError), + #[error("Scanout device: Could not create a copy from secondary")] + SodCreateCopyFromSecondary(#[source] CopyDeviceError), + #[error("Scanout device: Could not create a copy from render bo")] + SodCreateCopyFromRender(#[source] CopyDeviceError), + #[error("Render device: Could not create a copy to scanout device")] + RenderCreateCopyToSod(#[source] CopyDeviceError), + #[error("Render device: Copy buffer allocation failed")] + RenderCreateCopyBuffer(#[source] CopyDeviceError), } #[derive(Default, Debug)] @@ -446,13 +602,25 @@ pub struct ScanoutBufferError { #[derive(Copy, Clone, Linearize)] pub enum PrimeMethod { + DirectPull, Sampling, + IndirectPull, + Udmabuf, + // This does not work on AMD since use from another device will prevent the + // framebuffer from being pinned into video memory. It might be useful on other + // devices where the scanout device is CPU only and the render device can perform + // an accelerated copy. + DirectPush, } impl PrimeMethod { pub fn name(self) -> &'static str { match self { + PrimeMethod::DirectPull => "direct-pull", + PrimeMethod::IndirectPull => "indirect-pull", + PrimeMethod::DirectPush => "direct-push", PrimeMethod::Sampling => "direct-sampling", + PrimeMethod::Udmabuf => "udmabuf", } } } @@ -474,6 +642,10 @@ impl RenderBufferPrime { let method = match self { RenderBufferPrime::None => return None, RenderBufferPrime::Sampling { .. } => PrimeMethod::Sampling, + RenderBufferPrime::CopyUdmabuf { .. } => PrimeMethod::Udmabuf, + RenderBufferPrime::CopyDirectPull { .. } => PrimeMethod::DirectPull, + RenderBufferPrime::CopyDirectPush { .. } => PrimeMethod::DirectPush, + RenderBufferPrime::CopyIndirectPull { .. } => PrimeMethod::IndirectPull, }; Some(method) } @@ -618,7 +790,6 @@ impl BoAllocationSettings { ) } - #[expect(dead_code)] fn new2<'a>( common: &Builder<'_>, modifiers: impl IntoIterator + Clone, @@ -768,7 +939,7 @@ impl Builder<'_> { } fn copy_modifiers_iter(&self, support: &[CopyDeviceSupport]) -> impl Iterator { - let Builder { width, height, .. } = *self; + let Self { width, height, .. } = *self; support .iter() .filter(move |s| s.max_width >= width as _ && s.max_height >= height as _) @@ -779,12 +950,10 @@ impl Builder<'_> { self.copy_modifiers_iter(support).collect() } - #[expect(dead_code)] fn copy_src_modifiers(&self, dev: &CopyDevice) -> Vec { self.copy_modifiers(dev.src_support(self.format)) } - #[expect(dead_code)] fn copy_dst_modifiers(&self, dev: &CopyDevice) -> Vec { self.copy_modifiers(dev.dst_support(self.format)) } @@ -794,12 +963,10 @@ impl Builder<'_> { .any(|m| m == LINEAR_MODIFIER) } - #[expect(dead_code)] fn copy_src_supports_linear(&self, dev: &CopyDevice) -> bool { self.copy_supports_linear(dev.src_support(self.format)) } - #[expect(dead_code)] fn copy_dst_supports_linear(&self, dev: &CopyDevice) -> bool { self.copy_supports_linear(dev.dst_support(self.format)) } @@ -960,6 +1127,392 @@ impl Builder<'_> { }; self.create(drm, damage_queue, render, prime) } + + fn prepare_direct_copy_push( + &self, + dbg: &RefCell, + ) -> Result { + let dbg = &mut *dbg.borrow_mut(); + let Self { + dev, + render_fmt, + plane_modifiers, + render_ctx, + .. + } = *self; + let Some(render_copy_dev) = render_ctx.copy_device.get() else { + return Err(ScanoutBufferErrorKind::RenderNoCopyDevice); + }; + let Some(dev_copy_dev) = dev.copy_device.get() else { + return Err(ScanoutBufferErrorKind::SodNoCopyDevice); + }; + let render_copy_src_modifiers = self.copy_src_modifiers(&render_copy_dev); + let render_modifiers_possible = + intersect_render_modifiers(render_fmt, &render_copy_src_modifiers); + dbg.render_gfx_write_modifiers = Some(render_modifiers(render_fmt)); + dbg.render_gfx_read_modifiers = Some(sample_modifiers(render_fmt)); + dbg.render_copy_src_modifiers = Some(render_copy_src_modifiers); + dbg.render_modifiers_possible = Some(render_modifiers_possible.clone()); + if render_modifiers_possible.is_empty() { + return Err(ScanoutBufferErrorKind::RenderWriteReadCopySrcIntersection); + } + let render_copy_dst_modifiers = self.copy_dst_modifiers(&render_copy_dev); + let mut dev_modifiers = intersect_modifiers(plane_modifiers, &render_copy_dst_modifiers); + dbg.render_copy_dst_modifiers = Some(render_copy_dst_modifiers); + make_linear_only(&mut dev_modifiers); + dbg.dev_modifiers_possible = Some(dev_modifiers.clone()); + if dev_modifiers.is_empty() { + return Err(ScanoutBufferErrorKind::SodPlaneRenderCopyDstIntersection); + } + let render_allocation_settings = BoAllocationSettings::new2( + self, + &render_modifiers_possible, + render_fmt, + false, + true, + &mut dbg.render_usage, + ); + let dev_allocation_settings = + BoAllocationSettings::new3(self, &dev_modifiers, true, false, &mut dbg.dev_usage); + Ok(DirectCopyPush { + render_copy_dev, + dev_copy_dev, + render_allocation_settings, + dev_allocation_settings, + }) + } + + fn create_direct_copy_push( + &self, + prepared: &DirectCopyPush, + damage_queue: DamageQueue, + dbg: &RefCell, + ) -> Result { + let DirectCopyPush { + render_copy_dev, + dev_copy_dev, + render_allocation_settings, + dev_allocation_settings, + } = prepared; + let render = self.create_render_buffer_render(render_allocation_settings, dbg)?; + let send_render_modifier = on_drop(|| { + dbg.borrow_mut().render_modifier = Some(render.bo.dmabuf().modifier); + }); + let (dev_bo, drm) = self.create_dev_bo(dev_allocation_settings, dbg)?; + let send_dev_modifier = on_drop(|| { + dbg.borrow_mut().dev_modifier = Some(dev_bo.dmabuf().modifier); + }); + let render_copy = render_copy_dev + .create_copy(&render.bo.dmabuf(), &dev_bo.dmabuf()) + .map_err(ScanoutBufferErrorKind::RenderCreateCopyToSod)?; + send_dev_modifier.forget(); + send_render_modifier.forget(); + let prime = RenderBufferPrime::CopyDirectPush { + dev_copy_dev: dev_copy_dev.clone(), + render_copy, + dev_bo, + }; + self.create(drm, damage_queue, render, prime) + } + + fn prepare_direct_copy_pull( + &self, + dbg: &RefCell, + ) -> Result { + let dbg = &mut *dbg.borrow_mut(); + let Self { + dev, + render_fmt, + plane_modifiers, + .. + } = *self; + let Some(dev_copy_dev) = dev.copy_device.get() else { + return Err(ScanoutBufferErrorKind::SodNoCopyDevice); + }; + let dev_copy_src_modifiers = self.copy_src_modifiers(&dev_copy_dev); + let render_modifiers_possible = + intersect_render_modifiers(render_fmt, &dev_copy_src_modifiers); + dbg.render_gfx_write_modifiers = Some(render_modifiers(render_fmt)); + dbg.render_gfx_read_modifiers = Some(sample_modifiers(render_fmt)); + dbg.dev_copy_src_modifiers = Some(dev_copy_src_modifiers); + dbg.render_modifiers_possible = Some(render_modifiers_possible.clone()); + if render_modifiers_possible.is_empty() { + return Err(ScanoutBufferErrorKind::RenderWriteReadSodCopySrcIntersection); + } + let dev_copy_dst_modifiers = self.copy_dst_modifiers(&dev_copy_dev); + let mut dev_modifiers = intersect_modifiers(plane_modifiers, &dev_copy_dst_modifiers); + dbg.dev_copy_dst_modifiers = Some(dev_copy_dst_modifiers); + make_linear_only(&mut dev_modifiers); + dbg.dev_modifiers_possible = Some(dev_modifiers.clone()); + if dev_modifiers.is_empty() { + return Err(ScanoutBufferErrorKind::SodPlaneCopyDstIntersection); + } + let render_allocation_settings = BoAllocationSettings::new2( + self, + &render_modifiers_possible, + render_fmt, + false, + true, + &mut dbg.render_usage, + ); + let dev_allocation_settings = + BoAllocationSettings::new3(self, &dev_modifiers, true, false, &mut dbg.dev_usage); + Ok(DirectCopyPull { + dev_copy_dev, + render_allocation_settings, + dev_allocation_settings, + }) + } + + fn create_direct_copy_pull( + &self, + prepared: &DirectCopyPull, + damage_queue: DamageQueue, + dbg: &RefCell, + ) -> Result { + let DirectCopyPull { + dev_copy_dev, + render_allocation_settings, + dev_allocation_settings, + } = prepared; + let render = self.create_render_buffer_render(render_allocation_settings, dbg)?; + let send_render_modifier = on_drop(|| { + dbg.borrow_mut().render_modifier = Some(render.bo.dmabuf().modifier); + }); + let (dev_bo, drm) = self.create_dev_bo(dev_allocation_settings, dbg)?; + let send_dev_modifier = on_drop(|| { + dbg.borrow_mut().dev_modifier = Some(dev_bo.dmabuf().modifier); + }); + let dev_copy = dev_copy_dev + .create_copy(&render.bo.dmabuf(), &dev_bo.dmabuf()) + .map_err(ScanoutBufferErrorKind::SodCreateCopyFromRender)?; + send_dev_modifier.forget(); + send_render_modifier.forget(); + let prime = RenderBufferPrime::CopyDirectPull { + dev_copy_dev: dev_copy_dev.clone(), + dev_copy, + dev_bo, + }; + self.create(drm, damage_queue, render, prime) + } + + fn prepare_indirect_copy_pull( + &self, + dbg: &RefCell, + ) -> Result { + let dbg = &mut *dbg.borrow_mut(); + let Self { + dev, + render_fmt, + plane_modifiers, + render_ctx, + .. + } = *self; + let Some(render_copy_dev) = render_ctx.copy_device.get() else { + return Err(ScanoutBufferErrorKind::RenderNoCopyDevice); + }; + let Some(dev_copy_dev) = dev.copy_device.get() else { + return Err(ScanoutBufferErrorKind::SodNoCopyDevice); + }; + let render_copy_src_modifiers = self.copy_src_modifiers(&render_copy_dev); + let render_modifiers_possible = + intersect_render_modifiers(render_fmt, &render_copy_src_modifiers); + dbg.render_copy_src_modifiers = Some(render_copy_src_modifiers); + dbg.render_gfx_read_modifiers = Some(sample_modifiers(render_fmt)); + dbg.render_gfx_write_modifiers = Some(render_modifiers(render_fmt)); + dbg.render_modifiers_possible = Some(render_modifiers_possible.clone()); + if render_modifiers_possible.is_empty() { + return Err(ScanoutBufferErrorKind::RenderWriteReadCopySrcIntersection); + } + if !self.copy_dst_supports_linear(&render_copy_dev) { + return Err(ScanoutBufferErrorKind::RenderNoCopyToLinear); + } + if !self.copy_src_supports_linear(&dev_copy_dev) { + return Err(ScanoutBufferErrorKind::SodNoCopyFromLinear); + } + let dev_copy_dst_modifiers = self.copy_dst_modifiers(&dev_copy_dev); + let mut dev_modifiers = intersect_modifiers(plane_modifiers, &dev_copy_dst_modifiers); + dbg.dev_copy_dst_modifiers = Some(dev_copy_dst_modifiers); + make_linear_only(&mut dev_modifiers); + dbg.dev_modifiers_possible = Some(dev_modifiers.clone()); + if dev_modifiers.is_empty() { + return Err(ScanoutBufferErrorKind::SodPlaneCopyDstIntersection); + } + let render_allocation_settings = BoAllocationSettings::new2( + self, + &render_modifiers_possible, + render_fmt, + false, + true, + &mut dbg.render_usage, + ); + let dev_allocation_settings = + BoAllocationSettings::new3(self, &dev_modifiers, true, false, &mut dbg.dev_usage); + Ok(IndirectCopyPull { + render_allocation_settings, + render_copy_dev, + dev_copy_dev, + dev_allocation_settings, + }) + } + + fn create_indirect_copy_pull( + &self, + prepared: &IndirectCopyPull, + damage_queue: DamageQueue, + dbg: &RefCell, + ) -> Result { + let IndirectCopyPull { + render_allocation_settings, + render_copy_dev, + dev_copy_dev, + dev_allocation_settings, + } = prepared; + let Self { + format, + width, + height, + .. + } = *self; + let render_secondary_bo = render_copy_dev + .create_buffer(&self.slf.state.dma_buf_ids, width, height, format) + .map_err(ScanoutBufferErrorKind::RenderCreateCopyBuffer)?; + let render = self.create_render_buffer_render(render_allocation_settings, dbg)?; + let send_render_modifier = on_drop(|| { + dbg.borrow_mut().render_modifier = Some(render.bo.dmabuf().modifier); + }); + let (dev_bo, drm) = self.create_dev_bo(dev_allocation_settings, dbg)?; + let send_dev_modifier = on_drop(|| { + dbg.borrow_mut().dev_modifier = Some(dev_bo.dmabuf().modifier); + }); + let render_copy = render_copy_dev + .create_copy(render.bo.dmabuf(), render_secondary_bo.dmabuf()) + .map_err(ScanoutBufferErrorKind::RenderCreateCopyToSecondary)?; + let dev_copy = dev_copy_dev + .create_copy(render_secondary_bo.dmabuf(), dev_bo.dmabuf()) + .map_err(ScanoutBufferErrorKind::SodCreateCopyFromSecondary)?; + send_render_modifier.forget(); + send_dev_modifier.forget(); + let prime = RenderBufferPrime::CopyIndirectPull { + render_copy, + _render_secondary_bo: render_secondary_bo, + dev_copy_dev: dev_copy_dev.clone(), + dev_copy, + dev_bo, + }; + self.create(drm, damage_queue, render, prime) + } + + fn prepare_copy_udmabuf( + &self, + dbg: &RefCell, + ) -> Result { + let dbg = &mut *dbg.borrow_mut(); + let Self { + dev, + render_fmt, + plane_modifiers, + render_ctx, + .. + } = *self; + let Some(udmabuf) = self.slf.state.udmabuf.get() else { + return Err(ScanoutBufferErrorKind::UdmabufNotAvailable); + }; + let Some(render_copy_dev) = render_ctx.copy_device.get() else { + return Err(ScanoutBufferErrorKind::RenderNoCopyDevice); + }; + if !self.copy_dst_supports_linear(&render_copy_dev) { + return Err(ScanoutBufferErrorKind::RenderNoCopyToLinear); + } + let Some(dev_copy_dev) = dev.copy_device.get() else { + return Err(ScanoutBufferErrorKind::SodNoCopyDevice); + }; + if !self.copy_src_supports_linear(&dev_copy_dev) { + return Err(ScanoutBufferErrorKind::SodNoCopyFromLinear); + } + let render_copy_src_modifiers = self.copy_src_modifiers(&render_copy_dev); + let render_modifiers_possible = + intersect_render_modifiers(render_fmt, &render_copy_src_modifiers); + dbg.render_copy_src_modifiers = Some(render_copy_src_modifiers); + dbg.render_gfx_read_modifiers = Some(sample_modifiers(render_fmt)); + dbg.render_gfx_write_modifiers = Some(render_modifiers(render_fmt)); + dbg.render_modifiers_possible = Some(render_modifiers_possible.clone()); + if render_modifiers_possible.is_empty() { + return Err(ScanoutBufferErrorKind::RenderWriteReadCopySrcIntersection); + } + let dev_copy_dst_modifiers = self.copy_dst_modifiers(&dev_copy_dev); + let mut dev_modifiers = intersect_modifiers(plane_modifiers, &dev_copy_dst_modifiers); + dbg.dev_copy_dst_modifiers = Some(dev_copy_dst_modifiers); + make_linear_only(&mut dev_modifiers); + dbg.dev_modifiers_possible = Some(dev_modifiers.clone()); + if dev_modifiers.is_empty() { + return Err(ScanoutBufferErrorKind::SodPlaneCopyDstIntersection); + } + let render_allocation_settings = BoAllocationSettings::new2( + self, + &render_modifiers_possible, + render_fmt, + false, + true, + &mut dbg.render_usage, + ); + let dev_allocation_settings = + BoAllocationSettings::new3(self, &dev_modifiers, true, false, &mut dbg.dev_usage); + Ok(CopyUdmabuf { + udmabuf, + render_allocation_settings, + render_copy_dev, + dev_copy_dev, + dev_allocation_settings, + }) + } + + fn create_copy_udmabuf( + &self, + prepared: &CopyUdmabuf, + damage_queue: DamageQueue, + dbg: &RefCell, + ) -> Result { + let CopyUdmabuf { + udmabuf, + render_allocation_settings, + render_copy_dev, + dev_copy_dev, + dev_allocation_settings, + } = prepared; + let Self { + format, + width, + height, + .. + } = *self; + let udmabuf = udmabuf + .create_dmabuf(&self.slf.state.dma_buf_ids, width, height, format) + .map_err(ScanoutBufferErrorKind::CreateUdmabuf)?; + let render = self.create_render_buffer_render(render_allocation_settings, dbg)?; + let send_render_modifier = on_drop(|| { + dbg.borrow_mut().render_modifier = Some(render.bo.dmabuf().modifier); + }); + let (dev_bo, drm) = self.create_dev_bo(dev_allocation_settings, dbg)?; + let send_dev_modifier = on_drop(|| { + dbg.borrow_mut().dev_modifier = Some(dev_bo.dmabuf().modifier); + }); + let render_copy = render_copy_dev + .create_copy(&render.bo.dmabuf(), &udmabuf) + .map_err(ScanoutBufferErrorKind::RenderCreateCopyToUdmabuf)?; + let dev_copy = dev_copy_dev + .create_copy(&udmabuf, &dev_bo.dmabuf()) + .map_err(ScanoutBufferErrorKind::SodCreateCopyFromUdmabuf)?; + send_render_modifier.forget(); + send_dev_modifier.forget(); + let prime = RenderBufferPrime::CopyUdmabuf { + render_copy, + dev_copy_dev: dev_copy_dev.clone(), + dev_copy, + dev_bo, + }; + self.create(drm, damage_queue, render, prime) + } } const JAY_PRIME_METHODS: &str = "JAY_PRIME_METHODS"; @@ -1023,7 +1576,6 @@ fn intersect_modifiers<'a>( .collect() } -#[expect(dead_code)] fn intersect_render_modifiers<'a>( left: &'a GfxFormat, right: impl IntoIterator, @@ -1036,7 +1588,6 @@ fn intersect_render_modifiers<'a>( ) } -#[expect(dead_code)] fn make_linear_only(modifiers: &mut Vec) { if modifiers.contains(&LINEAR_MODIFIER) { *modifiers = vec![LINEAR_MODIFIER]; diff --git a/src/backends/metal/video.rs b/src/backends/metal/video.rs index 1598f478..20dde7b9 100644 --- a/src/backends/metal/video.rs +++ b/src/backends/metal/video.rs @@ -119,7 +119,6 @@ pub struct MetalDrmDevice { pub gbm: Rc, pub handle_events: HandleEvents, pub ctx: CloneCell>, - #[expect(dead_code)] pub copy_device: Rc, pub on_change: OnChange, pub direct_scanout_enabled: Cell>, @@ -2649,7 +2648,6 @@ impl MetalBackend { } impl CopyDeviceHolder { - #[expect(dead_code)] pub fn get(&self) -> Option> { self.dev .get_or_init( diff --git a/src/copy_device.rs b/src/copy_device.rs index 4acd68a8..effbbe1b 100644 --- a/src/copy_device.rs +++ b/src/copy_device.rs @@ -1135,7 +1135,6 @@ impl CopyDevice { }) } - #[expect(dead_code)] pub fn create_buffer( &self, dma_buf_ids: &DmaBufIds, @@ -1853,7 +1852,6 @@ impl Drop for Pending { } impl CopyDeviceBuffer { - #[expect(dead_code)] pub fn dmabuf(&self) -> &DmaBuf { &self.dmabuf } diff --git a/src/udmabuf.rs b/src/udmabuf.rs index e398d692..8948f5ca 100644 --- a/src/udmabuf.rs +++ b/src/udmabuf.rs @@ -116,7 +116,6 @@ impl Udmabuf { Ok(dmabuf) } - #[expect(dead_code)] pub fn create_dmabuf( &self, dma_buf_ids: &DmaBufIds,