use { crate::{ async_engine::{AsyncEngine, SpawnedFuture}, cmm::{ cmm_description::{ColorDescription, LinearColorDescription, LinearColorDescriptionId}, cmm_eotf::{Eotf, EotfPow, bt1886_eotf_args, bt1886_inv_eotf_args}, cmm_render_intent::RenderIntent, cmm_transform::ColorMatrix, }, cpu_worker::PendingJob, gfx_api::{ AcquireSync, AlphaMode, BufferResv, BufferResvUser, FdSync, GfxApiOpt, GfxBlendBuffer, GfxFormat, GfxTexture, GfxWriteModifier, ReleaseSync, }, gfx_apis::vulkan::{ VulkanError, VulkanSync, VulkanTimelineSemaphore, allocator::{VulkanAllocator, VulkanThreadedAllocator}, buffer_cache::{GenericBufferWriter, VulkanBuffer, VulkanBufferCache}, command::{VulkanCommandBuffer, VulkanCommandPool}, descriptor::VulkanDescriptorSetLayout, descriptor_buffer::VulkanDescriptorBufferWriter, device::VulkanDevice, eotfs::{EOTF_LINEAR, EotfExt, VulkanEotf}, image::{QueueFamily, QueueState, QueueTransfer, VulkanImage, VulkanImageMemory}, pipeline::{PipelineCreateInfo, VulkanPipeline}, sampler::VulkanSampler, semaphore::VulkanSemaphore, shaders::{ ColorManagementData, EotfArgs, FILL_FRAG, FILL_VERT, FillPushConstants, InvEotfArgs, LEGACY_FILL_FRAG, LEGACY_FILL_VERT, LEGACY_TEX_FRAG, LEGACY_TEX_VERT, LegacyFillPushConstants, LegacyTexPushConstants, OUT_FRAG, OUT_VERT, OutPushConstants, TEX_FRAG, TEX_VERT, TexPushConstants, TexVertex, VulkanShader, }, }, io_uring::IoUring, rect::{Rect, Region}, theme::Color, utils::{ copyhashmap::CopyHashMap, errorfmt::ErrorFmt, numcell::NumCell, ordered_float::F32, oserror::OsErrorExt2, stack::Stack, }, video::dmabuf::{DMA_BUF_SYNC_READ, DMA_BUF_SYNC_WRITE, dma_buf_export_sync_file}, vulkan_core::{ sync::VulkanDeviceSyncExt, timeline_semaphore::VulkanDeviceTimelineSemaphoreExt, }, }, ahash::AHashMap, arrayvec::ArrayVec, ash::{ Device, vk::{ self, AccessFlags2, AttachmentLoadOp, AttachmentStoreOp, BufferUsageFlags, ClearAttachment, ClearColorValue, ClearRect, ClearValue, CommandBuffer, CommandBufferBeginInfo, CommandBufferSubmitInfo, CommandBufferUsageFlags, CopyImageInfo2, DependencyInfoKHR, DescriptorAddressInfoEXT, DescriptorBufferBindingInfoEXT, DescriptorDataEXT, DescriptorGetInfoEXT, DescriptorImageInfo, DescriptorType, DeviceAddress, DeviceSize, Extent2D, Extent3D, ImageAspectFlags, ImageCopy2, ImageLayout, ImageMemoryBarrier2, ImageSubresourceLayers, ImageSubresourceRange, Offset2D, Offset3D, PipelineBindPoint, PipelineStageFlags2, QUEUE_FAMILY_FOREIGN_EXT, Rect2D, RenderingAttachmentInfo, RenderingInfo, SemaphoreSubmitInfo, SemaphoreSubmitInfoKHR, ShaderStageFlags, SubmitInfo2, Viewport, WriteDescriptorSet, }, }, isnt::std_1::{collections::IsntHashMapExt, primitive::IsntSliceExt}, jay_algorithms::rect::Tag, linearize::{Linearize, LinearizeExt, StaticMap, static_map}, std::{ any::Any, borrow::Cow, cell::{Cell, LazyCell, RefCell}, collections::hash_map::Entry, fmt::{Debug, Formatter}, mem, ops::Range, ptr, rc::{Rc, Weak}, slice, }, uapi::OwnedFd, }; pub struct VulkanRenderer { pub(super) formats: Rc>, pub(super) device: Rc, pub(super) fill_pipelines: CopyHashMap, pub(super) tex_pipelines: StaticMap>>, pub(super) out_pipelines: StaticMap>>, pub(super) gfx_command_buffers: CachedCommandBuffers, pub(super) transfer_command_buffers: Option, pub(super) wait_semaphores: Stack>, pub(super) memory: RefCell, pub(super) pending_frames: CopyHashMap>, pub(super) pending_submits: CopyHashMap>, pub(super) allocator: Rc, pub(super) last_point: NumCell, pub(super) buffer_resv_user: BufferResvUser, pub(super) eng: Rc, pub(super) ring: Rc, pub(super) fill_vert_shader: Rc, pub(super) fill_frag_shader: Rc, pub(super) tex_vert_shader: Rc, pub(super) tex_frag_shader: Rc, pub(super) out_vert_shader: Option>, pub(super) out_frag_shader: Option>, pub(super) tex_descriptor_set_layouts: ArrayVec, 2>, pub(super) out_descriptor_set_layout: Option>, pub(super) defunct: Cell, pub(super) pending_cpu_jobs: CopyHashMap, pub(super) shm_allocator: Rc, pub(super) _sampler: Rc, pub(super) sampler_descriptor: Box<[u8]>, pub(super) sampler_descriptor_buffer_cache: Rc, pub(super) resource_descriptor_buffer_cache: Rc, pub(super) blend_buffers: RefCell>>, pub(super) shader_buffer_cache: Rc, pub(super) uniform_buffer_cache: Rc, pub(super) render_tls: Option>, pub(super) transfer_tls: Option>, } pub(super) struct CachedCommandBuffers { pub(super) pool: Rc, pub(super) buffers: Stack>, pub(super) total_buffers: NumCell, } impl CachedCommandBuffers { pub(super) fn allocate(&self) -> Result, VulkanError> { zone!("allocate_command_buffer"); let buf = match self.buffers.pop() { Some(b) => b, _ => { self.total_buffers.fetch_add(1); self.pool.allocate_buffer()? } }; Ok(buf) } } pub(super) struct UsedTexture { tex: Rc, resv: Option>, acquire_sync: AcquireSync, release_sync: ReleaseSync, } #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, Linearize)] pub(super) enum TexCopyType { Identity, Multiply, } #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, Linearize)] pub(super) enum TexSourceType { Opaque, HasAlpha, } #[derive(Default)] pub(super) struct Memory { dmabuf_sample: Vec>, queue_transfer: Vec>, textures: Vec, image_barriers: Vec>, wait_semaphores: Vec>, wait_semaphore_infos: Vec>, release_vulkan_sync: Option, release_sync: Option, used_buffers: ArrayVec, paint_bounds: StaticMap>, paint_regions: StaticMap>, clear_rects: StaticMap>, image_copy_regions: Vec>, sampler_descriptor_buffer_writer: VulkanDescriptorBufferWriter, resource_descriptor_buffer_writer: VulkanDescriptorBufferWriter, regions_1: Vec, regions_2: Vec>, ops: StaticMap>, ops_tmp: StaticMap>, fill_targets: Vec, tex_targets: Vec<[Point; 2]>, data_buffer: Vec, out_address: DeviceAddress, color_transforms: ColorTransforms, eotf_args_cache: EotfArgsCache, uniform_buffer_writer: GenericBufferWriter, uniform_buffer_descriptor_cache: Option>, blend_buffer_inv_eotf_args_descriptor: Option>, fb_inv_eotf_args_descriptor: Option>, blend_buffer_descriptor_buffer_offset: DeviceAddress, blend_buffer_color_management_data_address: Option, blend_buffer_eotf_args_address: Option, blend_buffer_inv_eotf_args_address: Option, fb_inv_eotf_args_address: Option, } type Point = [[f32; 2]; 4]; enum VulkanOp { Fill(VulkanFillOp), Tex(VulkanTexOp), } struct VulkanTexOp { tex: Rc, index: usize, range: Range, buffer_resv: Option>, acquire_sync: Option, release_sync: ReleaseSync, alpha: f32, source_type: TexSourceType, copy_type: TexCopyType, alpha_mode: AlphaMode, range_address: DeviceAddress, instances: u32, tex_cd: Rc, color_management_data_address: Option, eotf_args_address: Option, resource_descriptor_buffer_offset: DeviceAddress, } struct VulkanFillOp { range: Range, color: [f32; 4], source_type: TexSourceType, range_address: DeviceAddress, instances: u32, } #[derive(Copy, Clone, Debug, Linearize, Eq, PartialEq)] pub(super) enum RenderPass { BlendBuffer, FrameBuffer, } #[derive(Copy, Clone)] struct PaintRegion { x1: f32, y1: f32, x2: f32, y2: f32, } pub(super) struct PendingFrame { point: u64, renderer: Rc, cmd: Cell>>, _fb: Rc, _bb: Option>, _textures: Vec, wait_semaphores: Cell>>, waiter: Cell>>, vulkan_sync: Option, _used_buffers: ArrayVec, } type FillPipelines = Rc>>; #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] struct TexPipelineKey { tex_copy_type: TexCopyType, tex_source_type: TexSourceType, tex_alpha_mode: AlphaMode, eotf: VulkanEotf, has_color_management_data: bool, } pub(super) struct TexPipelines { format: vk::Format, eotf: VulkanEotf, pipelines: CopyHashMap>, } #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] pub(super) struct OutPipelineKey { format: vk::Format, eotf: VulkanEotf, has_color_management_data: bool, } impl VulkanDevice { pub fn create_renderer( self: &Rc, eng: &Rc, ring: &Rc, ) -> Result, VulkanError> { let sampler = self.create_sampler()?; let fill_vert_shader; let fill_frag_shader; let tex_vert_shader; let tex_frag_shader; let out_vert_shader; let out_frag_shader; let mut tex_descriptor_set_layouts = ArrayVec::new(); if self.descriptor_buffer.is_some() { tex_vert_shader = self.create_shader(TEX_VERT)?; tex_frag_shader = self.create_shader(TEX_FRAG)?; fill_vert_shader = self.create_shader(FILL_VERT)?; fill_frag_shader = self.create_shader(FILL_FRAG)?; out_vert_shader = Some(self.create_shader(OUT_VERT)?); out_frag_shader = Some(self.create_shader(OUT_FRAG)?); tex_descriptor_set_layouts .push(self.create_tex_sampler_descriptor_set_layout(&sampler)?); tex_descriptor_set_layouts.push(self.create_tex_resource_descriptor_set_layout()?); } else { tex_vert_shader = self.create_shader(LEGACY_TEX_VERT)?; tex_frag_shader = self.create_shader(LEGACY_TEX_FRAG)?; fill_vert_shader = self.create_shader(LEGACY_FILL_VERT)?; fill_frag_shader = self.create_shader(LEGACY_FILL_FRAG)?; out_vert_shader = None; out_frag_shader = None; tex_descriptor_set_layouts .push(self.create_tex_legacy_descriptor_set_layout(&sampler)?); } let out_descriptor_set_layout = self .descriptor_buffer .as_ref() .map(|_| self.create_tex_resource_descriptor_set_layout()) .transpose()?; let gfx_command_buffers = self.create_command_pool(self.graphics_queue_idx)?; let transfer_command_buffers = self .distinct_transfer_queue_family_idx .map(|idx| self.create_command_pool(idx)) .transpose()?; let formats: AHashMap = self .formats .iter() .map(|(drm, vk)| { ( *drm, GfxFormat { format: vk.format, read_modifiers: vk .modifiers .values() .filter(|m| m.texture_limits.is_some()) .map(|m| m.modifier) .collect(), write_modifiers: vk .modifiers .values() .filter(|m| m.render_limits.is_some()) .map(|m| { ( m.modifier, GfxWriteModifier { needs_render_usage: !m.render_needs_bridge, }, ) }) .collect(), supports_shm: vk.shm.is_some(), }, ) }) .collect(); let allocator = self.create_allocator()?; let shm_allocator = self.create_threaded_allocator()?; let sampler_descriptor_buffer_cache = VulkanBufferCache::for_descriptor_buffer(self, &allocator, true); let resource_descriptor_buffer_cache = VulkanBufferCache::for_descriptor_buffer(self, &allocator, false); let shader_buffer_cache = { // TODO: https://github.com/KhronosGroup/Vulkan-Samples/issues/1286 let usage = BufferUsageFlags::SHADER_DEVICE_ADDRESS | BufferUsageFlags::STORAGE_BUFFER; let align = 8; VulkanBufferCache::new(self, &allocator, usage, align) }; let uniform_buffer_cache = { let usage = BufferUsageFlags::SHADER_DEVICE_ADDRESS | BufferUsageFlags::UNIFORM_BUFFER; let align = align_of::() .max(align_of::()) .max(align_of::()) as DeviceSize; VulkanBufferCache::new(self, &allocator, usage, align) }; let render = Rc::new(VulkanRenderer { formats: Rc::new(formats), device: self.clone(), fill_pipelines: Default::default(), tex_pipelines: Default::default(), out_pipelines: Default::default(), gfx_command_buffers, transfer_command_buffers, wait_semaphores: Default::default(), memory: Default::default(), pending_frames: Default::default(), pending_submits: Default::default(), allocator, last_point: Default::default(), buffer_resv_user: Default::default(), eng: eng.clone(), ring: ring.clone(), fill_vert_shader, fill_frag_shader, tex_vert_shader, tex_frag_shader, out_vert_shader, out_frag_shader, tex_descriptor_set_layouts, out_descriptor_set_layout, defunct: Cell::new(false), pending_cpu_jobs: Default::default(), shm_allocator, sampler_descriptor: self.create_sampler_descriptor(sampler.sampler), _sampler: sampler, sampler_descriptor_buffer_cache, resource_descriptor_buffer_cache, blend_buffers: Default::default(), shader_buffer_cache, uniform_buffer_cache, render_tls: self.create_timeline_semaphore_or_log(), transfer_tls: self.create_timeline_semaphore_or_log(), }); Ok(render) } } impl VulkanRenderer { fn get_or_create_fill_pipelines( &self, format: vk::Format, ) -> Result { if let Some(pl) = self.fill_pipelines.get(&format) { return Ok(pl); } let create_fill_pipeline = |src_has_alpha| { let push_size = if self.device.descriptor_buffer.is_some() { size_of::() } else { size_of::() }; let info = PipelineCreateInfo { format, vert: self.fill_vert_shader.clone(), frag: self.fill_frag_shader.clone(), blend: src_has_alpha, src_has_alpha, has_alpha_mult: false, alpha_mode: AlphaMode::PremultipliedOptical, // all transformations are applied in the compositor eotf: EOTF_LINEAR, inv_eotf: EOTF_LINEAR, descriptor_set_layouts: Default::default(), has_color_management_data: false, }; self.device.create_pipeline2(info, push_size) }; let fill_pipelines = Rc::new(static_map! { TexSourceType::HasAlpha => create_fill_pipeline(true)?, TexSourceType::Opaque => create_fill_pipeline(false)?, }); self.fill_pipelines.set(format, fill_pipelines.clone()); Ok(fill_pipelines) } fn get_or_create_tex_pipelines( &self, format: vk::Format, target_cd: &ColorDescription, ) -> Rc { let eotf = target_cd.eotf.to_vulkan(); let pipelines = &self.tex_pipelines[eotf]; match pipelines.get(&format) { Some(pl) => pl, _ => { let pl = Rc::new(TexPipelines { format, eotf, pipelines: Default::default(), }); pipelines.set(format, pl.clone()); pl } } } fn get_or_create_tex_pipeline( &self, pipelines: &TexPipelines, tex_cd: &ColorDescription, tex_copy_type: TexCopyType, tex_source_type: TexSourceType, mut tex_alpha_mode: AlphaMode, has_color_management_data: bool, ) -> Result, VulkanError> { if tex_source_type == TexSourceType::Opaque { tex_alpha_mode = AlphaMode::PremultipliedElectrical; } let key = TexPipelineKey { tex_copy_type, tex_source_type, tex_alpha_mode, eotf: tex_cd.eotf.to_vulkan(), has_color_management_data, }; if let Some(pl) = pipelines.pipelines.get(&key) { return Ok(pl); } let src_has_alpha = match tex_source_type { TexSourceType::Opaque => false, TexSourceType::HasAlpha => true, }; let has_alpha_mult = match tex_copy_type { TexCopyType::Identity => false, TexCopyType::Multiply => true, }; let push_size = if self.device.descriptor_buffer.is_some() { size_of::() } else { size_of::() }; let info = PipelineCreateInfo { format: pipelines.format, vert: self.tex_vert_shader.clone(), frag: self.tex_frag_shader.clone(), blend: src_has_alpha || has_alpha_mult, src_has_alpha, has_alpha_mult, alpha_mode: key.tex_alpha_mode, eotf: key.eotf.to_vulkan(), inv_eotf: pipelines.eotf.to_vulkan(), descriptor_set_layouts: self.tex_descriptor_set_layouts.clone(), has_color_management_data, }; let pl = self.device.create_pipeline2(info, push_size)?; pipelines.pipelines.set(key, pl.clone()); Ok(pl) } fn get_or_create_out_pipeline( &self, format: vk::Format, bb_cd: &ColorDescription, fb_cd: &ColorDescription, has_color_management_data: bool, ) -> Result, VulkanError> { let key = OutPipelineKey { format, eotf: bb_cd.eotf.to_vulkan(), has_color_management_data, }; let fb_eotf = fb_cd.eotf.to_vulkan(); let pipelines = &self.out_pipelines[fb_eotf]; if let Some(pl) = pipelines.get(&key) { return Ok(pl); } let mut descriptor_set_layouts = ArrayVec::new(); descriptor_set_layouts.push(self.out_descriptor_set_layout.clone().unwrap()); let out = self .device .create_pipeline::(PipelineCreateInfo { format: key.format, vert: self.out_vert_shader.clone().unwrap(), frag: self.out_frag_shader.clone().unwrap(), blend: false, src_has_alpha: true, has_alpha_mult: false, alpha_mode: AlphaMode::PremultipliedElectrical, eotf: key.eotf.to_vulkan(), inv_eotf: fb_eotf.to_vulkan(), descriptor_set_layouts, has_color_management_data, })?; pipelines.set(key, out.clone()); Ok(out) } pub(super) fn allocate_point(&self) -> u64 { self.last_point.fetch_add(1) + 1 } fn create_descriptor_buffers( &self, buf: CommandBuffer, bb: Option<&VulkanImage>, ) -> Result<(), VulkanError> { let Some(db) = &self.device.descriptor_buffer else { return Ok(()); }; zone!("create_descriptor_buffers"); let memory = &mut *self.memory.borrow_mut(); let sampler_writer = &mut memory.sampler_descriptor_buffer_writer; sampler_writer.clear(); { let mut writer = sampler_writer.add_set(&self.tex_descriptor_set_layouts[0]); writer.write( self.tex_descriptor_set_layouts[0].offsets[0], &self.sampler_descriptor, ); } let resource_writer = &mut memory.resource_descriptor_buffer_writer; resource_writer.clear(); macro_rules! ub_descriptor_cache { ($field:ident) => { memory.$field.get_or_insert_with(|| { vec![0u8; self.device.uniform_buffer_descriptor_size].into_boxed_slice() }) }; } let uniform_buffer_descriptor_cache = ub_descriptor_cache!(uniform_buffer_descriptor_cache); macro_rules! get_ub_descriptor { ($addr:expr, $ty:ty, $descriptor:expr $(,)?) => {{ let uniform_buffer = DescriptorAddressInfoEXT::default() .address($addr) .range(size_of::<$ty>() as _); let info = DescriptorGetInfoEXT::default() .ty(DescriptorType::UNIFORM_BUFFER) .data(DescriptorDataEXT { p_uniform_buffer: &uniform_buffer, }); unsafe { db.get_descriptor(&info, $descriptor); } &*$descriptor }}; ($addr:expr, $ty:ty $(,)?) => { get_ub_descriptor!($addr, $ty, uniform_buffer_descriptor_cache) }; } let mut blend_buffer_inv_eotf_args_descriptor = None; if let Some(addr) = memory.blend_buffer_inv_eotf_args_address { blend_buffer_inv_eotf_args_descriptor = Some(get_ub_descriptor!( addr, InvEotfArgs, ub_descriptor_cache!(blend_buffer_inv_eotf_args_descriptor), )); } let mut fb_inv_eotf_args_descriptor = None; if let Some(addr) = memory.fb_inv_eotf_args_address { fb_inv_eotf_args_descriptor = Some(get_ub_descriptor!( addr, InvEotfArgs, ub_descriptor_cache!(fb_inv_eotf_args_descriptor), )); } if let Some(bb) = bb { let layout = self.out_descriptor_set_layout.as_ref().unwrap(); memory.blend_buffer_descriptor_buffer_offset = resource_writer.next_offset(); let mut writer = resource_writer.add_set(layout); writer.write( layout.offsets[0], bb.sampled_image_descriptor.as_ref().unwrap(), ); if let Some(addr) = memory.blend_buffer_color_management_data_address { writer.write( layout.offsets[1], get_ub_descriptor!(addr, ColorManagementData), ); } if let Some(addr) = memory.blend_buffer_eotf_args_address { writer.write(layout.offsets[2], get_ub_descriptor!(addr, EotfArgs)); } if let Some(desc) = fb_inv_eotf_args_descriptor { writer.write(layout.offsets[3], desc); } } let tex_descriptor_set_layout = &self.tex_descriptor_set_layouts[1]; for pass in RenderPass::variants() { let inv_eotf_desc = match pass { RenderPass::BlendBuffer => blend_buffer_inv_eotf_args_descriptor, RenderPass::FrameBuffer => fb_inv_eotf_args_descriptor, }; for cmd in &mut memory.ops[pass] { let VulkanOp::Tex(c) = cmd else { continue; }; let tex = &c.tex; c.resource_descriptor_buffer_offset = resource_writer.next_offset(); let mut writer = resource_writer.add_set(tex_descriptor_set_layout); writer.write( tex_descriptor_set_layout.offsets[0], tex.sampled_image_descriptor.as_ref().unwrap(), ); if let Some(addr) = c.color_management_data_address { writer.write( tex_descriptor_set_layout.offsets[1], get_ub_descriptor!(addr, ColorManagementData), ); } if let Some(addr) = c.eotf_args_address { writer.write( tex_descriptor_set_layout.offsets[2], get_ub_descriptor!(addr, EotfArgs), ); } if let Some(desc) = inv_eotf_desc { writer.write(tex_descriptor_set_layout.offsets[3], desc); } } } let mut infos = ArrayVec::<_, 2>::new(); for (writer, cache) in [ (&sampler_writer, &self.sampler_descriptor_buffer_cache), (&resource_writer, &self.resource_descriptor_buffer_cache), ] { let buffer = cache.allocate(writer.len() as DeviceSize)?; buffer.buffer.allocation.upload(|ptr, _| unsafe { ptr::copy_nonoverlapping(writer.as_ptr(), ptr, writer.len()) })?; let info = DescriptorBufferBindingInfoEXT::default() .usage(cache.usage()) .address(buffer.buffer.address); infos.push(info); memory.used_buffers.push(buffer); } unsafe { db.cmd_bind_descriptor_buffers(buf, &infos); } Ok(()) } fn convert_ops( &self, opts: &[GfxApiOpt], blend_cd: &ColorDescription, fb_cd: &ColorDescription, ) -> Result<(), VulkanError> { zone!("convert_ops"); let memory = &mut *self.memory.borrow_mut(); for ops in memory.ops.values_mut() { ops.clear(); } for ops in memory.ops_tmp.values_mut() { ops.clear(); } memory.tex_targets.clear(); memory.fill_targets.clear(); memory.data_buffer.clear(); memory.uniform_buffer_writer.clear(); memory.color_transforms.map.clear(); memory.eotf_args_cache.map.clear(); let sync = |memory: &mut Memory| { for pass in RenderPass::variants() { let ops = &mut memory.ops_tmp[pass]; ops.sort_unstable_by_key(|o| { #[derive(Eq, PartialEq, PartialOrd, Ord)] enum Key { Fill { color: [u32; 4] }, Tex(usize), } match o { VulkanOp::Fill(f) => Key::Fill { color: f.color.map(|c| c.to_bits()), }, VulkanOp::Tex(t) => Key::Tex(t.index), } }); let mops = &mut memory.ops[pass]; if self.device.descriptor_buffer.is_none() { mops.append(ops); continue; } for (idx, op) in ops.drain(..).enumerate() { match op { VulkanOp::Fill(mut f) => { f.range_address = memory.data_buffer.len() as DeviceAddress; f.instances = f.range.len() as u32; for pos in &memory.fill_targets[f.range.clone()] { memory.data_buffer.extend_from_slice(uapi::as_bytes(pos)); } if let Some(VulkanOp::Fill(p)) = mops.last_mut() && p.color == f.color && idx > 0 { p.instances += f.instances; continue; } mops.push(VulkanOp::Fill(f)); } VulkanOp::Tex(mut c) => { c.range_address = memory.data_buffer.len() as DeviceAddress; c.instances = c.range.len() as u32; for &[pos, tex_pos] in &memory.tex_targets[c.range.clone()] { let vertex = TexVertex { pos, tex_pos }; memory .data_buffer .extend_from_slice(uapi::as_bytes(&vertex)); } mops.push(VulkanOp::Tex(c)); } } } } }; for (index, op) in opts.iter().enumerate() { match op { GfxApiOpt::Sync => { sync(memory); } GfxApiOpt::FillRect(fr) => { let target = fr.rect.to_points(); for pass in RenderPass::variants() { let Some(bounds) = memory.paint_bounds[pass] else { continue; }; if !bounds.intersects(&target) { continue; } let ops = &mut memory.ops_tmp[pass]; let lo = memory.fill_targets.len(); for region in &memory.paint_regions[pass] { let mut target = target; if !region.constrain(&mut target, None) { continue; } memory.fill_targets.push(target); } let hi = memory.fill_targets.len(); if lo == hi { continue; } let target_cd = match pass { RenderPass::BlendBuffer => blend_cd, RenderPass::FrameBuffer => fb_cd, }; let tf = target_cd.eotf; let color = memory.color_transforms.apply_to_color( &fr.cd, target_cd, fr.render_intent, fr.color, ); let color = color.to_array2(tf, fr.alpha); let source_type = match color[3] < 1.0 { false => TexSourceType::Opaque, true => TexSourceType::HasAlpha, }; ops.push(VulkanOp::Fill(VulkanFillOp { range: lo..hi, color, source_type, range_address: 0, instances: 0, })); } } GfxApiOpt::CopyTexture(ct) => { let tex = ct.tex.clone().into_vk(&self.device.device)?; if tex.contents_are_undefined.get() { log::warn!("Ignoring undefined texture"); continue; } if tex.queue_state.get().acquire(QueueFamily::Gfx) == QueueTransfer::Impossible { log::warn!("Ignoring texture owned by different queue"); continue; } let target = ct.target.to_points(); let source = ct.source.to_points(); for pass in RenderPass::variants() { let Some(bounds) = memory.paint_bounds[pass] else { continue; }; if !bounds.intersects(&target) { continue; } let ops = &mut memory.ops_tmp[pass]; let lo = memory.tex_targets.len(); for region in &memory.paint_regions[pass] { let mut target = target; let mut source = source; if !region.constrain(&mut target, Some(&mut source)) { continue; } memory.tex_targets.push([target, source]); } let hi = memory.tex_targets.len(); if lo == hi { continue; } let copy_type = match ct.alpha.is_some() { true => TexCopyType::Multiply, false => TexCopyType::Identity, }; let source_type = match tex.format.has_alpha && !ct.opaque { true => TexSourceType::HasAlpha, false => TexSourceType::Opaque, }; let target_cd = match pass { RenderPass::BlendBuffer => blend_cd, RenderPass::FrameBuffer => fb_cd, }; let color_management_data_address = memory.color_transforms.get_offset( &ct.cd.linear, target_cd, ct.render_intent, self.device.uniform_buffer_offset_mask, &mut memory.uniform_buffer_writer, ); let eotf_args_address = memory.eotf_args_cache.get_offset( &ct.cd, false, self.device.uniform_buffer_offset_mask, &mut memory.uniform_buffer_writer, ); ops.push(VulkanOp::Tex(VulkanTexOp { tex: tex.clone(), index, range: lo..hi, buffer_resv: ct.buffer_resv.clone(), acquire_sync: Some(ct.acquire_sync.clone()), release_sync: ct.release_sync, alpha: ct.alpha.unwrap_or_default(), source_type, copy_type, alpha_mode: ct.alpha_mode, range_address: 0, instances: 0, tex_cd: ct.cd.clone(), color_management_data_address, eotf_args_address, resource_descriptor_buffer_offset: 0, })); } } } } sync(memory); Ok(()) } fn create_fixed_cm_data( &self, bb: Option<&VulkanImage>, bb_cd: &ColorDescription, fb_cd: &ColorDescription, ) { zone!("create_fixed_cm_data"); let memory = &mut *self.memory.borrow_mut(); memory.blend_buffer_color_management_data_address = None; memory.blend_buffer_eotf_args_address = None; memory.blend_buffer_inv_eotf_args_address = None; memory.fb_inv_eotf_args_address = None; if bb.is_some() { memory.blend_buffer_color_management_data_address = memory.color_transforms.get_offset( &bb_cd.linear, fb_cd, RenderIntent::Perceptual, self.device.uniform_buffer_offset_mask, &mut memory.uniform_buffer_writer, ); memory.blend_buffer_eotf_args_address = memory.eotf_args_cache.get_offset( bb_cd, false, self.device.uniform_buffer_offset_mask, &mut memory.uniform_buffer_writer, ); memory.blend_buffer_inv_eotf_args_address = memory.eotf_args_cache.get_offset( bb_cd, true, self.device.uniform_buffer_offset_mask, &mut memory.uniform_buffer_writer, ); } memory.fb_inv_eotf_args_address = memory.eotf_args_cache.get_offset( fb_cd, true, self.device.uniform_buffer_offset_mask, &mut memory.uniform_buffer_writer, ); } fn create_data_buffer(&self) -> Result<(), VulkanError> { if self.device.descriptor_buffer.is_none() { return Ok(()); } zone!("create_data_buffer"); let memory = &mut *self.memory.borrow_mut(); let buf = &mut memory.data_buffer; { memory.out_address = buf.len() as _; for region in &memory.paint_regions[RenderPass::BlendBuffer] { buf.extend_from_slice(uapi::as_bytes(&[ [region.x2, region.y1], [region.x1, region.y1], [region.x2, region.y2], [region.x1, region.y2], ])); } } if buf.is_empty() { return Ok(()); } let buffer = self.shader_buffer_cache.allocate(buf.len() as _)?; buffer.buffer.allocation.upload(|ptr, _| unsafe { ptr::copy_nonoverlapping(buf.as_ptr(), ptr, buf.len()); })?; for ops in memory.ops.values_mut() { for op in ops { match op { VulkanOp::Fill(f) => { f.range_address += buffer.buffer.address; } VulkanOp::Tex(c) => { c.range_address += buffer.buffer.address; } } } } memory.out_address += buffer.buffer.address; memory.used_buffers.push(buffer); Ok(()) } fn create_uniform_buffer(&self) -> Result<(), VulkanError> { if self.device.descriptor_buffer.is_none() { return Ok(()); } zone!("create_uniform_buffer"); let memory = &mut *self.memory.borrow_mut(); let buf = &memory.uniform_buffer_writer; if buf.is_empty() { return Ok(()); } let buffer = self.uniform_buffer_cache.allocate(buf.len() as _)?; buffer.buffer.allocation.upload(|ptr, _| unsafe { ptr::copy_nonoverlapping(buf.as_ptr(), ptr, buf.len()); })?; macro_rules! adj { ($expr:expr) => { if let Some(addr) = $expr { *addr += buffer.buffer.address; } }; } for ops in memory.ops.values_mut() { for op in ops { if let VulkanOp::Tex(c) = op { adj!(&mut c.color_management_data_address); adj!(&mut c.eotf_args_address); } } } adj!(&mut memory.blend_buffer_color_management_data_address); adj!(&mut memory.blend_buffer_eotf_args_address); adj!(&mut memory.blend_buffer_inv_eotf_args_address); adj!(&mut memory.fb_inv_eotf_args_address); memory.used_buffers.push(buffer); Ok(()) } fn collect_memory(&self) { zone!("collect_memory"); let memory = &mut *self.memory.borrow_mut(); memory.dmabuf_sample.clear(); memory.queue_transfer.clear(); let execution = self.allocate_point(); for pass in RenderPass::variants() { for cmd in &mut memory.ops[pass] { if let VulkanOp::Tex(c) = cmd { let tex = &c.tex; if tex.execution_version.replace(execution) == execution { continue; } match tex.queue_state.get().acquire(QueueFamily::Gfx) { QueueTransfer::Unnecessary => {} QueueTransfer::Possible => memory.queue_transfer.push(tex.clone()), QueueTransfer::Impossible => continue, } if let VulkanImageMemory::DmaBuf(_) = &tex.ty { memory.dmabuf_sample.push(tex.clone()) } memory.textures.push(UsedTexture { tex: tex.clone(), resv: c.buffer_resv.take(), acquire_sync: c.acquire_sync.take().unwrap(), release_sync: c.release_sync, }); } } } } fn begin_command_buffer(&self, buf: CommandBuffer) -> Result<(), VulkanError> { zone!("begin_command_buffer"); let begin_info = CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT); unsafe { self.device .device .begin_command_buffer(buf, &begin_info) .map_err(VulkanError::BeginCommandBuffer) } } fn initial_barriers(&self, buf: CommandBuffer, fb: &VulkanImage) -> Result<(), VulkanError> { zone!("initial_barriers"); let mut memory = self.memory.borrow_mut(); let memory = &mut *memory; memory.image_barriers.clear(); let mut need_fb_barrier = true; if let VulkanImageMemory::Internal(..) = &fb.ty { need_fb_barrier = fb.is_undefined.get() || (self.device.distinct_transfer_queue_family_idx.is_some() && fb.queue_state.get().acquire(QueueFamily::Gfx) != QueueTransfer::Unnecessary); } if need_fb_barrier { let mut fb_image_memory_barrier = image_barrier() .image(fb.image) .new_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL) .dst_access_mask( AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ, ) .dst_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT); if fb.bridge.is_some() { fb_image_memory_barrier = fb_image_memory_barrier .src_access_mask(AccessFlags2::TRANSFER_READ) .src_stage_mask(PipelineStageFlags2::TRANSFER) .old_layout(if fb.is_undefined.get() { ImageLayout::UNDEFINED } else { ImageLayout::TRANSFER_SRC_OPTIMAL }); } else if let VulkanImageMemory::Internal(..) = &fb.ty { let mut queue_transfer = QueueTransfer::Unnecessary; if self.device.distinct_transfer_queue_family_idx.is_some() { queue_transfer = fb.queue_state.get().acquire(QueueFamily::Gfx); } match queue_transfer { QueueTransfer::Unnecessary => { fb_image_memory_barrier = fb_image_memory_barrier.old_layout(ImageLayout::UNDEFINED); } QueueTransfer::Possible => { if let Some(transfer_queue_idx) = self.device.distinct_transfer_queue_family_idx { fb_image_memory_barrier = fb_image_memory_barrier .src_queue_family_index(transfer_queue_idx) .dst_queue_family_index(self.device.graphics_queue_idx) .old_layout(ImageLayout::TRANSFER_SRC_OPTIMAL); } } QueueTransfer::Impossible => return Err(VulkanError::BusyInTransfer), } } else { fb_image_memory_barrier = fb_image_memory_barrier .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) .dst_queue_family_index(self.device.graphics_queue_idx) .old_layout(if fb.is_undefined.get() { ImageLayout::UNDEFINED } else { ImageLayout::GENERAL }); } memory.image_barriers.push(fb_image_memory_barrier); } for img in &memory.dmabuf_sample { let image_memory_barrier = image_barrier() .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) .dst_queue_family_index(self.device.graphics_queue_idx) .image(img.image) .old_layout(ImageLayout::GENERAL) .new_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL) .dst_access_mask(AccessFlags2::SHADER_SAMPLED_READ) .dst_stage_mask(PipelineStageFlags2::FRAGMENT_SHADER); memory.image_barriers.push(image_memory_barrier); } if let Some(family_idx) = self.device.distinct_transfer_queue_family_idx { for img in &memory.queue_transfer { let image_memory_barrier = image_barrier() .src_queue_family_index(family_idx) .dst_queue_family_index(self.device.graphics_queue_idx) .image(img.image) .dst_access_mask(AccessFlags2::SHADER_SAMPLED_READ) .dst_stage_mask(PipelineStageFlags2::FRAGMENT_SHADER) .old_layout(ImageLayout::TRANSFER_DST_OPTIMAL) .new_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL); memory.image_barriers.push(image_memory_barrier); } } let dep_info = DependencyInfoKHR::default().image_memory_barriers(&memory.image_barriers); unsafe { self.device.device.cmd_pipeline_barrier2(buf, &dep_info); } Ok(()) } fn begin_rendering( &self, buf: CommandBuffer, target: &VulkanImage, clear: Option<&Color>, clear_cd: &LinearColorDescription, pass: RenderPass, target_cd: &ColorDescription, ) { zone!("begin_rendering"); let memory = &mut *self.memory.borrow_mut(); let mut load_clear = None; let mut manual_clear = None; let clear_rects = &memory.clear_rects[pass]; if let Some(clear) = clear && clear_rects.is_not_empty() { let color = memory.color_transforms.apply_to_color( clear_cd, target_cd, RenderIntent::Perceptual, *clear, ); let clear_value = ClearValue { color: ClearColorValue { float32: color.to_array(target_cd.eotf), }, }; let use_load_clear = clear_rects.len() == 1 && { let rect = &clear_rects[0].rect; rect.offset.x == 0 && rect.offset.y == 0 && rect.extent.width == target.width && rect.extent.height == target.height }; if use_load_clear { load_clear = Some(clear_value); } else { manual_clear = Some(clear_value); } } let mut rendering_attachment_info = RenderingAttachmentInfo::default() .image_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL) .image_view(target.render_view.unwrap_or(target.texture_view)) .store_op(AttachmentStoreOp::STORE); let load_op = if let Some(clear) = load_clear { rendering_attachment_info = rendering_attachment_info.clear_value(clear); AttachmentLoadOp::CLEAR } else if pass == RenderPass::BlendBuffer { AttachmentLoadOp::DONT_CARE } else { AttachmentLoadOp::LOAD }; rendering_attachment_info = rendering_attachment_info.load_op(load_op); let rendering_info = RenderingInfo::default() .render_area(Rect2D { offset: Default::default(), extent: Extent2D { width: target.width, height: target.height, }, }) .layer_count(1) .color_attachments(slice::from_ref(&rendering_attachment_info)); unsafe { self.device.device.cmd_begin_rendering(buf, &rendering_info); } if clear_rects.is_not_empty() { if let Some(clear) = manual_clear { let clear_attachment = ClearAttachment::default() .color_attachment(0) .clear_value(clear) .aspect_mask(ImageAspectFlags::COLOR); unsafe { self.device .device .cmd_clear_attachments(buf, &[clear_attachment], clear_rects); } } } } fn set_viewport(&self, buf: CommandBuffer, fb: &VulkanImage) { zone!("set_viewport"); let viewport = Viewport { x: 0.0, y: 0.0, width: fb.width as _, height: fb.height as _, min_depth: 0.0, max_depth: 1.0, }; let scissor = Rect2D { offset: Default::default(), extent: Extent2D { width: fb.width, height: fb.height, }, }; unsafe { self.device .device .cmd_set_viewport(buf, 0, slice::from_ref(&viewport)); self.device .device .cmd_set_scissor(buf, 0, slice::from_ref(&scissor)); } } fn record_draws( &self, buf: CommandBuffer, target: &VulkanImage, pass: RenderPass, target_cd: &ColorDescription, ) -> Result<(), VulkanError> { zone!("record_draws"); let memory = &*self.memory.borrow(); let fill_pl = self.get_or_create_fill_pipelines(target.format.vk_format)?; let tex_pl = self.get_or_create_tex_pipelines(target.format.vk_format, target_cd); let dev = &self.device.device; let mut current_pipeline = None; let mut bind = |pipeline: &VulkanPipeline| { if current_pipeline != Some(pipeline.pipeline) { current_pipeline = Some(pipeline.pipeline); unsafe { dev.cmd_bind_pipeline(buf, PipelineBindPoint::GRAPHICS, pipeline.pipeline); } } }; for opt in &memory.ops[pass] { match opt { VulkanOp::Fill(r) => { let pipeline = &fill_pl[r.source_type]; bind(pipeline); if self.device.descriptor_buffer.is_some() { let push = FillPushConstants { color: r.color, vertices: r.range_address, _padding1: 0, _padding2: 0, }; unsafe { dev.cmd_push_constants( buf, pipeline.pipeline_layout, ShaderStageFlags::VERTEX | ShaderStageFlags::FRAGMENT, 0, uapi::as_bytes(&push), ); dev.cmd_draw(buf, 4, r.instances, 0, 0); } } else { for &pos in &memory.fill_targets[r.range.clone()] { let push = LegacyFillPushConstants { pos, color: r.color, }; unsafe { dev.cmd_push_constants( buf, pipeline.pipeline_layout, ShaderStageFlags::VERTEX | ShaderStageFlags::FRAGMENT, 0, uapi::as_bytes(&push), ); dev.cmd_draw(buf, 4, 1, 0, 0); } } } } VulkanOp::Tex(c) => { let tex = &c.tex; let pipeline = self.get_or_create_tex_pipeline( &tex_pl, &c.tex_cd, c.copy_type, c.source_type, c.alpha_mode, c.color_management_data_address.is_some(), )?; bind(&pipeline); let image_info = DescriptorImageInfo::default() .image_view(tex.texture_view) .image_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL); if let Some(db) = &self.device.descriptor_buffer { let push = TexPushConstants { vertices: c.range_address, alpha: c.alpha, }; unsafe { db.cmd_set_descriptor_buffer_offsets( buf, PipelineBindPoint::GRAPHICS, pipeline.pipeline_layout, 0, &[0, 1], &[0, c.resource_descriptor_buffer_offset], ); dev.cmd_push_constants( buf, pipeline.pipeline_layout, ShaderStageFlags::VERTEX | ShaderStageFlags::FRAGMENT, 0, uapi::as_bytes(&push), ); dev.cmd_draw(buf, 4, c.instances, 0, 0); } } else { let write_descriptor_set = WriteDescriptorSet::default() .descriptor_type(DescriptorType::COMBINED_IMAGE_SAMPLER) .image_info(slice::from_ref(&image_info)); unsafe { self.device.push_descriptor.cmd_push_descriptor_set( buf, PipelineBindPoint::GRAPHICS, pipeline.pipeline_layout, 0, slice::from_ref(&write_descriptor_set), ); } for &[pos, tex_pos] in &memory.tex_targets[c.range.clone()] { let push = LegacyTexPushConstants { pos, tex_pos, alpha: c.alpha, }; unsafe { dev.cmd_push_constants( buf, pipeline.pipeline_layout, ShaderStageFlags::VERTEX | ShaderStageFlags::FRAGMENT, 0, uapi::as_bytes(&push), ); dev.cmd_draw(buf, 4, 1, 0, 0); } } } } } } Ok(()) } fn blend_buffer_initial_barrier(&self, buf: CommandBuffer, bb: &VulkanImage) { zone!("blend_buffer_initial_barrier"); let memory = &mut *self.memory.borrow_mut(); memory.image_barriers.clear(); let barrier = image_barrier() .image(bb.image) .old_layout(if bb.is_undefined.get() { ImageLayout::UNDEFINED } else { ImageLayout::SHADER_READ_ONLY_OPTIMAL }) .new_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL) .src_stage_mask(PipelineStageFlags2::FRAGMENT_SHADER) .dst_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT) .src_access_mask(AccessFlags2::SHADER_READ) .dst_access_mask(AccessFlags2::COLOR_ATTACHMENT_WRITE); memory.image_barriers.push(barrier); let dep_info = DependencyInfoKHR::default().image_memory_barriers(&memory.image_barriers); unsafe { self.device.device.cmd_pipeline_barrier2(buf, &dep_info); } } fn blend_buffer_copy( &self, buf: CommandBuffer, fb: &VulkanImage, fb_cd: &ColorDescription, bb_cd: &ColorDescription, ) -> Result<(), VulkanError> { zone!("blend_buffer_copy"); let memory = &*self.memory.borrow(); let db = self.device.descriptor_buffer.as_ref().unwrap(); let pipeline = self.get_or_create_out_pipeline( fb.format.vk_format, bb_cd, fb_cd, memory.blend_buffer_color_management_data_address.is_some(), )?; let push = OutPushConstants { vertices: memory.out_address, }; let instances = memory.paint_regions[RenderPass::BlendBuffer].len() as u32; let dev = &self.device.device; unsafe { dev.cmd_bind_pipeline(buf, PipelineBindPoint::GRAPHICS, pipeline.pipeline); db.cmd_set_descriptor_buffer_offsets( buf, PipelineBindPoint::GRAPHICS, pipeline.pipeline_layout, 0, &[1], &[memory.blend_buffer_descriptor_buffer_offset], ); dev.cmd_push_constants( buf, pipeline.pipeline_layout, ShaderStageFlags::VERTEX | ShaderStageFlags::FRAGMENT, 0, uapi::as_bytes(&push), ); dev.cmd_draw(buf, 4, instances, 0, 0); } Ok(()) } fn blend_buffer_final_barrier(&self, buf: CommandBuffer, bb: &VulkanImage) { zone!("blend_buffer_final_barrier"); let image_barrier = image_barrier() .image(bb.image) .old_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL) .new_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL) .src_access_mask(AccessFlags2::COLOR_ATTACHMENT_WRITE) .dst_access_mask(AccessFlags2::SHADER_SAMPLED_READ) .src_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT) .dst_stage_mask(PipelineStageFlags2::FRAGMENT_SHADER); let dependency_info = DependencyInfoKHR::default().image_memory_barriers(slice::from_ref(&image_barrier)); unsafe { self.device .device .cmd_pipeline_barrier2(buf, &dependency_info); } } fn end_rendering(&self, buf: CommandBuffer) { zone!("end_rendering"); unsafe { self.device.device.cmd_end_rendering(buf); } } fn copy_bridge_to_dmabuf(&self, buf: CommandBuffer, fb: &VulkanImage, region: &Region) { zone!("copy_bridge_to_dmabuf"); let Some(bridge) = &fb.bridge else { return; }; let mut memory = self.memory.borrow_mut(); let memory = &mut *memory; memory.image_barriers.clear(); let bridge_image_memory_barrier = image_barrier() .image(fb.image) .old_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL) .new_layout(ImageLayout::TRANSFER_SRC_OPTIMAL) .src_access_mask( AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ, ) .src_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT) .dst_access_mask(AccessFlags2::TRANSFER_READ) .dst_stage_mask(PipelineStageFlags2::TRANSFER); memory.image_barriers.push(bridge_image_memory_barrier); let dmabuf_image_memory_barrier = image_barrier() .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) .dst_queue_family_index(self.device.graphics_queue_idx) .image(bridge.dmabuf_image) .old_layout(if fb.is_undefined.get() { ImageLayout::UNDEFINED } else { ImageLayout::GENERAL }) .new_layout(ImageLayout::TRANSFER_DST_OPTIMAL) .dst_access_mask(AccessFlags2::TRANSFER_WRITE) .dst_stage_mask(PipelineStageFlags2::TRANSFER); memory.image_barriers.push(dmabuf_image_memory_barrier); let dep_info = DependencyInfoKHR::default().image_memory_barriers(&memory.image_barriers); unsafe { self.device.device.cmd_pipeline_barrier2(buf, &dep_info); } let image_subresource_layers = ImageSubresourceLayers::default() .aspect_mask(ImageAspectFlags::COLOR) .layer_count(1) .base_array_layer(0) .mip_level(0); memory.image_copy_regions.clear(); for rect in region.rects() { let Some([x1, y1, x2, y2]) = constrain_to_fb(fb, rect) else { continue; }; let offset = Offset3D { x: x1, y: y1, z: 0 }; let extent = Extent3D { width: (x2 - x1) as _, height: (y2 - y1) as _, depth: 1, }; memory.image_copy_regions.push( ImageCopy2::default() .src_subresource(image_subresource_layers) .dst_subresource(image_subresource_layers) .src_offset(offset) .dst_offset(offset) .extent(extent), ); } if memory.image_copy_regions.is_not_empty() { let copy_image_info = CopyImageInfo2::default() .src_image(fb.image) .src_image_layout(ImageLayout::TRANSFER_SRC_OPTIMAL) .dst_image(bridge.dmabuf_image) .dst_image_layout(ImageLayout::TRANSFER_DST_OPTIMAL) .regions(&memory.image_copy_regions); unsafe { self.device.device.cmd_copy_image2(buf, ©_image_info); } } } fn final_barriers(&self, buf: CommandBuffer, fb: &VulkanImage) { zone!("final_barriers"); let mut memory = self.memory.borrow_mut(); let memory = &mut *memory; memory.image_barriers.clear(); if let VulkanImageMemory::DmaBuf(..) = fb.ty { let mut fb_image_memory_barrier = image_barrier() .src_queue_family_index(self.device.graphics_queue_idx) .dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) .new_layout(ImageLayout::GENERAL); if let Some(bridge) = &fb.bridge { fb_image_memory_barrier = fb_image_memory_barrier .image(bridge.dmabuf_image) .old_layout(ImageLayout::TRANSFER_DST_OPTIMAL) .src_access_mask(AccessFlags2::TRANSFER_WRITE) .src_stage_mask(PipelineStageFlags2::TRANSFER); } else { fb_image_memory_barrier = fb_image_memory_barrier .image(fb.image) .old_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL) .src_access_mask( AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ, ) .src_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT); } memory.image_barriers.push(fb_image_memory_barrier); } for img in &memory.dmabuf_sample { let image_memory_barrier = image_barrier() .src_queue_family_index(self.device.graphics_queue_idx) .dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) .old_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL) .new_layout(ImageLayout::GENERAL) .image(img.image) .src_access_mask(AccessFlags2::SHADER_SAMPLED_READ) .src_stage_mask(PipelineStageFlags2::FRAGMENT_SHADER); memory.image_barriers.push(image_memory_barrier); } let dep_info = DependencyInfoKHR::default().image_memory_barriers(&memory.image_barriers); unsafe { self.device.device.cmd_pipeline_barrier2(buf, &dep_info); } } fn end_command_buffer(&self, buf: CommandBuffer) -> Result<(), VulkanError> { zone!("end_command_buffer"); unsafe { self.device .device .end_command_buffer(buf) .map_err(VulkanError::EndCommandBuffer) } } fn create_wait_semaphores( &self, fb: &VulkanImage, fb_acquire_sync: &AcquireSync, ) -> Result<(), VulkanError> { zone!("create_wait_semaphores"); let mut memory = self.memory.borrow_mut(); let memory = &mut *memory; memory.wait_semaphore_infos.clear(); let import = |infos: &mut Vec, semaphores: &mut Vec>, img: &VulkanImage, sync: &AcquireSync, flag: u32| -> Result<(), VulkanError> { if let VulkanImageMemory::DmaBuf(buf) = &img.ty { let mut import_sync_file = |fd: OwnedFd| -> Result<(), VulkanError> { let semaphore = self.allocate_semaphore()?; semaphore.import_sync_file(fd)?; infos.push( SemaphoreSubmitInfo::default() .semaphore(semaphore.semaphore) .stage_mask(PipelineStageFlags2::TOP_OF_PIPE), ); semaphores.push(semaphore); Ok(()) }; match sync { AcquireSync::None => {} AcquireSync::Implicit => { zone!("import implicit"); for plane in &buf.template.dmabuf.planes { let fd = dma_buf_export_sync_file(&plane.fd, flag) .map_err(VulkanError::IoctlExportSyncFile)?; import_sync_file(fd)?; if buf.template.dmabuf.is_one_file() { break; } } } AcquireSync::FdSync(sync) => { if let Some(sync_file) = sync.get_sync_file() { let fd = uapi::fcntl_dupfd_cloexec(sync_file.raw(), 0) .map_os_err(VulkanError::Dupfd)?; import_sync_file(fd)?; } } AcquireSync::Unnecessary => {} } } Ok(()) }; for texture in &memory.textures { import( &mut memory.wait_semaphore_infos, &mut memory.wait_semaphores, &texture.tex, &texture.acquire_sync, DMA_BUF_SYNC_READ, )?; } import( &mut memory.wait_semaphore_infos, &mut memory.wait_semaphores, fb, fb_acquire_sync, DMA_BUF_SYNC_WRITE, )?; Ok(()) } fn import_release_semaphore(&self, fb: &VulkanImage, fb_release_sync: ReleaseSync) { zone!("import_release_semaphore"); let memory = &mut *self.memory.borrow_mut(); let fd_sync = match memory.release_sync.as_ref() { Some(sync) => sync, _ => return, }; let sync_file = LazyCell::new(|| fd_sync.get_sync_file()); let import = |img: &VulkanImage, sync: ReleaseSync, resv: Option>, flag: u32| { if sync == ReleaseSync::None { return; } if let Some(resv) = resv { resv.set_sync(self.buffer_resv_user, &fd_sync); } else if sync == ReleaseSync::Implicit { if let VulkanImageMemory::DmaBuf(buf) = &img.ty && let Some(fd) = &*sync_file && let Err(e) = buf.template.dmabuf.import_sync_file(flag, fd) { log::error!("Could not import sync file into dmabuf: {}", ErrorFmt(e)); log::warn!("Relying on implicit sync"); } } }; let attach_async_shm_sync_file = self.device.transfer_queue.is_some() && self.device.distinct_transfer_queue_family_idx.is_none(); for texture in &mut memory.textures { import( &texture.tex, texture.release_sync, texture.resv.take(), DMA_BUF_SYNC_READ, ); if attach_async_shm_sync_file && let VulkanImageMemory::Internal(shm) = &texture.tex.ty && let Some(data) = &shm.async_data { data.last_gfx_use.set(Some(fd_sync.clone())); } } if attach_async_shm_sync_file && let VulkanImageMemory::Internal(shm) = &fb.ty && let Some(data) = &shm.async_data { data.last_gfx_use.set(Some(fd_sync.clone())); } import(fb, fb_release_sync, None, DMA_BUF_SYNC_WRITE); } fn submit(&self, buf: CommandBuffer) -> Result<(), VulkanError> { zone!("submit"); let mut memory = self.memory.borrow_mut(); let mut semaphore_submit_info = SemaphoreSubmitInfo::default(); let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(buf); let mut submit_info = SubmitInfo2::default() .wait_semaphore_infos(&memory.wait_semaphore_infos) .command_buffer_infos(slice::from_ref(&command_buffer_info)); let vulkan_sync = self.device.create_sync( self.render_tls.as_ref(), &mut semaphore_submit_info, &mut submit_info, )?; unsafe { self.device .device .queue_submit2( self.device.graphics_queue, slice::from_ref(&submit_info), vulkan_sync.fence(), ) .inspect_err(self.device.idl()) .map_err(VulkanError::Submit)?; } memory.release_sync = vulkan_sync.to_sync(|| self.block()); memory.release_vulkan_sync = Some(vulkan_sync); Ok(()) } fn store_layouts(&self, fb: &VulkanImage, bb: Option<&VulkanImage>) { if let Some(bb) = bb { bb.is_undefined.set(false); } fb.is_undefined.set(false); fb.contents_are_undefined.set(false); fb.queue_state.set(QueueState::Acquired { family: QueueFamily::Gfx, }); let memory = self.memory.borrow(); for img in &*memory.queue_transfer { img.queue_state.set(QueueState::Acquired { family: QueueFamily::Gfx, }); } } fn create_pending_frame( self: &Rc, buf: Rc, fb: &Rc, bb: Option>, ) { zone!("create_pending_frame"); let point = self.allocate_point(); let mut memory = self.memory.borrow_mut(); let frame = Rc::new(PendingFrame { point, renderer: self.clone(), cmd: Cell::new(Some(buf)), _fb: fb.clone(), _bb: bb, _textures: mem::take(&mut memory.textures), wait_semaphores: Cell::new(mem::take(&mut memory.wait_semaphores)), waiter: Cell::new(None), vulkan_sync: memory.release_vulkan_sync.take(), _used_buffers: mem::take(&mut memory.used_buffers), }); self.pending_frames.set(frame.point, frame.clone()); let future = self.eng.spawn( "await release", await_release( memory.release_sync.clone(), self.ring.clone(), frame.clone(), self.clone(), ), ); frame.waiter.set(Some(future)); } pub fn execute( self: &Rc, fb: &Rc, fb_acquire_sync: AcquireSync, fb_release_sync: ReleaseSync, fb_cd: &Rc, opts: &[GfxApiOpt], clear: Option<&Color>, clear_cd: &Rc, region: &Region, blend_buffer: Option>, blend_cd: &Rc, ) -> Result, VulkanError> { zone!("execute"); let res = self.try_execute( fb, fb_acquire_sync, fb_release_sync, fb_cd, opts, clear, clear_cd, region, blend_buffer, blend_cd, ); let sync = { let mut memory = self.memory.borrow_mut(); memory.textures.clear(); memory.dmabuf_sample.clear(); memory.queue_transfer.clear(); memory.wait_semaphores.clear(); memory.release_vulkan_sync.take(); memory.used_buffers.clear(); memory.ops.clear(); memory.ops_tmp.clear(); memory.release_sync.take() }; res.map(|_| sync) } fn allocate_semaphore(&self) -> Result, VulkanError> { zone!("allocate_semaphore"); let semaphore = match self.wait_semaphores.pop() { Some(s) => s, _ => self.device.create_semaphore()?, }; Ok(semaphore) } fn create_regions( &self, fb: &VulkanImage, opts: &[GfxApiOpt], clear: Option<&Color>, region: &Region, bb: Option<&VulkanImage>, ) -> Result<(), VulkanError> { zone!("create_paint_regions"); let memory = &mut *self.memory.borrow_mut(); memory.regions_1.clear(); memory.regions_2.clear(); let width = fb.width as f32; let height = fb.height as f32; let mut tag = 0; for opt in opts.iter().rev() { let (opaque, fb_rect) = match opt { GfxApiOpt::Sync => continue, GfxApiOpt::FillRect(f) => (f.effective_color().is_opaque(), f.rect), GfxApiOpt::CopyTexture(c) => { let opaque = 'opaque: { if let Some(a) = c.alpha && a < 1.0 { break 'opaque false; } if !c.opaque { let tex = c.tex.as_vk(&self.device.device)?; if tex.format.has_alpha { break 'opaque false; } } true }; (opaque, c.target) } }; if opaque || bb.is_none() { tag |= 1; } else { tag += tag & 1; } let rect = fb_rect.to_rect(width, height); if opaque && clear.is_some() { memory.regions_1.push(rect); } memory.regions_2.push(rect.with_tag(tag)); } let clear_region = if clear.is_some() { let opaque_region = Region::from_rects2(&memory.regions_1); region.subtract_cow(&opaque_region) } else { Cow::Owned(Region::default()) }; let tagged_region = Region::from_rects_tagged(&memory.regions_2).intersect_tagged(region); memory.regions_1.clear(); memory.paint_regions[RenderPass::BlendBuffer].clear(); memory.paint_regions[RenderPass::FrameBuffer].clear(); let to_fb = |c: i32, max: u32| 2.0 * (c as f32 / max as f32) - 1.0; for rect in tagged_region.rects() { if rect.tag() == 0 && clear.is_some() { memory.regions_1.push(rect.untag()); } let Some([x1, y1, x2, y2]) = constrain_to_fb(fb, rect) else { continue; }; let region = match rect.tag() { 0 => &mut memory.paint_regions[RenderPass::BlendBuffer], _ => &mut memory.paint_regions[RenderPass::FrameBuffer], }; region.push(PaintRegion { x1: to_fb(x1, fb.width), x2: to_fb(x2, fb.width), y1: to_fb(y1, fb.height), y2: to_fb(y2, fb.height), }); } for pass in RenderPass::variants() { let regions = &memory.paint_regions[pass]; if regions.is_empty() { memory.paint_bounds[pass] = None; } else { let mut union = regions[0]; for region in ®ions[1..] { union = union.union(region); } memory.paint_bounds[pass] = Some(union); } } let blend_clear = clear_region.intersect(&Region::from_rects2(&memory.regions_1)); let opaque_clear = clear_region.subtract_cow(&blend_clear); // if bb.is_none() { // log::info!("blend_clear = {:?}", blend_clear); // log::info!("opaque_clear = {:?}", opaque_clear); // } for (pass, clear_region) in [ (RenderPass::BlendBuffer, &blend_clear), (RenderPass::FrameBuffer, &opaque_clear), ] { memory.clear_rects[pass].clear(); for rect in clear_region.rects() { let Some([x1, y1, x2, y2]) = constrain_to_fb(fb, rect) else { continue; }; memory.clear_rects[pass].push(ClearRect { rect: Rect2D { offset: Offset2D { x: x1 as _, y: y1 as _, }, extent: Extent2D { width: (x2 - x1) as u32, height: (y2 - y1) as u32, }, }, base_array_layer: 0, layer_count: 1, }); } } Ok(()) } fn elide_blend_buffer1( &self, blend_buffer: &mut Option>, bb_cd: &ColorDescription, fb_cd: &ColorDescription, ) { if blend_buffer.is_none() { return; } if bb_cd.embeds_into(fb_cd) { *blend_buffer = None; } } fn elide_blend_buffer2(&self, blend_buffer: &mut Option>) { if blend_buffer.is_none() { return; } let memory = &*self.memory.borrow(); if memory.paint_regions[RenderPass::BlendBuffer].is_empty() { *blend_buffer = None; } } fn try_execute( self: &Rc, fb: &Rc, fb_acquire_sync: AcquireSync, fb_release_sync: ReleaseSync, fb_cd: &Rc, opts: &[GfxApiOpt], clear: Option<&Color>, clear_cd: &Rc, region: &Region, mut blend_buffer: Option>, bb_cd: &Rc, ) -> Result<(), VulkanError> { self.check_defunct()?; self.elide_blend_buffer1(&mut blend_buffer, bb_cd, fb_cd); self.create_regions(fb, opts, clear, region, blend_buffer.as_deref())?; self.elide_blend_buffer2(&mut blend_buffer); let bb = blend_buffer.as_deref(); let buf = self.gfx_command_buffers.allocate()?; self.convert_ops(opts, bb_cd, fb_cd)?; self.create_fixed_cm_data(bb, bb_cd, fb_cd); self.create_data_buffer()?; self.create_uniform_buffer()?; self.collect_memory(); self.begin_command_buffer(buf.buffer)?; self.create_descriptor_buffers(buf.buffer, bb)?; self.initial_barriers(buf.buffer, fb)?; self.set_viewport(buf.buffer, fb); if let Some(bb) = bb { zone!("blend buffer pass"); let rp = RenderPass::BlendBuffer; self.blend_buffer_initial_barrier(buf.buffer, bb); self.begin_rendering(buf.buffer, bb, clear, clear_cd, rp, bb_cd); self.record_draws(buf.buffer, bb, rp, bb_cd)?; self.end_rendering(buf.buffer); self.blend_buffer_final_barrier(buf.buffer, bb); } { zone!("frame buffer pass"); let rp = RenderPass::FrameBuffer; self.begin_rendering(buf.buffer, fb, clear, clear_cd, rp, fb_cd); self.record_draws(buf.buffer, fb, rp, fb_cd)?; if bb.is_some() { self.blend_buffer_copy(buf.buffer, fb, fb_cd, bb_cd)?; } self.end_rendering(buf.buffer); } self.copy_bridge_to_dmabuf(buf.buffer, fb, region); self.final_barriers(buf.buffer, fb); self.end_command_buffer(buf.buffer)?; self.create_wait_semaphores(fb, &fb_acquire_sync)?; self.submit(buf.buffer)?; self.import_release_semaphore(fb, fb_release_sync); self.store_layouts(fb, bb); self.create_pending_frame(buf, fb, blend_buffer); Ok(()) } pub(super) fn block(&self) { log::warn!("Blocking."); unsafe { if let Err(e) = self.device.device.device_wait_idle() { self.device.idl()(&e); log::error!("Could not wait for device idle: {}", ErrorFmt(e)); } } } pub fn on_drop(&self) { self.defunct.set(true); let mut pending_frames = self.pending_frames.lock(); let mut pending_uploads = self.pending_submits.lock(); if pending_frames.is_not_empty() || pending_uploads.is_not_empty() { log::warn!("Context dropped with pending frames."); self.block(); } pending_frames.values().for_each(|f| { f.waiter.take(); }); pending_frames.clear(); pending_uploads.clear(); } pub(super) fn check_defunct(&self) -> Result<(), VulkanError> { match self.defunct.get() { true => Err(VulkanError::Defunct), false => Ok(()), } } } impl Debug for VulkanRenderer { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("VulkanRenderer").finish_non_exhaustive() } } impl VulkanImage { fn assert_device(&self, device: &Device) -> Result<(), VulkanError> { if self.renderer.device.device.handle() != device.handle() { match &self.ty { VulkanImageMemory::DmaBuf(v) => { log::warn!("Mixed device use with dmabuf {}", v.template.dmabuf.id); } VulkanImageMemory::Internal(_v) => { log::warn!("Mixed device use with internal image"); } VulkanImageMemory::Blend(_v) => { log::warn!("Mixed device use with blend buffer"); } } log::info!("Image address {:?}", ptr::from_ref(self)); return Err(VulkanError::MixedVulkanDeviceUse); } Ok(()) } } impl dyn GfxTexture { fn as_vk(&self, device: &Device) -> Result<&VulkanImage, VulkanError> { let img: &VulkanImage = (self as &dyn Any) .downcast_ref() .ok_or(VulkanError::NonVulkanBuffer)?; img.assert_device(device)?; Ok(img) } pub(super) fn into_vk(self: Rc, device: &Device) -> Result, VulkanError> { let img: Rc = (self as Rc) .downcast() .ok() .ok_or(VulkanError::NonVulkanBuffer)?; img.assert_device(device)?; Ok(img) } } impl dyn GfxBlendBuffer { pub(super) fn into_vk(self: Rc, device: &Device) -> Result, VulkanError> { let img: Rc = (self as Rc) .downcast() .ok() .ok_or(VulkanError::NonVulkanBuffer)?; img.assert_device(device)?; Ok(img) } } pub(super) fn image_barrier() -> ImageMemoryBarrier2<'static> { ImageMemoryBarrier2::default().subresource_range( ImageSubresourceRange::default() .aspect_mask(ImageAspectFlags::COLOR) .layer_count(1) .level_count(1), ) } async fn await_release( sync: Option, ring: Rc, frame: Rc, renderer: Rc, ) { if let Some(sync) = sync && let Err(e) = sync.try_signaled(&ring).await { log::error!( "Could not wait for release semaphore to be signaled: {}", ErrorFmt(e) ); frame.renderer.block(); } if let Some(vs) = &frame.vulkan_sync { vs.handle_validation(); } if let Some(buf) = frame.cmd.take() { frame.renderer.gfx_command_buffers.buffers.push(buf); } for wait_semaphore in frame.wait_semaphores.take() { frame.renderer.wait_semaphores.push(wait_semaphore); } renderer.pending_frames.remove(&frame.point); } impl PaintRegion { fn intersects(&self, pos: &Point) -> bool { let mut p = *pos; for [x, y] in &mut p { *x = x.clamp(self.x1, self.x2); *y = y.clamp(self.y1, self.y2); } if p[0] == p[1] && p[2] == p[3] { return false; } if p[0] == p[2] && p[1] == p[3] { return false; } true } fn union(&self, other: &Self) -> Self { Self { x1: self.x1.min(other.x1), y1: self.y1.min(other.y1), x2: self.x2.max(other.x2), y2: self.y2.max(other.y2), } } fn constrain(&self, pos: &mut Point, tex_pos: Option<&mut Point>) -> bool { zone!("constrain"); let mut npos = *pos; for [x, y] in &mut npos { *x = x.clamp(self.x1, self.x2); *y = y.clamp(self.y1, self.y2); } if npos == *pos { return true; } if npos[0] == npos[1] && npos[2] == npos[3] { return false; } if npos[0] == npos[2] && npos[1] == npos[3] { return false; } if let Some(tp) = tex_pos { let mut ntp = *tp; for i in 0..4 { if npos[i] == pos[i] { continue; } macro_rules! sub { ($l:expr, $r:expr) => { [$l[0] - $r[0], $l[1] - $r[1]] }; } let dx = sub!(npos[i], pos[i]); let dy = sub!(pos[(i + 1) & 3], pos[i]); let dz = sub!(pos[(i + 2) & 3], pos[i]); let det = 1.0 / (dy[0] * dz[1] - dy[1] * dz[0]); let alpha = [ (dx[0] * dz[1] - dx[1] * dz[0]) * det, (dx[1] * dy[0] - dx[0] * dy[1]) * det, ]; let dy = sub!(tp[(i + 1) & 3], tp[i]); let dz = sub!(tp[(i + 2) & 3], tp[i]); ntp[i][0] += alpha[0] * dy[0] + alpha[1] * dz[0]; ntp[i][1] += alpha[0] * dy[1] + alpha[1] * dz[1]; } *tp = ntp; } *pos = npos; true } } fn constrain_to_fb(fb: &VulkanImage, rect: &Rect) -> Option<[i32; 4]> where T: Tag, { let x1 = rect.x1().max(0); let y1 = rect.y1().max(0); let x2 = rect.x2(); let y2 = rect.y2(); if x1 as u32 > fb.width || y1 as u32 > fb.height || x2 <= 0 || y2 <= 0 { return None; } let x2 = x2.min(fb.width as i32); let y2 = y2.min(fb.height as i32); if x1 == x2 || y1 == y2 { return None; } Some([x1, y1, x2, y2]) } #[derive(Default)] struct ColorTransforms { map: AHashMap<([LinearColorDescriptionId; 2], RenderIntent), ColorTransform>, } struct ColorTransform { matrix: ColorMatrix, offset: Option, } impl ColorTransforms { fn get_or_create( &mut self, src: &LinearColorDescription, dst: &ColorDescription, intent: RenderIntent, ) -> Option<&mut ColorTransform> { if src.embeds_into(&dst.linear) { return None; } let ct = match self.map.entry(([src.id, dst.linear.id], intent)) { Entry::Occupied(o) => o.into_mut(), Entry::Vacant(e) => { let matrix = src.color_transform(&dst.linear, intent); let ct = ColorTransform { matrix, offset: None, }; e.insert(ct) } }; Some(ct) } fn apply_to_color( &mut self, src: &LinearColorDescription, dst: &ColorDescription, intent: RenderIntent, mut color: Color, ) -> Color { if let Some(ct) = self.get_or_create(src, dst, intent) { color = ct.matrix * color; }; color } fn get_offset( &mut self, src: &LinearColorDescription, dst: &ColorDescription, intent: RenderIntent, uniform_buffer_offset_mask: DeviceSize, writer: &mut GenericBufferWriter, ) -> Option { let ct = self.get_or_create(src, dst, intent)?; if ct.offset.is_none() { let data = ColorManagementData { matrix: ct.matrix.to_f32(), }; let offset = writer.write(uniform_buffer_offset_mask, &data); ct.offset = Some(offset); } ct.offset } } #[derive(Default)] struct EotfArgsCache { map: AHashMap<(EotfCacheKey, bool), EotfArg>, } #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] enum EotfCacheKey { Pow(EotfPow), Bt1886(F32), } struct EotfArg { offset: DeviceSize, } impl EotfArgsCache { fn get_offset( &mut self, desc: &ColorDescription, inv: bool, uniform_buffer_offset_mask: DeviceSize, writer: &mut GenericBufferWriter, ) -> Option { let key = match desc.eotf { Eotf::Bt1886(c) => EotfCacheKey::Bt1886(c), Eotf::Pow(pow) => EotfCacheKey::Pow(pow), _ => return None, }; let ct = match self.map.entry((key, inv)) { Entry::Occupied(o) => o.into_mut(), Entry::Vacant(e) => { #[expect(unused_assignments)] let [mut arg1, mut arg2, mut arg3, mut arg4] = [0.0; 4]; if inv { match key { EotfCacheKey::Pow(pow) => arg1 = pow.inv_eotf_f32(), EotfCacheKey::Bt1886(c) => { [arg1, arg2, arg3, arg4] = bt1886_inv_eotf_args(c); } } let data = InvEotfArgs { arg1, arg2, arg3, arg4, }; let offset = writer.write(uniform_buffer_offset_mask, &data); e.insert(EotfArg { offset }) } else { match key { EotfCacheKey::Pow(pow) => arg1 = pow.eotf_f32(), EotfCacheKey::Bt1886(c) => { [arg1, arg2, arg3, arg4] = bt1886_eotf_args(c); } } let data = EotfArgs { arg1, arg2, arg3, arg4, }; let offset = writer.write(uniform_buffer_offset_mask, &data); e.insert(EotfArg { offset }) } } }; Some(ct.offset) } }