diff --git a/src/gfx_apis/vulkan.rs b/src/gfx_apis/vulkan.rs index 21fc9792..45ac392e 100644 --- a/src/gfx_apis/vulkan.rs +++ b/src/gfx_apis/vulkan.rs @@ -2,6 +2,7 @@ mod allocator; mod bo_allocator; mod command; mod descriptor; +mod descriptor_buffer; mod device; mod fence; mod format; diff --git a/src/gfx_apis/vulkan/allocator.rs b/src/gfx_apis/vulkan/allocator.rs index bdc555ec..035b5db4 100644 --- a/src/gfx_apis/vulkan/allocator.rs +++ b/src/gfx_apis/vulkan/allocator.rs @@ -7,7 +7,7 @@ use { utils::{numcell::NumCell, ptr_ext::MutPtrExt}, }, ash::{ - vk::{DeviceMemory, DeviceSize, MemoryRequirements}, + vk::{DeviceMemory, DeviceSize, MappedMemoryRange, MemoryRequirements}, Device, }, gpu_alloc::{Config, GpuAllocator, MemoryBlock, MemoryPropertyFlags, Request, UsageFlags}, @@ -71,6 +71,49 @@ impl VulkanAllocation { do_free(gpu, &device.device, block, self.mem); } } + + pub fn upload(&self, f: F) -> Result + where + F: FnOnce(*mut u8, usize) -> T, + { + let t = f(self.mem.unwrap(), self.size as usize); + if let Some(mask) = self.coherency_mask { + let range = self.incoherent_range(mask); + let res = unsafe { self.device().device.flush_mapped_memory_ranges(&[range]) }; + res.map_err(VulkanError::FlushMemory)?; + } + Ok(t) + } + + pub fn download(&self, f: F) -> Result + where + F: FnOnce(*const u8, usize) -> T, + { + if let Some(mask) = self.coherency_mask { + let range = self.incoherent_range(mask); + let res = unsafe { + self.device() + .device + .invalidate_mapped_memory_ranges(&[range]) + }; + res.map_err(VulkanError::FlushMemory)?; + } + Ok(f(self.mem.unwrap(), self.size as usize)) + } + + fn incoherent_range(&self, mask: u64) -> MappedMemoryRange { + MappedMemoryRange::default() + .memory(self.memory) + .offset(self.offset & !mask) + .size((self.size + mask) & !mask) + } + + fn device(&self) -> &VulkanDevice { + match &self.allocator { + AllocatorType::Local(l) => &l.storage.device, + AllocatorType::Threaded { allocator, .. } => &allocator.storage.device, + } + } } impl Drop for VulkanAllocation { @@ -128,7 +171,7 @@ impl VulkanDevice { ) }; let mut props = props.map_err(VulkanError::GetDeviceProperties)?; - props.buffer_device_address = false; + props.buffer_device_address = self.descriptor_buffer.is_some(); let non_coherent_atom_size = props.non_coherent_atom_size; let allocator = GpuAllocator::new(config, props); Ok(Rc::new(VulkanAllocatorType { diff --git a/src/gfx_apis/vulkan/descriptor.rs b/src/gfx_apis/vulkan/descriptor.rs index 8282a1e8..90ea32b7 100644 --- a/src/gfx_apis/vulkan/descriptor.rs +++ b/src/gfx_apis/vulkan/descriptor.rs @@ -1,8 +1,9 @@ use { crate::gfx_apis::vulkan::{device::VulkanDevice, sampler::VulkanSampler, VulkanError}, + arrayvec::ArrayVec, ash::vk::{ DescriptorSetLayout, DescriptorSetLayoutBinding, DescriptorSetLayoutCreateFlags, - DescriptorSetLayoutCreateInfo, DescriptorType, ShaderStageFlags, + DescriptorSetLayoutCreateInfo, DescriptorType, DeviceSize, ShaderStageFlags, }, std::{rc::Rc, slice}, }; @@ -10,7 +11,10 @@ use { pub(super) struct VulkanDescriptorSetLayout { pub(super) device: Rc, pub(super) layout: DescriptorSetLayout, - pub(super) _sampler: Rc, + pub(super) size: DeviceSize, + pub(super) offsets: ArrayVec, + pub(super) _sampler: Option>, + pub(super) has_sampler: bool, } impl Drop for VulkanDescriptorSetLayout { @@ -25,7 +29,7 @@ impl Drop for VulkanDescriptorSetLayout { impl VulkanDevice { pub(super) fn create_descriptor_set_layout( - &self, + self: &Rc, sampler: &Rc, ) -> Result, VulkanError> { let immutable_sampler = [sampler.sampler]; @@ -34,15 +38,34 @@ impl VulkanDevice { .immutable_samplers(&immutable_sampler) .descriptor_count(1) .descriptor_type(DescriptorType::COMBINED_IMAGE_SAMPLER); + let mut flags = DescriptorSetLayoutCreateFlags::empty(); + if self.descriptor_buffer.is_some() { + flags |= DescriptorSetLayoutCreateFlags::DESCRIPTOR_BUFFER_EXT; + } else { + flags |= DescriptorSetLayoutCreateFlags::PUSH_DESCRIPTOR_KHR; + } let create_info = DescriptorSetLayoutCreateInfo::default() .bindings(slice::from_ref(&binding)) - .flags(DescriptorSetLayoutCreateFlags::PUSH_DESCRIPTOR_KHR); + .flags(flags); let layout = unsafe { self.device.create_descriptor_set_layout(&create_info, None) }; let layout = layout.map_err(VulkanError::CreateDescriptorSetLayout)?; + let mut size = 0; + let mut offsets = ArrayVec::new(); + if let Some(db) = &self.descriptor_buffer { + size = unsafe { db.get_descriptor_set_layout_size(layout) }; + size = + (size + self.descriptor_buffer_offset_mask) & !self.descriptor_buffer_offset_mask; + unsafe { + offsets.push(db.get_descriptor_set_layout_binding_offset(layout, 0)); + } + } Ok(Rc::new(VulkanDescriptorSetLayout { - device: sampler.device.clone(), + device: self.clone(), layout, - _sampler: sampler.clone(), + size, + offsets, + _sampler: Some(sampler.clone()), + has_sampler: true, })) } } diff --git a/src/gfx_apis/vulkan/descriptor_buffer.rs b/src/gfx_apis/vulkan/descriptor_buffer.rs new file mode 100644 index 00000000..6ea8434a --- /dev/null +++ b/src/gfx_apis/vulkan/descriptor_buffer.rs @@ -0,0 +1,206 @@ +use { + crate::{ + gfx_apis::vulkan::{ + allocator::{VulkanAllocation, VulkanAllocator}, + descriptor::VulkanDescriptorSetLayout, + device::VulkanDevice, + VulkanError, + }, + utils::on_drop::OnDrop, + }, + ash::vk::{ + Buffer, BufferCreateInfo, BufferDeviceAddressInfo, BufferUsageFlags, DeviceAddress, + DeviceSize, + }, + gpu_alloc::UsageFlags, + std::{cell::RefCell, mem::ManuallyDrop, ops::Deref, rc::Rc}, +}; + +pub struct VulkanDescriptorBufferCache { + device: Rc, + allocator: Rc, + buffers: RefCell>, + has_sampler: bool, +} + +pub struct VulkanDescriptorBuffer { + cache: Rc, + pub buffer: ManuallyDrop, +} + +pub struct VulkanDescriptorBufferUnused { + device: Rc, + pub size: DeviceSize, + pub buffer: Buffer, + pub allocation: VulkanAllocation, + pub address: DeviceAddress, +} + +pub struct VulkanDescriptorBufferWriter { + set_size: usize, + buffer: Vec, +} + +pub struct VulkanDescriptorBufferSetWriter<'a> { + set: &'a mut [u8], +} + +impl VulkanDescriptorBufferCache { + pub fn new( + device: &Rc, + allocator: &Rc, + layout: &VulkanDescriptorSetLayout, + ) -> Self { + Self { + device: device.clone(), + allocator: allocator.clone(), + buffers: Default::default(), + has_sampler: layout.has_sampler, + } + } + + pub fn allocate( + self: &Rc, + capacity: DeviceSize, + ) -> Result { + let mut smallest = None; + let mut smallest_size = DeviceSize::MAX; + let mut fitting = None; + let mut fitting_size = DeviceSize::MAX; + let buffers = &mut *self.buffers.borrow_mut(); + for (idx, buffer) in buffers.iter().enumerate() { + if buffer.size >= capacity { + if buffer.size < fitting_size { + fitting = Some(idx); + fitting_size = buffer.size; + } + } else { + if buffer.size < smallest_size { + smallest = Some(idx); + smallest_size = buffer.size; + } + } + } + if let Some(idx) = fitting { + return Ok(VulkanDescriptorBuffer { + cache: self.clone(), + buffer: ManuallyDrop::new(buffers.swap_remove(idx)), + }); + } + if let Some(idx) = smallest { + log::debug!("discarding size {}", smallest_size); + buffers.swap_remove(idx); + } + let size = capacity.checked_next_power_of_two().unwrap(); + log::debug!("allocating size {}", size); + let buffer = { + let usage = self.usage(); + let info = BufferCreateInfo::default().size(size).usage(usage); + unsafe { + self.device + .device + .create_buffer(&info, None) + .map_err(VulkanError::CreateBuffer)? + } + }; + let destroy_buffer = OnDrop(|| unsafe { self.device.device.destroy_buffer(buffer, None) }); + let memory_requirements = + unsafe { self.device.device.get_buffer_memory_requirements(buffer) }; + let allocation = { + let flags = UsageFlags::UPLOAD + | UsageFlags::FAST_DEVICE_ACCESS + | UsageFlags::HOST_ACCESS + | UsageFlags::DEVICE_ADDRESS; + self.allocator.alloc(&memory_requirements, flags, true)? + }; + unsafe { + self.device + .device + .bind_buffer_memory(buffer, allocation.memory, allocation.offset) + .map_err(VulkanError::BindBufferMemory)?; + } + destroy_buffer.forget(); + let address = { + let info = BufferDeviceAddressInfo::default().buffer(buffer); + unsafe { self.device.device.get_buffer_device_address(&info) } + }; + Ok(VulkanDescriptorBuffer { + cache: self.clone(), + buffer: ManuallyDrop::new(VulkanDescriptorBufferUnused { + device: self.device.clone(), + size: allocation.size, + buffer, + allocation, + address, + }), + }) + } + + pub fn usage(&self) -> BufferUsageFlags { + let mut usage = BufferUsageFlags::RESOURCE_DESCRIPTOR_BUFFER_EXT + | BufferUsageFlags::SHADER_DEVICE_ADDRESS; + if self.has_sampler { + usage |= BufferUsageFlags::SAMPLER_DESCRIPTOR_BUFFER_EXT; + } + usage + } +} + +impl Drop for VulkanDescriptorBuffer { + fn drop(&mut self) { + let buffer = unsafe { ManuallyDrop::take(&mut self.buffer) }; + self.cache.buffers.borrow_mut().push(buffer); + } +} + +impl Drop for VulkanDescriptorBufferUnused { + fn drop(&mut self) { + unsafe { + self.device.device.destroy_buffer(self.buffer, None); + } + } +} + +impl VulkanDescriptorBufferWriter { + pub fn new(layout: &VulkanDescriptorSetLayout) -> Self { + Self { + set_size: layout.size as usize, + buffer: Default::default(), + } + } + + pub fn clear(&mut self) { + self.buffer.clear(); + } + + pub fn next_offset(&self) -> DeviceSize { + self.buffer.len() as DeviceSize + } + + pub fn add_set(&mut self) -> VulkanDescriptorBufferSetWriter<'_> { + let buffer = &mut self.buffer; + let lo = buffer.len(); + buffer.resize(lo + self.set_size, 0); + VulkanDescriptorBufferSetWriter { + set: &mut buffer[lo..], + } + } +} + +impl VulkanDescriptorBufferSetWriter<'_> { + pub fn write(&mut self, offset: DeviceSize, data: &[u8]) { + let offset = offset as usize; + let set = &mut *self.set; + assert!(offset <= set.len()); + assert!(data.len() <= set.len() - offset); + set[offset..offset + data.len()].copy_from_slice(data); + } +} + +impl Deref for VulkanDescriptorBufferWriter { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.buffer + } +} diff --git a/src/gfx_apis/vulkan/device.rs b/src/gfx_apis/vulkan/device.rs index b882cbd1..5f7a6333 100644 --- a/src/gfx_apis/vulkan/device.rs +++ b/src/gfx_apis/vulkan/device.rs @@ -19,22 +19,23 @@ use { arrayvec::ArrayVec, ash::{ ext::{ - external_memory_dma_buf, image_drm_format_modifier, physical_device_drm, - queue_family_foreign, + descriptor_buffer, external_memory_dma_buf, image_drm_format_modifier, + physical_device_drm, queue_family_foreign, }, khr::{ driver_properties, external_fence_fd, external_memory_fd, external_semaphore_fd, push_descriptor, }, vk::{ - DeviceCreateInfo, DeviceQueueCreateInfo, ExternalSemaphoreFeatureFlags, + DeviceCreateInfo, DeviceQueueCreateInfo, DeviceSize, ExternalSemaphoreFeatureFlags, ExternalSemaphoreHandleTypeFlags, ExternalSemaphoreProperties, MemoryPropertyFlags, - MemoryType, PhysicalDevice, PhysicalDeviceDriverProperties, - PhysicalDeviceDriverPropertiesKHR, PhysicalDeviceDrmPropertiesEXT, - PhysicalDeviceDynamicRenderingFeatures, PhysicalDeviceExternalSemaphoreInfo, - PhysicalDeviceProperties, PhysicalDeviceProperties2, - PhysicalDeviceSynchronization2Features, PhysicalDeviceTimelineSemaphoreFeatures, Queue, - QueueFlags, MAX_MEMORY_TYPES, + MemoryType, PhysicalDevice, PhysicalDeviceBufferDeviceAddressFeatures, + PhysicalDeviceDescriptorBufferFeaturesEXT, PhysicalDeviceDescriptorBufferPropertiesEXT, + PhysicalDeviceDriverProperties, PhysicalDeviceDriverPropertiesKHR, + PhysicalDeviceDrmPropertiesEXT, PhysicalDeviceDynamicRenderingFeatures, + PhysicalDeviceExternalSemaphoreInfo, PhysicalDeviceProperties, + PhysicalDeviceProperties2, PhysicalDeviceSynchronization2Features, + PhysicalDeviceTimelineSemaphoreFeatures, Queue, QueueFlags, MAX_MEMORY_TYPES, }, Device, }, @@ -59,6 +60,7 @@ pub struct VulkanDevice { pub(super) external_fence_fd: external_fence_fd::Device, pub(super) push_descriptor: push_descriptor::Device, pub(super) image_drm_format_modifier: image_drm_format_modifier::Device, + pub(super) descriptor_buffer: Option, pub(super) formats: AHashMap, pub(super) memory_types: ArrayVec, pub(super) graphics_queue: Queue, @@ -66,6 +68,8 @@ pub struct VulkanDevice { pub(super) transfer_queue: Option, pub(super) distinct_transfer_queue_family_idx: Option, pub(super) transfer_granularity_mask: (u32, u32), + pub(super) descriptor_buffer_offset_mask: DeviceSize, + pub(super) combined_image_sampler_descriptor_size: usize, } impl Drop for VulkanDevice { @@ -266,6 +270,7 @@ impl VulkanInstance { return Err(VulkanError::MissingDeviceExtension(ext)); } } + let supports_descriptor_buffer = extensions.contains_key(descriptor_buffer::NAME); let (graphics_queue_family_idx, transfer_queue_family) = self.find_queues(phy_dev)?; let mut distinct_transfer_queue_family_idx = None; let mut transfer_granularity_mask = (0, 0); @@ -278,16 +283,23 @@ impl VulkanInstance { if !self.supports_semaphore_import(phy_dev) { return Err(VulkanError::SyncFileImport); } - let enabled_extensions: Vec<_> = REQUIRED_DEVICE_EXTENSIONS + let mut enabled_extensions: Vec<_> = REQUIRED_DEVICE_EXTENSIONS .iter() .map(|n| n.as_ptr()) .collect(); + if supports_descriptor_buffer { + enabled_extensions.push(descriptor_buffer::NAME.as_ptr()); + } let mut semaphore_features = PhysicalDeviceTimelineSemaphoreFeatures::default().timeline_semaphore(true); let mut synchronization2_features = PhysicalDeviceSynchronization2Features::default().synchronization2(true); let mut dynamic_rendering_features = PhysicalDeviceDynamicRenderingFeatures::default().dynamic_rendering(true); + let mut descriptor_buffer_features = + PhysicalDeviceDescriptorBufferFeaturesEXT::default().descriptor_buffer(true); + let mut buffer_device_address_features = + PhysicalDeviceBufferDeviceAddressFeatures::default().buffer_device_address(true); let mut queue_create_infos = ArrayVec::<_, 2>::new(); queue_create_infos.push( DeviceQueueCreateInfo::default() @@ -301,12 +313,17 @@ impl VulkanInstance { .queue_priorities(&[1.0]), ); } - let device_create_info = DeviceCreateInfo::default() + let mut device_create_info = DeviceCreateInfo::default() .push_next(&mut semaphore_features) .push_next(&mut synchronization2_features) .push_next(&mut dynamic_rendering_features) .queue_create_infos(&queue_create_infos) .enabled_extension_names(&enabled_extensions); + if supports_descriptor_buffer { + device_create_info = device_create_info + .push_next(&mut descriptor_buffer_features) + .push_next(&mut buffer_device_address_features); + } let device = unsafe { self.instance .create_device(phy_dev, &device_create_info, None) @@ -339,6 +356,27 @@ impl VulkanInstance { let push_descriptor = push_descriptor::Device::new(&self.instance, &device); let image_drm_format_modifier = image_drm_format_modifier::Device::new(&self.instance, &device); + let descriptor_buffer = supports_descriptor_buffer + .then(|| descriptor_buffer::Device::new(&self.instance, &device)); + let mut descriptor_buffer_offset_mask = 0; + let mut combined_image_sampler_descriptor_size = 0; + if supports_descriptor_buffer { + let mut descriptor_buffer_props = + PhysicalDeviceDescriptorBufferPropertiesEXT::default(); + let mut props = + PhysicalDeviceProperties2::default().push_next(&mut descriptor_buffer_props); + unsafe { + self.instance + .get_physical_device_properties2(phy_dev, &mut props); + } + descriptor_buffer_offset_mask = descriptor_buffer_props + .descriptor_buffer_offset_alignment + .checked_next_power_of_two() + .unwrap() + - 1; + combined_image_sampler_descriptor_size = + descriptor_buffer_props.combined_image_sampler_descriptor_size; + } let memory_properties = unsafe { self.instance.get_physical_device_memory_properties(phy_dev) }; let memory_types = memory_properties.memory_types @@ -366,6 +404,7 @@ impl VulkanInstance { external_fence_fd, push_descriptor, image_drm_format_modifier, + descriptor_buffer, formats, memory_types, graphics_queue, @@ -373,6 +412,8 @@ impl VulkanInstance { transfer_queue, distinct_transfer_queue_family_idx, transfer_granularity_mask, + descriptor_buffer_offset_mask, + combined_image_sampler_descriptor_size, })) } } diff --git a/src/gfx_apis/vulkan/image.rs b/src/gfx_apis/vulkan/image.rs index b7628ce6..088be323 100644 --- a/src/gfx_apis/vulkan/image.rs +++ b/src/gfx_apis/vulkan/image.rs @@ -19,7 +19,8 @@ use { }, ash::vk::{ BindImageMemoryInfo, BindImagePlaneMemoryInfo, ComponentMapping, ComponentSwizzle, - DeviceMemory, Extent3D, ExternalMemoryHandleTypeFlags, ExternalMemoryImageCreateInfo, + DescriptorDataEXT, DescriptorGetInfoEXT, DescriptorImageInfo, DescriptorType, DeviceMemory, + DeviceSize, Extent3D, ExternalMemoryHandleTypeFlags, ExternalMemoryImageCreateInfo, FormatFeatureFlags, Image, ImageAspectFlags, ImageCreateFlags, ImageCreateInfo, ImageDrmFormatModifierExplicitCreateInfoEXT, ImageLayout, ImageMemoryRequirementsInfo2, ImagePlaneMemoryRequirementsInfo, ImageSubresourceRange, ImageTiling, ImageType, @@ -62,6 +63,9 @@ pub struct VulkanImage { pub(super) queue_state: Cell, pub(super) ty: VulkanImageMemory, pub(super) bridge: Option, + pub(super) shader_read_only_optimal_descriptor: Box<[u8]>, + pub(super) descriptor_buffer_version: Cell, + pub(super) descriptor_buffer_offset: Cell, } #[derive(Copy, Clone, Eq, PartialEq, Debug)] @@ -228,6 +232,29 @@ impl VulkanDevice { } } +impl VulkanRenderer { + pub(super) fn sampler_read_only_descriptor(&self, view: ImageView) -> Box<[u8]> { + let Some(db) = &self.device.descriptor_buffer else { + return Box::new([]); + }; + let mut buf = + vec![0; self.device.combined_image_sampler_descriptor_size].into_boxed_slice(); + let image_info = DescriptorImageInfo::default() + .sampler(self.sampler.sampler) + .image_view(view) + .image_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL); + let info = DescriptorGetInfoEXT::default() + .ty(DescriptorType::COMBINED_IMAGE_SAMPLER) + .data(DescriptorDataEXT { + p_combined_image_sampler: &image_info, + }); + unsafe { + db.get_descriptor(&info, &mut buf); + } + buf + } +} + impl VulkanDmaBufImageTemplate { pub fn create_framebuffer(self: &Rc) -> Result, VulkanError> { self.create_image(true) @@ -420,6 +447,11 @@ impl VulkanDmaBufImageTemplate { family: QueueFamily::Gfx, }), bridge, + shader_read_only_optimal_descriptor: self + .renderer + .sampler_read_only_descriptor(texture_view), + descriptor_buffer_version: Cell::new(0), + descriptor_buffer_offset: Cell::new(0), })) } diff --git a/src/gfx_apis/vulkan/pipeline.rs b/src/gfx_apis/vulkan/pipeline.rs index c1382295..1721485c 100644 --- a/src/gfx_apis/vulkan/pipeline.rs +++ b/src/gfx_apis/vulkan/pipeline.rs @@ -12,7 +12,7 @@ use { vk::{ BlendFactor, BlendOp, ColorComponentFlags, CullModeFlags, DynamicState, FrontFace, GraphicsPipelineCreateInfo, Pipeline, PipelineCache, PipelineColorBlendAttachmentState, - PipelineColorBlendStateCreateInfo, PipelineDynamicStateCreateInfo, + PipelineColorBlendStateCreateInfo, PipelineCreateFlags, PipelineDynamicStateCreateInfo, PipelineInputAssemblyStateCreateInfo, PipelineLayout, PipelineLayoutCreateInfo, PipelineMultisampleStateCreateInfo, PipelineRasterizationStateCreateInfo, PipelineRenderingCreateInfo, PipelineShaderStageCreateInfo, @@ -136,8 +136,13 @@ impl VulkanDevice { .scissor_count(1); let mut pipeline_rendering_create_info = PipelineRenderingCreateInfo::default() .color_attachment_formats(slice::from_ref(&info.format)); + let mut flags = PipelineCreateFlags::empty(); + if self.descriptor_buffer.is_some() { + flags |= PipelineCreateFlags::DESCRIPTOR_BUFFER_EXT; + } let create_info = GraphicsPipelineCreateInfo::default() .push_next(&mut pipeline_rendering_create_info) + .flags(flags) .stages(&stages) .input_assembly_state(&input_assembly_state) .vertex_input_state(&vertex_input_state) diff --git a/src/gfx_apis/vulkan/renderer.rs b/src/gfx_apis/vulkan/renderer.rs index d8e44fca..3b06b10d 100644 --- a/src/gfx_apis/vulkan/renderer.rs +++ b/src/gfx_apis/vulkan/renderer.rs @@ -11,10 +11,14 @@ use { allocator::{VulkanAllocator, VulkanThreadedAllocator}, command::{VulkanCommandBuffer, VulkanCommandPool}, descriptor::VulkanDescriptorSetLayout, + descriptor_buffer::{ + VulkanDescriptorBuffer, VulkanDescriptorBufferCache, VulkanDescriptorBufferWriter, + }, device::VulkanDevice, fence::VulkanFence, image::{QueueFamily, QueueState, QueueTransfer, VulkanImage, VulkanImageMemory}, pipeline::{PipelineCreateInfo, VulkanPipeline}, + sampler::VulkanSampler, semaphore::VulkanSemaphore, shaders::{ FillPushConstants, TexPushConstants, VulkanShader, FILL_FRAG, FILL_VERT, TEX_FRAG, @@ -33,12 +37,13 @@ use { vk::{ AccessFlags2, AttachmentLoadOp, AttachmentStoreOp, ClearColorValue, ClearValue, CommandBuffer, CommandBufferBeginInfo, CommandBufferSubmitInfo, - CommandBufferUsageFlags, CopyImageInfo2, DependencyInfoKHR, DescriptorImageInfo, - DescriptorType, Extent2D, Extent3D, ImageAspectFlags, ImageCopy2, ImageLayout, - ImageMemoryBarrier2, ImageSubresourceLayers, ImageSubresourceRange, PipelineBindPoint, - PipelineStageFlags2, Rect2D, RenderingAttachmentInfo, RenderingInfo, - SemaphoreSubmitInfo, SemaphoreSubmitInfoKHR, ShaderStageFlags, SubmitInfo2, Viewport, - WriteDescriptorSet, QUEUE_FAMILY_FOREIGN_EXT, + CommandBufferUsageFlags, CopyImageInfo2, DependencyInfoKHR, + DescriptorBufferBindingInfoEXT, DescriptorImageInfo, DescriptorType, DeviceSize, + Extent2D, Extent3D, ImageAspectFlags, ImageCopy2, ImageLayout, ImageMemoryBarrier2, + ImageSubresourceLayers, ImageSubresourceRange, PipelineBindPoint, PipelineStageFlags2, + Rect2D, RenderingAttachmentInfo, RenderingInfo, SemaphoreSubmitInfo, + SemaphoreSubmitInfoKHR, ShaderStageFlags, SubmitInfo2, Viewport, WriteDescriptorSet, + QUEUE_FAMILY_FOREIGN_EXT, }, Device, }, @@ -47,7 +52,7 @@ use { std::{ cell::{Cell, RefCell}, fmt::{Debug, Formatter}, - mem, + mem, ptr, rc::Rc, slice, }, @@ -77,6 +82,10 @@ pub struct VulkanRenderer { pub(super) defunct: Cell, pub(super) pending_cpu_jobs: CopyHashMap, pub(super) shm_allocator: Rc, + pub(super) sampler: Rc, + pub(super) tex_sampler_descriptor_buffer_cache: Rc, + pub(super) descriptor_buffer_version: NumCell, + pub(super) tex_descriptor_buffer_writer: RefCell, } pub(super) struct CachedCommandBuffers { @@ -128,6 +137,7 @@ pub(super) struct Memory { wait_semaphore_infos: Vec>, release_fence: Option>, release_sync_file: Option, + descriptor_buffer: Option, } pub(super) struct PendingFrame { @@ -138,6 +148,7 @@ pub(super) struct PendingFrame { wait_semaphores: Cell>>, waiter: Cell>>, _release_fence: Option>, + _descriptor_buffer: Option, } pub(super) struct VulkanFormatPipelines { @@ -195,6 +206,14 @@ impl VulkanDevice { .collect(); let allocator = self.create_allocator()?; let shm_allocator = self.create_threaded_allocator()?; + let tex_descriptor_buffer_cache = Rc::new(VulkanDescriptorBufferCache::new( + self, + &allocator, + &tex_descriptor_set_layout, + )); + let tex_descriptor_buffer_writer = RefCell::new(VulkanDescriptorBufferWriter::new( + &tex_descriptor_set_layout, + )); let render = Rc::new(VulkanRenderer { formats: Rc::new(formats), device: self.clone(), @@ -218,6 +237,10 @@ impl VulkanDevice { defunct: Cell::new(false), pending_cpu_jobs: Default::default(), shm_allocator, + sampler, + tex_sampler_descriptor_buffer_cache: tex_descriptor_buffer_cache, + descriptor_buffer_version: Default::default(), + tex_descriptor_buffer_writer, }); render.get_or_create_pipelines(XRGB8888.vk_format)?; Ok(render) @@ -280,6 +303,51 @@ impl VulkanRenderer { self.last_point.fetch_add(1) + 1 } + fn create_descriptor_buffer( + &self, + buf: CommandBuffer, + opts: &[GfxApiOpt], + ) -> Result<(), VulkanError> { + let Some(db) = &self.device.descriptor_buffer else { + return Ok(()); + }; + zone!("create_descriptor_buffer"); + let version = self.descriptor_buffer_version.add_fetch(1); + let memory = &mut *self.memory.borrow_mut(); + let writer = &mut *self.tex_descriptor_buffer_writer.borrow_mut(); + writer.clear(); + for cmd in opts { + let GfxApiOpt::CopyTexture(c) = cmd else { + continue; + }; + let tex = c.tex.clone().into_vk(&self.device.device); + if tex.descriptor_buffer_version.replace(version) == version { + continue; + } + let offset = writer.next_offset(); + tex.descriptor_buffer_offset.set(offset); + let mut writer = writer.add_set(); + writer.write( + self.tex_descriptor_set_layout.offsets[0], + &tex.shader_read_only_optimal_descriptor, + ); + } + let buffer = self + .tex_sampler_descriptor_buffer_cache + .allocate(writer.len() as DeviceSize)?; + buffer.buffer.allocation.upload(|ptr, _| unsafe { + ptr::copy_nonoverlapping(writer.as_ptr(), ptr, writer.len()) + })?; + let info = DescriptorBufferBindingInfoEXT::default() + .usage(self.tex_sampler_descriptor_buffer_cache.usage()) + .address(buffer.buffer.address); + unsafe { + db.cmd_bind_descriptor_buffers(buf, slice::from_ref(&info)); + } + memory.descriptor_buffer = Some(buffer); + Ok(()) + } + fn collect_memory(&self, opts: &[GfxApiOpt]) { zone!("collect_memory"); let mut memory = self.memory.borrow_mut(); @@ -543,17 +611,28 @@ impl VulkanRenderer { let image_info = DescriptorImageInfo::default() .image_view(tex.texture_view) .image_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL); - let write_descriptor_set = WriteDescriptorSet::default() - .descriptor_type(DescriptorType::COMBINED_IMAGE_SAMPLER) - .image_info(slice::from_ref(&image_info)); unsafe { - self.device.push_descriptor.cmd_push_descriptor_set( - buf, - PipelineBindPoint::GRAPHICS, - pipeline.pipeline_layout, - 0, - slice::from_ref(&write_descriptor_set), - ); + if let Some(db) = &self.device.descriptor_buffer { + db.cmd_set_descriptor_buffer_offsets( + buf, + PipelineBindPoint::GRAPHICS, + pipeline.pipeline_layout, + 0, + &[0], + &[tex.descriptor_buffer_offset.get()], + ); + } else { + let write_descriptor_set = WriteDescriptorSet::default() + .descriptor_type(DescriptorType::COMBINED_IMAGE_SAMPLER) + .image_info(slice::from_ref(&image_info)); + self.device.push_descriptor.cmd_push_descriptor_set( + buf, + PipelineBindPoint::GRAPHICS, + pipeline.pipeline_layout, + 0, + slice::from_ref(&write_descriptor_set), + ); + } dev.cmd_push_constants( buf, pipeline.pipeline_layout, @@ -864,6 +943,7 @@ impl VulkanRenderer { wait_semaphores: Cell::new(mem::take(&mut memory.wait_semaphores)), waiter: Cell::new(None), _release_fence: memory.release_fence.take(), + _descriptor_buffer: memory.descriptor_buffer.take(), }); self.pending_frames.set(frame.point, frame.clone()); let future = self.eng.spawn( @@ -921,6 +1001,7 @@ impl VulkanRenderer { let buf = self.gfx_command_buffers.allocate()?; self.collect_memory(opts); self.begin_command_buffer(buf.buffer)?; + self.create_descriptor_buffer(buf.buffer, opts)?; self.initial_barriers(buf.buffer, fb)?; self.begin_rendering(buf.buffer, fb, clear); self.set_viewport(buf.buffer, fb); diff --git a/src/gfx_apis/vulkan/shm_image.rs b/src/gfx_apis/vulkan/shm_image.rs index 8f84c467..8b08a2a7 100644 --- a/src/gfx_apis/vulkan/shm_image.rs +++ b/src/gfx_apis/vulkan/shm_image.rs @@ -459,6 +459,9 @@ impl VulkanRenderer { }), ty: VulkanImageMemory::Internal(shm), bridge: None, + shader_read_only_optimal_descriptor: self.sampler_read_only_descriptor(view), + descriptor_buffer_version: Cell::new(0), + descriptor_buffer_offset: Cell::new(0), }); let shm = match &img.ty { VulkanImageMemory::DmaBuf(_) => unreachable!(), diff --git a/src/gfx_apis/vulkan/staging.rs b/src/gfx_apis/vulkan/staging.rs index d8fa685b..d168e64e 100644 --- a/src/gfx_apis/vulkan/staging.rs +++ b/src/gfx_apis/vulkan/staging.rs @@ -14,7 +14,7 @@ use { }, }, ash::{ - vk::{Buffer, BufferCreateInfo, BufferUsageFlags, MappedMemoryRange}, + vk::{Buffer, BufferCreateInfo, BufferUsageFlags}, Device, }, gpu_alloc::UsageFlags, @@ -146,32 +146,14 @@ impl VulkanStagingBuffer { where F: FnOnce(*mut u8, usize) -> T, { - let t = f(self.allocation.mem.unwrap(), self.size as usize); - if let Some(mask) = self.allocation.coherency_mask { - let range = self.incoherent_range(mask); - let res = unsafe { self.device.device.flush_mapped_memory_ranges(&[range]) }; - res.map_err(VulkanError::FlushMemory)?; - } - Ok(t) + self.allocation.upload(f) } pub fn download(&self, f: F) -> Result where F: FnOnce(*const u8, usize) -> T, { - if let Some(mask) = self.allocation.coherency_mask { - let range = self.incoherent_range(mask); - let res = unsafe { self.device.device.invalidate_mapped_memory_ranges(&[range]) }; - res.map_err(VulkanError::FlushMemory)?; - } - Ok(f(self.allocation.mem.unwrap(), self.size as usize)) - } - - fn incoherent_range(&self, mask: u64) -> MappedMemoryRange { - MappedMemoryRange::default() - .memory(self.allocation.memory) - .offset(self.allocation.offset & !mask) - .size((self.allocation.size + mask) & !mask) + self.allocation.download(f) } }