diff --git a/src/copy_device.rs b/src/copy_device.rs index 70191dd0..3f34a533 100644 --- a/src/copy_device.rs +++ b/src/copy_device.rs @@ -5,7 +5,6 @@ use { format::{FORMATS, Format}, gfx_api::FdSync, io_uring::IoUring, - rect::{Rect, Region}, utils::{ clonecell::CloneCell, errorfmt::ErrorFmt, @@ -21,12 +20,10 @@ use { }, vulkan_core::{ self, VULKAN_API_VERSION, VulkanCoreError, VulkanCoreInstance, device::VulkanDeviceInf, - map_extension_properties, sync::VulkanDeviceSyncExt, - timeline_semaphore::VulkanDeviceTimelineSemaphoreExt, + map_extension_properties, timeline_semaphore::VulkanDeviceTimelineSemaphoreExt, }, }, ahash::AHashMap, - arrayvec::ArrayVec, ash::{ Device, ext::{ @@ -35,12 +32,9 @@ use { }, khr::{external_fence_fd, external_memory_fd, external_semaphore_fd}, vk::{ - self, AccessFlags2, BindImageMemoryInfo, BindImagePlaneMemoryInfo, BlitImageInfo2, - BufferCopy2, BufferCreateInfo, BufferImageCopy2, BufferMemoryBarrier2, - BufferUsageFlags, CommandBuffer, CommandBufferAllocateInfo, CommandBufferBeginInfo, - CommandBufferSubmitInfo, CommandBufferUsageFlags, CommandPoolCreateFlags, - CommandPoolCreateInfo, CopyBufferInfo2, CopyBufferToImageInfo2, CopyImageInfo2, - CopyImageToBufferInfo2, DependencyInfo, DeviceCreateInfo, DeviceMemory, + self, BindImageMemoryInfo, BindImagePlaneMemoryInfo, BufferCopy2, BufferCreateInfo, + BufferImageCopy2, BufferUsageFlags, CommandBuffer, CommandBufferAllocateInfo, + CommandPoolCreateFlags, CommandPoolCreateInfo, DeviceCreateInfo, DeviceMemory, DeviceQueueCreateInfo, DrmFormatModifierPropertiesEXT, DrmFormatModifierPropertiesListEXT, ExportMemoryAllocateInfo, Extent3D, ExternalBufferProperties, ExternalFenceFeatureFlags, ExternalFenceHandleTypeFlags, @@ -48,23 +42,21 @@ use { ExternalMemoryBufferCreateInfo, ExternalMemoryBufferCreateInfoKHR, ExternalMemoryFeatureFlags, ExternalMemoryHandleTypeFlags, ExternalMemoryImageCreateInfo, ExternalSemaphoreFeatureFlags, - ExternalSemaphoreHandleTypeFlags, ExternalSemaphoreProperties, Filter, - FormatFeatureFlags, FormatProperties2, ImageAspectFlags, ImageBlit2, ImageCopy2, - ImageCreateFlags, ImageCreateInfo, ImageDrmFormatModifierExplicitCreateInfoEXT, - ImageFormatProperties2, ImageLayout, ImageMemoryBarrier2, ImageMemoryRequirementsInfo2, - ImagePlaneMemoryRequirementsInfo, ImageSubresourceLayers, ImageSubresourceRange, - ImageTiling, ImageType, ImageUsageFlags, ImportMemoryFdInfoKHR, - ImportSemaphoreFdInfoKHR, MemoryAllocateInfo, MemoryDedicatedAllocateInfo, - MemoryFdPropertiesKHR, MemoryGetFdInfoKHR, MemoryPropertyFlags, MemoryRequirements2, - MemoryType, Offset3D, PhysicalDevice, PhysicalDeviceDrmPropertiesEXT, + ExternalSemaphoreHandleTypeFlags, ExternalSemaphoreProperties, FormatFeatureFlags, + FormatProperties2, ImageAspectFlags, ImageBlit2, ImageCopy2, ImageCreateFlags, + ImageCreateInfo, ImageDrmFormatModifierExplicitCreateInfoEXT, ImageFormatProperties2, + ImageLayout, ImageMemoryRequirementsInfo2, ImagePlaneMemoryRequirementsInfo, + ImageTiling, ImageType, ImageUsageFlags, ImportMemoryFdInfoKHR, ImportSemaphoreFdInfoKHR, + MemoryAllocateInfo, MemoryDedicatedAllocateInfo, MemoryFdPropertiesKHR, + MemoryGetFdInfoKHR, MemoryPropertyFlags, MemoryRequirements2, MemoryType, + PhysicalDevice, PhysicalDeviceDrmPropertiesEXT, PhysicalDeviceExternalBufferInfo, PhysicalDeviceExternalFenceInfo, PhysicalDeviceExternalImageFormatInfoKHR, PhysicalDeviceExternalSemaphoreInfo, PhysicalDeviceFeatures2, PhysicalDeviceImageDrmFormatModifierInfoEXT, PhysicalDeviceImageFormatInfo2, PhysicalDeviceProperties2, PhysicalDeviceSynchronization2Features, PhysicalDeviceTimelineSemaphoreFeatures, - PipelineStageFlags2, QUEUE_FAMILY_FOREIGN_EXT, Queue, QueueFlags, SampleCountFlags, - SemaphoreCreateInfo, SemaphoreImportFlags, SemaphoreSubmitInfo, SharingMode, - SubmitInfo2, SubresourceLayout, WHOLE_SIZE, + Queue, QueueFlags, SampleCountFlags, SemaphoreCreateInfo, SemaphoreImportFlags, + SharingMode, SubresourceLayout, }, }, bstr::ByteSlice, @@ -85,6 +77,7 @@ use { vk::{Buffer, CommandPool, Image, Semaphore}, }; +mod execute; mod queue_allocation; mod registry; @@ -1275,429 +1268,6 @@ impl CopyDeviceInner { } } -impl CopyDeviceCopy { - fn ensure_not_busy(&self) -> Result<(), CopyDeviceError> { - let slf = &*self.inner; - if let Some(sync) = slf.busy.get() - && sync.is_unsignaled() - { - return Err(CopyDeviceError::Busy); - } - slf.busy.take(); - Ok(()) - } - - pub fn execute( - &self, - sync: Option<&FdSync>, - region: Option<&Region>, - ) -> Result, CopyDeviceError> { - self.ensure_not_busy()?; - let slf = &*self.inner; - let tt = slf.tt; - let dev = &slf.dev.dev; - let cmd = slf.command_buffer; - let queue_family = slf.dev.phy.queues[tt].family; - let region_buf; - let width = slf.width; - let height = slf.height; - let region = match region { - Some(r) => r, - _ => { - region_buf = Region::new(Rect::new_saturating(0, 0, width as i32, height as i32)); - ®ion_buf - } - }; - let (x_mask, y_mask) = slf.dev.phy.queues[tt].transfer_granularity_mask; - let rects = &mut *slf.dev.phy.rects.borrow_mut(); - rects.clear(); - for rect in region.iter() { - let x1 = (rect.x1().max(0) as u32 & !x_mask).min(width); - let y1 = (rect.y1().max(0) as u32 & !y_mask).min(height); - let x2 = ((rect.x2().max(0) as u32 + x_mask) & !x_mask).min(width); - let y2 = ((rect.y2().max(0) as u32 + y_mask) & !y_mask).min(height); - let width = x2 - x1; - let height = y2 - y1; - if width == 0 || height == 0 { - continue; - } - rects.push((x1 as i32, y1 as i32, width, height)); - } - if rects.is_empty() { - return Ok(None); - } - let begin_info = - CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT); - unsafe { - dev.begin_command_buffer(cmd, &begin_info) - .map_err(CopyDeviceError::BeginCommandBuffer)?; - } - macro_rules! initial_buffer_barriers { - ($($buf:expr, $access:expr;)*) => { - [$( - BufferMemoryBarrier2::default() - .dst_stage_mask(PipelineStageFlags2::TRANSFER) - .dst_access_mask($access) - .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) - .dst_queue_family_index(queue_family) - .buffer($buf.buf) - .size(WHOLE_SIZE), - )*] - }; - } - macro_rules! final_buffer_barriers { - ($($buf:expr, $access:expr;)*) => { - [$( - BufferMemoryBarrier2::default() - .src_stage_mask(PipelineStageFlags2::TRANSFER) - .src_access_mask($access) - .src_queue_family_index(queue_family) - .dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) - .buffer($buf.buf) - .size(WHOLE_SIZE), - )*] - }; - } - let image_subresource_range = ImageSubresourceRange { - aspect_mask: ImageAspectFlags::COLOR, - base_mip_level: 0, - level_count: 1, - base_array_layer: 0, - layer_count: 1, - }; - let image_subresource = ImageSubresourceLayers { - aspect_mask: ImageAspectFlags::COLOR, - mip_level: 0, - base_array_layer: 0, - layer_count: 1, - }; - macro_rules! initial_image_barriers { - ($($img:expr, $layout:expr, $access:expr;)*) => { - [$( - ImageMemoryBarrier2::default() - .dst_stage_mask(PipelineStageFlags2::TRANSFER) - .dst_access_mask($access) - .old_layout(ImageLayout::GENERAL) - .new_layout(ImageLayout::GENERAL) - .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) - .dst_queue_family_index(queue_family) - .image($img.img) - .subresource_range(image_subresource_range), - ImageMemoryBarrier2::default() - .src_stage_mask(PipelineStageFlags2::TRANSFER) - .src_access_mask($access) - .dst_stage_mask(PipelineStageFlags2::TRANSFER) - .dst_access_mask($access) - .old_layout(ImageLayout::GENERAL) - .new_layout($layout) - .src_queue_family_index(queue_family) - .dst_queue_family_index(queue_family) - .image($img.img) - .subresource_range(image_subresource_range), - )*] - }; - } - macro_rules! final_image_barriers { - ($($img:expr, $layout:expr, $access:expr;)*) => { - [$( - ImageMemoryBarrier2::default() - .src_stage_mask(PipelineStageFlags2::TRANSFER) - .src_access_mask($access) - .dst_stage_mask(PipelineStageFlags2::TRANSFER) - .dst_access_mask($access) - .old_layout($layout) - .new_layout(ImageLayout::GENERAL) - .src_queue_family_index(queue_family) - .dst_queue_family_index(queue_family) - .image($img.img) - .subresource_range(image_subresource_range), - ImageMemoryBarrier2::default() - .src_stage_mask(PipelineStageFlags2::TRANSFER) - .src_access_mask($access) - .old_layout(ImageLayout::GENERAL) - .new_layout(ImageLayout::GENERAL) - .src_queue_family_index(queue_family) - .dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) - .image($img.img) - .subresource_range(image_subresource_range), - )*] - }; - } - match &slf.ty { - CopyDeviceCopyType::BufferToBuffer { - src, - dst, - stride, - bpp, - } => { - let regions = &mut *slf.dev.phy.buffer_copy_2.borrow_mut(); - regions.clear(); - let stride = *stride as u64; - let bpp = *bpp as u64; - for &mut (x, y, width, height) in rects { - let lo = y as u64 * stride + x as u64 * bpp; - let size = (height as u64 - 1) * stride + width as u64 * bpp; - let region = BufferCopy2::default() - .src_offset(lo) - .dst_offset(lo) - .size(size); - regions.push(region); - } - use AccessFlags2 as A; - let initial_barriers = initial_buffer_barriers![ - src, A::TRANSFER_READ; - dst, A::TRANSFER_WRITE; - ]; - let final_barriers = final_buffer_barriers![ - src, A::TRANSFER_READ; - dst, A::TRANSFER_WRITE; - ]; - let initial_dependency_info = - DependencyInfo::default().buffer_memory_barriers(&initial_barriers); - let final_dependency_info = - DependencyInfo::default().buffer_memory_barriers(&final_barriers); - let copy_buffer_info = CopyBufferInfo2::default() - .src_buffer(src.buf) - .dst_buffer(dst.buf) - .regions(regions); - unsafe { - dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info); - dev.cmd_copy_buffer2(cmd, ©_buffer_info); - dev.cmd_pipeline_barrier2(cmd, &final_dependency_info); - } - } - CopyDeviceCopyType::BufferToImage { - buf, - buf_format, - buf_stride, - img, - } - | CopyDeviceCopyType::ImageToBuffer { - img, - buf, - buf_format, - buf_stride, - } => { - let regions = &mut *slf.dev.phy.buffer_image_copy_2.borrow_mut(); - regions.clear(); - for &mut (x, y, width, height) in rects { - let offset = y as u64 * *buf_stride as u64 + x as u64 * buf_format.bpp as u64; - let region = BufferImageCopy2::default() - .buffer_offset(offset) - .buffer_row_length(*buf_stride / buf_format.bpp) - .buffer_image_height(slf.height) - .image_subresource(image_subresource) - .image_offset(Offset3D { x, y, z: 0 }) - .image_extent(Extent3D { - width, - height, - depth: 1, - }); - regions.push(region); - } - let buffer_to_image = match &slf.ty { - CopyDeviceCopyType::BufferToImage { .. } => true, - CopyDeviceCopyType::ImageToBuffer { .. } => false, - _ => unreachable!(), - }; - let image_access_mask; - let image_layout; - let buffer_access_mask; - match buffer_to_image { - true => { - image_access_mask = AccessFlags2::TRANSFER_WRITE; - image_layout = ImageLayout::TRANSFER_DST_OPTIMAL; - buffer_access_mask = AccessFlags2::TRANSFER_READ; - } - false => { - image_access_mask = AccessFlags2::TRANSFER_READ; - image_layout = ImageLayout::TRANSFER_SRC_OPTIMAL; - buffer_access_mask = AccessFlags2::TRANSFER_WRITE; - } - } - let initial_image_barriers = initial_image_barriers![ - img, image_layout, image_access_mask; - ]; - let final_image_barriers = final_image_barriers![ - img, image_layout, image_access_mask; - ]; - let initial_buffer_barriers = initial_buffer_barriers![ - buf, buffer_access_mask; - ]; - let final_buffer_barriers = final_buffer_barriers![ - buf, buffer_access_mask; - ]; - let initial_dependency_info = DependencyInfo::default() - .buffer_memory_barriers(&initial_buffer_barriers) - .image_memory_barriers(&initial_image_barriers); - let final_dependency_info = DependencyInfo::default() - .buffer_memory_barriers(&final_buffer_barriers) - .image_memory_barriers(&final_image_barriers); - unsafe { - dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info); - match buffer_to_image { - true => { - let copy = CopyBufferToImageInfo2::default() - .src_buffer(buf.buf) - .dst_image(img.img) - .dst_image_layout(image_layout) - .regions(®ions); - dev.cmd_copy_buffer_to_image2(cmd, ©); - } - false => { - let copy = CopyImageToBufferInfo2::default() - .src_image(img.img) - .src_image_layout(image_layout) - .dst_buffer(buf.buf) - .regions(®ions); - dev.cmd_copy_image_to_buffer2(cmd, ©); - } - } - dev.cmd_pipeline_barrier2(cmd, &final_dependency_info); - } - } - CopyDeviceCopyType::ImageToImage { src, dst } => { - let regions = &mut *slf.dev.phy.image_copy_2.borrow_mut(); - regions.clear(); - for &mut (x, y, width, height) in rects { - let region = ImageCopy2::default() - .src_subresource(image_subresource) - .src_offset(Offset3D { x, y, z: 0 }) - .dst_subresource(image_subresource) - .dst_offset(Offset3D { x, y, z: 0 }) - .extent(Extent3D { - width, - height, - depth: 1, - }); - regions.push(region); - } - use {AccessFlags2 as A, ImageLayout as L}; - let initial_barriers = initial_image_barriers![ - src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ; - dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE; - ]; - let final_barriers = final_image_barriers![ - src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ; - dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE; - ]; - let initial_dependency_info = - DependencyInfo::default().image_memory_barriers(&initial_barriers); - let final_dependency_info = - DependencyInfo::default().image_memory_barriers(&final_barriers); - let copy_image_info = CopyImageInfo2::default() - .src_image(src.img) - .src_image_layout(L::TRANSFER_SRC_OPTIMAL) - .dst_image(dst.img) - .dst_image_layout(L::TRANSFER_DST_OPTIMAL) - .regions(regions); - unsafe { - dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info); - dev.cmd_copy_image2(cmd, ©_image_info); - dev.cmd_pipeline_barrier2(cmd, &final_dependency_info); - } - } - CopyDeviceCopyType::Blit { src, dst } => { - let regions = &mut *slf.dev.phy.image_blit_2.borrow_mut(); - regions.clear(); - for &mut (x, y, width, height) in rects { - let x1 = x; - let y1 = y; - let x2 = x1 + width as i32; - let y2 = y1 + height as i32; - let offsets = [ - Offset3D { x: x1, y: y1, z: 0 }, - Offset3D { x: x2, y: y2, z: 1 }, - ]; - let region = ImageBlit2::default() - .src_subresource(image_subresource) - .src_offsets(offsets) - .dst_subresource(image_subresource) - .dst_offsets(offsets); - regions.push(region); - } - use {AccessFlags2 as A, ImageLayout as L}; - let initial_barriers = initial_image_barriers![ - src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ; - dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE; - ]; - let final_barriers = final_image_barriers![ - src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ; - dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE; - ]; - let initial_dependency_info = - DependencyInfo::default().image_memory_barriers(&initial_barriers); - let final_dependency_info = - DependencyInfo::default().image_memory_barriers(&final_barriers); - let blit_image_info = BlitImageInfo2::default() - .src_image(src.img) - .src_image_layout(L::TRANSFER_SRC_OPTIMAL) - .dst_image(dst.img) - .dst_image_layout(L::TRANSFER_DST_OPTIMAL) - .regions(regions) - .filter(Filter::NEAREST); - unsafe { - dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info); - dev.cmd_blit_image2(cmd, &blit_image_info); - dev.cmd_pipeline_barrier2(cmd, &final_dependency_info); - } - } - }; - unsafe { - dev.end_command_buffer(cmd) - .map_err(CopyDeviceError::EndCommandBuffer)?; - } - let mut wait_semaphore = None; - let mut wait_semaphores = ArrayVec::<_, 1>::new(); - if let Some(sync) = sync - && let Some(sync_file) = sync.get_sync_file() - { - let semaphore = match slf.dev.semaphores.pop() { - Some(s) => s, - _ => slf.dev.create_semaphore()?, - }; - semaphore.import(sync_file)?; - let info = SemaphoreSubmitInfo::default() - .semaphore(semaphore.semaphore) - .stage_mask(PipelineStageFlags2::TRANSFER); - wait_semaphores.push(info); - wait_semaphore = Some(semaphore); - } - let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd); - let mut semaphore_submit_info = SemaphoreSubmitInfo::default(); - let mut submit_info = SubmitInfo2::default() - .command_buffer_infos(slice::from_ref(&command_buffer_info)) - .wait_semaphore_infos(&wait_semaphores); - let vulkan_sync = slf.dev.create_sync( - self.dev.timeline_semaphore.as_ref(), - &mut semaphore_submit_info, - &mut submit_info, - )?; - unsafe { - slf.dev - .dev - .queue_submit2( - slf.dev.queues[tt], - slice::from_ref(&submit_info), - vulkan_sync.fence(), - ) - .map_err(CopyDeviceError::SubmitCopy)?; - } - let sync = vulkan_sync.to_sync(|| slf.dev.wait_idle()); - slf.busy.set(sync.clone()); - let pending = Pending { - dev: slf.dev.clone(), - busy_id: slf.busy_id.add_fetch(1), - sync: sync.clone(), - copy: self.inner.clone(), - semaphore: wait_semaphore, - vulkan_sync, - }; - slf.dev.submissions[tt].pending.push(pending); - Ok(sync) - } -} - impl VulkanSemaphore { fn import(&self, sync_file: &OwnedFd) -> Result<(), CopyDeviceError> { let fd = uapi::fcntl_dupfd_cloexec(sync_file.raw(), 0) diff --git a/src/copy_device/execute.rs b/src/copy_device/execute.rs new file mode 100644 index 00000000..7f1544e7 --- /dev/null +++ b/src/copy_device/execute.rs @@ -0,0 +1,442 @@ +use { + super::{CopyDeviceCopy, CopyDeviceCopyType, CopyDeviceError, Pending}, + crate::{ + gfx_api::FdSync, + rect::{Rect, Region}, + vulkan_core::sync::VulkanDeviceSyncExt, + }, + arrayvec::ArrayVec, + ash::vk::{ + AccessFlags2, BlitImageInfo2, BufferCopy2, BufferImageCopy2, BufferMemoryBarrier2, + CommandBufferBeginInfo, CommandBufferSubmitInfo, CommandBufferUsageFlags, + CopyBufferInfo2, CopyBufferToImageInfo2, CopyImageInfo2, CopyImageToBufferInfo2, + DependencyInfo, Extent3D, Filter, ImageAspectFlags, ImageBlit2, ImageCopy2, ImageLayout, + ImageMemoryBarrier2, ImageSubresourceLayers, ImageSubresourceRange, Offset3D, + PipelineStageFlags2, QUEUE_FAMILY_FOREIGN_EXT, SemaphoreSubmitInfo, SubmitInfo2, + WHOLE_SIZE, + }, + std::slice, +}; + +impl CopyDeviceCopy { + fn ensure_not_busy(&self) -> Result<(), CopyDeviceError> { + let slf = &*self.inner; + if let Some(sync) = slf.busy.get() + && sync.is_unsignaled() + { + return Err(CopyDeviceError::Busy); + } + slf.busy.take(); + Ok(()) + } + + pub fn execute( + &self, + sync: Option<&FdSync>, + region: Option<&Region>, + ) -> Result, CopyDeviceError> { + self.ensure_not_busy()?; + let slf = &*self.inner; + let tt = slf.tt; + let dev = &slf.dev.dev; + let cmd = slf.command_buffer; + let queue_family = slf.dev.phy.queues[tt].family; + let region_buf; + let width = slf.width; + let height = slf.height; + let region = match region { + Some(r) => r, + _ => { + region_buf = Region::new(Rect::new_saturating(0, 0, width as i32, height as i32)); + ®ion_buf + } + }; + let (x_mask, y_mask) = slf.dev.phy.queues[tt].transfer_granularity_mask; + let rects = &mut *slf.dev.phy.rects.borrow_mut(); + rects.clear(); + for rect in region.iter() { + let x1 = (rect.x1().max(0) as u32 & !x_mask).min(width); + let y1 = (rect.y1().max(0) as u32 & !y_mask).min(height); + let x2 = ((rect.x2().max(0) as u32 + x_mask) & !x_mask).min(width); + let y2 = ((rect.y2().max(0) as u32 + y_mask) & !y_mask).min(height); + let width = x2 - x1; + let height = y2 - y1; + if width == 0 || height == 0 { + continue; + } + rects.push((x1 as i32, y1 as i32, width, height)); + } + if rects.is_empty() { + return Ok(None); + } + let begin_info = + CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT); + unsafe { + dev.begin_command_buffer(cmd, &begin_info) + .map_err(CopyDeviceError::BeginCommandBuffer)?; + } + macro_rules! initial_buffer_barriers { + ($($buf:expr, $access:expr;)*) => { + [$( + BufferMemoryBarrier2::default() + .dst_stage_mask(PipelineStageFlags2::TRANSFER) + .dst_access_mask($access) + .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) + .dst_queue_family_index(queue_family) + .buffer($buf.buf) + .size(WHOLE_SIZE), + )*] + }; + } + macro_rules! final_buffer_barriers { + ($($buf:expr, $access:expr;)*) => { + [$( + BufferMemoryBarrier2::default() + .src_stage_mask(PipelineStageFlags2::TRANSFER) + .src_access_mask($access) + .src_queue_family_index(queue_family) + .dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) + .buffer($buf.buf) + .size(WHOLE_SIZE), + )*] + }; + } + let image_subresource_range = ImageSubresourceRange { + aspect_mask: ImageAspectFlags::COLOR, + base_mip_level: 0, + level_count: 1, + base_array_layer: 0, + layer_count: 1, + }; + let image_subresource = ImageSubresourceLayers { + aspect_mask: ImageAspectFlags::COLOR, + mip_level: 0, + base_array_layer: 0, + layer_count: 1, + }; + macro_rules! initial_image_barriers { + ($($img:expr, $layout:expr, $access:expr;)*) => { + [$( + ImageMemoryBarrier2::default() + .dst_stage_mask(PipelineStageFlags2::TRANSFER) + .dst_access_mask($access) + .old_layout(ImageLayout::GENERAL) + .new_layout(ImageLayout::GENERAL) + .src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) + .dst_queue_family_index(queue_family) + .image($img.img) + .subresource_range(image_subresource_range), + ImageMemoryBarrier2::default() + .src_stage_mask(PipelineStageFlags2::TRANSFER) + .src_access_mask($access) + .dst_stage_mask(PipelineStageFlags2::TRANSFER) + .dst_access_mask($access) + .old_layout(ImageLayout::GENERAL) + .new_layout($layout) + .src_queue_family_index(queue_family) + .dst_queue_family_index(queue_family) + .image($img.img) + .subresource_range(image_subresource_range), + )*] + }; + } + macro_rules! final_image_barriers { + ($($img:expr, $layout:expr, $access:expr;)*) => { + [$( + ImageMemoryBarrier2::default() + .src_stage_mask(PipelineStageFlags2::TRANSFER) + .src_access_mask($access) + .dst_stage_mask(PipelineStageFlags2::TRANSFER) + .dst_access_mask($access) + .old_layout($layout) + .new_layout(ImageLayout::GENERAL) + .src_queue_family_index(queue_family) + .dst_queue_family_index(queue_family) + .image($img.img) + .subresource_range(image_subresource_range), + ImageMemoryBarrier2::default() + .src_stage_mask(PipelineStageFlags2::TRANSFER) + .src_access_mask($access) + .old_layout(ImageLayout::GENERAL) + .new_layout(ImageLayout::GENERAL) + .src_queue_family_index(queue_family) + .dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT) + .image($img.img) + .subresource_range(image_subresource_range), + )*] + }; + } + match &slf.ty { + CopyDeviceCopyType::BufferToBuffer { + src, + dst, + stride, + bpp, + } => { + let regions = &mut *slf.dev.phy.buffer_copy_2.borrow_mut(); + regions.clear(); + let stride = *stride as u64; + let bpp = *bpp as u64; + for &mut (x, y, width, height) in rects { + let lo = y as u64 * stride + x as u64 * bpp; + let size = (height as u64 - 1) * stride + width as u64 * bpp; + let region = BufferCopy2::default() + .src_offset(lo) + .dst_offset(lo) + .size(size); + regions.push(region); + } + use AccessFlags2 as A; + let initial_barriers = initial_buffer_barriers![ + src, A::TRANSFER_READ; + dst, A::TRANSFER_WRITE; + ]; + let final_barriers = final_buffer_barriers![ + src, A::TRANSFER_READ; + dst, A::TRANSFER_WRITE; + ]; + let initial_dependency_info = + DependencyInfo::default().buffer_memory_barriers(&initial_barriers); + let final_dependency_info = + DependencyInfo::default().buffer_memory_barriers(&final_barriers); + let copy_buffer_info = CopyBufferInfo2::default() + .src_buffer(src.buf) + .dst_buffer(dst.buf) + .regions(regions); + unsafe { + dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info); + dev.cmd_copy_buffer2(cmd, ©_buffer_info); + dev.cmd_pipeline_barrier2(cmd, &final_dependency_info); + } + } + CopyDeviceCopyType::BufferToImage { + buf, + buf_format, + buf_stride, + img, + } + | CopyDeviceCopyType::ImageToBuffer { + img, + buf, + buf_format, + buf_stride, + } => { + let regions = &mut *slf.dev.phy.buffer_image_copy_2.borrow_mut(); + regions.clear(); + for &mut (x, y, width, height) in rects { + let offset = y as u64 * *buf_stride as u64 + x as u64 * buf_format.bpp as u64; + let region = BufferImageCopy2::default() + .buffer_offset(offset) + .buffer_row_length(*buf_stride / buf_format.bpp) + .buffer_image_height(slf.height) + .image_subresource(image_subresource) + .image_offset(Offset3D { x, y, z: 0 }) + .image_extent(Extent3D { + width, + height, + depth: 1, + }); + regions.push(region); + } + let buffer_to_image = match &slf.ty { + CopyDeviceCopyType::BufferToImage { .. } => true, + CopyDeviceCopyType::ImageToBuffer { .. } => false, + _ => unreachable!(), + }; + let image_access_mask; + let image_layout; + let buffer_access_mask; + match buffer_to_image { + true => { + image_access_mask = AccessFlags2::TRANSFER_WRITE; + image_layout = ImageLayout::TRANSFER_DST_OPTIMAL; + buffer_access_mask = AccessFlags2::TRANSFER_READ; + } + false => { + image_access_mask = AccessFlags2::TRANSFER_READ; + image_layout = ImageLayout::TRANSFER_SRC_OPTIMAL; + buffer_access_mask = AccessFlags2::TRANSFER_WRITE; + } + } + let initial_image_barriers = initial_image_barriers![ + img, image_layout, image_access_mask; + ]; + let final_image_barriers = final_image_barriers![ + img, image_layout, image_access_mask; + ]; + let initial_buffer_barriers = initial_buffer_barriers![ + buf, buffer_access_mask; + ]; + let final_buffer_barriers = final_buffer_barriers![ + buf, buffer_access_mask; + ]; + let initial_dependency_info = DependencyInfo::default() + .buffer_memory_barriers(&initial_buffer_barriers) + .image_memory_barriers(&initial_image_barriers); + let final_dependency_info = DependencyInfo::default() + .buffer_memory_barriers(&final_buffer_barriers) + .image_memory_barriers(&final_image_barriers); + unsafe { + dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info); + match buffer_to_image { + true => { + let copy = CopyBufferToImageInfo2::default() + .src_buffer(buf.buf) + .dst_image(img.img) + .dst_image_layout(image_layout) + .regions(®ions); + dev.cmd_copy_buffer_to_image2(cmd, ©); + } + false => { + let copy = CopyImageToBufferInfo2::default() + .src_image(img.img) + .src_image_layout(image_layout) + .dst_buffer(buf.buf) + .regions(®ions); + dev.cmd_copy_image_to_buffer2(cmd, ©); + } + } + dev.cmd_pipeline_barrier2(cmd, &final_dependency_info); + } + } + CopyDeviceCopyType::ImageToImage { src, dst } => { + let regions = &mut *slf.dev.phy.image_copy_2.borrow_mut(); + regions.clear(); + for &mut (x, y, width, height) in rects { + let region = ImageCopy2::default() + .src_subresource(image_subresource) + .src_offset(Offset3D { x, y, z: 0 }) + .dst_subresource(image_subresource) + .dst_offset(Offset3D { x, y, z: 0 }) + .extent(Extent3D { + width, + height, + depth: 1, + }); + regions.push(region); + } + use {AccessFlags2 as A, ImageLayout as L}; + let initial_barriers = initial_image_barriers![ + src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ; + dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE; + ]; + let final_barriers = final_image_barriers![ + src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ; + dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE; + ]; + let initial_dependency_info = + DependencyInfo::default().image_memory_barriers(&initial_barriers); + let final_dependency_info = + DependencyInfo::default().image_memory_barriers(&final_barriers); + let copy_image_info = CopyImageInfo2::default() + .src_image(src.img) + .src_image_layout(L::TRANSFER_SRC_OPTIMAL) + .dst_image(dst.img) + .dst_image_layout(L::TRANSFER_DST_OPTIMAL) + .regions(regions); + unsafe { + dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info); + dev.cmd_copy_image2(cmd, ©_image_info); + dev.cmd_pipeline_barrier2(cmd, &final_dependency_info); + } + } + CopyDeviceCopyType::Blit { src, dst } => { + let regions = &mut *slf.dev.phy.image_blit_2.borrow_mut(); + regions.clear(); + for &mut (x, y, width, height) in rects { + let x1 = x; + let y1 = y; + let x2 = x1 + width as i32; + let y2 = y1 + height as i32; + let offsets = [ + Offset3D { x: x1, y: y1, z: 0 }, + Offset3D { x: x2, y: y2, z: 1 }, + ]; + let region = ImageBlit2::default() + .src_subresource(image_subresource) + .src_offsets(offsets) + .dst_subresource(image_subresource) + .dst_offsets(offsets); + regions.push(region); + } + use {AccessFlags2 as A, ImageLayout as L}; + let initial_barriers = initial_image_barriers![ + src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ; + dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE; + ]; + let final_barriers = final_image_barriers![ + src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ; + dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE; + ]; + let initial_dependency_info = + DependencyInfo::default().image_memory_barriers(&initial_barriers); + let final_dependency_info = + DependencyInfo::default().image_memory_barriers(&final_barriers); + let blit_image_info = BlitImageInfo2::default() + .src_image(src.img) + .src_image_layout(L::TRANSFER_SRC_OPTIMAL) + .dst_image(dst.img) + .dst_image_layout(L::TRANSFER_DST_OPTIMAL) + .regions(regions) + .filter(Filter::NEAREST); + unsafe { + dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info); + dev.cmd_blit_image2(cmd, &blit_image_info); + dev.cmd_pipeline_barrier2(cmd, &final_dependency_info); + } + } + }; + unsafe { + dev.end_command_buffer(cmd) + .map_err(CopyDeviceError::EndCommandBuffer)?; + } + let mut wait_semaphore = None; + let mut wait_semaphores = ArrayVec::<_, 1>::new(); + if let Some(sync) = sync + && let Some(sync_file) = sync.get_sync_file() + { + let semaphore = match slf.dev.semaphores.pop() { + Some(s) => s, + _ => slf.dev.create_semaphore()?, + }; + semaphore.import(sync_file)?; + let info = SemaphoreSubmitInfo::default() + .semaphore(semaphore.semaphore) + .stage_mask(PipelineStageFlags2::TRANSFER); + wait_semaphores.push(info); + wait_semaphore = Some(semaphore); + } + let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd); + let mut semaphore_submit_info = SemaphoreSubmitInfo::default(); + let mut submit_info = SubmitInfo2::default() + .command_buffer_infos(slice::from_ref(&command_buffer_info)) + .wait_semaphore_infos(&wait_semaphores); + let vulkan_sync = slf.dev.create_sync( + self.dev.timeline_semaphore.as_ref(), + &mut semaphore_submit_info, + &mut submit_info, + )?; + unsafe { + slf.dev + .dev + .queue_submit2( + slf.dev.queues[tt], + slice::from_ref(&submit_info), + vulkan_sync.fence(), + ) + .map_err(CopyDeviceError::SubmitCopy)?; + } + let sync = vulkan_sync.to_sync(|| slf.dev.wait_idle()); + slf.busy.set(sync.clone()); + let pending = Pending { + dev: slf.dev.clone(), + busy_id: slf.busy_id.add_fetch(1), + sync: sync.clone(), + copy: self.inner.clone(), + semaphore: wait_semaphore, + vulkan_sync, + }; + slf.dev.submissions[tt].pending.push(pending); + Ok(sync) + } +}