kossLAN/wry
1
0
Fork 0
forked from wry/wry
Code Pull requests Activity

copy-device: split copy execution

Browse source
This commit is contained in:
kossLAN 2026-05-29 21:44:06 -04:00
parent 100c657050
commit 112b1a8e5f
No known key found for this signature in database
2 changed files with 457 additions and 445 deletions
Download patch file Download diff file Expand all files Collapse all files

460
src/copy_device.rs
View file

@ -5,7 +5,6 @@ use {
format::{FORMATS, Format}, format::{FORMATS, Format},
gfx_api::FdSync, gfx_api::FdSync,
io_uring::IoUring, io_uring::IoUring,
rect::{Rect, Region},
utils::{ utils::{
clonecell::CloneCell, clonecell::CloneCell,
errorfmt::ErrorFmt, errorfmt::ErrorFmt,
@ -21,12 +20,10 @@ use {
}, },
vulkan_core::{ vulkan_core::{
self, VULKAN_API_VERSION, VulkanCoreError, VulkanCoreInstance, device::VulkanDeviceInf, self, VULKAN_API_VERSION, VulkanCoreError, VulkanCoreInstance, device::VulkanDeviceInf,
map_extension_properties, sync::VulkanDeviceSyncExt, map_extension_properties, timeline_semaphore::VulkanDeviceTimelineSemaphoreExt,
timeline_semaphore::VulkanDeviceTimelineSemaphoreExt,
}, },
}, },
ahash::AHashMap, ahash::AHashMap,
arrayvec::ArrayVec,
ash::{ ash::{
Device, Device,
ext::{ ext::{
@ -35,12 +32,9 @@ use {
}, },
khr::{external_fence_fd, external_memory_fd, external_semaphore_fd}, khr::{external_fence_fd, external_memory_fd, external_semaphore_fd},
vk::{ vk::{
self, AccessFlags2, BindImageMemoryInfo, BindImagePlaneMemoryInfo, BlitImageInfo2, self, BindImageMemoryInfo, BindImagePlaneMemoryInfo, BufferCopy2, BufferCreateInfo,
BufferCopy2, BufferCreateInfo, BufferImageCopy2, BufferMemoryBarrier2, BufferImageCopy2, BufferUsageFlags, CommandBuffer, CommandBufferAllocateInfo,
BufferUsageFlags, CommandBuffer, CommandBufferAllocateInfo, CommandBufferBeginInfo, CommandPoolCreateFlags, CommandPoolCreateInfo, DeviceCreateInfo, DeviceMemory,
CommandBufferSubmitInfo, CommandBufferUsageFlags, CommandPoolCreateFlags,
CommandPoolCreateInfo, CopyBufferInfo2, CopyBufferToImageInfo2, CopyImageInfo2,
CopyImageToBufferInfo2, DependencyInfo, DeviceCreateInfo, DeviceMemory,
DeviceQueueCreateInfo, DrmFormatModifierPropertiesEXT, DeviceQueueCreateInfo, DrmFormatModifierPropertiesEXT,
DrmFormatModifierPropertiesListEXT, ExportMemoryAllocateInfo, Extent3D, DrmFormatModifierPropertiesListEXT, ExportMemoryAllocateInfo, Extent3D,
ExternalBufferProperties, ExternalFenceFeatureFlags, ExternalFenceHandleTypeFlags, ExternalBufferProperties, ExternalFenceFeatureFlags, ExternalFenceHandleTypeFlags,
@ -48,23 +42,21 @@ use {
ExternalMemoryBufferCreateInfo, ExternalMemoryBufferCreateInfoKHR, ExternalMemoryBufferCreateInfo, ExternalMemoryBufferCreateInfoKHR,
ExternalMemoryFeatureFlags, ExternalMemoryHandleTypeFlags, ExternalMemoryFeatureFlags, ExternalMemoryHandleTypeFlags,
ExternalMemoryImageCreateInfo, ExternalSemaphoreFeatureFlags, ExternalMemoryImageCreateInfo, ExternalSemaphoreFeatureFlags,
ExternalSemaphoreHandleTypeFlags, ExternalSemaphoreProperties, Filter, ExternalSemaphoreHandleTypeFlags, ExternalSemaphoreProperties, FormatFeatureFlags,
FormatFeatureFlags, FormatProperties2, ImageAspectFlags, ImageBlit2, ImageCopy2, FormatProperties2, ImageAspectFlags, ImageBlit2, ImageCopy2, ImageCreateFlags,
ImageCreateFlags, ImageCreateInfo, ImageDrmFormatModifierExplicitCreateInfoEXT, ImageCreateInfo, ImageDrmFormatModifierExplicitCreateInfoEXT, ImageFormatProperties2,
ImageFormatProperties2, ImageLayout, ImageMemoryBarrier2, ImageMemoryRequirementsInfo2, ImageLayout, ImageMemoryRequirementsInfo2, ImagePlaneMemoryRequirementsInfo,
ImagePlaneMemoryRequirementsInfo, ImageSubresourceLayers, ImageSubresourceRange, ImageTiling, ImageType, ImageUsageFlags, ImportMemoryFdInfoKHR, ImportSemaphoreFdInfoKHR,
ImageTiling, ImageType, ImageUsageFlags, ImportMemoryFdInfoKHR, MemoryAllocateInfo, MemoryDedicatedAllocateInfo, MemoryFdPropertiesKHR,
ImportSemaphoreFdInfoKHR, MemoryAllocateInfo, MemoryDedicatedAllocateInfo, MemoryGetFdInfoKHR, MemoryPropertyFlags, MemoryRequirements2, MemoryType,
MemoryFdPropertiesKHR, MemoryGetFdInfoKHR, MemoryPropertyFlags, MemoryRequirements2, PhysicalDevice, PhysicalDeviceDrmPropertiesEXT,
MemoryType, Offset3D, PhysicalDevice, PhysicalDeviceDrmPropertiesEXT,
PhysicalDeviceExternalBufferInfo, PhysicalDeviceExternalFenceInfo, PhysicalDeviceExternalBufferInfo, PhysicalDeviceExternalFenceInfo,
PhysicalDeviceExternalImageFormatInfoKHR, PhysicalDeviceExternalSemaphoreInfo, PhysicalDeviceExternalImageFormatInfoKHR, PhysicalDeviceExternalSemaphoreInfo,
PhysicalDeviceFeatures2, PhysicalDeviceImageDrmFormatModifierInfoEXT, PhysicalDeviceFeatures2, PhysicalDeviceImageDrmFormatModifierInfoEXT,
PhysicalDeviceImageFormatInfo2, PhysicalDeviceProperties2, PhysicalDeviceImageFormatInfo2, PhysicalDeviceProperties2,
PhysicalDeviceSynchronization2Features, PhysicalDeviceTimelineSemaphoreFeatures, PhysicalDeviceSynchronization2Features, PhysicalDeviceTimelineSemaphoreFeatures,
PipelineStageFlags2, QUEUE_FAMILY_FOREIGN_EXT, Queue, QueueFlags, SampleCountFlags, Queue, QueueFlags, SampleCountFlags, SemaphoreCreateInfo, SemaphoreImportFlags,
SemaphoreCreateInfo, SemaphoreImportFlags, SemaphoreSubmitInfo, SharingMode, SharingMode, SubresourceLayout,
SubmitInfo2, SubresourceLayout, WHOLE_SIZE,
}, },
}, },
bstr::ByteSlice, bstr::ByteSlice,
@ -85,6 +77,7 @@ use {
vk::{Buffer, CommandPool, Image, Semaphore}, vk::{Buffer, CommandPool, Image, Semaphore},
}; };
mod execute;
mod queue_allocation; mod queue_allocation;
mod registry; mod registry;
@ -1275,429 +1268,6 @@ impl CopyDeviceInner {
} }
} }
impl CopyDeviceCopy {
fn ensure_not_busy(&self) -> Result<(), CopyDeviceError> {
let slf = &*self.inner;
if let Some(sync) = slf.busy.get()
&& sync.is_unsignaled()
{
return Err(CopyDeviceError::Busy);
}
slf.busy.take();
Ok(())
}
pub fn execute(
&self,
sync: Option<&FdSync>,
region: Option<&Region>,
) -> Result<Option<FdSync>, CopyDeviceError> {
self.ensure_not_busy()?;
let slf = &*self.inner;
let tt = slf.tt;
let dev = &slf.dev.dev;
let cmd = slf.command_buffer;
let queue_family = slf.dev.phy.queues[tt].family;
let region_buf;
let width = slf.width;
let height = slf.height;
let region = match region {
Some(r) => r,
_ => {
region_buf = Region::new(Rect::new_saturating(0, 0, width as i32, height as i32));
&region_buf
}
};
let (x_mask, y_mask) = slf.dev.phy.queues[tt].transfer_granularity_mask;
let rects = &mut *slf.dev.phy.rects.borrow_mut();
rects.clear();
for rect in region.iter() {
let x1 = (rect.x1().max(0) as u32 & !x_mask).min(width);
let y1 = (rect.y1().max(0) as u32 & !y_mask).min(height);
let x2 = ((rect.x2().max(0) as u32 + x_mask) & !x_mask).min(width);
let y2 = ((rect.y2().max(0) as u32 + y_mask) & !y_mask).min(height);
let width = x2 - x1;
let height = y2 - y1;
if width == 0 || height == 0 {
continue;
}
rects.push((x1 as i32, y1 as i32, width, height));
}
if rects.is_empty() {
return Ok(None);
}
let begin_info =
CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT);
unsafe {
dev.begin_command_buffer(cmd, &begin_info)
.map_err(CopyDeviceError::BeginCommandBuffer)?;
}
macro_rules! initial_buffer_barriers {
($($buf:expr, $access:expr;)*) => {
[$(
BufferMemoryBarrier2::default()
.dst_stage_mask(PipelineStageFlags2::TRANSFER)
.dst_access_mask($access)
.src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
.dst_queue_family_index(queue_family)
.buffer($buf.buf)
.size(WHOLE_SIZE),
)*]
};
}
macro_rules! final_buffer_barriers {
($($buf:expr, $access:expr;)*) => {
[$(
BufferMemoryBarrier2::default()
.src_stage_mask(PipelineStageFlags2::TRANSFER)
.src_access_mask($access)
.src_queue_family_index(queue_family)
.dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
.buffer($buf.buf)
.size(WHOLE_SIZE),
)*]
};
}
let image_subresource_range = ImageSubresourceRange {
aspect_mask: ImageAspectFlags::COLOR,
base_mip_level: 0,
level_count: 1,
base_array_layer: 0,
layer_count: 1,
};
let image_subresource = ImageSubresourceLayers {
aspect_mask: ImageAspectFlags::COLOR,
mip_level: 0,
base_array_layer: 0,
layer_count: 1,
};
macro_rules! initial_image_barriers {
($($img:expr, $layout:expr, $access:expr;)*) => {
[$(
ImageMemoryBarrier2::default()
.dst_stage_mask(PipelineStageFlags2::TRANSFER)
.dst_access_mask($access)
.old_layout(ImageLayout::GENERAL)
.new_layout(ImageLayout::GENERAL)
.src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
.dst_queue_family_index(queue_family)
.image($img.img)
.subresource_range(image_subresource_range),
ImageMemoryBarrier2::default()
.src_stage_mask(PipelineStageFlags2::TRANSFER)
.src_access_mask($access)
.dst_stage_mask(PipelineStageFlags2::TRANSFER)
.dst_access_mask($access)
.old_layout(ImageLayout::GENERAL)
.new_layout($layout)
.src_queue_family_index(queue_family)
.dst_queue_family_index(queue_family)
.image($img.img)
.subresource_range(image_subresource_range),
)*]
};
}
macro_rules! final_image_barriers {
($($img:expr, $layout:expr, $access:expr;)*) => {
[$(
ImageMemoryBarrier2::default()
.src_stage_mask(PipelineStageFlags2::TRANSFER)
.src_access_mask($access)
.dst_stage_mask(PipelineStageFlags2::TRANSFER)
.dst_access_mask($access)
.old_layout($layout)
.new_layout(ImageLayout::GENERAL)
.src_queue_family_index(queue_family)
.dst_queue_family_index(queue_family)
.image($img.img)
.subresource_range(image_subresource_range),
ImageMemoryBarrier2::default()
.src_stage_mask(PipelineStageFlags2::TRANSFER)
.src_access_mask($access)
.old_layout(ImageLayout::GENERAL)
.new_layout(ImageLayout::GENERAL)
.src_queue_family_index(queue_family)
.dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
.image($img.img)
.subresource_range(image_subresource_range),
)*]
};
}
match &slf.ty {
CopyDeviceCopyType::BufferToBuffer {
src,
dst,
stride,
bpp,
} => {
let regions = &mut *slf.dev.phy.buffer_copy_2.borrow_mut();
regions.clear();
let stride = *stride as u64;
let bpp = *bpp as u64;
for &mut (x, y, width, height) in rects {
let lo = y as u64 * stride + x as u64 * bpp;
let size = (height as u64 - 1) * stride + width as u64 * bpp;
let region = BufferCopy2::default()
.src_offset(lo)
.dst_offset(lo)
.size(size);
regions.push(region);
}
use AccessFlags2 as A;
let initial_barriers = initial_buffer_barriers![
src, A::TRANSFER_READ;
dst, A::TRANSFER_WRITE;
];
let final_barriers = final_buffer_barriers![
src, A::TRANSFER_READ;
dst, A::TRANSFER_WRITE;
];
let initial_dependency_info =
DependencyInfo::default().buffer_memory_barriers(&initial_barriers);
let final_dependency_info =
DependencyInfo::default().buffer_memory_barriers(&final_barriers);
let copy_buffer_info = CopyBufferInfo2::default()
.src_buffer(src.buf)
.dst_buffer(dst.buf)
.regions(regions);
unsafe {
dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info);
dev.cmd_copy_buffer2(cmd, &copy_buffer_info);
dev.cmd_pipeline_barrier2(cmd, &final_dependency_info);
}
}
CopyDeviceCopyType::BufferToImage {
buf,
buf_format,
buf_stride,
img,
}
| CopyDeviceCopyType::ImageToBuffer {
img,
buf,
buf_format,
buf_stride,
} => {
let regions = &mut *slf.dev.phy.buffer_image_copy_2.borrow_mut();
regions.clear();
for &mut (x, y, width, height) in rects {
let offset = y as u64 * *buf_stride as u64 + x as u64 * buf_format.bpp as u64;
let region = BufferImageCopy2::default()
.buffer_offset(offset)
.buffer_row_length(*buf_stride / buf_format.bpp)
.buffer_image_height(slf.height)
.image_subresource(image_subresource)
.image_offset(Offset3D { x, y, z: 0 })
.image_extent(Extent3D {
width,
height,
depth: 1,
});
regions.push(region);
}
let buffer_to_image = match &slf.ty {
CopyDeviceCopyType::BufferToImage { .. } => true,
CopyDeviceCopyType::ImageToBuffer { .. } => false,
_ => unreachable!(),
};
let image_access_mask;
let image_layout;
let buffer_access_mask;
match buffer_to_image {
true => {
image_access_mask = AccessFlags2::TRANSFER_WRITE;
image_layout = ImageLayout::TRANSFER_DST_OPTIMAL;
buffer_access_mask = AccessFlags2::TRANSFER_READ;
}
false => {
image_access_mask = AccessFlags2::TRANSFER_READ;
image_layout = ImageLayout::TRANSFER_SRC_OPTIMAL;
buffer_access_mask = AccessFlags2::TRANSFER_WRITE;
}
}
let initial_image_barriers = initial_image_barriers![
img, image_layout, image_access_mask;
];
let final_image_barriers = final_image_barriers![
img, image_layout, image_access_mask;
];
let initial_buffer_barriers = initial_buffer_barriers![
buf, buffer_access_mask;
];
let final_buffer_barriers = final_buffer_barriers![
buf, buffer_access_mask;
];
let initial_dependency_info = DependencyInfo::default()
.buffer_memory_barriers(&initial_buffer_barriers)
.image_memory_barriers(&initial_image_barriers);
let final_dependency_info = DependencyInfo::default()
.buffer_memory_barriers(&final_buffer_barriers)
.image_memory_barriers(&final_image_barriers);
unsafe {
dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info);
match buffer_to_image {
true => {
let copy = CopyBufferToImageInfo2::default()
.src_buffer(buf.buf)
.dst_image(img.img)
.dst_image_layout(image_layout)
.regions(&regions);
dev.cmd_copy_buffer_to_image2(cmd, &copy);
}
false => {
let copy = CopyImageToBufferInfo2::default()
.src_image(img.img)
.src_image_layout(image_layout)
.dst_buffer(buf.buf)
.regions(&regions);
dev.cmd_copy_image_to_buffer2(cmd, &copy);
}
}
dev.cmd_pipeline_barrier2(cmd, &final_dependency_info);
}
}
CopyDeviceCopyType::ImageToImage { src, dst } => {
let regions = &mut *slf.dev.phy.image_copy_2.borrow_mut();
regions.clear();
for &mut (x, y, width, height) in rects {
let region = ImageCopy2::default()
.src_subresource(image_subresource)
.src_offset(Offset3D { x, y, z: 0 })
.dst_subresource(image_subresource)
.dst_offset(Offset3D { x, y, z: 0 })
.extent(Extent3D {
width,
height,
depth: 1,
});
regions.push(region);
}
use {AccessFlags2 as A, ImageLayout as L};
let initial_barriers = initial_image_barriers![
src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ;
dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE;
];
let final_barriers = final_image_barriers![
src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ;
dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE;
];
let initial_dependency_info =
DependencyInfo::default().image_memory_barriers(&initial_barriers);
let final_dependency_info =
DependencyInfo::default().image_memory_barriers(&final_barriers);
let copy_image_info = CopyImageInfo2::default()
.src_image(src.img)
.src_image_layout(L::TRANSFER_SRC_OPTIMAL)
.dst_image(dst.img)
.dst_image_layout(L::TRANSFER_DST_OPTIMAL)
.regions(regions);
unsafe {
dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info);
dev.cmd_copy_image2(cmd, &copy_image_info);
dev.cmd_pipeline_barrier2(cmd, &final_dependency_info);
}
}
CopyDeviceCopyType::Blit { src, dst } => {
let regions = &mut *slf.dev.phy.image_blit_2.borrow_mut();
regions.clear();
for &mut (x, y, width, height) in rects {
let x1 = x;
let y1 = y;
let x2 = x1 + width as i32;
let y2 = y1 + height as i32;
let offsets = [
Offset3D { x: x1, y: y1, z: 0 },
Offset3D { x: x2, y: y2, z: 1 },
];
let region = ImageBlit2::default()
.src_subresource(image_subresource)
.src_offsets(offsets)
.dst_subresource(image_subresource)
.dst_offsets(offsets);
regions.push(region);
}
use {AccessFlags2 as A, ImageLayout as L};
let initial_barriers = initial_image_barriers![
src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ;
dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE;
];
let final_barriers = final_image_barriers![
src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ;
dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE;
];
let initial_dependency_info =
DependencyInfo::default().image_memory_barriers(&initial_barriers);
let final_dependency_info =
DependencyInfo::default().image_memory_barriers(&final_barriers);
let blit_image_info = BlitImageInfo2::default()
.src_image(src.img)
.src_image_layout(L::TRANSFER_SRC_OPTIMAL)
.dst_image(dst.img)
.dst_image_layout(L::TRANSFER_DST_OPTIMAL)
.regions(regions)
.filter(Filter::NEAREST);
unsafe {
dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info);
dev.cmd_blit_image2(cmd, &blit_image_info);
dev.cmd_pipeline_barrier2(cmd, &final_dependency_info);
}
}
};
unsafe {
dev.end_command_buffer(cmd)
.map_err(CopyDeviceError::EndCommandBuffer)?;
}
let mut wait_semaphore = None;
let mut wait_semaphores = ArrayVec::<_, 1>::new();
if let Some(sync) = sync
&& let Some(sync_file) = sync.get_sync_file()
{
let semaphore = match slf.dev.semaphores.pop() {
Some(s) => s,
_ => slf.dev.create_semaphore()?,
};
semaphore.import(sync_file)?;
let info = SemaphoreSubmitInfo::default()
.semaphore(semaphore.semaphore)
.stage_mask(PipelineStageFlags2::TRANSFER);
wait_semaphores.push(info);
wait_semaphore = Some(semaphore);
}
let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd);
let mut semaphore_submit_info = SemaphoreSubmitInfo::default();
let mut submit_info = SubmitInfo2::default()
.command_buffer_infos(slice::from_ref(&command_buffer_info))
.wait_semaphore_infos(&wait_semaphores);
let vulkan_sync = slf.dev.create_sync(
self.dev.timeline_semaphore.as_ref(),
&mut semaphore_submit_info,
&mut submit_info,
)?;
unsafe {
slf.dev
.dev
.queue_submit2(
slf.dev.queues[tt],
slice::from_ref(&submit_info),
vulkan_sync.fence(),
)
.map_err(CopyDeviceError::SubmitCopy)?;
}
let sync = vulkan_sync.to_sync(|| slf.dev.wait_idle());
slf.busy.set(sync.clone());
let pending = Pending {
dev: slf.dev.clone(),
busy_id: slf.busy_id.add_fetch(1),
sync: sync.clone(),
copy: self.inner.clone(),
semaphore: wait_semaphore,
vulkan_sync,
};
slf.dev.submissions[tt].pending.push(pending);
Ok(sync)
}
}
impl VulkanSemaphore { impl VulkanSemaphore {
fn import(&self, sync_file: &OwnedFd) -> Result<(), CopyDeviceError> { fn import(&self, sync_file: &OwnedFd) -> Result<(), CopyDeviceError> {
let fd = uapi::fcntl_dupfd_cloexec(sync_file.raw(), 0) let fd = uapi::fcntl_dupfd_cloexec(sync_file.raw(), 0)

442
src/copy_device/execute.rs Normal file
View file

@ -0,0 +1,442 @@
use {
super::{CopyDeviceCopy, CopyDeviceCopyType, CopyDeviceError, Pending},
crate::{
gfx_api::FdSync,
rect::{Rect, Region},
vulkan_core::sync::VulkanDeviceSyncExt,
},
arrayvec::ArrayVec,
ash::vk::{
AccessFlags2, BlitImageInfo2, BufferCopy2, BufferImageCopy2, BufferMemoryBarrier2,
CommandBufferBeginInfo, CommandBufferSubmitInfo, CommandBufferUsageFlags,
CopyBufferInfo2, CopyBufferToImageInfo2, CopyImageInfo2, CopyImageToBufferInfo2,
DependencyInfo, Extent3D, Filter, ImageAspectFlags, ImageBlit2, ImageCopy2, ImageLayout,
ImageMemoryBarrier2, ImageSubresourceLayers, ImageSubresourceRange, Offset3D,
PipelineStageFlags2, QUEUE_FAMILY_FOREIGN_EXT, SemaphoreSubmitInfo, SubmitInfo2,
WHOLE_SIZE,
},
std::slice,
};
impl CopyDeviceCopy {
fn ensure_not_busy(&self) -> Result<(), CopyDeviceError> {
let slf = &*self.inner;
if let Some(sync) = slf.busy.get()
&& sync.is_unsignaled()
{
return Err(CopyDeviceError::Busy);
}
slf.busy.take();
Ok(())
}
pub fn execute(
&self,
sync: Option<&FdSync>,
region: Option<&Region>,
) -> Result<Option<FdSync>, CopyDeviceError> {
self.ensure_not_busy()?;
let slf = &*self.inner;
let tt = slf.tt;
let dev = &slf.dev.dev;
let cmd = slf.command_buffer;
let queue_family = slf.dev.phy.queues[tt].family;
let region_buf;
let width = slf.width;
let height = slf.height;
let region = match region {
Some(r) => r,
_ => {
region_buf = Region::new(Rect::new_saturating(0, 0, width as i32, height as i32));
&region_buf
}
};
let (x_mask, y_mask) = slf.dev.phy.queues[tt].transfer_granularity_mask;
let rects = &mut *slf.dev.phy.rects.borrow_mut();
rects.clear();
for rect in region.iter() {
let x1 = (rect.x1().max(0) as u32 & !x_mask).min(width);
let y1 = (rect.y1().max(0) as u32 & !y_mask).min(height);
let x2 = ((rect.x2().max(0) as u32 + x_mask) & !x_mask).min(width);
let y2 = ((rect.y2().max(0) as u32 + y_mask) & !y_mask).min(height);
let width = x2 - x1;
let height = y2 - y1;
if width == 0 || height == 0 {
continue;
}
rects.push((x1 as i32, y1 as i32, width, height));
}
if rects.is_empty() {
return Ok(None);
}
let begin_info =
CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT);
unsafe {
dev.begin_command_buffer(cmd, &begin_info)
.map_err(CopyDeviceError::BeginCommandBuffer)?;
}
macro_rules! initial_buffer_barriers {
($($buf:expr, $access:expr;)*) => {
[$(
BufferMemoryBarrier2::default()
.dst_stage_mask(PipelineStageFlags2::TRANSFER)
.dst_access_mask($access)
.src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
.dst_queue_family_index(queue_family)
.buffer($buf.buf)
.size(WHOLE_SIZE),
)*]
};
}
macro_rules! final_buffer_barriers {
($($buf:expr, $access:expr;)*) => {
[$(
BufferMemoryBarrier2::default()
.src_stage_mask(PipelineStageFlags2::TRANSFER)
.src_access_mask($access)
.src_queue_family_index(queue_family)
.dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
.buffer($buf.buf)
.size(WHOLE_SIZE),
)*]
};
}
let image_subresource_range = ImageSubresourceRange {
aspect_mask: ImageAspectFlags::COLOR,
base_mip_level: 0,
level_count: 1,
base_array_layer: 0,
layer_count: 1,
};
let image_subresource = ImageSubresourceLayers {
aspect_mask: ImageAspectFlags::COLOR,
mip_level: 0,
base_array_layer: 0,
layer_count: 1,
};
macro_rules! initial_image_barriers {
($($img:expr, $layout:expr, $access:expr;)*) => {
[$(
ImageMemoryBarrier2::default()
.dst_stage_mask(PipelineStageFlags2::TRANSFER)
.dst_access_mask($access)
.old_layout(ImageLayout::GENERAL)
.new_layout(ImageLayout::GENERAL)
.src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
.dst_queue_family_index(queue_family)
.image($img.img)
.subresource_range(image_subresource_range),
ImageMemoryBarrier2::default()
.src_stage_mask(PipelineStageFlags2::TRANSFER)
.src_access_mask($access)
.dst_stage_mask(PipelineStageFlags2::TRANSFER)
.dst_access_mask($access)
.old_layout(ImageLayout::GENERAL)
.new_layout($layout)
.src_queue_family_index(queue_family)
.dst_queue_family_index(queue_family)
.image($img.img)
.subresource_range(image_subresource_range),
)*]
};
}
macro_rules! final_image_barriers {
($($img:expr, $layout:expr, $access:expr;)*) => {
[$(
ImageMemoryBarrier2::default()
.src_stage_mask(PipelineStageFlags2::TRANSFER)
.src_access_mask($access)
.dst_stage_mask(PipelineStageFlags2::TRANSFER)
.dst_access_mask($access)
.old_layout($layout)
.new_layout(ImageLayout::GENERAL)
.src_queue_family_index(queue_family)
.dst_queue_family_index(queue_family)
.image($img.img)
.subresource_range(image_subresource_range),
ImageMemoryBarrier2::default()
.src_stage_mask(PipelineStageFlags2::TRANSFER)
.src_access_mask($access)
.old_layout(ImageLayout::GENERAL)
.new_layout(ImageLayout::GENERAL)
.src_queue_family_index(queue_family)
.dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
.image($img.img)
.subresource_range(image_subresource_range),
)*]
};
}
match &slf.ty {
CopyDeviceCopyType::BufferToBuffer {
src,
dst,
stride,
bpp,
} => {
let regions = &mut *slf.dev.phy.buffer_copy_2.borrow_mut();
regions.clear();
let stride = *stride as u64;
let bpp = *bpp as u64;
for &mut (x, y, width, height) in rects {
let lo = y as u64 * stride + x as u64 * bpp;
let size = (height as u64 - 1) * stride + width as u64 * bpp;
let region = BufferCopy2::default()
.src_offset(lo)
.dst_offset(lo)
.size(size);
regions.push(region);
}
use AccessFlags2 as A;
let initial_barriers = initial_buffer_barriers![
src, A::TRANSFER_READ;
dst, A::TRANSFER_WRITE;
];
let final_barriers = final_buffer_barriers![
src, A::TRANSFER_READ;
dst, A::TRANSFER_WRITE;
];
let initial_dependency_info =
DependencyInfo::default().buffer_memory_barriers(&initial_barriers);
let final_dependency_info =
DependencyInfo::default().buffer_memory_barriers(&final_barriers);
let copy_buffer_info = CopyBufferInfo2::default()
.src_buffer(src.buf)
.dst_buffer(dst.buf)
.regions(regions);
unsafe {
dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info);
dev.cmd_copy_buffer2(cmd, &copy_buffer_info);
dev.cmd_pipeline_barrier2(cmd, &final_dependency_info);
}
}
CopyDeviceCopyType::BufferToImage {
buf,
buf_format,
buf_stride,
img,
}
| CopyDeviceCopyType::ImageToBuffer {
img,
buf,
buf_format,
buf_stride,
} => {
let regions = &mut *slf.dev.phy.buffer_image_copy_2.borrow_mut();
regions.clear();
for &mut (x, y, width, height) in rects {
let offset = y as u64 * *buf_stride as u64 + x as u64 * buf_format.bpp as u64;
let region = BufferImageCopy2::default()
.buffer_offset(offset)
.buffer_row_length(*buf_stride / buf_format.bpp)
.buffer_image_height(slf.height)
.image_subresource(image_subresource)
.image_offset(Offset3D { x, y, z: 0 })
.image_extent(Extent3D {
width,
height,
depth: 1,
});
regions.push(region);
}
let buffer_to_image = match &slf.ty {
CopyDeviceCopyType::BufferToImage { .. } => true,
CopyDeviceCopyType::ImageToBuffer { .. } => false,
_ => unreachable!(),
};
let image_access_mask;
let image_layout;
let buffer_access_mask;
match buffer_to_image {
true => {
image_access_mask = AccessFlags2::TRANSFER_WRITE;
image_layout = ImageLayout::TRANSFER_DST_OPTIMAL;
buffer_access_mask = AccessFlags2::TRANSFER_READ;
}
false => {
image_access_mask = AccessFlags2::TRANSFER_READ;
image_layout = ImageLayout::TRANSFER_SRC_OPTIMAL;
buffer_access_mask = AccessFlags2::TRANSFER_WRITE;
}
}
let initial_image_barriers = initial_image_barriers![
img, image_layout, image_access_mask;
];
let final_image_barriers = final_image_barriers![
img, image_layout, image_access_mask;
];
let initial_buffer_barriers = initial_buffer_barriers![
buf, buffer_access_mask;
];
let final_buffer_barriers = final_buffer_barriers![
buf, buffer_access_mask;
];
let initial_dependency_info = DependencyInfo::default()
.buffer_memory_barriers(&initial_buffer_barriers)
.image_memory_barriers(&initial_image_barriers);
let final_dependency_info = DependencyInfo::default()
.buffer_memory_barriers(&final_buffer_barriers)
.image_memory_barriers(&final_image_barriers);
unsafe {
dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info);
match buffer_to_image {
true => {
let copy = CopyBufferToImageInfo2::default()
.src_buffer(buf.buf)
.dst_image(img.img)
.dst_image_layout(image_layout)
.regions(&regions);
dev.cmd_copy_buffer_to_image2(cmd, &copy);
}
false => {
let copy = CopyImageToBufferInfo2::default()
.src_image(img.img)
.src_image_layout(image_layout)
.dst_buffer(buf.buf)
.regions(&regions);
dev.cmd_copy_image_to_buffer2(cmd, &copy);
}
}
dev.cmd_pipeline_barrier2(cmd, &final_dependency_info);
}
}
CopyDeviceCopyType::ImageToImage { src, dst } => {
let regions = &mut *slf.dev.phy.image_copy_2.borrow_mut();
regions.clear();
for &mut (x, y, width, height) in rects {
let region = ImageCopy2::default()
.src_subresource(image_subresource)
.src_offset(Offset3D { x, y, z: 0 })
.dst_subresource(image_subresource)
.dst_offset(Offset3D { x, y, z: 0 })
.extent(Extent3D {
width,
height,
depth: 1,
});
regions.push(region);
}
use {AccessFlags2 as A, ImageLayout as L};
let initial_barriers = initial_image_barriers![
src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ;
dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE;
];
let final_barriers = final_image_barriers![
src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ;
dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE;
];
let initial_dependency_info =
DependencyInfo::default().image_memory_barriers(&initial_barriers);
let final_dependency_info =
DependencyInfo::default().image_memory_barriers(&final_barriers);
let copy_image_info = CopyImageInfo2::default()
.src_image(src.img)
.src_image_layout(L::TRANSFER_SRC_OPTIMAL)
.dst_image(dst.img)
.dst_image_layout(L::TRANSFER_DST_OPTIMAL)
.regions(regions);
unsafe {
dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info);
dev.cmd_copy_image2(cmd, &copy_image_info);
dev.cmd_pipeline_barrier2(cmd, &final_dependency_info);
}
}
CopyDeviceCopyType::Blit { src, dst } => {
let regions = &mut *slf.dev.phy.image_blit_2.borrow_mut();
regions.clear();
for &mut (x, y, width, height) in rects {
let x1 = x;
let y1 = y;
let x2 = x1 + width as i32;
let y2 = y1 + height as i32;
let offsets = [
Offset3D { x: x1, y: y1, z: 0 },
Offset3D { x: x2, y: y2, z: 1 },
];
let region = ImageBlit2::default()
.src_subresource(image_subresource)
.src_offsets(offsets)
.dst_subresource(image_subresource)
.dst_offsets(offsets);
regions.push(region);
}
use {AccessFlags2 as A, ImageLayout as L};
let initial_barriers = initial_image_barriers![
src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ;
dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE;
];
let final_barriers = final_image_barriers![
src, L::TRANSFER_SRC_OPTIMAL, A::TRANSFER_READ;
dst, L::TRANSFER_DST_OPTIMAL, A::TRANSFER_WRITE;
];
let initial_dependency_info =
DependencyInfo::default().image_memory_barriers(&initial_barriers);
let final_dependency_info =
DependencyInfo::default().image_memory_barriers(&final_barriers);
let blit_image_info = BlitImageInfo2::default()
.src_image(src.img)
.src_image_layout(L::TRANSFER_SRC_OPTIMAL)
.dst_image(dst.img)
.dst_image_layout(L::TRANSFER_DST_OPTIMAL)
.regions(regions)
.filter(Filter::NEAREST);
unsafe {
dev.cmd_pipeline_barrier2(cmd, &initial_dependency_info);
dev.cmd_blit_image2(cmd, &blit_image_info);
dev.cmd_pipeline_barrier2(cmd, &final_dependency_info);
}
}
};
unsafe {
dev.end_command_buffer(cmd)
.map_err(CopyDeviceError::EndCommandBuffer)?;
}
let mut wait_semaphore = None;
let mut wait_semaphores = ArrayVec::<_, 1>::new();
if let Some(sync) = sync
&& let Some(sync_file) = sync.get_sync_file()
{
let semaphore = match slf.dev.semaphores.pop() {
Some(s) => s,
_ => slf.dev.create_semaphore()?,
};
semaphore.import(sync_file)?;
let info = SemaphoreSubmitInfo::default()
.semaphore(semaphore.semaphore)
.stage_mask(PipelineStageFlags2::TRANSFER);
wait_semaphores.push(info);
wait_semaphore = Some(semaphore);
}
let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd);
let mut semaphore_submit_info = SemaphoreSubmitInfo::default();
let mut submit_info = SubmitInfo2::default()
.command_buffer_infos(slice::from_ref(&command_buffer_info))
.wait_semaphore_infos(&wait_semaphores);
let vulkan_sync = slf.dev.create_sync(
self.dev.timeline_semaphore.as_ref(),
&mut semaphore_submit_info,
&mut submit_info,
)?;
unsafe {
slf.dev
.dev
.queue_submit2(
slf.dev.queues[tt],
slice::from_ref(&submit_info),
vulkan_sync.fence(),
)
.map_err(CopyDeviceError::SubmitCopy)?;
}
let sync = vulkan_sync.to_sync(|| slf.dev.wait_idle());
slf.busy.set(sync.clone());
let pending = Pending {
dev: slf.dev.clone(),
busy_id: slf.busy_id.add_fetch(1),
sync: sync.clone(),
copy: self.inner.clone(),
semaphore: wait_semaphore,
vulkan_sync,
};
slf.dev.submissions[tt].pending.push(pending);
Ok(sync)
}
}