diff --git a/src/copy_device.rs b/src/copy_device.rs index a3aee40f..70191dd0 100644 --- a/src/copy_device.rs +++ b/src/copy_device.rs @@ -8,7 +8,6 @@ use { rect::{Rect, Region}, utils::{ clonecell::CloneCell, - copyhashmap::CopyHashMap, errorfmt::ErrorFmt, numcell::NumCell, oserror::{OsError, OsErrorExt2}, @@ -26,7 +25,7 @@ use { timeline_semaphore::VulkanDeviceTimelineSemaphoreExt, }, }, - ahash::{AHashMap, AHashSet}, + ahash::AHashMap, arrayvec::ArrayVec, ash::{ Device, @@ -70,7 +69,7 @@ use { }, bstr::ByteSlice, isnt::std_1::collections::IsntHashMapExt, - linearize::{Linearize, LinearizeExt, StaticCopyMap, StaticMap, static_copy_map, static_map}, + linearize::{Linearize, LinearizeExt, StaticCopyMap, StaticMap, static_map}, log::Level, run_on_drop::on_drop, std::{ @@ -86,6 +85,13 @@ use { vk::{Buffer, CommandPool, Image, Semaphore}, }; +mod queue_allocation; +mod registry; + +pub use registry::CopyDeviceRegistry; + +use queue_allocation::{QueueIndex, QueueToAllocate, allocate_queues}; + #[derive(Debug, Error)] pub enum CopyDeviceError { #[error(transparent)] @@ -197,20 +203,6 @@ pub struct PhysicalCopyDevice { image_blit_2: RefCell>>, } -#[derive(Debug)] -struct QueueToAllocate { - family: u32, - num: usize, -} - -#[derive(Copy, Clone, Default, Debug)] -struct QueueIndex { - allocate_idx: usize, - family: u32, - idx_within_family: u32, - transfer_granularity_mask: (u32, u32), -} - pub struct CopyDevice { _tasks: Vec>, dev: Rc, @@ -349,13 +341,6 @@ struct ClassifiedDmabuf<'a> { format: &'a CopyDeviceSupport, } -pub struct CopyDeviceRegistry { - ring: Rc, - eng: Rc, - eventfd_cache: Rc, - devs: CopyHashMap>>, -} - const DEVICE_EXTENSIONS: [&CStr; 6] = [ external_semaphore_fd::NAME, external_fence_fd::NAME, @@ -1733,47 +1718,6 @@ impl VulkanSemaphore { } } -impl CopyDeviceRegistry { - pub fn new( - ring: &Rc, - eng: &Rc, - eventfd_cache: &Rc, - ) -> Self { - Self { - ring: ring.clone(), - eng: eng.clone(), - eventfd_cache: eventfd_cache.clone(), - devs: Default::default(), - } - } - - pub fn remove(&self, dev: c::dev_t) { - self.devs.remove(&dev); - } - - pub fn get(&self, dev: c::dev_t) -> Option> { - if let Some(dev) = self.devs.get(&dev) { - return dev; - } - match PhysicalCopyDevice::new(&self.ring, &self.eng, &self.eventfd_cache, dev).map(Some) { - Ok(cd) => { - self.devs.set(dev, cd.clone()); - cd - } - Err(e) => { - let maj = uapi::major(dev); - let min = uapi::minor(dev); - log::warn!( - "Could not create physical copy device for {maj}:{min}: {}", - ErrorFmt(e), - ); - self.devs.set(dev, None); - None - } - } - } -} - impl Drop for VulkanSemaphore { fn drop(&mut self) { unsafe { @@ -1898,125 +1842,6 @@ impl Deref for CopyDevice { } } -type QueueInfo = (u32, (u32, u32), u32); - -fn allocate_queues( - gfx: QueueInfo, - compute_only: Option, - transfer_only: Option, -) -> (Vec, KeyedCopy) { - let intra = compute_only.unwrap_or(gfx); - let cross = transfer_only.unwrap_or(intra); - let mut distinct_families = AHashSet::default(); - distinct_families.insert(cross); - distinct_families.insert(intra); - distinct_families.insert(gfx); - let mut queues_to_allocate = vec![]; - macro_rules! index { - ($qi:expr, $within:expr) => { - QueueIndex { - allocate_idx: queues_to_allocate.len(), - family: $qi.0, - idx_within_family: $within as u32, - transfer_granularity_mask: $qi.1, - } - }; - } - macro_rules! alloc { - ($qi:expr, $num:expr) => { - QueueToAllocate { - family: $qi.0, - num: $num as usize, - } - }; - } - let (blit, intra_idx, download, upload); - if distinct_families.len() == 3 { - let num_cross = cross.2.min(2) as usize; - blit = index!(gfx, 0); - queues_to_allocate.push(alloc!(gfx, 1)); - intra_idx = index!(intra, 0); - queues_to_allocate.push(alloc!(intra, 1)); - download = index!(cross, 0); - upload = index!(cross, num_cross - 1); - queues_to_allocate.push(alloc!(cross, num_cross)); - } else if distinct_families.len() == 1 { - let qi = cross; - let num = qi.2.min(4); - match num { - 1 => { - blit = index!(qi, 0); - intra_idx = index!(qi, 0); - download = index!(qi, 0); - upload = index!(qi, 0); - } - 2 => { - blit = index!(qi, 0); - intra_idx = index!(qi, 0); - download = index!(qi, 0); - upload = index!(qi, 1); - } - 3 => { - blit = index!(qi, 0); - intra_idx = index!(qi, 0); - download = index!(qi, 1); - upload = index!(qi, 2); - } - 4 => { - blit = index!(qi, 0); - intra_idx = index!(qi, 1); - download = index!(qi, 2); - upload = index!(qi, 3); - } - _ => unreachable!(), - } - queues_to_allocate.push(alloc!(qi, num)); - } else { - if gfx == intra { - let num_gfx = gfx.2.min(2); - blit = index!(gfx, 0); - intra_idx = index!(gfx, num_gfx - 1); - queues_to_allocate.push(alloc!(gfx, num_gfx)); - let num_cross = cross.2.min(2); - download = index!(cross, 0); - upload = index!(cross, num_cross - 1); - queues_to_allocate.push(alloc!(cross, num_cross)); - } else { - // if cross == gfx then intra == gfx - assert_eq!(intra, cross); - blit = index!(gfx, 0); - queues_to_allocate.push(alloc!(gfx, 1)); - let num_intra = intra.2.min(3); - match num_intra { - 1 => { - intra_idx = index!(intra, 0); - download = index!(intra, 0); - upload = index!(intra, 0); - } - 2 => { - intra_idx = index!(intra, 0); - download = index!(intra, 0); - upload = index!(intra, 1); - } - 3 => { - intra_idx = index!(intra, 0); - download = index!(intra, 1); - upload = index!(intra, 2); - } - _ => unreachable!(), - } - queues_to_allocate.push(alloc!(intra, num_intra)); - } - } - let queue_indices = static_copy_map! { - TransferType::Blit => blit, - TransferType::Intra => intra_idx, - TransferType::Download => download, - TransferType::Upload => upload, - }; - (queues_to_allocate, queue_indices) -} - impl VulkanDeviceInf for CopyDeviceInner { fn instance(&self) -> &VulkanCoreInstance { &self.phy.instance diff --git a/src/copy_device/queue_allocation.rs b/src/copy_device/queue_allocation.rs new file mode 100644 index 00000000..8d3d91d9 --- /dev/null +++ b/src/copy_device/queue_allocation.rs @@ -0,0 +1,138 @@ +use { + super::{KeyedCopy, TransferType}, + ahash::AHashSet, + linearize::static_copy_map, +}; + +#[derive(Debug)] +pub(super) struct QueueToAllocate { + pub(super) family: u32, + pub(super) num: usize, +} + +#[derive(Copy, Clone, Default, Debug)] +pub(super) struct QueueIndex { + pub(super) allocate_idx: usize, + pub(super) family: u32, + pub(super) idx_within_family: u32, + pub(super) transfer_granularity_mask: (u32, u32), +} + +type QueueInfo = (u32, (u32, u32), u32); + +pub(super) fn allocate_queues( + gfx: QueueInfo, + compute_only: Option, + transfer_only: Option, +) -> (Vec, KeyedCopy) { + let intra = compute_only.unwrap_or(gfx); + let cross = transfer_only.unwrap_or(intra); + let mut distinct_families = AHashSet::default(); + distinct_families.insert(cross); + distinct_families.insert(intra); + distinct_families.insert(gfx); + let mut queues_to_allocate = vec![]; + macro_rules! index { + ($qi:expr, $within:expr) => { + QueueIndex { + allocate_idx: queues_to_allocate.len(), + family: $qi.0, + idx_within_family: $within as u32, + transfer_granularity_mask: $qi.1, + } + }; + } + macro_rules! alloc { + ($qi:expr, $num:expr) => { + QueueToAllocate { + family: $qi.0, + num: $num as usize, + } + }; + } + let (blit, intra_idx, download, upload); + if distinct_families.len() == 3 { + let num_cross = cross.2.min(2) as usize; + blit = index!(gfx, 0); + queues_to_allocate.push(alloc!(gfx, 1)); + intra_idx = index!(intra, 0); + queues_to_allocate.push(alloc!(intra, 1)); + download = index!(cross, 0); + upload = index!(cross, num_cross - 1); + queues_to_allocate.push(alloc!(cross, num_cross)); + } else if distinct_families.len() == 1 { + let qi = cross; + let num = qi.2.min(4); + match num { + 1 => { + blit = index!(qi, 0); + intra_idx = index!(qi, 0); + download = index!(qi, 0); + upload = index!(qi, 0); + } + 2 => { + blit = index!(qi, 0); + intra_idx = index!(qi, 0); + download = index!(qi, 0); + upload = index!(qi, 1); + } + 3 => { + blit = index!(qi, 0); + intra_idx = index!(qi, 0); + download = index!(qi, 1); + upload = index!(qi, 2); + } + 4 => { + blit = index!(qi, 0); + intra_idx = index!(qi, 1); + download = index!(qi, 2); + upload = index!(qi, 3); + } + _ => unreachable!(), + } + queues_to_allocate.push(alloc!(qi, num)); + } else { + if gfx == intra { + let num_gfx = gfx.2.min(2); + blit = index!(gfx, 0); + intra_idx = index!(gfx, num_gfx - 1); + queues_to_allocate.push(alloc!(gfx, num_gfx)); + let num_cross = cross.2.min(2); + download = index!(cross, 0); + upload = index!(cross, num_cross - 1); + queues_to_allocate.push(alloc!(cross, num_cross)); + } else { + // if cross == gfx then intra == gfx + assert_eq!(intra, cross); + blit = index!(gfx, 0); + queues_to_allocate.push(alloc!(gfx, 1)); + let num_intra = intra.2.min(3); + match num_intra { + 1 => { + intra_idx = index!(intra, 0); + download = index!(intra, 0); + upload = index!(intra, 0); + } + 2 => { + intra_idx = index!(intra, 0); + download = index!(intra, 0); + upload = index!(intra, 1); + } + 3 => { + intra_idx = index!(intra, 0); + download = index!(intra, 1); + upload = index!(intra, 2); + } + _ => unreachable!(), + } + queues_to_allocate.push(alloc!(intra, num_intra)); + } + } + let queue_indices = static_copy_map! { + TransferType::Blit => blit, + TransferType::Intra => intra_idx, + TransferType::Download => download, + TransferType::Upload => upload, + }; + (queues_to_allocate, queue_indices) +} diff --git a/src/copy_device/registry.rs b/src/copy_device/registry.rs new file mode 100644 index 00000000..c2518a68 --- /dev/null +++ b/src/copy_device/registry.rs @@ -0,0 +1,59 @@ +use { + super::PhysicalCopyDevice, + crate::{ + async_engine::AsyncEngine, + eventfd_cache::EventfdCache, + io_uring::IoUring, + utils::{copyhashmap::CopyHashMap, errorfmt::ErrorFmt}, + }, + std::rc::Rc, + uapi::c, +}; + +pub struct CopyDeviceRegistry { + ring: Rc, + eng: Rc, + eventfd_cache: Rc, + devs: CopyHashMap>>, +} + +impl CopyDeviceRegistry { + pub fn new( + ring: &Rc, + eng: &Rc, + eventfd_cache: &Rc, + ) -> Self { + Self { + ring: ring.clone(), + eng: eng.clone(), + eventfd_cache: eventfd_cache.clone(), + devs: Default::default(), + } + } + + pub fn remove(&self, dev: c::dev_t) { + self.devs.remove(&dev); + } + + pub fn get(&self, dev: c::dev_t) -> Option> { + if let Some(dev) = self.devs.get(&dev) { + return dev; + } + match PhysicalCopyDevice::new(&self.ring, &self.eng, &self.eventfd_cache, dev).map(Some) { + Ok(cd) => { + self.devs.set(dev, cd.clone()); + cd + } + Err(e) => { + let maj = uapi::major(dev); + let min = uapi::minor(dev); + log::warn!( + "Could not create physical copy device for {maj}:{min}: {}", + ErrorFmt(e), + ); + self.devs.set(dev, None); + None + } + } + } +}