copy-device: split support modules
This commit is contained in:
parent
c482a2b99d
commit
d36ec3b7d3
3 changed files with 206 additions and 184 deletions
|
|
@ -8,7 +8,6 @@ use {
|
|||
rect::{Rect, Region},
|
||||
utils::{
|
||||
clonecell::CloneCell,
|
||||
copyhashmap::CopyHashMap,
|
||||
errorfmt::ErrorFmt,
|
||||
numcell::NumCell,
|
||||
oserror::{OsError, OsErrorExt2},
|
||||
|
|
@ -26,7 +25,7 @@ use {
|
|||
timeline_semaphore::VulkanDeviceTimelineSemaphoreExt,
|
||||
},
|
||||
},
|
||||
ahash::{AHashMap, AHashSet},
|
||||
ahash::AHashMap,
|
||||
arrayvec::ArrayVec,
|
||||
ash::{
|
||||
Device,
|
||||
|
|
@ -70,7 +69,7 @@ use {
|
|||
},
|
||||
bstr::ByteSlice,
|
||||
isnt::std_1::collections::IsntHashMapExt,
|
||||
linearize::{Linearize, LinearizeExt, StaticCopyMap, StaticMap, static_copy_map, static_map},
|
||||
linearize::{Linearize, LinearizeExt, StaticCopyMap, StaticMap, static_map},
|
||||
log::Level,
|
||||
run_on_drop::on_drop,
|
||||
std::{
|
||||
|
|
@ -86,6 +85,13 @@ use {
|
|||
vk::{Buffer, CommandPool, Image, Semaphore},
|
||||
};
|
||||
|
||||
mod queue_allocation;
|
||||
mod registry;
|
||||
|
||||
pub use registry::CopyDeviceRegistry;
|
||||
|
||||
use queue_allocation::{QueueIndex, QueueToAllocate, allocate_queues};
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum CopyDeviceError {
|
||||
#[error(transparent)]
|
||||
|
|
@ -197,20 +203,6 @@ pub struct PhysicalCopyDevice {
|
|||
image_blit_2: RefCell<Vec<ImageBlit2<'static>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct QueueToAllocate {
|
||||
family: u32,
|
||||
num: usize,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Default, Debug)]
|
||||
struct QueueIndex {
|
||||
allocate_idx: usize,
|
||||
family: u32,
|
||||
idx_within_family: u32,
|
||||
transfer_granularity_mask: (u32, u32),
|
||||
}
|
||||
|
||||
pub struct CopyDevice {
|
||||
_tasks: Vec<SpawnedFuture<()>>,
|
||||
dev: Rc<CopyDeviceInner>,
|
||||
|
|
@ -349,13 +341,6 @@ struct ClassifiedDmabuf<'a> {
|
|||
format: &'a CopyDeviceSupport,
|
||||
}
|
||||
|
||||
pub struct CopyDeviceRegistry {
|
||||
ring: Rc<IoUring>,
|
||||
eng: Rc<AsyncEngine>,
|
||||
eventfd_cache: Rc<EventfdCache>,
|
||||
devs: CopyHashMap<c::dev_t, Option<Rc<PhysicalCopyDevice>>>,
|
||||
}
|
||||
|
||||
const DEVICE_EXTENSIONS: [&CStr; 6] = [
|
||||
external_semaphore_fd::NAME,
|
||||
external_fence_fd::NAME,
|
||||
|
|
@ -1733,47 +1718,6 @@ impl VulkanSemaphore {
|
|||
}
|
||||
}
|
||||
|
||||
impl CopyDeviceRegistry {
|
||||
pub fn new(
|
||||
ring: &Rc<IoUring>,
|
||||
eng: &Rc<AsyncEngine>,
|
||||
eventfd_cache: &Rc<EventfdCache>,
|
||||
) -> Self {
|
||||
Self {
|
||||
ring: ring.clone(),
|
||||
eng: eng.clone(),
|
||||
eventfd_cache: eventfd_cache.clone(),
|
||||
devs: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remove(&self, dev: c::dev_t) {
|
||||
self.devs.remove(&dev);
|
||||
}
|
||||
|
||||
pub fn get(&self, dev: c::dev_t) -> Option<Rc<PhysicalCopyDevice>> {
|
||||
if let Some(dev) = self.devs.get(&dev) {
|
||||
return dev;
|
||||
}
|
||||
match PhysicalCopyDevice::new(&self.ring, &self.eng, &self.eventfd_cache, dev).map(Some) {
|
||||
Ok(cd) => {
|
||||
self.devs.set(dev, cd.clone());
|
||||
cd
|
||||
}
|
||||
Err(e) => {
|
||||
let maj = uapi::major(dev);
|
||||
let min = uapi::minor(dev);
|
||||
log::warn!(
|
||||
"Could not create physical copy device for {maj}:{min}: {}",
|
||||
ErrorFmt(e),
|
||||
);
|
||||
self.devs.set(dev, None);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for VulkanSemaphore {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
|
|
@ -1898,125 +1842,6 @@ impl Deref for CopyDevice {
|
|||
}
|
||||
}
|
||||
|
||||
type QueueInfo = (u32, (u32, u32), u32);
|
||||
|
||||
fn allocate_queues(
|
||||
gfx: QueueInfo,
|
||||
compute_only: Option<QueueInfo>,
|
||||
transfer_only: Option<QueueInfo>,
|
||||
) -> (Vec<QueueToAllocate>, KeyedCopy<QueueIndex>) {
|
||||
let intra = compute_only.unwrap_or(gfx);
|
||||
let cross = transfer_only.unwrap_or(intra);
|
||||
let mut distinct_families = AHashSet::default();
|
||||
distinct_families.insert(cross);
|
||||
distinct_families.insert(intra);
|
||||
distinct_families.insert(gfx);
|
||||
let mut queues_to_allocate = vec![];
|
||||
macro_rules! index {
|
||||
($qi:expr, $within:expr) => {
|
||||
QueueIndex {
|
||||
allocate_idx: queues_to_allocate.len(),
|
||||
family: $qi.0,
|
||||
idx_within_family: $within as u32,
|
||||
transfer_granularity_mask: $qi.1,
|
||||
}
|
||||
};
|
||||
}
|
||||
macro_rules! alloc {
|
||||
($qi:expr, $num:expr) => {
|
||||
QueueToAllocate {
|
||||
family: $qi.0,
|
||||
num: $num as usize,
|
||||
}
|
||||
};
|
||||
}
|
||||
let (blit, intra_idx, download, upload);
|
||||
if distinct_families.len() == 3 {
|
||||
let num_cross = cross.2.min(2) as usize;
|
||||
blit = index!(gfx, 0);
|
||||
queues_to_allocate.push(alloc!(gfx, 1));
|
||||
intra_idx = index!(intra, 0);
|
||||
queues_to_allocate.push(alloc!(intra, 1));
|
||||
download = index!(cross, 0);
|
||||
upload = index!(cross, num_cross - 1);
|
||||
queues_to_allocate.push(alloc!(cross, num_cross));
|
||||
} else if distinct_families.len() == 1 {
|
||||
let qi = cross;
|
||||
let num = qi.2.min(4);
|
||||
match num {
|
||||
1 => {
|
||||
blit = index!(qi, 0);
|
||||
intra_idx = index!(qi, 0);
|
||||
download = index!(qi, 0);
|
||||
upload = index!(qi, 0);
|
||||
}
|
||||
2 => {
|
||||
blit = index!(qi, 0);
|
||||
intra_idx = index!(qi, 0);
|
||||
download = index!(qi, 0);
|
||||
upload = index!(qi, 1);
|
||||
}
|
||||
3 => {
|
||||
blit = index!(qi, 0);
|
||||
intra_idx = index!(qi, 0);
|
||||
download = index!(qi, 1);
|
||||
upload = index!(qi, 2);
|
||||
}
|
||||
4 => {
|
||||
blit = index!(qi, 0);
|
||||
intra_idx = index!(qi, 1);
|
||||
download = index!(qi, 2);
|
||||
upload = index!(qi, 3);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
queues_to_allocate.push(alloc!(qi, num));
|
||||
} else {
|
||||
if gfx == intra {
|
||||
let num_gfx = gfx.2.min(2);
|
||||
blit = index!(gfx, 0);
|
||||
intra_idx = index!(gfx, num_gfx - 1);
|
||||
queues_to_allocate.push(alloc!(gfx, num_gfx));
|
||||
let num_cross = cross.2.min(2);
|
||||
download = index!(cross, 0);
|
||||
upload = index!(cross, num_cross - 1);
|
||||
queues_to_allocate.push(alloc!(cross, num_cross));
|
||||
} else {
|
||||
// if cross == gfx then intra == gfx
|
||||
assert_eq!(intra, cross);
|
||||
blit = index!(gfx, 0);
|
||||
queues_to_allocate.push(alloc!(gfx, 1));
|
||||
let num_intra = intra.2.min(3);
|
||||
match num_intra {
|
||||
1 => {
|
||||
intra_idx = index!(intra, 0);
|
||||
download = index!(intra, 0);
|
||||
upload = index!(intra, 0);
|
||||
}
|
||||
2 => {
|
||||
intra_idx = index!(intra, 0);
|
||||
download = index!(intra, 0);
|
||||
upload = index!(intra, 1);
|
||||
}
|
||||
3 => {
|
||||
intra_idx = index!(intra, 0);
|
||||
download = index!(intra, 1);
|
||||
upload = index!(intra, 2);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
queues_to_allocate.push(alloc!(intra, num_intra));
|
||||
}
|
||||
}
|
||||
let queue_indices = static_copy_map! {
|
||||
TransferType::Blit => blit,
|
||||
TransferType::Intra => intra_idx,
|
||||
TransferType::Download => download,
|
||||
TransferType::Upload => upload,
|
||||
};
|
||||
(queues_to_allocate, queue_indices)
|
||||
}
|
||||
|
||||
impl VulkanDeviceInf for CopyDeviceInner {
|
||||
fn instance(&self) -> &VulkanCoreInstance {
|
||||
&self.phy.instance
|
||||
|
|
|
|||
138
src/copy_device/queue_allocation.rs
Normal file
138
src/copy_device/queue_allocation.rs
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
use {
|
||||
super::{KeyedCopy, TransferType},
|
||||
ahash::AHashSet,
|
||||
linearize::static_copy_map,
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) struct QueueToAllocate {
|
||||
pub(super) family: u32,
|
||||
pub(super) num: usize,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Default, Debug)]
|
||||
pub(super) struct QueueIndex {
|
||||
pub(super) allocate_idx: usize,
|
||||
pub(super) family: u32,
|
||||
pub(super) idx_within_family: u32,
|
||||
pub(super) transfer_granularity_mask: (u32, u32),
|
||||
}
|
||||
|
||||
type QueueInfo = (u32, (u32, u32), u32);
|
||||
|
||||
pub(super) fn allocate_queues(
|
||||
gfx: QueueInfo,
|
||||
compute_only: Option<QueueInfo>,
|
||||
transfer_only: Option<QueueInfo>,
|
||||
) -> (Vec<QueueToAllocate>, KeyedCopy<QueueIndex>) {
|
||||
let intra = compute_only.unwrap_or(gfx);
|
||||
let cross = transfer_only.unwrap_or(intra);
|
||||
let mut distinct_families = AHashSet::default();
|
||||
distinct_families.insert(cross);
|
||||
distinct_families.insert(intra);
|
||||
distinct_families.insert(gfx);
|
||||
let mut queues_to_allocate = vec![];
|
||||
macro_rules! index {
|
||||
($qi:expr, $within:expr) => {
|
||||
QueueIndex {
|
||||
allocate_idx: queues_to_allocate.len(),
|
||||
family: $qi.0,
|
||||
idx_within_family: $within as u32,
|
||||
transfer_granularity_mask: $qi.1,
|
||||
}
|
||||
};
|
||||
}
|
||||
macro_rules! alloc {
|
||||
($qi:expr, $num:expr) => {
|
||||
QueueToAllocate {
|
||||
family: $qi.0,
|
||||
num: $num as usize,
|
||||
}
|
||||
};
|
||||
}
|
||||
let (blit, intra_idx, download, upload);
|
||||
if distinct_families.len() == 3 {
|
||||
let num_cross = cross.2.min(2) as usize;
|
||||
blit = index!(gfx, 0);
|
||||
queues_to_allocate.push(alloc!(gfx, 1));
|
||||
intra_idx = index!(intra, 0);
|
||||
queues_to_allocate.push(alloc!(intra, 1));
|
||||
download = index!(cross, 0);
|
||||
upload = index!(cross, num_cross - 1);
|
||||
queues_to_allocate.push(alloc!(cross, num_cross));
|
||||
} else if distinct_families.len() == 1 {
|
||||
let qi = cross;
|
||||
let num = qi.2.min(4);
|
||||
match num {
|
||||
1 => {
|
||||
blit = index!(qi, 0);
|
||||
intra_idx = index!(qi, 0);
|
||||
download = index!(qi, 0);
|
||||
upload = index!(qi, 0);
|
||||
}
|
||||
2 => {
|
||||
blit = index!(qi, 0);
|
||||
intra_idx = index!(qi, 0);
|
||||
download = index!(qi, 0);
|
||||
upload = index!(qi, 1);
|
||||
}
|
||||
3 => {
|
||||
blit = index!(qi, 0);
|
||||
intra_idx = index!(qi, 0);
|
||||
download = index!(qi, 1);
|
||||
upload = index!(qi, 2);
|
||||
}
|
||||
4 => {
|
||||
blit = index!(qi, 0);
|
||||
intra_idx = index!(qi, 1);
|
||||
download = index!(qi, 2);
|
||||
upload = index!(qi, 3);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
queues_to_allocate.push(alloc!(qi, num));
|
||||
} else {
|
||||
if gfx == intra {
|
||||
let num_gfx = gfx.2.min(2);
|
||||
blit = index!(gfx, 0);
|
||||
intra_idx = index!(gfx, num_gfx - 1);
|
||||
queues_to_allocate.push(alloc!(gfx, num_gfx));
|
||||
let num_cross = cross.2.min(2);
|
||||
download = index!(cross, 0);
|
||||
upload = index!(cross, num_cross - 1);
|
||||
queues_to_allocate.push(alloc!(cross, num_cross));
|
||||
} else {
|
||||
// if cross == gfx then intra == gfx
|
||||
assert_eq!(intra, cross);
|
||||
blit = index!(gfx, 0);
|
||||
queues_to_allocate.push(alloc!(gfx, 1));
|
||||
let num_intra = intra.2.min(3);
|
||||
match num_intra {
|
||||
1 => {
|
||||
intra_idx = index!(intra, 0);
|
||||
download = index!(intra, 0);
|
||||
upload = index!(intra, 0);
|
||||
}
|
||||
2 => {
|
||||
intra_idx = index!(intra, 0);
|
||||
download = index!(intra, 0);
|
||||
upload = index!(intra, 1);
|
||||
}
|
||||
3 => {
|
||||
intra_idx = index!(intra, 0);
|
||||
download = index!(intra, 1);
|
||||
upload = index!(intra, 2);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
queues_to_allocate.push(alloc!(intra, num_intra));
|
||||
}
|
||||
}
|
||||
let queue_indices = static_copy_map! {
|
||||
TransferType::Blit => blit,
|
||||
TransferType::Intra => intra_idx,
|
||||
TransferType::Download => download,
|
||||
TransferType::Upload => upload,
|
||||
};
|
||||
(queues_to_allocate, queue_indices)
|
||||
}
|
||||
59
src/copy_device/registry.rs
Normal file
59
src/copy_device/registry.rs
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
use {
|
||||
super::PhysicalCopyDevice,
|
||||
crate::{
|
||||
async_engine::AsyncEngine,
|
||||
eventfd_cache::EventfdCache,
|
||||
io_uring::IoUring,
|
||||
utils::{copyhashmap::CopyHashMap, errorfmt::ErrorFmt},
|
||||
},
|
||||
std::rc::Rc,
|
||||
uapi::c,
|
||||
};
|
||||
|
||||
pub struct CopyDeviceRegistry {
|
||||
ring: Rc<IoUring>,
|
||||
eng: Rc<AsyncEngine>,
|
||||
eventfd_cache: Rc<EventfdCache>,
|
||||
devs: CopyHashMap<c::dev_t, Option<Rc<PhysicalCopyDevice>>>,
|
||||
}
|
||||
|
||||
impl CopyDeviceRegistry {
|
||||
pub fn new(
|
||||
ring: &Rc<IoUring>,
|
||||
eng: &Rc<AsyncEngine>,
|
||||
eventfd_cache: &Rc<EventfdCache>,
|
||||
) -> Self {
|
||||
Self {
|
||||
ring: ring.clone(),
|
||||
eng: eng.clone(),
|
||||
eventfd_cache: eventfd_cache.clone(),
|
||||
devs: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remove(&self, dev: c::dev_t) {
|
||||
self.devs.remove(&dev);
|
||||
}
|
||||
|
||||
pub fn get(&self, dev: c::dev_t) -> Option<Rc<PhysicalCopyDevice>> {
|
||||
if let Some(dev) = self.devs.get(&dev) {
|
||||
return dev;
|
||||
}
|
||||
match PhysicalCopyDevice::new(&self.ring, &self.eng, &self.eventfd_cache, dev).map(Some) {
|
||||
Ok(cd) => {
|
||||
self.devs.set(dev, cd.clone());
|
||||
cd
|
||||
}
|
||||
Err(e) => {
|
||||
let maj = uapi::major(dev);
|
||||
let min = uapi::minor(dev);
|
||||
log::warn!(
|
||||
"Could not create physical copy device for {maj}:{min}: {}",
|
||||
ErrorFmt(e),
|
||||
);
|
||||
self.devs.set(dev, None);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue