kossLAN/wry
1
0
Fork 0
forked from wry/wry
Code Pull requests Activity

Merge pull request #285 from mahkoh/jorth/async-download

Browse source 
Implement async shm downloads
This commit is contained in:
mahkoh 2024-10-07 13:47:05 +02:00 • committed by GitHub
commit cc8db84289
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 1299 additions and 1094 deletions
Download patch file Download diff file Expand all files Collapse all files

89
src/gfx_api.rs
View file

@ -265,20 +265,23 @@ pub trait GfxFramebuffer: Debug {
clear: Option<&Color>,
) -> Result<Option<SyncFile>, GfxError>;
fn copy_to_shm(
self: Rc<Self>,
x: i32,
y: i32,
width: i32,
height: i32,
stride: i32,
format: &'static Format,
shm: &[Cell<u8>],
) -> Result<(), GfxError>;
fn format(&self) -> &'static Format;
}
pub trait GfxInternalFramebuffer: GfxFramebuffer {
fn into_fb(self: Rc<Self>) -> Rc<dyn GfxFramebuffer>;
fn staging_size(&self) -> usize;
fn download(
self: Rc<Self>,
staging: &Rc<dyn GfxStagingBuffer>,
callback: Rc<dyn AsyncShmGfxTextureCallback>,
mem: Rc<dyn ShmMemory>,
damage: Region,
) -> Result<Option<PendingShmTransfer>, GfxError>;
}
impl dyn GfxFramebuffer {
pub fn clear(
&self,
@ -481,16 +484,6 @@ pub trait GfxTexture: Debug {
fn size(&self) -> (i32, i32);
fn as_any(&self) -> &dyn Any;
fn into_any(self: Rc<Self>) -> Rc<dyn Any>;
fn read_pixels(
self: Rc<Self>,
x: i32,
y: i32,
width: i32,
height: i32,
stride: i32,
format: &'static Format,
shm: &[Cell<u8>],
) -> Result<(), GfxError>;
fn dmabuf(&self) -> Option<&DmaBuf>;
fn format(&self) -> &'static Format;
}
@ -503,12 +496,23 @@ pub trait AsyncShmGfxTextureCallback {
fn completed(self: Rc<Self>, res: Result<(), GfxError>);
}
pub trait AsyncShmGfxTextureUploadCancellable {
bitflags! {
StagingBufferUsecase: u32;
STAGING_UPLOAD = 1 << 0,
STAGING_DOWNLOAD = 1 << 1,
}
pub trait GfxStagingBuffer {
fn size(&self) -> usize;
fn into_any(self: Rc<Self>) -> Rc<dyn Any>;
}
pub trait AsyncShmGfxTextureTransferCancellable {
fn cancel(&self, id: u64);
}
pub struct PendingShmUpload {
cancel: Rc<dyn AsyncShmGfxTextureUploadCancellable>,
pub struct PendingShmTransfer {
cancel: Rc<dyn AsyncShmGfxTextureTransferCancellable>,
id: u64,
}
@ -539,12 +543,17 @@ impl ShmMemory for Vec<Cell<u8>> {
}
pub trait AsyncShmGfxTexture: GfxTexture {
fn staging_size(&self) -> usize {
0
}
fn async_upload(
self: Rc<Self>,
staging: &Rc<dyn GfxStagingBuffer>,
callback: Rc<dyn AsyncShmGfxTextureCallback>,
mem: Rc<dyn ShmMemory>,
damage: Region,
) -> Result<Option<PendingShmUpload>, GfxError>;
) -> Result<Option<PendingShmTransfer>, GfxError>;
fn sync_upload(self: Rc<Self>, shm: &[Cell<u8>], damage: Region) -> Result<(), GfxError>;
@ -596,15 +605,35 @@ pub trait GfxContext: Debug {
fn gfx_api(&self) -> GfxApi;
fn create_fb(
fn create_internal_fb(
self: Rc<Self>,
cpu_worker: &Rc<CpuWorker>,
width: i32,
height: i32,
stride: i32,
format: &'static Format,
) -> Result<Rc<dyn GfxFramebuffer>, GfxError>;
) -> Result<Rc<dyn GfxInternalFramebuffer>, GfxError>;
fn sync_obj_ctx(&self) -> Option<&Rc<SyncObjCtx>>;
fn create_staging_buffer(
&self,
size: usize,
usecase: StagingBufferUsecase,
) -> Rc<dyn GfxStagingBuffer> {
let _ = usecase;
struct Dummy(usize);
impl GfxStagingBuffer for Dummy {
fn size(&self) -> usize {
self.0
}
fn into_any(self: Rc<Self>) -> Rc<dyn Any> {
self
}
}
Rc::new(Dummy(size))
}
}
#[derive(Clone, Debug)]
@ -671,13 +700,13 @@ pub fn cross_intersect_formats(
res
}
impl PendingShmUpload {
pub fn new(cancel: Rc<dyn AsyncShmGfxTextureUploadCancellable>, id: u64) -> Self {
impl PendingShmTransfer {
pub fn new(cancel: Rc<dyn AsyncShmGfxTextureTransferCancellable>, id: u64) -> Self {
Self { cancel, id }
}
}
impl Drop for PendingShmUpload {
impl Drop for PendingShmTransfer {
fn drop(&mut self) {
self.cancel.cancel(self.id);
}

2
src/gfx_apis/gl.rs
View file

@ -185,8 +185,6 @@ enum RenderError {
ExternalUnsupported,
#[error("OpenGL context does not support any formats")]
NoSupportedFormats,
#[error("Cannot convert a shm texture into a framebuffer")]
ShmTextureToFb,
#[error("Could not create EGLSyncKHR")]
CreateEglSync,
#[error("Could not destroy EGLSyncKHR")]

7
src/gfx_apis/gl/renderer/context.rs
View file

@ -5,7 +5,7 @@ use {
format::{Format, XRGB8888},
gfx_api::{
AsyncShmGfxTexture, BufferResvUser, GfxContext, GfxError, GfxFormat, GfxFramebuffer,
GfxImage, ResetStatus, ShmGfxTexture,
GfxImage, GfxInternalFramebuffer, ResetStatus, ShmGfxTexture,
},
gfx_apis::gl::{
egl::{context::EglContext, display::EglDisplay, image::EglImage},
@ -316,13 +316,14 @@ impl GfxContext for GlRenderContext {
GfxApi::OpenGl
}
fn create_fb(
fn create_internal_fb(
self: Rc<Self>,
_cpu_worker: &Rc<CpuWorker>,
width: i32,
height: i32,
_stride: i32,
format: &'static Format,
) -> Result<Rc<dyn GfxFramebuffer>, GfxError> {
) -> Result<Rc<dyn GfxInternalFramebuffer>, GfxError> {
let fb = self.ctx.with_current(|| unsafe {
GlRenderBuffer::new(&self.ctx, width, height, format)?.create_framebuffer()
})?;

66
src/gfx_apis/gl/renderer/framebuffer.rs
View file

@ -1,7 +1,11 @@
use {
crate::{
format::Format,
gfx_api::{AcquireSync, GfxApiOpt, GfxError, GfxFramebuffer, ReleaseSync, SyncFile},
gfx_api::{
AcquireSync, AsyncShmGfxTextureCallback, GfxApiOpt, GfxError, GfxFramebuffer,
GfxInternalFramebuffer, GfxStagingBuffer, PendingShmTransfer, ReleaseSync, ShmMemory,
SyncFile,
},
gfx_apis::gl::{
gl::{
frame_buffer::GlFrameBuffer,
@ -13,6 +17,7 @@ use {
sys::{GL_ONE, GL_ONE_MINUS_SRC_ALPHA},
RenderError,
},
rect::Region,
theme::Color,
},
std::{
@ -34,29 +39,21 @@ impl Debug for Framebuffer {
}
impl Framebuffer {
pub fn copy_to_shm(
&self,
x: i32,
y: i32,
width: i32,
height: i32,
format: &Format,
shm: &[Cell<u8>],
) -> Result<(), RenderError> {
pub fn copy_to_shm(&self, shm: &[Cell<u8>]) -> Result<(), RenderError> {
let format = self.gl.rb.format;
let Some(shm_info) = &format.shm_info else {
return Err(RenderError::UnsupportedShmFormat(format.name));
};
let gles = self.ctx.ctx.dpy.gles;
let y = self.gl.height - y - height;
let _ = self.ctx.ctx.with_current(|| {
unsafe {
(gles.glBindFramebuffer)(GL_FRAMEBUFFER, self.gl.fbo);
(gles.glViewport)(0, 0, self.gl.width, self.gl.height);
(gles.glReadnPixels)(
x,
y,
width,
height,
0,
0,
self.gl.width,
self.gl.height,
shm_info.gl_format as _,
shm_info.gl_type as _,
shm.len() as _,
@ -112,22 +109,31 @@ impl GfxFramebuffer for Framebuffer {
self.render(acquire_sync, ops, clear).map_err(|e| e.into())
}
fn copy_to_shm(
self: Rc<Self>,
x: i32,
y: i32,
width: i32,
height: i32,
_stride: i32,
format: &'static Format,
shm: &[Cell<u8>],
) -> Result<(), GfxError> {
(*self)
.copy_to_shm(x, y, width, height, format, shm)
.map_err(|e| e.into())
}
fn format(&self) -> &'static Format {
self.gl.rb.format
}
}
impl GfxInternalFramebuffer for Framebuffer {
fn into_fb(self: Rc<Self>) -> Rc<dyn GfxFramebuffer> {
self
}
fn staging_size(&self) -> usize {
0
}
fn download(
self: Rc<Self>,
_staging: &Rc<dyn GfxStagingBuffer>,
_callback: Rc<dyn AsyncShmGfxTextureCallback>,
mem: Rc<dyn ShmMemory>,
_damage: Region,
) -> Result<Option<PendingShmTransfer>, GfxError> {
let mut res = Ok(());
mem.access(&mut |mem| res = self.copy_to_shm(mem))
.map_err(RenderError::AccessFailed)?;
res?;
Ok(None)
}
}

31
src/gfx_apis/gl/renderer/texture.rs
View file

@ -2,12 +2,12 @@ use {
crate::{
format::Format,
gfx_api::{
AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxError, GfxTexture, PendingShmUpload,
ShmGfxTexture, ShmMemory,
AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxError, GfxStagingBuffer, GfxTexture,
PendingShmTransfer, ShmGfxTexture, ShmMemory,
},
gfx_apis::gl::{
gl::texture::GlTexture,
renderer::{context::GlRenderContext, framebuffer::Framebuffer},
renderer::context::GlRenderContext,
sys::{
GLint, GL_CLAMP_TO_EDGE, GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_TEXTURE_WRAP_T,
GL_UNPACK_ROW_LENGTH_EXT,
@ -45,13 +45,6 @@ impl Texture {
pub fn height(&self) -> i32 {
self.gl.height
}
pub fn to_framebuffer(&self) -> Result<Rc<Framebuffer>, RenderError> {
match &self.gl.img {
Some(img) => self.ctx.image_to_fb(img),
_ => Err(RenderError::ShmTextureToFb),
}
}
}
impl GfxTexture for Texture {
@ -67,21 +60,6 @@ impl GfxTexture for Texture {
self
}
fn read_pixels(
self: Rc<Self>,
x: i32,
y: i32,
width: i32,
height: i32,
_stride: i32,
format: &Format,
shm: &[Cell<u8>],
) -> Result<(), GfxError> {
self.to_framebuffer()?
.copy_to_shm(x, y, width, height, format, shm)
.map_err(|e| e.into())
}
fn dmabuf(&self) -> Option<&DmaBuf> {
self.gl.img.as_ref().map(|i| &i.dmabuf)
}
@ -100,10 +78,11 @@ impl ShmGfxTexture for Texture {
impl AsyncShmGfxTexture for Texture {
fn async_upload(
self: Rc<Self>,
_staging: &Rc<dyn GfxStagingBuffer>,
_callback: Rc<dyn AsyncShmGfxTextureCallback>,
mem: Rc<dyn ShmMemory>,
_damage: Region,
) -> Result<Option<PendingShmUpload>, GfxError> {
) -> Result<Option<PendingShmTransfer>, GfxError> {
let mut res = Ok(());
mem.access(&mut |data| {
res = self.clone().sync_upload(data, Region::default());

55
src/gfx_apis/vulkan.rs
View file

@ -14,6 +14,7 @@ mod semaphore;
mod shaders;
mod shm_image;
mod staging;
mod transfer;
use {
crate::{
@ -22,8 +23,9 @@ use {
cpu_worker::{jobs::read_write::ReadWriteJobError, CpuWorker},
format::Format,
gfx_api::{
AsyncShmGfxTexture, GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage,
ResetStatus, ShmGfxTexture,
AsyncShmGfxTexture, GfxContext, GfxError, GfxFormat, GfxImage, GfxInternalFramebuffer,
GfxStagingBuffer, ResetStatus, ShmGfxTexture, StagingBufferUsecase, STAGING_DOWNLOAD,
STAGING_UPLOAD,
},
gfx_apis::vulkan::{
image::VulkanImageMemory, instance::VulkanInstance, renderer::VulkanRenderer,
@ -171,16 +173,6 @@ pub enum VulkanError {
InvalidStride,
#[error("Shm stride and height do not match buffer size")]
InvalidBufferSize,
#[error("The shm parameters are invalid x={x}, y={y}, width={width}, height={height}, stride={stride}")]
InvalidShmParameters {
x: i32,
y: i32,
width: i32,
height: i32,
stride: i32,
},
#[error(transparent)]
GfxError(GfxError),
#[error("Buffer format {0} is not supported for shm buffers in Vulkan context")]
UnsupportedShmFormat(&'static str),
#[error("Only BO_USE_RENDERING and BO_USE_WRITE are supported")]
@ -203,6 +195,16 @@ pub enum VulkanError {
AsyncCopyToStaging(#[source] ReadWriteJobError),
#[error("The async shm texture is busy")]
AsyncCopyBusy,
#[error("The staging buffer is busy")]
StagingBufferBusy,
#[error("The staging buffer does not support uploads")]
StagingBufferNoUpload,
#[error("The staging buffer does not support downloads")]
StagingBufferNoDownload,
#[error("Image contents are undefined")]
UndefinedContents,
#[error("The framebuffer is being used by the transfer queue")]
BusyInTransfer,
}
impl From<VulkanError> for GfxError {
@ -314,22 +316,41 @@ impl GfxContext for Context {
GfxApi::Vulkan
}
fn create_fb(
fn create_internal_fb(
self: Rc<Self>,
cpu_worker: &Rc<CpuWorker>,
width: i32,
height: i32,
stride: i32,
format: &'static Format,
) -> Result<Rc<dyn GfxFramebuffer>, GfxError> {
let fb = self
.0
.create_shm_texture(format, width, height, stride, &[], true, None)?;
) -> Result<Rc<dyn GfxInternalFramebuffer>, GfxError> {
let fb = self.0.create_shm_texture(
format,
width,
height,
stride,
&[],
true,
Some(cpu_worker),
)?;
Ok(fb)
}
fn sync_obj_ctx(&self) -> Option<&Rc<SyncObjCtx>> {
Some(&self.0.device.sync_ctx)
}
fn create_staging_buffer(
&self,
size: usize,
usecase: StagingBufferUsecase,
) -> Rc<dyn GfxStagingBuffer> {
let upload = usecase.contains(STAGING_UPLOAD);
let download = usecase.contains(STAGING_DOWNLOAD);
self.0
.device
.create_staging_shell(size as u64, upload, download)
}
}
impl Drop for Context {

99
src/gfx_apis/vulkan/image.rs
View file

@ -3,12 +3,14 @@ use {
format::Format,
gfx_api::{
AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback,
AsyncShmGfxTextureUploadCancellable, GfxApiOpt, GfxError, GfxFramebuffer, GfxImage,
GfxTexture, PendingShmUpload, ReleaseSync, ShmGfxTexture, ShmMemory, SyncFile,
AsyncShmGfxTextureTransferCancellable, GfxApiOpt, GfxError, GfxFramebuffer, GfxImage,
GfxInternalFramebuffer, GfxStagingBuffer, GfxTexture, PendingShmTransfer, ReleaseSync,
ShmGfxTexture, ShmMemory, SyncFile,
},
gfx_apis::vulkan::{
allocator::VulkanAllocation, device::VulkanDevice, format::VulkanModifierLimits,
renderer::VulkanRenderer, shm_image::VulkanShmImage, VulkanError,
renderer::VulkanRenderer, shm_image::VulkanShmImage, transfer::TransferType,
VulkanError,
},
rect::Region,
theme::Color,
@ -508,26 +510,46 @@ impl GfxFramebuffer for VulkanImage {
.map_err(|e| e.into())
}
fn copy_to_shm(
self: Rc<Self>,
x: i32,
y: i32,
width: i32,
height: i32,
stride: i32,
format: &'static Format,
shm: &[Cell<u8>],
) -> Result<(), GfxError> {
self.renderer
.read_pixels(&self, x, y, width, height, stride, format, shm)
.map_err(|e| e.into())
}
fn format(&self) -> &'static Format {
self.format
}
}
impl GfxInternalFramebuffer for VulkanImage {
fn into_fb(self: Rc<Self>) -> Rc<dyn GfxFramebuffer> {
self
}
fn staging_size(&self) -> usize {
let VulkanImageMemory::Internal(shm) = &self.ty else {
unreachable!();
};
shm.size as _
}
fn download(
self: Rc<Self>,
staging: &Rc<dyn GfxStagingBuffer>,
callback: Rc<dyn AsyncShmGfxTextureCallback>,
mem: Rc<dyn ShmMemory>,
damage: Region,
) -> Result<Option<PendingShmTransfer>, GfxError> {
let VulkanImageMemory::Internal(shm) = &self.ty else {
unreachable!();
};
let staging = staging.clone().into_vk(&self.renderer.device.device);
let pending = shm.async_transfer(
&self,
staging,
&mem,
damage,
callback,
TransferType::Download,
)?;
Ok(pending)
}
}
impl GfxTexture for VulkanImage {
fn size(&self) -> (i32, i32) {
(self.width as _, self.height as _)
@ -541,21 +563,6 @@ impl GfxTexture for VulkanImage {
self
}
fn read_pixels(
self: Rc<Self>,
x: i32,
y: i32,
width: i32,
height: i32,
stride: i32,
format: &'static Format,
shm: &[Cell<u8>],
) -> Result<(), GfxError> {
self.renderer
.read_pixels(&self, x, y, width, height, stride, format, shm)
.map_err(|e| e.into())
}
fn dmabuf(&self) -> Option<&DmaBuf> {
match &self.ty {
VulkanImageMemory::DmaBuf(b) => Some(&b.template.dmabuf),
@ -575,16 +582,26 @@ impl ShmGfxTexture for VulkanImage {
}
impl AsyncShmGfxTexture for VulkanImage {
fn async_upload(
self: Rc<Self>,
callback: Rc<dyn AsyncShmGfxTextureCallback>,
mem: Rc<dyn ShmMemory>,
damage: Region,
) -> Result<Option<PendingShmUpload>, GfxError> {
fn staging_size(&self) -> usize {
let VulkanImageMemory::Internal(shm) = &self.ty else {
unreachable!();
};
let pending = shm.async_upload(&self, &mem, damage, callback)?;
shm.size as _
}
fn async_upload(
self: Rc<Self>,
staging: &Rc<dyn GfxStagingBuffer>,
callback: Rc<dyn AsyncShmGfxTextureCallback>,
mem: Rc<dyn ShmMemory>,
damage: Region,
) -> Result<Option<PendingShmTransfer>, GfxError> {
let VulkanImageMemory::Internal(shm) = &self.ty else {
unreachable!();
};
let staging = staging.clone().into_vk(&self.renderer.device.device);
let pending =
shm.async_transfer(&self, staging, &mem, damage, callback, TransferType::Upload)?;
Ok(pending)
}
@ -617,7 +634,7 @@ impl AsyncShmGfxTexture for VulkanImage {
}
}
impl AsyncShmGfxTextureUploadCancellable for VulkanImage {
impl AsyncShmGfxTextureTransferCancellable for VulkanImage {
fn cancel(&self, id: u64) {
let VulkanImageMemory::Internal(shm) = &self.ty else {
unreachable!();

340
src/gfx_apis/vulkan/renderer.rs
View file

@ -2,10 +2,10 @@ use {
crate::{
async_engine::{AsyncEngine, SpawnedFuture},
cpu_worker::PendingJob,
format::{Format, XRGB8888},
format::XRGB8888,
gfx_api::{
AcquireSync, BufferResv, BufferResvUser, GfxApiOpt, GfxFormat, GfxFramebuffer,
GfxTexture, GfxWriteModifier, ReleaseSync, SyncFile,
AcquireSync, BufferResv, BufferResvUser, GfxApiOpt, GfxFormat, GfxTexture,
GfxWriteModifier, ReleaseSync, SyncFile,
},
gfx_apis::vulkan::{
allocator::{VulkanAllocator, VulkanThreadedAllocator},
@ -32,11 +32,10 @@ use {
ash::{
vk,
vk::{
AccessFlags2, AttachmentLoadOp, AttachmentStoreOp, BufferImageCopy,
BufferMemoryBarrier2, ClearColorValue, ClearValue, CommandBuffer,
CommandBufferBeginInfo, CommandBufferSubmitInfo, CommandBufferUsageFlags,
CopyImageInfo2, DependencyInfo, DependencyInfoKHR, DescriptorImageInfo, DescriptorType,
Extent2D, Extent3D, Fence, ImageAspectFlags, ImageCopy2, ImageLayout,
AccessFlags2, AttachmentLoadOp, AttachmentStoreOp, ClearColorValue, ClearValue,
CommandBuffer, CommandBufferBeginInfo, CommandBufferSubmitInfo,
CommandBufferUsageFlags, CopyImageInfo2, DependencyInfoKHR, DescriptorImageInfo,
DescriptorType, Extent2D, Extent3D, ImageAspectFlags, ImageCopy2, ImageLayout,
ImageMemoryBarrier2, ImageSubresourceLayers, ImageSubresourceRange, PipelineBindPoint,
PipelineStageFlags2, Rect2D, RenderingAttachmentInfo, RenderingInfo,
SemaphoreSubmitInfo, SemaphoreSubmitInfoKHR, ShaderStageFlags, SubmitInfo2, Viewport,
@ -49,7 +48,7 @@ use {
std::{
cell::{Cell, RefCell},
fmt::{Debug, Formatter},
mem, ptr,
mem,
rc::Rc,
slice,
},
@ -337,38 +336,69 @@ impl VulkanRenderer {
}
}
fn initial_barriers(&self, buf: CommandBuffer, fb: &VulkanImage) {
fn initial_barriers(&self, buf: CommandBuffer, fb: &VulkanImage) -> Result<(), VulkanError> {
zone!("initial_barriers");
let mut memory = self.memory.borrow_mut();
let memory = &mut *memory;
memory.image_barriers.clear();
let mut fb_image_memory_barrier = image_barrier()
.image(fb.image)
.new_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL)
.dst_access_mask(
AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ,
)
.dst_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT);
if fb.bridge.is_some() {
fb_image_memory_barrier = fb_image_memory_barrier
.src_access_mask(AccessFlags2::TRANSFER_READ)
.src_stage_mask(PipelineStageFlags2::TRANSFER)
.old_layout(if fb.is_undefined.get() {
ImageLayout::UNDEFINED
} else {
ImageLayout::TRANSFER_SRC_OPTIMAL
});
} else {
fb_image_memory_barrier = fb_image_memory_barrier
.src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
.dst_queue_family_index(self.device.graphics_queue_idx)
.old_layout(if fb.is_undefined.get() {
ImageLayout::UNDEFINED
} else {
ImageLayout::GENERAL
});
let mut need_fb_barrier = true;
if let VulkanImageMemory::Internal(..) = &fb.ty {
need_fb_barrier = fb.is_undefined.get()
|| (self.device.distinct_transfer_queue_family_idx.is_some()
&& fb.queue_state.get().acquire(QueueFamily::Gfx)
!= QueueTransfer::Unnecessary);
}
if need_fb_barrier {
let mut fb_image_memory_barrier = image_barrier()
.image(fb.image)
.new_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL)
.dst_access_mask(
AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ,
)
.dst_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT);
if fb.bridge.is_some() {
fb_image_memory_barrier = fb_image_memory_barrier
.src_access_mask(AccessFlags2::TRANSFER_READ)
.src_stage_mask(PipelineStageFlags2::TRANSFER)
.old_layout(if fb.is_undefined.get() {
ImageLayout::UNDEFINED
} else {
ImageLayout::TRANSFER_SRC_OPTIMAL
});
} else if let VulkanImageMemory::Internal(..) = &fb.ty {
let mut queue_transfer = QueueTransfer::Unnecessary;
if self.device.distinct_transfer_queue_family_idx.is_some() {
queue_transfer = fb.queue_state.get().acquire(QueueFamily::Gfx);
}
match queue_transfer {
QueueTransfer::Unnecessary => {
fb_image_memory_barrier =
fb_image_memory_barrier.old_layout(ImageLayout::UNDEFINED);
}
QueueTransfer::Possible => {
if let Some(transfer_queue_idx) =
self.device.distinct_transfer_queue_family_idx
{
fb_image_memory_barrier = fb_image_memory_barrier
.src_queue_family_index(transfer_queue_idx)
.dst_queue_family_index(self.device.graphics_queue_idx)
.old_layout(ImageLayout::TRANSFER_SRC_OPTIMAL);
}
}
QueueTransfer::Impossible => return Err(VulkanError::BusyInTransfer),
}
} else {
fb_image_memory_barrier = fb_image_memory_barrier
.src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
.dst_queue_family_index(self.device.graphics_queue_idx)
.old_layout(if fb.is_undefined.get() {
ImageLayout::UNDEFINED
} else {
ImageLayout::GENERAL
});
}
memory.image_barriers.push(fb_image_memory_barrier);
}
memory.image_barriers.push(fb_image_memory_barrier);
for img in &memory.dmabuf_sample {
let image_memory_barrier = image_barrier()
.src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
@ -397,6 +427,7 @@ impl VulkanRenderer {
unsafe {
self.device.device.cmd_pipeline_barrier2(buf, &dep_info);
}
Ok(())
}
fn begin_rendering(&self, buf: CommandBuffer, fb: &VulkanImage, clear: Option<&Color>) {
@ -643,26 +674,28 @@ impl VulkanRenderer {
let mut memory = self.memory.borrow_mut();
let memory = &mut *memory;
memory.image_barriers.clear();
let mut fb_image_memory_barrier = image_barrier()
.src_queue_family_index(self.device.graphics_queue_idx)
.dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
.new_layout(ImageLayout::GENERAL);
if let Some(bridge) = &fb.bridge {
fb_image_memory_barrier = fb_image_memory_barrier
.image(bridge.dmabuf_image)
.old_layout(ImageLayout::TRANSFER_DST_OPTIMAL)
.src_access_mask(AccessFlags2::TRANSFER_WRITE)
.src_stage_mask(PipelineStageFlags2::TRANSFER);
} else {
fb_image_memory_barrier = fb_image_memory_barrier
.image(fb.image)
.old_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL)
.src_access_mask(
AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ,
)
.src_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT);
if let VulkanImageMemory::DmaBuf(..) = fb.ty {
let mut fb_image_memory_barrier = image_barrier()
.src_queue_family_index(self.device.graphics_queue_idx)
.dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
.new_layout(ImageLayout::GENERAL);
if let Some(bridge) = &fb.bridge {
fb_image_memory_barrier = fb_image_memory_barrier
.image(bridge.dmabuf_image)
.old_layout(ImageLayout::TRANSFER_DST_OPTIMAL)
.src_access_mask(AccessFlags2::TRANSFER_WRITE)
.src_stage_mask(PipelineStageFlags2::TRANSFER);
} else {
fb_image_memory_barrier = fb_image_memory_barrier
.image(fb.image)
.old_layout(ImageLayout::COLOR_ATTACHMENT_OPTIMAL)
.src_access_mask(
AccessFlags2::COLOR_ATTACHMENT_WRITE | AccessFlags2::COLOR_ATTACHMENT_READ,
)
.src_stage_mask(PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT);
}
memory.image_barriers.push(fb_image_memory_barrier);
}
memory.image_barriers.push(fb_image_memory_barrier);
for img in &memory.dmabuf_sample {
let image_memory_barrier = image_barrier()
.src_queue_family_index(self.device.graphics_queue_idx)
@ -833,6 +866,10 @@ impl VulkanRenderer {
fn store_layouts(&self, fb: &VulkanImage) {
fb.is_undefined.set(false);
fb.contents_are_undefined.set(false);
fb.queue_state.set(QueueState::Acquired {
family: QueueFamily::Gfx,
});
let memory = self.memory.borrow();
for img in &*memory.queue_transfer {
img.queue_state.set(QueueState::Acquired {
@ -867,197 +904,6 @@ impl VulkanRenderer {
frame.waiter.set(Some(future));
}
pub fn read_pixels(
self: &Rc<Self>,
tex: &Rc<VulkanImage>,
x: i32,
y: i32,
width: i32,
height: i32,
stride: i32,
format: &'static Format,
dst: &[Cell<u8>],
) -> Result<(), VulkanError> {
if x < 0 || y < 0 || width <= 0 || height <= 0 || stride <= 0 {
return Err(VulkanError::InvalidShmParameters {
x,
y,
width,
height,
stride,
});
}
let width = width as u32;
let height = height as u32;
let stride = stride as u32;
if x == 0 && y == 0 && width == tex.width && height == tex.height && format == tex.format {
return self.read_all_pixels(tex, stride, dst);
}
let tmp_tex = self.create_shm_texture(
format,
width as i32,
height as i32,
stride as i32,
&[],
true,
None,
)?;
(&*tmp_tex as &dyn GfxFramebuffer)
.copy_texture(
AcquireSync::None,
ReleaseSync::None,
&(tex.clone() as _),
None,
AcquireSync::None,
ReleaseSync::None,
x,
y,
)
.map_err(VulkanError::GfxError)?;
self.read_all_pixels(&tmp_tex, stride, dst)
}
fn read_all_pixels(
self: &Rc<Self>,
tex: &VulkanImage,
stride: u32,
dst: &[Cell<u8>],
) -> Result<(), VulkanError> {
let Some(shm_info) = &tex.format.shm_info else {
return Err(VulkanError::UnsupportedShmFormat(tex.format.name));
};
if stride < tex.width * shm_info.bpp || stride % shm_info.bpp != 0 {
return Err(VulkanError::InvalidStride);
}
let size = stride as u64 * tex.height as u64;
if size != dst.len() as u64 {
return Err(VulkanError::InvalidBufferSize);
}
let region = BufferImageCopy::default()
.buffer_row_length(stride / shm_info.bpp)
.buffer_image_height(tex.height)
.image_subresource(ImageSubresourceLayers {
aspect_mask: ImageAspectFlags::COLOR,
mip_level: 0,
base_array_layer: 0,
layer_count: 1,
})
.image_extent(Extent3D {
width: tex.width,
height: tex.height,
depth: 1,
});
let staging =
self.device
.create_staging_buffer(&self.allocator, size, false, true, true)?;
let initial_tex_barrier;
let initial_buffer_barrier = BufferMemoryBarrier2::default()
.buffer(staging.buffer)
.offset(0)
.size(staging.size)
.dst_access_mask(AccessFlags2::TRANSFER_WRITE)
.dst_stage_mask(PipelineStageFlags2::TRANSFER);
let mut initial_barriers = DependencyInfo::default()
.buffer_memory_barriers(slice::from_ref(&initial_buffer_barrier));
if tex.bridge.is_none() {
initial_tex_barrier = image_barrier()
.src_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
.dst_queue_family_index(self.device.graphics_queue_idx)
.image(tex.image)
.old_layout(ImageLayout::GENERAL)
.new_layout(ImageLayout::TRANSFER_SRC_OPTIMAL)
.dst_access_mask(AccessFlags2::TRANSFER_READ)
.dst_stage_mask(PipelineStageFlags2::TRANSFER);
initial_barriers =
initial_barriers.image_memory_barriers(slice::from_ref(&initial_tex_barrier));
}
let final_tex_barrier;
let final_buffer_barrier = BufferMemoryBarrier2::default()
.buffer(staging.buffer)
.offset(0)
.size(staging.size)
.src_access_mask(AccessFlags2::TRANSFER_WRITE)
.src_stage_mask(PipelineStageFlags2::TRANSFER)
.dst_access_mask(AccessFlags2::HOST_READ)
.dst_stage_mask(PipelineStageFlags2::HOST);
let mut final_barriers = DependencyInfo::default()
.buffer_memory_barriers(slice::from_ref(&final_buffer_barrier));
if tex.bridge.is_none() {
final_tex_barrier = image_barrier()
.src_queue_family_index(self.device.graphics_queue_idx)
.dst_queue_family_index(QUEUE_FAMILY_FOREIGN_EXT)
.image(tex.image)
.old_layout(ImageLayout::TRANSFER_SRC_OPTIMAL)
.new_layout(ImageLayout::GENERAL)
.src_access_mask(AccessFlags2::TRANSFER_READ)
.src_stage_mask(PipelineStageFlags2::TRANSFER);
final_barriers =
final_barriers.image_memory_barriers(slice::from_ref(&final_tex_barrier));
}
let buf = self.gfx_command_buffers.allocate()?;
let mut semaphores = vec![];
let mut semaphore_infos = vec![];
if let VulkanImageMemory::DmaBuf(buf) = &tex.ty {
for plane in &buf.template.dmabuf.planes {
let fd = dma_buf_export_sync_file(&plane.fd, DMA_BUF_SYNC_READ)
.map_err(VulkanError::IoctlExportSyncFile)?;
let semaphore = self.allocate_semaphore()?;
semaphore.import_sync_file(fd)?;
let semaphore_info = SemaphoreSubmitInfo::default()
.semaphore(semaphore.semaphore)
.stage_mask(PipelineStageFlags2::TOP_OF_PIPE);
semaphores.push(semaphore);
semaphore_infos.push(semaphore_info);
}
}
let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(buf.buffer);
let submit_info = SubmitInfo2::default()
.wait_semaphore_infos(&semaphore_infos)
.command_buffer_infos(slice::from_ref(&command_buffer_info));
let begin_info =
CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT);
unsafe {
self.device
.device
.begin_command_buffer(buf.buffer, &begin_info)
.map_err(VulkanError::BeginCommandBuffer)?;
self.device
.device
.cmd_pipeline_barrier2(buf.buffer, &initial_barriers);
self.device.device.cmd_copy_image_to_buffer(
buf.buffer,
tex.image,
ImageLayout::TRANSFER_SRC_OPTIMAL,
staging.buffer,
&[region],
);
self.device
.device
.cmd_pipeline_barrier2(buf.buffer, &final_barriers);
self.device
.device
.end_command_buffer(buf.buffer)
.map_err(VulkanError::EndCommandBuffer)?;
self.device
.device
.queue_submit2(
self.device.graphics_queue,
slice::from_ref(&submit_info),
Fence::null(),
)
.map_err(VulkanError::Submit)?;
}
self.block();
self.gfx_command_buffers.buffers.push(buf);
for semaphore in semaphores {
self.wait_semaphores.push(semaphore);
}
staging.download(|mem, size| unsafe {
ptr::copy_nonoverlapping(mem, dst.as_ptr() as _, size);
})?;
Ok(())
}
pub fn execute(
self: &Rc<Self>,
fb: &VulkanImage,
@ -1101,7 +947,7 @@ impl VulkanRenderer {
let buf = self.gfx_command_buffers.allocate()?;
self.collect_memory(opts);
self.begin_command_buffer(buf.buffer)?;
self.initial_barriers(buf.buffer, fb);
self.initial_barriers(buf.buffer, fb)?;
self.begin_rendering(buf.buffer, fb, clear);
self.set_viewport(buf.buffer, fb);
self.record_draws(buf.buffer, fb, opts)?;

597
src/gfx_apis/vulkan/shm_image.rs
View file

@ -1,46 +1,32 @@
use {
crate::{
cpu_worker::{
jobs::{
img_copy::ImgCopyWork,
read_write::{ReadWriteJobError, ReadWriteWork},
},
CpuJob, CpuWork, CpuWorker,
},
cpu_worker::CpuWorker,
format::{Format, FormatShmInfo},
gfx_api::{
AsyncShmGfxTextureCallback, PendingShmUpload, ShmMemory, ShmMemoryBacking, SyncFile,
},
gfx_api::SyncFile,
gfx_apis::vulkan::{
allocator::VulkanAllocation,
command::VulkanCommandBuffer,
fence::VulkanFence,
image::{QueueFamily, QueueState, QueueTransfer, VulkanImage, VulkanImageMemory},
image::{QueueFamily, QueueState, VulkanImage, VulkanImageMemory},
renderer::{image_barrier, VulkanRenderer},
staging::VulkanStagingBuffer,
transfer::{TransferType, VulkanShmImageAsyncData},
VulkanError,
},
rect::{Rect, Region},
utils::{clonecell::CloneCell, errorfmt::ErrorFmt, on_drop::OnDrop},
rect::Rect,
utils::{errorfmt::ErrorFmt, on_drop::OnDrop},
},
arrayvec::ArrayVec,
ash::vk::{
AccessFlags2, BufferImageCopy2, BufferMemoryBarrier2, CommandBufferBeginInfo,
CommandBufferSubmitInfo, CommandBufferUsageFlags, CopyBufferToImageInfo2, DependencyInfo,
DependencyInfoKHR, DeviceSize, Extent3D, ImageAspectFlags, ImageCreateInfo, ImageLayout,
ImageSubresourceLayers, ImageSubresourceRange, ImageTiling, ImageType, ImageUsageFlags,
ImageViewCreateInfo, ImageViewType, Offset3D, PipelineStageFlags2, SampleCountFlags,
SharingMode, SubmitInfo2,
CommandBufferSubmitInfo, CommandBufferUsageFlags, CopyBufferToImageInfo2,
CopyImageToBufferInfo2, DependencyInfoKHR, DeviceSize, Extent3D, ImageAspectFlags,
ImageCreateInfo, ImageLayout, ImageSubresourceLayers, ImageSubresourceRange, ImageTiling,
ImageType, ImageUsageFlags, ImageViewCreateInfo, ImageViewType, Offset3D,
PipelineStageFlags2, SampleCountFlags, SharingMode, SubmitInfo2,
},
gpu_alloc::UsageFlags,
isnt::std_1::primitive::IsntSliceExt,
std::{
cell::{Cell, RefCell},
ptr,
rc::Rc,
slice,
},
uapi::OwnedFd,
std::{cell::Cell, ptr, rc::Rc, slice},
};
pub struct VulkanShmImage {
@ -51,19 +37,6 @@ pub struct VulkanShmImage {
pub(super) async_data: Option<VulkanShmImageAsyncData>,
}
pub struct VulkanShmImageAsyncData {
pub(super) busy: Cell<bool>,
pub(super) io_job: Cell<Option<Box<IoUploadJob>>>,
pub(super) copy_job: Cell<Option<Box<CopyUploadJob>>>,
pub(super) staging: CloneCell<Option<Rc<VulkanStagingBuffer>>>,
pub(super) callback: Cell<Option<Rc<dyn AsyncShmGfxTextureCallback>>>,
pub(super) callback_id: Cell<u64>,
pub(super) regions: RefCell<Vec<BufferImageCopy2<'static>>>,
pub(super) cpu: Rc<CpuWorker>,
pub(super) last_sample: Cell<Option<SyncFile>>,
pub(super) data_copied: Cell<bool>,
}
impl VulkanShmImage {
pub fn upload(
&self,
@ -164,7 +137,7 @@ impl VulkanShmImage {
}
})?;
let Some((cmd, fence, sync_file, point)) =
self.submit_buffer_to_image_copy(img, &staging, cpy, false)?
self.submit_buffer_image_copy(img, &staging, cpy, false, TransferType::Upload)?
else {
return Ok(());
};
@ -176,12 +149,13 @@ impl VulkanShmImage {
Ok(())
}
fn submit_buffer_to_image_copy(
pub(super) fn submit_buffer_image_copy(
&self,
img: &Rc<VulkanImage>,
staging: &VulkanStagingBuffer,
regions: &[BufferImageCopy2],
use_transfer_queue: bool,
tt: TransferType,
) -> Result<Option<(Rc<VulkanCommandBuffer>, Rc<VulkanFence>, SyncFile, u64)>, VulkanError>
{
let memory_barrier = |sam, ssm, dam, dsm| {
@ -209,18 +183,30 @@ impl VulkanShmImage {
.old_layout(if img.is_undefined.get() {
ImageLayout::UNDEFINED
} else {
ImageLayout::SHADER_READ_ONLY_OPTIMAL
match tt {
TransferType::Upload => ImageLayout::SHADER_READ_ONLY_OPTIMAL,
TransferType::Download => ImageLayout::COLOR_ATTACHMENT_OPTIMAL,
}
})
.new_layout(ImageLayout::TRANSFER_DST_OPTIMAL);
.new_layout(match tt {
TransferType::Upload => ImageLayout::TRANSFER_DST_OPTIMAL,
TransferType::Download => ImageLayout::TRANSFER_SRC_OPTIMAL,
});
if transfer_queue_family_idx == img.renderer.device.graphics_queue_idx {
initial_image_barrier = initial_image_barrier
.src_access_mask(AccessFlags2::SHADER_SAMPLED_READ)
.src_stage_mask(PipelineStageFlags2::FRAGMENT_SHADER)
}
let initial_buffer_barrier = memory_barrier(
AccessFlags2::HOST_WRITE,
match tt {
TransferType::Upload => AccessFlags2::HOST_WRITE,
TransferType::Download => AccessFlags2::HOST_READ,
},
PipelineStageFlags2::HOST,
AccessFlags2::TRANSFER_READ,
match tt {
TransferType::Upload => AccessFlags2::TRANSFER_READ,
TransferType::Download => AccessFlags2::TRANSFER_WRITE,
},
PipelineStageFlags2::TRANSFER,
);
let initial_dep_info = DependencyInfoKHR::default()
@ -230,29 +216,42 @@ impl VulkanShmImage {
.image(img.image)
.src_queue_family_index(transfer_queue_family_idx)
.dst_queue_family_index(img.renderer.device.graphics_queue_idx)
.src_access_mask(AccessFlags2::TRANSFER_WRITE)
.src_access_mask(match tt {
TransferType::Upload => AccessFlags2::TRANSFER_WRITE,
TransferType::Download => AccessFlags2::TRANSFER_READ,
})
.src_stage_mask(PipelineStageFlags2::TRANSFER)
.old_layout(ImageLayout::TRANSFER_DST_OPTIMAL)
.new_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL);
.old_layout(match tt {
TransferType::Upload => ImageLayout::TRANSFER_DST_OPTIMAL,
TransferType::Download => ImageLayout::TRANSFER_SRC_OPTIMAL,
})
.new_layout(match tt {
TransferType::Upload => ImageLayout::SHADER_READ_ONLY_OPTIMAL,
TransferType::Download => ImageLayout::COLOR_ATTACHMENT_OPTIMAL,
});
if transfer_queue_family_idx == img.renderer.device.graphics_queue_idx {
final_image_barrier = final_image_barrier
.dst_access_mask(AccessFlags2::SHADER_SAMPLED_READ)
.dst_access_mask(match tt {
TransferType::Upload => AccessFlags2::SHADER_SAMPLED_READ,
TransferType::Download => AccessFlags2::COLOR_ATTACHMENT_WRITE,
})
.dst_stage_mask(PipelineStageFlags2::FRAGMENT_SHADER);
}
let final_buffer_barrier = memory_barrier(
AccessFlags2::TRANSFER_READ,
match tt {
TransferType::Upload => AccessFlags2::TRANSFER_READ,
TransferType::Download => AccessFlags2::TRANSFER_WRITE,
},
PipelineStageFlags2::TRANSFER,
AccessFlags2::HOST_WRITE,
match tt {
TransferType::Upload => AccessFlags2::HOST_WRITE,
TransferType::Download => AccessFlags2::HOST_READ,
},
PipelineStageFlags2::HOST,
);
let final_dep_info = DependencyInfoKHR::default()
.buffer_memory_barriers(slice::from_ref(&final_buffer_barrier))
.image_memory_barriers(slice::from_ref(&final_image_barrier));
let cpy_info = CopyBufferToImageInfo2::default()
.src_buffer(staging.buffer)
.dst_image(img.image)
.dst_image_layout(ImageLayout::TRANSFER_DST_OPTIMAL)
.regions(regions);
let cmd = match &img.renderer.transfer_command_buffers {
Some(b) if use_transfer_queue => b.allocate()?,
_ => img.renderer.gfx_command_buffers.allocate()?,
@ -268,7 +267,24 @@ impl VulkanShmImage {
dev.begin_command_buffer(cmd.buffer, &begin_info)
.map_err(VulkanError::BeginCommandBuffer)?;
dev.cmd_pipeline_barrier2(cmd.buffer, &initial_dep_info);
dev.cmd_copy_buffer_to_image2(cmd.buffer, &cpy_info);
match tt {
TransferType::Upload => {
let cpy_info = CopyBufferToImageInfo2::default()
.src_buffer(staging.buffer)
.dst_image(img.image)
.dst_image_layout(ImageLayout::TRANSFER_DST_OPTIMAL)
.regions(regions);
dev.cmd_copy_buffer_to_image2(cmd.buffer, &cpy_info);
}
TransferType::Download => {
let cpy_info = CopyImageToBufferInfo2::default()
.dst_buffer(staging.buffer)
.src_image(img.image)
.src_image_layout(ImageLayout::TRANSFER_SRC_OPTIMAL)
.regions(regions);
dev.cmd_copy_image_to_buffer2(cmd.buffer, &cpy_info);
}
}
dev.cmd_pipeline_barrier2(cmd.buffer, &final_dep_info);
dev.end_command_buffer(cmd.buffer)
.map_err(VulkanError::EndCommandBuffer)?;
@ -282,8 +298,10 @@ impl VulkanShmImage {
)
.map_err(VulkanError::Submit)?;
}
img.is_undefined.set(false);
img.contents_are_undefined.set(false);
if tt == TransferType::Upload {
img.is_undefined.set(false);
img.contents_are_undefined.set(false);
}
let release_sync_file = match release_fence.export_sync_file() {
Ok(s) => s,
Err(e) => {
@ -317,466 +335,6 @@ async fn await_upload(
img.renderer.pending_submits.remove(&id);
}
impl VulkanShmImageAsyncData {
fn complete(&self, result: Result<(), VulkanError>) {
self.busy.set(false);
if let Some(cb) = self.callback.take() {
cb.completed(result.map_err(|e| e.into()));
}
}
}
impl VulkanShmImage {
pub fn async_upload(
&self,
img: &Rc<VulkanImage>,
client_mem: &Rc<dyn ShmMemory>,
damage: Region,
callback: Rc<dyn AsyncShmGfxTextureCallback>,
) -> Result<Option<PendingShmUpload>, VulkanError> {
let data = self.async_data.as_ref().unwrap();
let res = self.try_async_upload(img, data, client_mem, damage);
match res {
Ok(()) => {
let id = img.renderer.allocate_point();
data.callback_id.set(id);
data.callback.set(Some(callback));
Ok(Some(PendingShmUpload::new(img.clone(), id)))
}
Err(e) => Err(e),
}
}
fn try_async_upload(
&self,
img: &Rc<VulkanImage>,
data: &VulkanShmImageAsyncData,
client_mem: &Rc<dyn ShmMemory>,
mut damage: Region,
) -> Result<(), VulkanError> {
if data.busy.get() {
return Err(VulkanError::AsyncCopyBusy);
}
if self.size > client_mem.len() as u64 {
return Err(VulkanError::InvalidBufferSize);
}
data.busy.set(true);
data.data_copied.set(false);
if img.contents_are_undefined.get() {
damage = Region::new2(Rect::new_sized(0, 0, img.width as _, img.height as _).unwrap());
}
let copies = &mut *data.regions.borrow_mut();
copies.clear();
let mut copy = |x, y, width, height| {
let buffer_offset = (y as u32 * img.stride + x as u32 * self.shm_info.bpp) as u64;
let copy = BufferImageCopy2::default()
.buffer_offset(buffer_offset)
.image_offset(Offset3D { x, y, z: 0 })
.image_extent(Extent3D {
width,
height,
depth: 1,
})
.image_subresource(ImageSubresourceLayers {
aspect_mask: ImageAspectFlags::COLOR,
mip_level: 0,
base_array_layer: 0,
layer_count: 1,
})
.buffer_image_height(img.height)
.buffer_row_length(img.stride / self.shm_info.bpp);
copies.push(copy);
};
let (width_mask, height_mask) = img.renderer.device.transfer_granularity_mask;
let width_mask = width_mask as i32;
let height_mask = height_mask as i32;
for damage in damage.rects() {
if damage.x2() < 0 || damage.y2() < 0 {
continue;
}
let x1 = damage.x1().max(0) & !width_mask;
let y1 = damage.y1().max(0) & !height_mask;
let x2 = ((damage.x2() + width_mask) & !width_mask).min(img.width as i32);
let y2 = ((damage.y2() + height_mask) & !height_mask).min(img.height as i32);
let Some(damage) = Rect::new(x1, y1, x2, y2) else {
continue;
};
if damage.is_empty() {
continue;
}
copy(
damage.x1(),
damage.y1(),
damage.width() as u32,
damage.height() as u32,
);
}
self.async_release_from_gfx_queue(img, data)?;
if let Some(staging) = data.staging.get() {
return self.async_upload_initiate_copy(img, data, &staging, copies, client_mem);
}
let img2 = img.clone();
let client_mem = client_mem.clone();
img.renderer.device.create_shm_staging(
&img.renderer,
&data.cpu,
self.size,
true,
false,
move |res| {
let VulkanImageMemory::Internal(shm) = &img2.ty else {
unreachable!();
};
if let Err(e) = shm.async_upload_after_allocation(&img2, &client_mem, res) {
shm.async_data.as_ref().unwrap().complete(Err(e));
}
},
)
}
fn async_release_from_gfx_queue(
&self,
img: &Rc<VulkanImage>,
data: &VulkanShmImageAsyncData,
) -> Result<(), VulkanError> {
img.renderer.check_defunct()?;
let Some(transfer_queue_idx) = img.renderer.device.distinct_transfer_queue_family_idx
else {
let Some(sync_file) = data.last_sample.take() else {
img.queue_state.set(QueueState::Released {
to: QueueFamily::Transfer,
});
return Ok(());
};
let id = img.renderer.allocate_point();
let pending = img.renderer.eng.spawn(
"await_transfer_to_transfer",
await_gfx_queue_release(id, img.clone(), None, None, sync_file),
);
img.renderer.pending_submits.set(id, pending);
img.queue_state.set(QueueState::Releasing);
return Ok(());
};
let mut barriers = ArrayVec::<_, 2>::new();
match img.queue_state.get() {
QueueState::Acquired { family } => {
assert_eq!(family, QueueFamily::Gfx);
}
QueueState::Releasing => {
unreachable!();
}
QueueState::Released { to } => {
assert_eq!(to, QueueFamily::Gfx);
let barrier = image_barrier()
.image(img.image)
.src_queue_family_index(transfer_queue_idx)
.dst_queue_family_index(img.renderer.device.graphics_queue_idx)
.dst_stage_mask(PipelineStageFlags2::ALL_COMMANDS)
.old_layout(ImageLayout::TRANSFER_DST_OPTIMAL)
.new_layout(ImageLayout::SHADER_READ_ONLY_OPTIMAL);
barriers.push(barrier);
}
}
let barrier = image_barrier()
.image(img.image)
.src_queue_family_index(img.renderer.device.graphics_queue_idx)
.dst_queue_family_index(transfer_queue_idx)
.src_access_mask(AccessFlags2::SHADER_SAMPLED_READ)
.src_stage_mask(PipelineStageFlags2::ALL_COMMANDS)
.old_layout(if img.is_undefined.get() {
ImageLayout::UNDEFINED
} else {
ImageLayout::SHADER_READ_ONLY_OPTIMAL
})
.new_layout(ImageLayout::TRANSFER_DST_OPTIMAL);
barriers.push(barrier);
let dep_info = DependencyInfo::default().image_memory_barriers(&barriers);
let release_fence = img.renderer.device.create_fence()?;
let dev = &img.renderer.device.device;
let begin_info =
CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT);
let cmd = img.renderer.gfx_command_buffers.allocate()?;
let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd.buffer);
let submit_info =
SubmitInfo2::default().command_buffer_infos(slice::from_ref(&command_buffer_info));
unsafe {
dev.begin_command_buffer(cmd.buffer, &begin_info)
.map_err(VulkanError::BeginCommandBuffer)?;
dev.cmd_pipeline_barrier2(cmd.buffer, &dep_info);
dev.end_command_buffer(cmd.buffer)
.map_err(VulkanError::EndCommandBuffer)?;
dev.queue_submit2(
img.renderer.device.graphics_queue,
slice::from_ref(&submit_info),
release_fence.fence,
)
.map_err(VulkanError::Submit)?;
}
let sync_file = release_fence.export_sync_file()?;
let id = img.renderer.allocate_point();
let pending = img.renderer.eng.spawn(
"await_transfer_to_transfer",
await_gfx_queue_release(id, img.clone(), Some(cmd), Some(release_fence), sync_file),
);
img.renderer.pending_submits.set(id, pending);
img.queue_state.set(QueueState::Releasing);
Ok(())
}
fn async_upload_after_allocation(
&self,
img: &Rc<VulkanImage>,
client_mem: &Rc<dyn ShmMemory>,
res: Result<VulkanStagingBuffer, VulkanError>,
) -> Result<(), VulkanError> {
let staging = Rc::new(res?);
let data = self.async_data.as_ref().unwrap();
data.staging.set(Some(staging.clone()));
let copies = &*data.regions.borrow();
self.async_upload_initiate_copy(img, data, &staging, copies, client_mem)
}
fn async_upload_initiate_copy(
&self,
img: &Rc<VulkanImage>,
data: &VulkanShmImageAsyncData,
staging: &VulkanStagingBuffer,
copies: &[BufferImageCopy2],
client_mem: &Rc<dyn ShmMemory>,
) -> Result<(), VulkanError> {
img.renderer.check_defunct()?;
let id = img.renderer.allocate_point();
let pending;
match client_mem.safe_access() {
ShmMemoryBacking::Ptr(ptr) => {
let mut job = data.copy_job.take().unwrap_or_else(|| {
Box::new(CopyUploadJob {
img: None,
id,
_mem: None,
work: unsafe { ImgCopyWork::new() },
})
});
job.id = id;
job.img = Some(img.clone());
job._mem = Some(client_mem.clone());
job.work.src = ptr as _;
job.work.dst = staging.allocation.mem.unwrap();
job.work.width = img.width as _;
job.work.stride = img.stride as _;
job.work.bpp = self.shm_info.bpp as _;
job.work.rects.clear();
for copy in copies {
job.work.rects.push(
Rect::new_sized(
copy.image_offset.x as _,
copy.image_offset.y as _,
copy.image_extent.width as _,
copy.image_extent.height as _,
)
.unwrap(),
);
}
pending = data.cpu.submit(job);
}
ShmMemoryBacking::Fd(fd, offset) => {
let mut min_offset = client_mem.len() as u64;
let mut max_offset = 0;
for copy in copies {
min_offset = min_offset.min(copy.buffer_offset);
let len = img.stride * (copy.image_extent.height - 1)
+ copy.image_extent.width * self.shm_info.bpp;
max_offset = max_offset.max(copy.buffer_offset + len as u64);
}
let mut job = data.io_job.take().unwrap_or_else(|| {
Box::new(IoUploadJob {
img: None,
id,
_mem: None,
work: unsafe { ReadWriteWork::new() },
fd: None,
})
});
job.id = id;
job.img = Some(img.clone());
job._mem = Some(client_mem.clone());
job.fd = Some(fd.clone());
unsafe {
let config = job.work.config();
config.fd = fd.raw();
config.offset = offset + min_offset as usize;
config.ptr = staging.allocation.mem.unwrap().add(min_offset as _);
config.len = max_offset.saturating_sub(min_offset) as usize;
config.write = false;
}
pending = data.cpu.submit(job);
}
}
img.renderer.pending_cpu_jobs.set(id, pending);
Ok(())
}
fn async_upload_copy_buffer_to_image(
&self,
img: &Rc<VulkanImage>,
data: &VulkanShmImageAsyncData,
) -> Result<(), VulkanError> {
if !data.data_copied.get() {
return Ok(());
}
if img.queue_state.get().acquire(QueueFamily::Transfer) == QueueTransfer::Impossible {
return Ok(());
}
img.renderer.check_defunct()?;
let regions = &*data.regions.borrow();
let staging = data.staging.get().unwrap();
staging.upload(|_, _| ())?;
let Some((cmd, fence, sync_file, point)) =
self.submit_buffer_to_image_copy(img, &staging, regions, true)?
else {
return Ok(());
};
img.queue_state.set(QueueState::Releasing);
let future = img.renderer.eng.spawn(
"await async upload",
await_async_upload(point, img.clone(), cmd, fence, sync_file),
);
img.renderer.pending_submits.set(point, future);
Ok(())
}
}
pub(super) struct IoUploadJob {
img: Option<Rc<VulkanImage>>,
id: u64,
_mem: Option<Rc<dyn ShmMemory>>,
fd: Option<Rc<OwnedFd>>,
work: ReadWriteWork,
}
pub(super) struct CopyUploadJob {
img: Option<Rc<VulkanImage>>,
id: u64,
_mem: Option<Rc<dyn ShmMemory>>,
work: ImgCopyWork,
}
impl CpuJob for IoUploadJob {
fn work(&mut self) -> &mut dyn CpuWork {
&mut self.work
}
fn completed(mut self: Box<Self>) {
self._mem = None;
self.fd = None;
let img = self.img.take().unwrap();
let res = self.work.config().result.take().unwrap();
complete_async_upload(&img, self.id, res, |data| data.io_job.set(Some(self)));
}
}
impl CpuJob for CopyUploadJob {
fn work(&mut self) -> &mut dyn CpuWork {
&mut self.work
}
fn completed(mut self: Box<Self>) {
self._mem = None;
let img = self.img.take().unwrap();
complete_async_upload(&img, self.id, Ok(()), |data| data.copy_job.set(Some(self)));
}
}
fn complete_async_upload(
img: &Rc<VulkanImage>,
id: u64,
res: Result<(), ReadWriteJobError>,
store: impl FnOnce(&VulkanShmImageAsyncData),
) {
img.renderer.pending_cpu_jobs.remove(&id);
let VulkanImageMemory::Internal(shm) = &img.ty else {
unreachable!();
};
let data = shm.async_data.as_ref().unwrap();
store(data);
if let Err(e) = res {
data.complete(Err(VulkanError::AsyncCopyToStaging(e)));
}
data.data_copied.set(true);
if let Err(e) = shm.async_upload_copy_buffer_to_image(img, data) {
data.complete(Err(e));
}
}
async fn await_gfx_queue_release(
id: u64,
img: Rc<VulkanImage>,
buf: Option<Rc<VulkanCommandBuffer>>,
_fence: Option<Rc<VulkanFence>>,
sync_file: SyncFile,
) {
let res = img.renderer.ring.readable(&sync_file.0).await;
if let Err(e) = res {
log::error!(
"Could not wait for sync file to become readable: {}",
ErrorFmt(e)
);
img.renderer.block();
}
if let Some(buf) = buf {
img.renderer.gfx_command_buffers.buffers.push(buf);
}
img.renderer.pending_submits.remove(&id);
img.queue_state.set(QueueState::Released {
to: QueueFamily::Transfer,
});
let VulkanImageMemory::Internal(shm) = &img.ty else {
unreachable!();
};
let data = shm.async_data.as_ref().unwrap();
if let Err(e) = shm.async_upload_copy_buffer_to_image(&img, data) {
data.complete(Err(e));
}
}
async fn await_async_upload(
id: u64,
img: Rc<VulkanImage>,
buf: Rc<VulkanCommandBuffer>,
_fence: Rc<VulkanFence>,
sync_file: SyncFile,
) {
let res = img.renderer.ring.readable(&sync_file.0).await;
if let Err(e) = res {
log::error!(
"Could not wait for sync file to become readable: {}",
ErrorFmt(e)
);
img.renderer.block();
}
match &img.renderer.transfer_command_buffers {
Some(b) => b.buffers.push(buf),
None => img.renderer.gfx_command_buffers.buffers.push(buf),
}
img.queue_state.set(QueueState::Released {
to: QueueFamily::Gfx,
});
img.renderer.pending_submits.remove(&id);
let VulkanImageMemory::Internal(shm) = &img.ty else {
unreachable!();
};
let data = shm.async_data.as_ref().unwrap();
data.complete(Ok(()));
}
impl VulkanRenderer {
pub fn create_shm_texture(
self: &Rc<Self>,
@ -868,6 +426,7 @@ impl VulkanRenderer {
io_job: Default::default(),
copy_job: Default::default(),
staging: Default::default(),
client_mem: Default::default(),
callback: Default::default(),
callback_id: Cell::new(0),
regions: Default::default(),

89
src/gfx_apis/vulkan/staging.rs
View file

@ -1,17 +1,24 @@
use {
crate::{
cpu_worker::CpuWorker,
gfx_api::GfxStagingBuffer,
gfx_apis::vulkan::{
allocator::{VulkanAllocation, VulkanAllocator},
device::VulkanDevice,
renderer::VulkanRenderer,
VulkanError,
},
utils::on_drop::{OnDrop, OnDrop2},
utils::{
clonecell::CloneCell,
on_drop::{OnDrop, OnDrop2},
},
},
ash::{
vk::{Buffer, BufferCreateInfo, BufferUsageFlags, MappedMemoryRange},
Device,
},
ash::vk::{Buffer, BufferCreateInfo, BufferUsageFlags, MappedMemoryRange},
gpu_alloc::UsageFlags,
std::rc::Rc,
std::{any::Any, cell::Cell, rc::Rc},
};
pub struct VulkanStagingBuffer {
@ -22,6 +29,22 @@ pub struct VulkanStagingBuffer {
}
impl VulkanDevice {
pub(super) fn create_staging_shell(
self: &Rc<Self>,
size: u64,
upload: bool,
download: bool,
) -> Rc<VulkanStagingShell> {
Rc::new(VulkanStagingShell {
device: self.clone(),
staging: Default::default(),
size,
download,
upload,
busy: Cell::new(false),
})
}
pub(super) fn create_staging_buffer(
self: &Rc<Self>,
allocator: &Rc<VulkanAllocator>,
@ -51,17 +74,15 @@ impl VulkanDevice {
})
}
pub(super) fn create_shm_staging(
pub(super) fn fill_staging_shell(
self: &Rc<Self>,
renderer: &Rc<VulkanRenderer>,
cpu: &Rc<CpuWorker>,
size: u64,
upload: bool,
download: bool,
cb: impl FnOnce(Result<VulkanStagingBuffer, VulkanError>) + 'static,
shell: Rc<VulkanStagingShell>,
cb: impl FnOnce(Result<Rc<VulkanStagingBuffer>, VulkanError>) + 'static,
) -> Result<(), VulkanError> {
let (vk_usage, usage) = get_usage(upload, download, false);
let buffer = self.create_buffer(size, vk_usage)?;
let (vk_usage, usage) = get_usage(shell.upload, shell.download, false);
let buffer = self.create_buffer(shell.size, vk_usage)?;
let memory_requirements = unsafe { self.device.get_buffer_memory_requirements(buffer) };
let slf = self.clone();
let destroy_buffer =
@ -77,12 +98,14 @@ impl VulkanDevice {
res.map_err(VulkanError::BindBufferMemory)?;
}
destroy_buffer.forget();
Ok(VulkanStagingBuffer {
let buffer = Rc::new(VulkanStagingBuffer {
device: slf.clone(),
allocation,
buffer,
size,
})
size: shell.size,
});
shell.staging.set(Some(buffer.clone()));
Ok(buffer)
};
renderer.shm_allocator.async_alloc(
renderer,
@ -159,3 +182,43 @@ impl Drop for VulkanStagingBuffer {
}
}
}
pub(super) struct VulkanStagingShell {
pub(super) device: Rc<VulkanDevice>,
pub(super) staging: CloneCell<Option<Rc<VulkanStagingBuffer>>>,
pub(super) size: u64,
pub(super) download: bool,
pub(super) upload: bool,
pub(super) busy: Cell<bool>,
}
impl GfxStagingBuffer for VulkanStagingShell {
fn size(&self) -> usize {
self.size as _
}
fn into_any(self: Rc<Self>) -> Rc<dyn Any> {
self
}
}
impl VulkanStagingShell {
fn assert_device(&self, device: &Device) {
assert_eq!(
self.device.device.handle(),
device.handle(),
"Mixed vulkan device use"
);
}
}
impl dyn GfxStagingBuffer {
pub(super) fn into_vk(self: Rc<Self>, device: &Device) -> Rc<VulkanStagingShell> {
let shell: Rc<VulkanStagingShell> = self
.into_any()
.downcast()
.expect("Non-vulkan staging buffer passed into vulkan");
shell.assert_device(device);
shell
}
}

684
src/gfx_apis/vulkan/transfer.rs Normal file
View file

@ -0,0 +1,684 @@
use {
crate::{
cpu_worker::{
jobs::{
img_copy::ImgCopyWork,
read_write::{ReadWriteJobError, ReadWriteWork},
},
CpuJob, CpuWork, CpuWorker,
},
gfx_api::{
AsyncShmGfxTextureCallback, PendingShmTransfer, ShmMemory, ShmMemoryBacking, SyncFile,
},
gfx_apis::vulkan::{
command::VulkanCommandBuffer,
fence::VulkanFence,
image::{QueueFamily, QueueState, QueueTransfer, VulkanImage, VulkanImageMemory},
renderer::image_barrier,
shm_image::VulkanShmImage,
staging::{VulkanStagingBuffer, VulkanStagingShell},
VulkanError,
},
rect::{Rect, Region},
utils::{clonecell::CloneCell, errorfmt::ErrorFmt},
},
arrayvec::ArrayVec,
ash::vk::{
AccessFlags2, BufferImageCopy2, CommandBufferBeginInfo, CommandBufferSubmitInfo,
CommandBufferUsageFlags, DependencyInfo, Extent3D, ImageAspectFlags, ImageLayout,
ImageSubresourceLayers, Offset3D, PipelineStageFlags2, SubmitInfo2,
},
std::{
cell::{Cell, RefCell},
rc::Rc,
slice,
},
uapi::OwnedFd,
};
pub struct VulkanShmImageAsyncData {
pub(super) busy: Cell<bool>,
pub(super) io_job: Cell<Option<Box<IoTransferJob>>>,
pub(super) copy_job: Cell<Option<Box<CopyTransferJob>>>,
pub(super) staging: CloneCell<Option<Rc<VulkanStagingShell>>>,
pub(super) client_mem: CloneCell<Option<Rc<dyn ShmMemory>>>,
pub(super) callback: Cell<Option<Rc<dyn AsyncShmGfxTextureCallback>>>,
pub(super) callback_id: Cell<u64>,
pub(super) regions: RefCell<Vec<BufferImageCopy2<'static>>>,
pub(super) cpu: Rc<CpuWorker>,
pub(super) last_sample: Cell<Option<SyncFile>>,
pub(super) data_copied: Cell<bool>,
}
impl VulkanShmImageAsyncData {
fn complete(&self, result: Result<(), VulkanError>) {
self.busy.set(false);
self.staging.take().unwrap().busy.set(false);
self.client_mem.take();
if let Some(cb) = self.callback.take() {
cb.completed(result.map_err(|e| e.into()));
}
}
}
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub(super) enum TransferType {
Upload,
Download,
}
impl VulkanShmImage {
pub fn async_transfer(
&self,
img: &Rc<VulkanImage>,
staging: Rc<VulkanStagingShell>,
client_mem: &Rc<dyn ShmMemory>,
damage: Region,
callback: Rc<dyn AsyncShmGfxTextureCallback>,
tt: TransferType,
) -> Result<Option<PendingShmTransfer>, VulkanError> {
let data = self.async_data.as_ref().unwrap();
let res = self.try_async_transfer(img, staging, data, client_mem, damage, tt);
match res {
Ok(()) => {
let id = img.renderer.allocate_point();
data.callback_id.set(id);
data.callback.set(Some(callback));
Ok(Some(PendingShmTransfer::new(img.clone(), id)))
}
Err(e) => Err(e),
}
}
fn try_async_transfer(
&self,
img: &Rc<VulkanImage>,
staging: Rc<VulkanStagingShell>,
data: &VulkanShmImageAsyncData,
client_mem: &Rc<dyn ShmMemory>,
mut damage: Region,
tt: TransferType,
) -> Result<(), VulkanError> {
if data.busy.get() {
return Err(VulkanError::AsyncCopyBusy);
}
if staging.busy.get() {
return Err(VulkanError::StagingBufferBusy);
}
match tt {
TransferType::Upload => {
if !staging.upload {
return Err(VulkanError::StagingBufferNoUpload);
}
}
TransferType::Download => {
if !staging.download {
return Err(VulkanError::StagingBufferNoDownload);
}
}
}
if self.size > client_mem.len() as u64 {
return Err(VulkanError::InvalidBufferSize);
}
data.busy.set(true);
data.data_copied.set(false);
staging.busy.set(true);
data.staging.set(Some(staging.clone()));
data.client_mem.set(Some(client_mem.clone()));
if img.contents_are_undefined.get() {
if tt == TransferType::Download {
return Err(VulkanError::UndefinedContents);
}
damage = Region::new2(Rect::new_sized(0, 0, img.width as _, img.height as _).unwrap());
}
let copies = &mut *data.regions.borrow_mut();
copies.clear();
let mut copy = |x, y, width, height| {
let buffer_offset = (y as u32 * img.stride + x as u32 * self.shm_info.bpp) as u64;
let copy = BufferImageCopy2::default()
.buffer_offset(buffer_offset)
.image_offset(Offset3D { x, y, z: 0 })
.image_extent(Extent3D {
width,
height,
depth: 1,
})
.image_subresource(ImageSubresourceLayers {
aspect_mask: ImageAspectFlags::COLOR,
mip_level: 0,
base_array_layer: 0,
layer_count: 1,
})
.buffer_image_height(img.height)
.buffer_row_length(img.stride / self.shm_info.bpp);
copies.push(copy);
};
let (width_mask, height_mask) = img.renderer.device.transfer_granularity_mask;
let width_mask = width_mask as i32;
let height_mask = height_mask as i32;
for damage in damage.rects() {
if damage.x2() < 0 || damage.y2() < 0 {
continue;
}
let x1 = damage.x1().max(0) & !width_mask;
let y1 = damage.y1().max(0) & !height_mask;
let x2 = ((damage.x2() + width_mask) & !width_mask).min(img.width as i32);
let y2 = ((damage.y2() + height_mask) & !height_mask).min(img.height as i32);
let Some(damage) = Rect::new(x1, y1, x2, y2) else {
continue;
};
if damage.is_empty() {
continue;
}
copy(
damage.x1(),
damage.y1(),
damage.width() as u32,
damage.height() as u32,
);
}
self.async_release_from_gfx_queue(img, data, tt)?;
if let Some(staging) = staging.staging.get() {
return match tt {
TransferType::Upload => self
.async_transfer_initiate_host_copy(img, data, &staging, copies, client_mem, tt),
TransferType::Download => {
self.async_download_copy_image_to_buffer(img, &staging, copies)
}
};
}
let img2 = img.clone();
let client_mem = client_mem.clone();
img.renderer
.device
.fill_staging_shell(&img.renderer, &data.cpu, staging, move |res| {
let VulkanImageMemory::Internal(shm) = &img2.ty else {
unreachable!();
};
if let Err(e) = shm.async_transfer_after_allocation(&img2, &client_mem, res, tt) {
shm.async_data.as_ref().unwrap().complete(Err(e));
}
})
}
fn async_release_from_gfx_queue(
&self,
img: &Rc<VulkanImage>,
data: &VulkanShmImageAsyncData,
tt: TransferType,
) -> Result<(), VulkanError> {
img.renderer.check_defunct()?;
let Some(transfer_queue_idx) = img.renderer.device.distinct_transfer_queue_family_idx
else {
let Some(sync_file) = data.last_sample.take() else {
img.queue_state.set(QueueState::Released {
to: QueueFamily::Transfer,
});
return Ok(());
};
let id = img.renderer.allocate_point();
let pending = img.renderer.eng.spawn(
"await_transfer_to_transfer",
await_gfx_queue_release(id, img.clone(), None, None, sync_file, tt),
);
img.renderer.pending_submits.set(id, pending);
img.queue_state.set(QueueState::Releasing);
return Ok(());
};
let (gfx_access_mask, gfx_layout, transfer_layout) = match tt {
TransferType::Upload => (
AccessFlags2::SHADER_SAMPLED_READ,
ImageLayout::SHADER_READ_ONLY_OPTIMAL,
ImageLayout::TRANSFER_DST_OPTIMAL,
),
TransferType::Download => (
AccessFlags2::COLOR_ATTACHMENT_WRITE,
ImageLayout::COLOR_ATTACHMENT_OPTIMAL,
ImageLayout::TRANSFER_SRC_OPTIMAL,
),
};
let mut barriers = ArrayVec::<_, 2>::new();
match img.queue_state.get() {
QueueState::Acquired { family } => {
assert_eq!(family, QueueFamily::Gfx);
}
QueueState::Releasing => {
unreachable!();
}
QueueState::Released { to } => {
assert_eq!(to, QueueFamily::Gfx);
let barrier = image_barrier()
.image(img.image)
.src_queue_family_index(transfer_queue_idx)
.dst_queue_family_index(img.renderer.device.graphics_queue_idx)
.dst_stage_mask(PipelineStageFlags2::ALL_COMMANDS)
.old_layout(transfer_layout)
.new_layout(gfx_layout);
barriers.push(barrier);
}
}
let barrier = image_barrier()
.image(img.image)
.src_queue_family_index(img.renderer.device.graphics_queue_idx)
.dst_queue_family_index(transfer_queue_idx)
.src_access_mask(gfx_access_mask)
.src_stage_mask(PipelineStageFlags2::ALL_COMMANDS)
.old_layout(if img.is_undefined.get() {
ImageLayout::UNDEFINED
} else {
gfx_layout
})
.new_layout(transfer_layout);
barriers.push(barrier);
let dep_info = DependencyInfo::default().image_memory_barriers(&barriers);
let release_fence = img.renderer.device.create_fence()?;
let dev = &img.renderer.device.device;
let begin_info =
CommandBufferBeginInfo::default().flags(CommandBufferUsageFlags::ONE_TIME_SUBMIT);
let cmd = img.renderer.gfx_command_buffers.allocate()?;
let command_buffer_info = CommandBufferSubmitInfo::default().command_buffer(cmd.buffer);
let submit_info =
SubmitInfo2::default().command_buffer_infos(slice::from_ref(&command_buffer_info));
unsafe {
dev.begin_command_buffer(cmd.buffer, &begin_info)
.map_err(VulkanError::BeginCommandBuffer)?;
dev.cmd_pipeline_barrier2(cmd.buffer, &dep_info);
dev.end_command_buffer(cmd.buffer)
.map_err(VulkanError::EndCommandBuffer)?;
dev.queue_submit2(
img.renderer.device.graphics_queue,
slice::from_ref(&submit_info),
release_fence.fence,
)
.map_err(VulkanError::Submit)?;
}
let sync_file = release_fence.export_sync_file()?;
let id = img.renderer.allocate_point();
let pending = img.renderer.eng.spawn(
"await_transfer_to_transfer",
await_gfx_queue_release(
id,
img.clone(),
Some(cmd),
Some(release_fence),
sync_file,
tt,
),
);
img.renderer.pending_submits.set(id, pending);
img.queue_state.set(QueueState::Releasing);
Ok(())
}
fn async_transfer_after_allocation(
&self,
img: &Rc<VulkanImage>,
client_mem: &Rc<dyn ShmMemory>,
res: Result<Rc<VulkanStagingBuffer>, VulkanError>,
tt: TransferType,
) -> Result<(), VulkanError> {
let staging = res?;
let data = self.async_data.as_ref().unwrap();
let copies = &*data.regions.borrow();
match tt {
TransferType::Upload => {
self.async_transfer_initiate_host_copy(img, data, &staging, copies, client_mem, tt)
}
TransferType::Download => {
self.async_download_copy_image_to_buffer(img, &staging, copies)
}
}
}
pub(super) fn async_transfer_initiate_host_copy(
&self,
img: &Rc<VulkanImage>,
data: &VulkanShmImageAsyncData,
staging: &VulkanStagingBuffer,
copies: &[BufferImageCopy2],
client_mem: &Rc<dyn ShmMemory>,
tt: TransferType,
) -> Result<(), VulkanError> {
img.renderer.check_defunct()?;
if tt == TransferType::Download {
staging.download(|_, _| ())?;
}
let id = img.renderer.allocate_point();
let pending;
match client_mem.safe_access() {
ShmMemoryBacking::Ptr(ptr) => {
let mut job = data.copy_job.take().unwrap_or_else(|| {
Box::new(CopyTransferJob {
img: None,
id,
_mem: None,
work: unsafe { ImgCopyWork::new() },
tt,
})
});
job.id = id;
job.img = Some(img.clone());
job._mem = Some(client_mem.clone());
job.tt = tt;
match tt {
TransferType::Upload => {
job.work.src = ptr as _;
job.work.dst = staging.allocation.mem.unwrap();
}
TransferType::Download => {
job.work.src = staging.allocation.mem.unwrap();
job.work.dst = ptr as _;
}
}
job.work.width = img.width as _;
job.work.stride = img.stride as _;
job.work.bpp = self.shm_info.bpp as _;
job.work.rects.clear();
for copy in copies {
job.work.rects.push(
Rect::new_sized(
copy.image_offset.x as _,
copy.image_offset.y as _,
copy.image_extent.width as _,
copy.image_extent.height as _,
)
.unwrap(),
);
}
pending = data.cpu.submit(job);
}
ShmMemoryBacking::Fd(fd, offset) => {
let mut min_offset = client_mem.len() as u64;
let mut max_offset = 0;
for copy in copies {
min_offset = min_offset.min(copy.buffer_offset);
let len = img.stride * (copy.image_extent.height - 1)
+ copy.image_extent.width * self.shm_info.bpp;
max_offset = max_offset.max(copy.buffer_offset + len as u64);
}
let mut job = data.io_job.take().unwrap_or_else(|| {
Box::new(IoTransferJob {
img: None,
id,
_mem: None,
work: unsafe { ReadWriteWork::new() },
fd: None,
tt,
})
});
job.id = id;
job.img = Some(img.clone());
job._mem = Some(client_mem.clone());
job.fd = Some(fd.clone());
job.tt = tt;
unsafe {
let config = job.work.config();
config.fd = fd.raw();
config.offset = offset + min_offset as usize;
config.ptr = staging.allocation.mem.unwrap().add(min_offset as _);
config.len = max_offset.saturating_sub(min_offset) as usize;
config.write = tt == TransferType::Download;
}
pending = data.cpu.submit(job);
}
}
img.renderer.pending_cpu_jobs.set(id, pending);
Ok(())
}
fn async_upload_copy_buffer_to_image(
&self,
img: &Rc<VulkanImage>,
data: &VulkanShmImageAsyncData,
) -> Result<(), VulkanError> {
if !data.data_copied.get() {
return Ok(());
}
if img.queue_state.get().acquire(QueueFamily::Transfer) == QueueTransfer::Impossible {
return Ok(());
}
img.renderer.check_defunct()?;
let regions = &*data.regions.borrow();
let staging = data.staging.get().unwrap().staging.get().unwrap();
staging.upload(|_, _| ())?;
let Some((cmd, fence, sync_file, point)) =
self.submit_buffer_image_copy(img, &staging, regions, true, TransferType::Upload)?
else {
return Ok(());
};
img.queue_state.set(QueueState::Releasing);
let future = img.renderer.eng.spawn(
"await async upload",
await_async_transfer_release_to_gfx(
point,
img.clone(),
cmd,
fence,
sync_file,
TransferType::Upload,
),
);
img.renderer.pending_submits.set(point, future);
Ok(())
}
fn async_download_copy_image_to_buffer(
&self,
img: &Rc<VulkanImage>,
staging: &VulkanStagingBuffer,
copies: &[BufferImageCopy2],
) -> Result<(), VulkanError> {
if img.queue_state.get().acquire(QueueFamily::Transfer) == QueueTransfer::Impossible {
return Ok(());
}
img.renderer.check_defunct()?;
let Some((cmd, fence, sync_file, point)) =
self.submit_buffer_image_copy(img, &staging, copies, true, TransferType::Download)?
else {
img.queue_state.set(QueueState::Released {
to: QueueFamily::Gfx,
});
let data = self.async_data.as_ref().unwrap();
let client_mem = data.client_mem.get().unwrap();
return self.async_transfer_initiate_host_copy(
&img,
data,
&staging,
copies,
&client_mem,
TransferType::Download,
);
};
img.queue_state.set(QueueState::Releasing);
let future = img.renderer.eng.spawn(
"await async image to buffer copy",
await_async_transfer_release_to_gfx(
point,
img.clone(),
cmd,
fence,
sync_file,
TransferType::Download,
),
);
img.renderer.pending_submits.set(point, future);
Ok(())
}
}
pub(super) struct IoTransferJob {
img: Option<Rc<VulkanImage>>,
id: u64,
_mem: Option<Rc<dyn ShmMemory>>,
fd: Option<Rc<OwnedFd>>,
work: ReadWriteWork,
tt: TransferType,
}
pub(super) struct CopyTransferJob {
img: Option<Rc<VulkanImage>>,
id: u64,
_mem: Option<Rc<dyn ShmMemory>>,
work: ImgCopyWork,
tt: TransferType,
}
impl CpuJob for IoTransferJob {
fn work(&mut self) -> &mut dyn CpuWork {
&mut self.work
}
fn completed(mut self: Box<Self>) {
self._mem = None;
self.fd = None;
let img = self.img.take().unwrap();
let res = self.work.config().result.take().unwrap();
complete_async_host_copy(&img, self.id, res, self.tt, |data| {
data.io_job.set(Some(self))
});
}
}
impl CpuJob for CopyTransferJob {
fn work(&mut self) -> &mut dyn CpuWork {
&mut self.work
}
fn completed(mut self: Box<Self>) {
self._mem = None;
let img = self.img.take().unwrap();
complete_async_host_copy(&img, self.id, Ok(()), self.tt, |data| {
data.copy_job.set(Some(self))
});
}
}
fn complete_async_host_copy(
img: &Rc<VulkanImage>,
id: u64,
res: Result<(), ReadWriteJobError>,
tt: TransferType,
store: impl FnOnce(&VulkanShmImageAsyncData),
) {
img.renderer.pending_cpu_jobs.remove(&id);
let VulkanImageMemory::Internal(shm) = &img.ty else {
unreachable!();
};
let data = shm.async_data.as_ref().unwrap();
store(data);
if let Err(e) = res {
data.complete(Err(VulkanError::AsyncCopyToStaging(e)));
}
data.data_copied.set(true);
match tt {
TransferType::Upload => {
let res = shm.async_upload_copy_buffer_to_image(img, data);
if let Err(e) = res {
data.complete(Err(e));
}
}
TransferType::Download => data.complete(Ok(())),
}
}
async fn await_gfx_queue_release(
id: u64,
img: Rc<VulkanImage>,
buf: Option<Rc<VulkanCommandBuffer>>,
_fence: Option<Rc<VulkanFence>>,
sync_file: SyncFile,
tt: TransferType,
) {
let res = img.renderer.ring.readable(&sync_file.0).await;
if let Err(e) = res {
log::error!(
"Could not wait for sync file to become readable: {}",
ErrorFmt(e)
);
img.renderer.block();
}
if let Some(buf) = buf {
img.renderer.gfx_command_buffers.buffers.push(buf);
}
img.renderer.pending_submits.remove(&id);
img.queue_state.set(QueueState::Released {
to: QueueFamily::Transfer,
});
let VulkanImageMemory::Internal(shm) = &img.ty else {
unreachable!();
};
let data = shm.async_data.as_ref().unwrap();
let res = match tt {
TransferType::Upload => shm.async_upload_copy_buffer_to_image(&img, data),
TransferType::Download => match data.staging.get().unwrap().staging.get() {
Some(staging) => {
let copies = &*data.regions.borrow();
shm.async_download_copy_image_to_buffer(&img, &staging, copies)
}
None => Ok(()),
},
};
if let Err(e) = res {
data.complete(Err(e));
}
}
pub async fn await_async_transfer_release_to_gfx(
id: u64,
img: Rc<VulkanImage>,
buf: Rc<VulkanCommandBuffer>,
_fence: Rc<VulkanFence>,
sync_file: SyncFile,
tt: TransferType,
) {
let res = img.renderer.ring.readable(&sync_file.0).await;
if let Err(e) = res {
log::error!(
"Could not wait for sync file to become readable: {}",
ErrorFmt(e)
);
img.renderer.block();
}
match &img.renderer.transfer_command_buffers {
Some(b) => b.buffers.push(buf),
None => img.renderer.gfx_command_buffers.buffers.push(buf),
}
img.queue_state.set(QueueState::Released {
to: QueueFamily::Gfx,
});
img.renderer.pending_submits.remove(&id);
let VulkanImageMemory::Internal(shm) = &img.ty else {
unreachable!();
};
let data = shm.async_data.as_ref().unwrap();
match tt {
TransferType::Upload => {
data.complete(Ok(()));
}
TransferType::Download => {
let data = shm.async_data.as_ref().unwrap();
let staging = data.staging.get().unwrap().staging.get().unwrap();
let client_mem = data.client_mem.get().unwrap();
let copies = &*data.regions.borrow();
let res = shm.async_transfer_initiate_host_copy(
&img,
data,
&staging,
copies,
&client_mem,
tt,
);
if let Err(e) = res {
data.complete(Err(e));
}
}
}
}

1
src/ifs/wl_buffer.rs
View file

@ -175,6 +175,7 @@ impl WlBuffer {
WlBufferStorage::Shm { .. } => {
return match surface {
Some(s) => {
s.shm_staging.take();
s.shm_textures.back().tex.take();
s.shm_textures.front().tex.take().is_some()
}

7
src/ifs/wl_surface.rs
View file

@ -23,8 +23,8 @@ use {
drm_feedback::DrmFeedback,
fixed::Fixed,
gfx_api::{
AsyncShmGfxTexture, BufferResv, BufferResvUser, GfxError, ReleaseSync, SampleRect,
SyncFile,
AsyncShmGfxTexture, BufferResv, BufferResvUser, GfxError, GfxStagingBuffer,
ReleaseSync, SampleRect, SyncFile,
},
ifs::{
wl_buffer::WlBuffer,
@ -276,6 +276,7 @@ pub struct WlSurface {
pub buffer_abs_pos: Cell<Rect>,
pub need_extents_update: Cell<bool>,
pub buffer: CloneCell<Option<Rc<SurfaceBuffer>>>,
pub shm_staging: CloneCell<Option<Rc<dyn GfxStagingBuffer>>>,
pub shm_textures: DoubleBuffered<SurfaceShmTexture>,
pub buf_x: NumCell<i32>,
pub buf_y: NumCell<i32>,
@ -593,6 +594,7 @@ impl WlSurface {
buffer_abs_pos: Cell::new(Default::default()),
need_extents_update: Default::default(),
buffer: Default::default(),
shm_staging: Default::default(),
shm_textures: DoubleBuffered::new(DamageQueue::new().map(|damage| SurfaceShmTexture {
tex: Default::default(),
damage,
@ -1327,6 +1329,7 @@ impl WlSurface {
}
pub fn reset_shm_textures(&self) {
self.shm_staging.take();
for tex in &*self.shm_textures {
tex.tex.take();
tex.damage.clear();

30
src/ifs/wl_surface/commit_timeline.rs
View file

@ -1,6 +1,6 @@
use {
crate::{
gfx_api::{AsyncShmGfxTextureCallback, GfxError, PendingShmUpload},
gfx_api::{AsyncShmGfxTextureCallback, GfxError, PendingShmTransfer, STAGING_UPLOAD},
ifs::{
wl_buffer::WlBufferStorage,
wl_surface::{PendingState, WlSurface, WlSurfaceError},
@ -310,7 +310,7 @@ enum EntryKind {
enum ShmUploadState {
None,
Todo(Rc<NodeRef<Entry>>),
Scheduled(#[expect(dead_code)] SmallVec<[PendingShmUpload; 1]>),
Scheduled(#[expect(dead_code)] SmallVec<[PendingShmTransfer; 1]>),
}
struct Commit {
@ -395,7 +395,7 @@ fn schedule_async_uploads(
node_ref: &Rc<NodeRef<Entry>>,
surface: &WlSurface,
pending: &PendingState,
uploads: &mut SmallVec<[PendingShmUpload; 1]>,
uploads: &mut SmallVec<[PendingShmTransfer; 1]>,
) -> Result<(), WlSurfaceError> {
if let Some(pending) = schedule_async_upload(node_ref, surface, pending)? {
uploads.push(pending);
@ -412,7 +412,7 @@ fn schedule_async_upload(
node_ref: &Rc<NodeRef<Entry>>,
surface: &WlSurface,
pending: &PendingState,
) -> Result<Option<PendingShmUpload>, WlSurfaceError> {
) -> Result<Option<PendingShmTransfer>, WlSurfaceError> {
let Some(Some(buf)) = &pending.buffer else {
return Ok(None);
};
@ -459,8 +459,28 @@ fn schedule_async_upload(
back_tex
}
};
let mut staging_opt = surface.shm_staging.get();
if let Some(staging) = &staging_opt {
if staging.size() != back_tex.staging_size() {
staging_opt = None;
}
}
let staging = match staging_opt {
Some(s) => s,
None => {
let s = surface
.client
.state
.render_ctx
.get()
.ok_or(WlSurfaceError::NoRenderContext)?
.create_staging_buffer(back_tex.staging_size(), STAGING_UPLOAD);
surface.shm_staging.set(Some(s.clone()));
s
}
};
back_tex
.async_upload(node_ref.clone(), mem.clone(), back.damage.get())
.async_upload(&staging, node_ref.clone(), mem.clone(), back.damage.get())
.map_err(WlSurfaceError::PrepareAsyncUpload)
}

20
src/ifs/zwlr_screencopy_frame_v1.rs
View file

@ -2,6 +2,7 @@ use {
crate::{
client::{Client, ClientError},
format::XRGB8888,
gfx_api::{AsyncShmGfxTextureCallback, GfxError, PendingShmTransfer},
ifs::{
wl_buffer::{WlBuffer, WlBufferError, WlBufferStorage},
wl_output::OutputGlobalOpt,
@ -9,6 +10,7 @@ use {
leaks::Tracker,
object::{Object, Version},
rect::Rect,
utils::errorfmt::ErrorFmt,
wire::{zwlr_screencopy_frame_v1::*, WlBufferId, ZwlrScreencopyFrameV1Id},
},
std::{cell::Cell, ops::Deref, rc::Rc},
@ -29,6 +31,7 @@ pub struct ZwlrScreencopyFrameV1 {
pub with_damage: Cell<bool>,
pub buffer: Cell<Option<Rc<WlBuffer>>>,
pub version: Version,
pub pending: Cell<Option<PendingShmTransfer>>,
}
impl ZwlrScreencopyFrameV1 {
@ -132,6 +135,7 @@ impl ZwlrScreencopyFrameV1 {
node.screencopies.remove(&(self.client.id, self.id));
node.screencast_changed();
}
self.pending.take();
}
}
@ -153,6 +157,22 @@ impl ZwlrScreencopyFrameV1RequestHandler for ZwlrScreencopyFrameV1 {
}
}
impl AsyncShmGfxTextureCallback for ZwlrScreencopyFrameV1 {
fn completed(self: Rc<Self>, res: Result<(), GfxError>) {
self.pending.take();
match res {
Ok(_) => {
let now = self.client.state.now();
self.send_ready(now.0.tv_sec as _, now.0.tv_nsec as _);
}
Err(e) => {
log::warn!("Could not perform shm screencopy: {}", ErrorFmt(e));
self.send_failed();
}
}
}
}
object_base! {
self = ZwlrScreencopyFrameV1;
version = self.version;

1
src/ifs/zwlr_screencopy_manager_v1.rs
View file

@ -130,6 +130,7 @@ impl ZwlrScreencopyManagerV1 {
with_damage: Cell::new(false),
buffer: Cell::new(None),
version: self.version,
pending: Default::default(),
});
track!(self.client, frame);
self.client.add_client_obj(&frame)?;

105
src/it/test_gfx_api.rs
View file

@ -6,8 +6,8 @@ use {
gfx_api::{
AcquireSync, AsyncShmGfxTexture, AsyncShmGfxTextureCallback, CopyTexture, FillRect,
FramebufferRect, GfxApiOpt, GfxContext, GfxError, GfxFormat, GfxFramebuffer, GfxImage,
GfxTexture, GfxWriteModifier, PendingShmUpload, ReleaseSync, ResetStatus,
ShmGfxTexture, ShmMemory, SyncFile,
GfxInternalFramebuffer, GfxStagingBuffer, GfxTexture, GfxWriteModifier,
PendingShmTransfer, ReleaseSync, ResetStatus, ShmGfxTexture, ShmMemory, SyncFile,
},
rect::{Rect, Region},
theme::Color,
@ -165,13 +165,14 @@ impl GfxContext for TestGfxCtx {
GfxApi::OpenGl
}
fn create_fb(
fn create_internal_fb(
self: Rc<Self>,
_cpu_worker: &Rc<CpuWorker>,
width: i32,
height: i32,
stride: i32,
format: &'static Format,
) -> Result<Rc<dyn GfxFramebuffer>, GfxError> {
) -> Result<Rc<dyn GfxInternalFramebuffer>, GfxError> {
assert!(stride >= width * 4);
Ok(Rc::new(TestGfxFb {
img: Rc::new(TestGfxImage::Shm(TestShmGfxImage {
@ -214,44 +215,17 @@ struct TestDmaBufGfxImage {
}
impl TestGfxImage {
fn read_pixels(
&self,
x: i32,
y: i32,
width: i32,
height: i32,
stride: i32,
format: &'static Format,
shm: &[Cell<u8>],
) -> Result<(), GfxError> {
assert!(x >= 0);
assert!(y >= 0);
assert!(width >= 0);
assert!(height >= 0);
assert!(stride >= 0);
assert!(x + width <= self.width());
assert!(y + height <= self.height());
assert!(stride >= width * 4);
let size = (stride * height) as usize;
assert!(shm.len() >= size);
let copy = |src_stride: i32, src_format: &Format, mut src: *const u8, mut dst: *mut u8| unsafe {
src = src.add((y * src_stride + x * 4) as usize);
for _ in 0..height {
ptr::copy_nonoverlapping(src, dst, (width * 4) as usize);
if !src_format.has_alpha && format.has_alpha {
for i in 0..width {
*dst.add((i * 4 + 3) as usize) = 255;
}
}
src = src.add(src_stride as usize);
dst = dst.add(stride as usize);
}
fn read_pixels(&self, shm: &[Cell<u8>]) -> Result<(), GfxError> {
let copy = |height: i32, stride: i32, src: *const u8, dst: *mut u8| unsafe {
let size = (height * stride) as usize;
assert!(shm.len() >= size);
ptr::copy_nonoverlapping(src, dst, size);
};
match self {
TestGfxImage::Shm(s) => {
copy(
s.height,
s.stride,
s.format,
s.data.borrow().as_ptr(),
shm.as_ptr() as _,
);
@ -260,8 +234,8 @@ impl TestGfxImage {
let map = d.bo.clone().map_read().map_err(TestGfxError::MapDmaBuf)?;
unsafe {
copy(
d.buf.height,
map.stride(),
d.buf.format,
map.data().as_ptr(),
shm.as_ptr() as _,
);
@ -300,20 +274,6 @@ impl GfxTexture for TestGfxImage {
self
}
fn read_pixels(
self: Rc<Self>,
x: i32,
y: i32,
width: i32,
height: i32,
stride: i32,
format: &'static Format,
shm: &[Cell<u8>],
) -> Result<(), GfxError> {
self.deref()
.read_pixels(x, y, width, height, stride, format, shm)
}
fn dmabuf(&self) -> Option<&DmaBuf> {
match self {
TestGfxImage::Shm(_) => None,
@ -335,10 +295,11 @@ impl ShmGfxTexture for TestGfxImage {
impl AsyncShmGfxTexture for TestGfxImage {
fn async_upload(
self: Rc<Self>,
_staging: &Rc<dyn GfxStagingBuffer>,
_callback: Rc<dyn AsyncShmGfxTextureCallback>,
mem: Rc<dyn ShmMemory>,
_damage: Region,
) -> Result<Option<PendingShmUpload>, GfxError> {
) -> Result<Option<PendingShmTransfer>, GfxError> {
let mut res = Ok(());
mem.access(&mut |d| {
res = self.clone().sync_upload(d, Region::default());
@ -588,26 +549,34 @@ impl GfxFramebuffer for TestGfxFb {
Ok(None)
}
fn copy_to_shm(
self: Rc<Self>,
x: i32,
y: i32,
width: i32,
height: i32,
stride: i32,
format: &'static Format,
shm: &[Cell<u8>],
) -> Result<(), GfxError> {
self.img
.deref()
.read_pixels(x, y, width, height, stride, format, shm)
}
fn format(&self) -> &'static Format {
&ARGB8888
}
}
impl GfxInternalFramebuffer for TestGfxFb {
fn into_fb(self: Rc<Self>) -> Rc<dyn GfxFramebuffer> {
self
}
fn staging_size(&self) -> usize {
0
}
fn download(
self: Rc<Self>,
_staging: &Rc<dyn GfxStagingBuffer>,
_callback: Rc<dyn AsyncShmGfxTextureCallback>,
mem: Rc<dyn ShmMemory>,
_damage: Region,
) -> Result<Option<PendingShmTransfer>, GfxError> {
let mut res = Ok(());
mem.access(&mut |mem| res = self.img.deref().read_pixels(mem))
.map_err(TestGfxError::AccessFailed)?;
res.map(|_| None)
}
}
impl dyn GfxTexture {
fn as_native(&self) -> &TestGfxImage {
self.as_any()

129
src/state.rs
View file

@ -27,8 +27,8 @@ use {
forker::ForkerProxy,
format::Format,
gfx_api::{
AcquireSync, BufferResv, GfxContext, GfxError, GfxFramebuffer, GfxTexture, ReleaseSync,
SampleRect, SyncFile,
AcquireSync, BufferResv, GfxContext, GfxError, GfxFramebuffer, GfxTexture,
PendingShmTransfer, ReleaseSync, SampleRect, SyncFile, STAGING_DOWNLOAD,
},
gfx_apis::create_gfx_context,
globals::{Globals, GlobalsError, RemovableWaylandGlobal, WaylandGlobal},
@ -62,7 +62,7 @@ use {
io_uring::IoUring,
leaks::Tracker,
logger::Logger,
rect::Rect,
rect::{Rect, Region},
renderer::Renderer,
scale::Scale,
security_context_acceptor::SecurityContextAcceptors,
@ -1006,17 +1006,6 @@ impl State {
)
}
fn have_hardware_cursor(&self) -> bool {
if let Some(group) = self.cursor_user_group_hardware_cursor.get() {
if let Some(user) = group.active() {
if user.get().is_some() {
return true;
}
}
}
false
}
pub fn perform_shm_screencopy(
&self,
src: &Rc<dyn GfxTexture>,
@ -1025,80 +1014,54 @@ impl State {
x_off: i32,
y_off: i32,
size: Option<(i32, i32)>,
capture: &ZwlrScreencopyFrameV1,
mem: &ClientMemOffset,
capture: &Rc<ZwlrScreencopyFrameV1>,
mem: &Rc<ClientMemOffset>,
stride: i32,
format: &'static Format,
transform: Transform,
scale: Scale,
) -> Result<(), ShmScreencopyError> {
let (src_width, src_height) = src.size();
let mut needs_copy = capture.rect.x1() < x_off
|| capture.rect.x2() > x_off + src_width
|| capture.rect.y1() < y_off
|| capture.rect.y2() > y_off + src_height
|| self.have_hardware_cursor();
if let Some((target_width, target_height)) = size {
if (target_width, target_height) != (src_width, src_height) {
needs_copy = true;
}
}
let acc = if needs_copy {
let Some(ctx) = self.render_ctx.get() else {
return Err(ShmScreencopyError::NoRenderContext);
};
let fb = ctx
.create_fb(capture.rect.width(), capture.rect.height(), stride, format)
.map_err(ShmScreencopyError::CreateTemporaryFb)?;
self.perform_screencopy(
src,
None,
acquire_sync,
ReleaseSync::None,
&fb,
AcquireSync::Unnecessary,
ReleaseSync::None,
transform,
position,
true,
x_off - capture.rect.x1(),
y_off - capture.rect.y1(),
size,
transform,
scale,
)
.map_err(ShmScreencopyError::CopyToTemporary)?;
mem.access(|mem| {
fb.copy_to_shm(
0,
0,
capture.rect.width(),
capture.rect.height(),
stride,
format,
mem,
)
})
} else {
mem.access(|mem| {
src.clone().read_pixels(
capture.rect.x1() - x_off,
capture.rect.y1() - y_off,
capture.rect.width(),
capture.rect.height(),
stride,
format,
mem,
)
})
) -> Result<Option<PendingShmTransfer>, ShmScreencopyError> {
let Some(ctx) = self.render_ctx.get() else {
return Err(ShmScreencopyError::NoRenderContext);
};
match acc {
Ok(res) => res.map_err(ShmScreencopyError::ReadPixels),
Err(e) => {
capture.client.error(e);
Ok(())
}
}
let fb = ctx
.clone()
.create_internal_fb(
&self.cpu_worker,
capture.rect.width(),
capture.rect.height(),
stride,
format,
)
.map_err(ShmScreencopyError::CreateTemporaryFb)?;
self.perform_screencopy(
src,
None,
acquire_sync,
ReleaseSync::None,
&fb.clone().into_fb(),
AcquireSync::Unnecessary,
ReleaseSync::None,
transform,
position,
true,
x_off - capture.rect.x1(),
y_off - capture.rect.y1(),
size,
transform,
scale,
)
.map_err(ShmScreencopyError::CopyToTemporary)?;
let staging = ctx.create_staging_buffer(fb.staging_size(), STAGING_DOWNLOAD);
let pending = fb
.download(
&staging,
capture.clone(),
mem.clone(),
Region::new2(capture.rect.at_point(0, 0)),
)
.map_err(ShmScreencopyError::ReadPixels)?;
Ok(pending)
}
pub fn create_seat(self: &Rc<Self>, name: &str) -> Rc<WlSeatGlobal> {

22
src/text.rs
View file

@ -3,8 +3,8 @@ use {
cpu_worker::{AsyncCpuWork, CpuJob, CpuWork, CpuWorker, PendingJob},
format::ARGB8888,
gfx_api::{
AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxContext, GfxError, GfxTexture,
PendingShmUpload,
AsyncShmGfxTexture, AsyncShmGfxTextureCallback, GfxContext, GfxError, GfxStagingBuffer,
GfxTexture, PendingShmTransfer, STAGING_UPLOAD,
},
pango::{
consts::{
@ -302,9 +302,10 @@ impl Drop for TextTexture {
struct Shared {
cpu_worker: Rc<CpuWorker>,
ctx: Rc<dyn GfxContext>,
staging: CloneCell<Option<Rc<dyn GfxStagingBuffer>>>,
textures: DoubleBuffered<TextBuffer>,
pending_render: Cell<Option<PendingJob>>,
pending_upload: Cell<Option<PendingShmUpload>>,
pending_upload: Cell<Option<PendingShmTransfer>>,
render_job: Cell<Option<Box<RenderJob>>>,
result: Cell<Option<Result<(), TextError>>>,
waiter: Cell<Option<Rc<dyn OnCompleted>>>,
@ -367,6 +368,7 @@ impl TextTexture {
let data = Rc::new(Shared {
cpu_worker: cpu_worker.clone(),
ctx: ctx.clone(),
staging: Default::default(),
textures: Default::default(),
pending_render: Default::default(),
pending_upload: Default::default(),
@ -528,9 +530,22 @@ impl CpuJob for RenderJob {
}
}
};
let mut staging_opt = data.staging.take();
if let Some(staging) = &staging_opt {
if staging.size() != tex.staging_size() {
staging_opt = None;
}
}
let staging = match staging_opt {
Some(s) => s,
None => data
.ctx
.create_staging_buffer(tex.staging_size(), STAGING_UPLOAD),
};
let pending = tex
.clone()
.async_upload(
&staging,
data.clone(),
Rc::new(rt.data),
Region::new2(Rect::new_sized_unchecked(0, 0, rt.width, rt.height)),
@ -538,6 +553,7 @@ impl CpuJob for RenderJob {
.map_err(TextError::Upload);
if pending.is_ok() {
data.textures.back().tex.set(Some(tex));
data.staging.set(Some(staging));
}
match pending {
Ok(Some(p)) => data.pending_upload.set(Some(p)),

19
src/tree/output.rs
View file

@ -267,6 +267,7 @@ impl OutputNode {
capture.send_failed();
continue;
}
let mut ready = true;
if let Some(storage) = wl_buffer.storage.borrow_mut().deref() {
match storage {
WlBufferStorage::Shm { mem, stride } => {
@ -284,10 +285,16 @@ impl OutputNode {
self.global.persistent.transform.get(),
self.global.persistent.scale.get(),
);
if let Err(e) = res {
log::warn!("Could not perform shm screencopy: {}", ErrorFmt(e));
capture.send_failed();
continue;
match res {
Ok(p) => {
ready = p.is_none();
capture.pending.set(p);
}
Err(e) => {
log::warn!("Could not perform shm screencopy: {}", ErrorFmt(e));
capture.send_failed();
continue;
}
}
}
WlBufferStorage::Dmabuf { fb, .. } => {
@ -327,7 +334,9 @@ impl OutputNode {
if capture.with_damage.get() {
capture.send_damage();
}
capture.send_ready(now.0.tv_sec as _, now.0.tv_nsec as _);
if ready {
capture.send_ready(now.0.tv_sec as _, now.0.tv_nsec as _);
}
}
self.screencast_changed();
}