Merge pull request #106 from mahkoh/jorth/direct-scanout-culling

Try direct scanout in more cases

src/backends/metal/video.rs

@@ -9,10 +9,14 @@ use {
         drm_feedback::DrmFeedback,
         edid::Descriptor,
         format::{Format, ARGB8888, XRGB8888},
-        gfx_api::{BufferPoints, GfxApiOpt, GfxContext, GfxFramebuffer, GfxRenderPass, GfxTexture},
+        gfx_api::{
+            AbsoluteRect, BufferPoints, GfxApiOpt, GfxContext, GfxFramebuffer, GfxRenderPass,
+            GfxTexture,
+        },
         ifs::wp_presentation_feedback::{KIND_HW_COMPLETION, KIND_VSYNC},
         renderer::RenderResult,
         state::State,
+        theme::Color,
         tree::OutputNode,
         udev::UdevDevice,
         utils::{
@@ -345,6 +349,17 @@ pub struct DirectScanoutData {
     fb: Rc<DrmFramebuffer>,
     dma_buf_id: DmaBufId,
     acquired: Cell<bool>,
+    position: DirectScanoutPosition,
+}
+
+#[derive(Debug)]
+pub struct DirectScanoutPosition {
+    pub src_width: i32,
+    pub src_height: i32,
+    pub crtc_x: i32,
+    pub crtc_y: i32,
+    pub crtc_width: i32,
+    pub crtc_height: i32,
 }
 
 impl Drop for DirectScanoutData {
@@ -413,25 +428,96 @@ impl MetalConnector {
         pass: &GfxRenderPass,
         plane: &Rc<MetalPlane>,
     ) -> Option<DirectScanoutData> {
-        if pass.ops.len() != 1 {
+        let plane_w = plane.mode_w.get();
-            return None;
+        let plane_h = plane.mode_h.get();
-        }
+        let ct = 'ct: {
-        let GfxApiOpt::CopyTexture(ct) = &pass.ops[0] else {
+            let mut ops = pass.ops.iter().rev();
-            return None;
+            let ct = 'ct2: {
+                for opt in &mut ops {
+                    match opt {
+                        GfxApiOpt::Sync => {}
+                        GfxApiOpt::FillRect(_) => {
+                            // Top-most layer must be a texture.
+                            return None;
+                        }
+                        GfxApiOpt::CopyTexture(ct) => break 'ct2 ct,
+                    }
+                }
+                return None;
+            };
+            let plane_rect = AbsoluteRect {
+                x1: 0.0,
+                x2: plane_w as f32,
+                y1: 0.0,
+                y2: plane_h as f32,
+            };
+            if !ct.tex.format().has_alpha && ct.target == plane_rect {
+                // Texture covers the entire screen and is opaque.
+                break 'ct ct;
+            }
+            for opt in ops {
+                match opt {
+                    GfxApiOpt::Sync => {}
+                    GfxApiOpt::FillRect(fr) => {
+                        if fr.color == Color::SOLID_BLACK {
+                            // Black fills can be ignored because this is the CRTC background color.
+                            if fr.rect == plane_rect {
+                                // If fill covers the entire screen, we don't have to look further.
+                                break 'ct ct;
+                            }
+                        } else {
+                            // Fill could be visible.
+                            return None;
+                        }
+                    }
+                    GfxApiOpt::CopyTexture(_) => {
+                        // Texture could be visible.
+                        return None;
+                    }
+                }
+            }
+            if let Some(clear) = pass.clear {
+                if clear != Color::SOLID_BLACK {
+                    // Background could be visible.
+                    return None;
+                }
+            }
+            ct
         };
         if ct.source != BufferPoints::identity() {
+            // Non-trivial transforms are not supported.
             return None;
         }
-        if ct.target.x1 != 0.0
+        if ct.target.x1 < 0.0
-            || ct.target.y1 != 0.0
+            || ct.target.y1 < 0.0
-            || ct.target.x2 != plane.mode_w.get() as f32
+            || ct.target.x2 > plane_w as f32
-            || ct.target.y2 != plane.mode_h.get() as f32
+            || ct.target.y2 > plane_h as f32
         {
+            // Rendering outside the screen is not supported.
+            return None;
+        }
+        let (tex_w, tex_h) = ct.tex.size();
+        let (crtc_w, crtc_h) = (ct.target.x2 - ct.target.x1, ct.target.y2 - ct.target.y1);
+        if crtc_w < 0.0 || crtc_h < 0.0 {
+            // Flipping x or y axis is not supported.
+            return None;
+        }
+        if self.cursor_enabled.get() && (tex_w as f32, tex_h as f32) != (crtc_w, crtc_h) {
+            // If hardware cursors are used, we cannot scale the texture.
             return None;
         }
         let Some(dmabuf) = ct.tex.dmabuf() else {
+            // Shm buffers cannot be scanned out.
             return None;
         };
+        let position = DirectScanoutPosition {
+            src_width: tex_w,
+            src_height: tex_h,
+            crtc_x: ct.target.x1 as _,
+            crtc_y: ct.target.y1 as _,
+            crtc_width: crtc_w as _,
+            crtc_height: crtc_h as _,
+        };
         let mut cache = self.scanout_buffers.borrow_mut();
         if let Some(buffer) = cache.get(&dmabuf.id) {
             return buffer.fb.as_ref().map(|fb| DirectScanoutData {
@@ -439,12 +525,14 @@ impl MetalConnector {
                 fb: fb.clone(),
                 dma_buf_id: dmabuf.id,
                 acquired: Default::default(),
+                position,
             });
         }
         let format = 'format: {
             if let Some(f) = plane.formats.get(&dmabuf.format.drm) {
                 break 'format f;
             }
+            // Try opaque format if possible.
             if let Some(opaque) = dmabuf.format.opaque {
                 if let Some(f) = plane.formats.get(&opaque.drm) {
                     break 'format f;
@@ -461,6 +549,7 @@ impl MetalConnector {
                 fb: Rc::new(fb),
                 dma_buf_id: dmabuf.id,
                 acquired: Default::default(),
+                position,
             }),
             Err(e) => {
                 log::debug!(
@@ -505,6 +594,7 @@ impl MetalConnector {
             Some(rr),
             output.global.preferred_scale.get(),
             render_hw_cursor,
+            output.has_fullscreen(),
         );
         let try_direct_scanout = try_direct_scanout
             && !output.global.have_shm_screencopies()
@@ -518,7 +608,14 @@ impl MetalConnector {
         let mut direct_scanout_data = None;
         if try_direct_scanout {
             if let Some(dsd) = self.prepare_direct_scanout(&pass, plane) {
-                output.perform_screencopies(None, &dsd.tex, !render_hw_cursor);
+                output.perform_screencopies(
+                    None,
+                    &dsd.tex,
+                    !render_hw_cursor,
+                    dsd.position.crtc_x,
+                    dsd.position.crtc_y,
+                    Some((dsd.position.crtc_width, dsd.position.crtc_height)),
+                );
                 direct_scanout_data = Some(dsd);
             }
         }
@@ -541,6 +638,9 @@ impl MetalConnector {
                     Some(&*buffer_fb),
                     &buffer.render_tex,
                     !render_hw_cursor,
+                    0,
+                    0,
+                    None,
                 );
                 buffer.drm.clone()
             }
@@ -584,8 +684,33 @@ impl MetalConnector {
                 let fb =
                     self.prepare_present_fb(&mut rr, buffer, &plane, &node, try_direct_scanout);
                 rr.dispatch_frame_requests();
+                let (crtc_x, crtc_y, crtc_w, crtc_h, src_width, src_height) =
+                    match &fb.direct_scanout_data {
+                        None => {
+                            let plane_w = plane.mode_w.get();
+                            let plane_h = plane.mode_h.get();
+                            (0, 0, plane_w, plane_h, plane_w, plane_h)
+                        }
+                        Some(dsd) => {
+                            let p = &dsd.position;
+                            (
+                                p.crtc_x,
+                                p.crtc_y,
+                                p.crtc_width,
+                                p.crtc_height,
+                                p.src_width,
+                                p.src_height,
+                            )
+                        }
+                    };
                 changes.change_object(plane.id, |c| {
                     c.change(plane.fb_id, fb.fb.id().0 as _);
+                    c.change(plane.src_w.id, (src_width as u64) << 16);
+                    c.change(plane.src_h.id, (src_height as u64) << 16);
+                    c.change(plane.crtc_x.id, crtc_x as u64);
+                    c.change(plane.crtc_y.id, crtc_y as u64);
+                    c.change(plane.crtc_w.id, crtc_w as u64);
+                    c.change(plane.crtc_h.id, crtc_h as u64);
                 });
                 new_fb = Some(fb);
             }

src/gfx_api.rs

@@ -8,7 +8,7 @@ use {
         scale::Scale,
         state::State,
         theme::Color,
-        tree::Node,
+        tree::{Node, OutputNode},
         utils::numcell::NumCell,
         video::{dmabuf::DmaBuf, gbm::GbmDevice, Modifier},
     },
@@ -112,7 +112,7 @@ impl BufferPoints {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub struct AbsoluteRect {
     pub x1: f32,
     pub x2: f32,
@@ -211,6 +211,7 @@ impl dyn GfxFramebuffer {
         result: Option<&mut RenderResult>,
         scale: Scale,
         render_hardware_cursor: bool,
+        black_background: bool,
     ) -> GfxRenderPass {
         let mut ops = self.take_render_ops();
         let (width, height) = self.size();
@@ -251,7 +252,10 @@ impl dyn GfxFramebuffer {
                 }
             }
         }
-        let c = state.theme.colors.background.get();
+        let c = match black_background {
+            true => Color::SOLID_BLACK,
+            false => state.theme.colors.background.get(),
+        };
         GfxRenderPass {
             ops,
             clear: Some(c),
@@ -262,6 +266,26 @@ impl dyn GfxFramebuffer {
         self.render(pass.ops, pass.clear.as_ref())
     }
 
+    pub fn render_output(
+        &self,
+        node: &OutputNode,
+        state: &State,
+        cursor_rect: Option<Rect>,
+        result: Option<&mut RenderResult>,
+        scale: Scale,
+        render_hardware_cursor: bool,
+    ) {
+        self.render_node(
+            node,
+            state,
+            cursor_rect,
+            result,
+            scale,
+            render_hardware_cursor,
+            node.has_fullscreen(),
+        )
+    }
+
     pub fn render_node(
         &self,
         node: &dyn Node,
@@ -270,6 +294,7 @@ impl dyn GfxFramebuffer {
         result: Option<&mut RenderResult>,
         scale: Scale,
         render_hardware_cursor: bool,
+        black_background: bool,
     ) {
         let pass = self.create_render_pass(
             node,
@@ -278,6 +303,7 @@ impl dyn GfxFramebuffer {
             result,
             scale,
             render_hardware_cursor,
+            black_background,
         );
         self.perform_render_pass(pass);
     }
@@ -359,6 +385,7 @@ pub trait GfxTexture: Debug {
     ) -> Result<(), GfxError>;
     fn dmabuf(&self) -> Option<&DmaBuf>;
     fn reservations(&self) -> &TextureReservations;
+    fn format(&self) -> &'static Format;
 }
 
 pub trait GfxContext: Debug {

src/gfx_apis/gl/renderer/context.rs

@@ -187,6 +187,7 @@ impl GlRenderContext {
             ctx: self.clone(),
             gl,
             resv: Default::default(),
+            format,
         }))
     }
 }

src/gfx_apis/gl/renderer/image.rs

@@ -29,6 +29,7 @@ impl Image {
             ctx: self.ctx.clone(),
             gl: GlTexture::import_img(&self.ctx.ctx, &self.gl)?,
             resv: Default::default(),
+            format: self.gl.dmabuf.format,
         }))
     }
 

src/gfx_apis/gl/renderer/texture.rs

@@ -17,6 +17,7 @@ pub struct Texture {
     pub(in crate::gfx_apis::gl) ctx: Rc<GlRenderContext>,
     pub(in crate::gfx_apis::gl) gl: GlTexture,
     pub(in crate::gfx_apis::gl) resv: TextureReservations,
+    pub(in crate::gfx_apis::gl) format: &'static Format,
 }
 
 impl Debug for Texture {
@@ -68,4 +69,8 @@ impl GfxTexture for Texture {
     fn reservations(&self) -> &TextureReservations {
         &self.resv
     }
+
+    fn format(&self) -> &'static Format {
+        self.format
+    }
 }

src/gfx_apis/vulkan/image.rs

@@ -587,4 +587,8 @@ impl GfxTexture for VulkanImage {
     fn reservations(&self) -> &TextureReservations {
         &self.resv
     }
+
+    fn format(&self) -> &'static Format {
+        self.format
+    }
 }

src/gfx_apis/vulkan/sampler.rs

@@ -17,9 +17,9 @@ impl VulkanDevice {
             .mag_filter(Filter::LINEAR)
             .min_filter(Filter::LINEAR)
             .mipmap_mode(SamplerMipmapMode::NEAREST)
-            .address_mode_u(SamplerAddressMode::REPEAT)
+            .address_mode_u(SamplerAddressMode::CLAMP_TO_EDGE)
-            .address_mode_v(SamplerAddressMode::REPEAT)
+            .address_mode_v(SamplerAddressMode::CLAMP_TO_EDGE)
-            .address_mode_w(SamplerAddressMode::REPEAT)
+            .address_mode_w(SamplerAddressMode::CLAMP_TO_EDGE)
             .max_anisotropy(1.0)
             .min_lod(0.0)
             .max_lod(0.25)

src/ifs/jay_screencast.rs

@@ -154,6 +154,9 @@ impl JayScreencast {
         on: &OutputNode,
         texture: &Rc<dyn GfxTexture>,
         render_hardware_cursors: bool,
+        x_off: i32,
+        y_off: i32,
+        size: Option<(i32, i32)>,
     ) {
         if !self.running.get() {
             return;
@@ -176,8 +179,9 @@ impl JayScreencast {
                     on.global.preferred_scale.get(),
                     on.global.pos.get(),
                     render_hardware_cursors,
-                    0,
+                    x_off,
-                    0,
+                    y_off,
+                    size,
                 );
                 self.client.event(Ready {
                     self_id: self.id,

src/ifs/wl_output.rs

@@ -211,6 +211,9 @@ impl WlOutputGlobal {
         fb: Option<&dyn GfxFramebuffer>,
         tex: &Rc<dyn GfxTexture>,
         render_hardware_cursors: bool,
+        x_off: i32,
+        y_off: i32,
+        size: Option<(i32, i32)>,
     ) {
         if self.pending_captures.is_empty() {
             return;
@@ -295,8 +298,9 @@ impl WlOutputGlobal {
                     self.preferred_scale.get(),
                     self.pos.get(),
                     render_hardware_cursors,
-                    -capture.rect.x1(),
+                    x_off - capture.rect.x1(),
-                    -capture.rect.y1(),
+                    y_off - capture.rect.y1(),
+                    size,
                 );
             }
             if capture.with_damage.get() {

src/renderer/renderer_base.rs

@@ -65,7 +65,7 @@ impl RendererBase<'_> {
     }
 
     pub fn fill_boxes2(&mut self, boxes: &[Rect], color: &Color, dx: i32, dy: i32) {
-        if boxes.is_empty() {
+        if boxes.is_empty() || *color == Color::TRANSPARENT {
             return;
         }
         let (dx, dy) = self.scale_point(dx, dy);
@@ -94,7 +94,7 @@ impl RendererBase<'_> {
         dx: f32,
         dy: f32,
     ) {
-        if boxes.is_empty() {
+        if boxes.is_empty() || *color == Color::TRANSPARENT {
             return;
         }
         let (dx, dy) = self.scale_point_f(dx, dy);

src/screenshoter.rs

@@ -75,6 +75,7 @@ pub fn take_screenshot(state: &State) -> Result<Screenshot, ScreenshooterError>
         None,
         Scale::from_int(1),
         true,
+        false,
     );
     let drm = gbm.drm.dup_render()?.fd().clone();
     Ok(Screenshot { drm, bo })

src/state.rs

@@ -743,7 +743,7 @@ impl State {
         rr: &mut RenderResult,
         render_hw_cursor: bool,
     ) {
-        fb.render_node(
+        fb.render_output(
             output,
             self,
             Some(output.global.pos.get()),
@@ -751,7 +751,7 @@ impl State {
             output.global.preferred_scale.get(),
             render_hw_cursor,
         );
-        output.perform_screencopies(Some(&**fb), tex, !render_hw_cursor);
+        output.perform_screencopies(Some(&**fb), tex, !render_hw_cursor, 0, 0, None);
         rr.dispatch_frame_requests();
     }
 
@@ -764,6 +764,7 @@ impl State {
         render_hardware_cursors: bool,
         x_off: i32,
         y_off: i32,
+        size: Option<(i32, i32)>,
     ) {
         let mut ops = target.take_render_ops();
         let (width, height) = target.size();
@@ -781,7 +782,7 @@ impl State {
         };
         renderer
             .base
-            .render_texture(src, x_off, y_off, None, None, scale, None);
+            .render_texture(src, x_off, y_off, None, size, scale, None);
         if render_hardware_cursors {
             for seat in self.globals.lock_seats().values() {
                 if let Some(cursor) = seat.get_cursor() {
@@ -794,7 +795,6 @@ impl State {
                 }
             }
         }
-        let clear = target.format().has_alpha.then_some(&Color::TRANSPARENT);
+        target.render(ops, Some(&Color::SOLID_BLACK));
-        target.render(ops, clear);
     }
 }

src/theme.rs

@@ -45,6 +45,13 @@ impl Color {
         a: 0.0,
     };
 
+    pub const SOLID_BLACK: Self = Self {
+        r: 0.0,
+        g: 0.0,
+        b: 0.0,
+        a: 1.0,
+    };
+
     pub fn from_gray(g: u8) -> Self {
         Self::from_rgb(g, g, g)
     }

src/tree/output.rs

@@ -83,6 +83,9 @@ impl OutputNode {
         fb: Option<&dyn GfxFramebuffer>,
         tex: &Rc<dyn GfxTexture>,
         render_hardware_cursor: bool,
+        x_off: i32,
+        y_off: i32,
+        size: Option<(i32, i32)>,
     ) {
         if let Some(workspace) = self.workspace.get() {
             if !workspace.capture.get() {
@@ -90,9 +93,9 @@ impl OutputNode {
             }
         }
         self.global
-            .perform_screencopies(fb, tex, render_hardware_cursor);
+            .perform_screencopies(fb, tex, render_hardware_cursor, x_off, y_off, size);
         for sc in self.screencasts.lock().values() {
-            sc.copy_texture(self, tex, render_hardware_cursor);
+            sc.copy_texture(self, tex, render_hardware_cursor, x_off, y_off, size);
         }
     }
 
@@ -477,6 +480,13 @@ impl OutputNode {
     fn pointer_move(self: &Rc<Self>, seat: &Rc<WlSeatGlobal>, x: i32, y: i32) {
         self.pointer_positions.set(seat.id(), (x, y));
     }
+
+    pub fn has_fullscreen(&self) -> bool {
+        self.workspace
+            .get()
+            .map(|w| w.fullscreen.get().is_some())
+            .unwrap_or(false)
+    }
 }
 
 pub struct OutputTitle {