diff --git a/src/compositor.rs b/src/compositor.rs
index 39df57bd..c6dce04d 100644
--- a/src/compositor.rs
+++ b/src/compositor.rs
@@ -359,6 +359,7 @@ fn start_compositor2(
         enable_primary_selection: Cell::new(true),
         xdg_surface_configure_events: Default::default(),
         workspace_display_order: Cell::new(WorkspaceDisplayOrder::Manual),
+        outputs_without_hc: Default::default(),
     });
     state.tracker.register(ClientId::from_raw(0));
     create_dummy_output(&state);
diff --git a/src/cursor_user.rs b/src/cursor_user.rs
index 0f945d39..f983e577 100644
--- a/src/cursor_user.rs
+++ b/src/cursor_user.rs
@@ -85,7 +85,7 @@ impl CursorUserGroup {
             let x_int = x.round_down();
             let y_int = y.round_down();
             let extents = cursor.extents_at_scale(Scale::default());
-            self.state.damage2(true, extents.move_(x_int, y_int));
+            self.damage(extents.move_(x_int, y_int));
         }
     }
 
@@ -248,6 +248,10 @@ impl CursorUserGroup {
         };
         active.present_hardware_cursor(output, hc);
     }
+
+    fn damage(&self, rect: Rect) {
+        self.state.damage2(true, self.hardware_cursor.get(), rect);
+    }
 }
 
 impl CursorUser {
@@ -416,10 +420,8 @@ impl CursorUser {
             let old_x_int = old_x.round_down();
             let old_y_int = old_y.round_down();
             let extents = cursor.extents_at_scale(Scale::default());
-            self.group
-                .state
-                .damage2(true, extents.move_(old_x_int, old_y_int));
-            self.group.state.damage2(true, extents.move_(x_int, y_int));
+            self.group.damage(extents.move_(old_x_int, old_y_int));
+            self.group.damage(extents.move_(x_int, y_int));
         }
         self.pos.set((x, y));
         self.update_hardware_cursor_(false);
@@ -435,7 +437,8 @@ impl CursorUser {
     }
 
     pub fn software_cursor(&self) -> bool {
-        self.is_active() && !self.group.hardware_cursor.get()
+        self.is_active()
+            && (!self.group.hardware_cursor.get() || self.group.state.outputs_without_hc.get() > 0)
     }
 
     fn update_hardware_cursor_(&self, render: bool) {
diff --git a/src/state.rs b/src/state.rs
index 421d77e9..56f38271 100644
--- a/src/state.rs
+++ b/src/state.rs
@@ -277,6 +277,7 @@ pub struct State {
     pub enable_primary_selection: Cell<bool>,
     pub xdg_surface_configure_events: AsyncQueue<XdgSurfaceConfigureEvent>,
     pub workspace_display_order: Cell<WorkspaceDisplayOrder>,
+    pub outputs_without_hc: NumCell<usize>,
 }
 
 // impl Drop for State {
@@ -648,7 +649,7 @@ impl State {
                 fn visit_output(&mut self, node: &Rc<OutputNode>) {
                     node.render_data.borrow_mut().titles.clear();
                     node.render_data.borrow_mut().status.take();
-                    node.hardware_cursor.set(None);
+                    node.set_hardware_cursor(None);
                     node.node_visit_children(self);
                 }
                 fn visit_float(&mut self, node: &Rc<FloatNode>) {
@@ -968,16 +969,19 @@ impl State {
     }
 
     pub fn damage(&self, rect: Rect) {
-        self.damage2(false, rect);
+        self.damage2(false, false, rect);
     }
 
-    pub fn damage2(&self, cursor: bool, rect: Rect) {
+    pub fn damage2(&self, cursor: bool, skip_hc: bool, rect: Rect) {
         if rect.is_empty() {
             return;
         }
         self.damage_visualizer.add(rect);
         for output in self.root.outputs.lock().values() {
             if output.global.pos.get().intersects(&rect) {
+                if skip_hc && output.hardware_cursor.is_some() {
+                    continue;
+                }
                 output.global.add_damage_area(&rect);
                 if cursor && output.schedule.defer_cursor_updates() {
                     output.schedule.software_cursor_changed();
diff --git a/src/tasks/connector.rs b/src/tasks/connector.rs
index f7960960..2ca0f8d8 100644
--- a/src/tasks/connector.rs
+++ b/src/tasks/connector.rs
@@ -271,6 +271,7 @@ impl ConnectorHandler {
         self.state.outputs.set(self.id, output_data.clone());
         on.schedule_update_render_data();
         self.state.root.outputs.set(self.id, on.clone());
+        self.state.outputs_without_hc.fetch_add(1);
         self.state.output_extents_changed();
         global.opt.node.set(Some(on.clone()));
         global.opt.global.set(Some(global.clone()));
@@ -330,7 +331,7 @@ impl ConnectorHandler {
                     ConnectorEvent::Disconnected => break 'outer,
                     ConnectorEvent::HardwareCursor(hc) => {
                         on.schedule.set_hardware_cursor(&hc);
-                        on.hardware_cursor.set(hc);
+                        on.set_hardware_cursor(hc);
                         self.state.refresh_hardware_cursors();
                     }
                     ConnectorEvent::FormatsChanged(formats) => {
@@ -363,6 +364,9 @@ impl ConnectorHandler {
             sc.stop();
         }
         global.destroyed.set(true);
+        if on.hardware_cursor.is_none() {
+            self.state.outputs_without_hc.fetch_sub(1);
+        }
         self.state.root.outputs.remove(&self.id);
         self.state.output_extents_changed();
         self.state.outputs.remove(&self.id);
diff --git a/src/tree/output.rs b/src/tree/output.rs
index c7842d64..c0f49100 100644
--- a/src/tree/output.rs
+++ b/src/tree/output.rs
@@ -1433,6 +1433,19 @@ impl OutputNode {
                 .handle_tearing_mode_change(mode.to_config());
         }
     }
+
+    pub fn set_hardware_cursor(&self, hc: Option<Rc<dyn HardwareCursor>>) {
+        let is_none = hc.is_none();
+        let old = self.hardware_cursor.set(hc);
+        let was_none = old.is_none();
+        if was_none != is_none {
+            if is_none {
+                self.state.outputs_without_hc.fetch_add(1);
+            } else {
+                self.state.outputs_without_hc.fetch_sub(1);
+            }
+        }
+    }
 }
 
 pub struct OutputTitle {