diff --git a/release-notes.md b/release-notes.md
index 44409299..a5b2254a 100644
--- a/release-notes.md
+++ b/release-notes.md
@@ -1,5 +1,6 @@
 # Unreleased
 
+- Various bugfixes.
 - Floating windows can now be configured to be shown above fullscreen windows
   by using the `enable-float-above-fullscreen` action.
 - Implement xdg-toplevel-tag-v1.
diff --git a/src/cursor_user.rs b/src/cursor_user.rs
index bb8e96c9..e191f48c 100644
--- a/src/cursor_user.rs
+++ b/src/cursor_user.rs
@@ -106,7 +106,6 @@ impl CursorUserGroup {
     fn remove_hardware_cursor(&self) {
         self.state.hardware_tick_cursor.push(None);
         self.state.damage_hardware_cursors(false);
-        self.state.cursor_user_group_hardware_cursor.take();
     }
 
     pub fn detach(&self) {
@@ -182,6 +181,7 @@ impl CursorUserGroup {
             }
         } else {
             self.remove_hardware_cursor();
+            self.state.cursor_user_group_hardware_cursor.take();
         }
     }