rect: fix overflow in dist_squared

src/rect.rs

@@ -222,19 +222,27 @@ where
         !self.contains(x, y)
     }
 
-    pub fn dist_squared(&self, x: i32, y: i32) -> i32 {
+    pub fn dist_squared(&self, x: i32, y: i32) -> i128 {
+        let x = x as i64;
+        let y = y as i64;
+        let x1 = self.raw.x1 as i64;
+        let x2 = self.raw.x2 as i64;
+        let y1 = self.raw.y1 as i64;
+        let y2 = self.raw.y2 as i64;
         let mut dx = 0;
-        if self.raw.x1 > x {
-            dx = self.raw.x1 - x;
-        } else if self.raw.x2 < x {
-            dx = x - self.raw.x2;
+        if x1 > x {
+            dx = x1 - x;
+        } else if x2 < x {
+            dx = x - x2;
         }
         let mut dy = 0;
-        if self.raw.y1 > y {
-            dy = self.raw.y1 - y;
-        } else if self.raw.y2 < y {
-            dy = y - self.raw.y2;
+        if y1 > y {
+            dy = y1 - y;
+        } else if y2 < y {
+            dy = y - y2;
         }
+        let dx = dx as i128;
+        let dy = dy as i128;
         dx * dx + dy * dy
     }
 

src/rect/tests.rs

@@ -704,3 +704,12 @@ fn new_saturating() {
     let r3 = Rect::new_sized_saturating(i32::MAX - 10, 0, 100, 10);
     assert_eq!(r3.x2(), i32::MAX);
 }
+
+#[test]
+fn dist_squared() {
+    let r1 = Rect::new_sized_saturating(i32::MIN, i32::MIN, 0, 0);
+    assert_eq!(
+        r1.dist_squared(i32::MAX, i32::MAX),
+        (1 << 65) + 2 - (1 << 34),
+    );
+}

src/state.rs

@@ -1372,7 +1372,7 @@ impl State {
     }
 
     pub fn find_closest_output(&self, mut x: i32, mut y: i32) -> (Rc<OutputNode>, i32, i32) {
-        let mut optimal_dist = i32::MAX;
+        let mut optimal_dist = i128::MAX;
         let mut optimal_output = None;
         let outputs = self.root.outputs.lock();
         for output in outputs.values() {