diff --git a/src/main.c b/src/main.c
index 3fa9b5d..c2e41da 100644
--- a/src/main.c
+++ b/src/main.c
@@ -303,6 +303,12 @@ static void resize_to_fit(struct wd_state *state) {
   }
   GtkRequisition natural;
   gtk_widget_get_preferred_size(state->window, NULL, &natural);
+  bool grew = natural.width > state->window_natural.width
+      || natural.height > state->window_natural.height;
+  state->window_natural = natural;
+  if (!grew) {
+    return;
+  }
   int grow_width = natural.width - gtk_widget_get_allocated_width(state->window);
   int grow_height = natural.height - gtk_widget_get_allocated_height(state->window);
   if (grow_width > 0 || grow_height > 0) {
diff --git a/src/wdisplays.h b/src/wdisplays.h
index f0c919f..2a91195 100644
--- a/src/wdisplays.h
+++ b/src/wdisplays.h
@@ -201,6 +201,7 @@ struct wd_state {
   struct wd_point pan_start;
 
   GtkWidget *window;
+  GtkRequisition window_natural;
   GtkWidget *main_box;
   GtkWidget *header_stack;
   GtkWidget *stack_switcher;