diff --git a/src/main.c b/src/main.c
index 083a28f..86052cb 100644
--- a/src/main.c
+++ b/src/main.c
@@ -405,6 +405,10 @@ void wd_ui_apply_done(struct wd_state *state, struct wl_list *outputs) {
 }
 
 void wd_ui_show_error(struct wd_state *state, const char *message) {
+  if (gtk_widget_in_destruction(gtk_widget_get_toplevel(state->info_bar))) {
+    fprintf(stderr, "%s\n", message);
+    return;
+  }
   gtk_label_set_text(GTK_LABEL(state->info_label), message);
   gtk_widget_show(state->info_bar);
   gtk_info_bar_set_revealed(GTK_INFO_BAR(state->info_bar), TRUE);
@@ -421,6 +425,9 @@ static void cleanup(GtkWidget *window, gpointer data) {
   g_object_unref(state->grabbing_cursor);
   g_object_unref(state->move_cursor);
   g_clear_object(&state->settings);
+#ifdef WITH_KANSHI
+  wd_store_config_if_dirty(state);
+#endif
   wd_state_destroy(state);
 }
 
@@ -866,6 +873,11 @@ static void auto_apply_selected(GSimpleAction *action, GVariant *param, gpointer
   }
   state->autoapply = g_variant_get_boolean(param);
   g_simple_action_set_state(action, param);
+#ifdef WITH_KANSHI
+  if (!state->autoapply) {
+    wd_store_config_if_dirty(state);
+  }
+#endif
 }
 
 static gboolean redraw_canvas(GtkWidget *widget, GdkFrameClock *frame_clock, gpointer data) {
@@ -913,6 +925,7 @@ static void save_kanshi_config_selected(GSimpleAction *action, GVariant *param,
     g_settings_set_boolean(state->settings, "save-kanshi-config", g_variant_get_boolean(param));
   }
   state->save_kanshi_config = g_variant_get_boolean(param);
+  state->kanshi_dirty = false;
   g_simple_action_set_state(action, param);
 }
 #endif
diff --git a/src/outputs.c b/src/outputs.c
index 6cec53a..9d0ce60 100644
--- a/src/outputs.c
+++ b/src/outputs.c
@@ -54,8 +54,11 @@ static void config_handle_succeeded(void *data,
   zwlr_output_configuration_v1_destroy(config);
   wd_ui_apply_done(pending->state, pending->outputs);
 #ifdef WITH_KANSHI
-  if (pending->manual && pending->state->save_kanshi_config) {
-    wd_store_config(pending->state, pending->outputs);
+  if (pending->state->save_kanshi_config) {
+    if (pending->manual) {
+      wd_store_config(pending->state, pending->outputs);
+    }
+    pending->state->kanshi_dirty = !pending->manual;
   }
 #endif
   destroy_pending(pending);
diff --git a/src/store.c b/src/store.c
index 4e6a166..8b9fe8a 100644
--- a/src/store.c
+++ b/src/store.c
@@ -634,6 +634,37 @@ static void reload_kanshi(void) {
   }
 }
 
+void wd_store_config_if_dirty(struct wd_state *state) {
+  if (!state->kanshi_dirty) {
+    return;
+  }
+  state->kanshi_dirty = false;
+
+  struct wl_list outputs;
+  wl_list_init(&outputs);
+  struct wd_head *head;
+  wl_list_for_each(head, &state->heads, link) {
+    struct wd_head_config *output = g_new0(struct wd_head_config, 1);
+    output->head = head;
+    output->enabled = head->enabled;
+    output->width = head->mode != NULL ? head->mode->width : head->custom_mode.width;
+    output->height = head->mode != NULL ? head->mode->height : head->custom_mode.height;
+    output->refresh = head->mode != NULL ? head->mode->refresh : head->custom_mode.refresh;
+    output->x = head->x;
+    output->y = head->y;
+    output->scale = head->scale;
+    output->transform = head->transform;
+    wl_list_insert(&outputs, &output->link);
+  }
+  wd_store_config(state, &outputs);
+
+  struct wd_head_config *output, *tmp;
+  wl_list_for_each_safe(output, tmp, &outputs, link) {
+    wl_list_remove(&output->link);
+    g_free(output);
+  }
+}
+
 void wd_store_config(struct wd_state *state, struct wl_list *outputs) {
   g_autofree char *path = get_config_path();
   g_autoptr(GPtrArray) files = g_ptr_array_new_with_free_func(kanshi_file_free);
diff --git a/src/wdisplays.h b/src/wdisplays.h
index dbdb03d..ffec9bf 100644
--- a/src/wdisplays.h
+++ b/src/wdisplays.h
@@ -192,6 +192,7 @@ struct wd_state {
   bool capture;
   bool show_overlay;
   bool save_kanshi_config;
+  bool kanshi_dirty;
   double zoom;
 
   unsigned int apply_idle;
@@ -352,6 +353,12 @@ void wd_destroy_overlay(struct wd_output *output);
  * Updates kanshi config
  */
 void wd_store_config(struct wd_state *state, struct wl_list *outputs);
+
+/*
+ * Saves the current layout to the kanshi config if auto-apply changed it
+ * since the last save.
+ */
+void wd_store_config_if_dirty(struct wd_state *state);
 #endif
 // SPDX-SnippetEnd
 #endif