diff --git a/autoenv.zsh b/autoenv.zsh
index 12e058d..ee0daa0 100644
--- a/autoenv.zsh
+++ b/autoenv.zsh
@@ -260,8 +260,12 @@ _autoenv_source() {
   local autoenv_to_dir=$PWD
 
   # Source varstash library once.
+  # XXX: pollutes environment with e.g. `stash`, and `autostash` will cause
+  # an overwritten `stash` function to be called!
   if [[ -z "$functions[(I)autostash]" ]]; then
-    source ${${funcsourcetrace[1]%:*}:h}/lib/varstash
+    if \grep -qE '\b(autostash|autounstash|stash)\b' $autoenv_env_file; then
+      source ${${funcsourcetrace[1]%:*}:h}/lib/varstash
+    fi
     # NOTE: Varstash uses $PWD as default for varstash_dir, we might set it to
     # ${autoenv_env_file:h}.
   fi
@@ -331,7 +335,9 @@ _autoenv_chpwd_handler() {
         fi
 
         # Unstash any autostashed stuff.
-        varstash_dir=$prev_dir autounstash
+        if [[ -n "$functions[(I)autostash]" ]]; then
+          varstash_dir=$prev_dir autounstash
+        fi
 
         _autoenv_stack_entered_remove $prev_file
       fi
diff --git a/tests/varstash.t b/tests/varstash.t
index b75aa59..dd730c4 100644
--- a/tests/varstash.t
+++ b/tests/varstash.t
@@ -6,11 +6,22 @@ Setup test environment.
 
   $ mkdir sub
   $ cd sub
+
+The varstash library should not get loaded always.
+
+  $ echo 'echo ENTER' > $AUTOENV_FILE_ENTER
+  $ echo 'echo LEAVE' > $AUTOENV_FILE_LEAVE
+  $ test_autoenv_auth_env_files
+  $ cd .
+  ENTER
+  $ type -w autostash
+  autostash: none
+  [1]
+
+Now on to some stashing.
+
   $ echo 'echo ENTER; autostash FOO=changed' > $AUTOENV_FILE_ENTER
   $ echo 'echo LEAVE; autounstash' > $AUTOENV_FILE_LEAVE
-
-Manually create auth file
-
   $ test_autoenv_auth_env_files
 
 Set environment variable.
@@ -19,8 +30,12 @@ Set environment variable.
 
 Activating the env stashes it and applies a new value.
 
-  $ cd .
+  $ cd ..
+  LEAVE
+  $ cd sub
   ENTER
+  $ type -w autostash
+  autostash: function
   $ echo $FOO
   changed