diff --git a/autoenv.zsh b/autoenv.zsh
index aa44141..94425f1 100644
--- a/autoenv.zsh
+++ b/autoenv.zsh
@@ -162,13 +162,6 @@ _autoenv_hash_pair() {
   echo ":${env_file}:${env_shasum}:1"
 }
 
-_autoenv_authorized_env_file() {
-  local env_file=$1
-  local pair=$(_autoenv_hash_pair $env_file)
-  test -f $AUTOENV_ENV_FILENAME \
-    && \grep -qF $pair $AUTOENV_ENV_FILENAME
-}
-
 _autoenv_authorize() {
   local env_file=${1:A}
   _autoenv_deauthorize $env_file
@@ -196,30 +189,46 @@ _autoenv_ask_for_yes() {
   fi
 }
 
+typeset -g -A _autoenv_asked_already
+_autoenv_asked_already=()
+
 # Args: 1: absolute path to env file (resolved symlinks).
 _autoenv_check_authorized_env_file() {
-  if ! [[ -f $1 ]]; then
+  local env_file=${1:A}
+  if ! [[ -f $env_file ]]; then
     return 1
   fi
-  if ! _autoenv_authorized_env_file $1; then
+
+  local pair=$(_autoenv_hash_pair $env_file)
+
+  if [[ -n ${_autoenv_asked_already[$pair]} ]]; then
+    _autoenv_debug "Asked already: ${_autoenv_asked_already[$pair]}"
+    return ${_autoenv_asked_already[$pair]}
+  fi
+
+  if [[ -f $AUTOENV_ENV_FILENAME ]] && \grep -qF $pair $AUTOENV_ENV_FILENAME; then
+    _autoenv_asked_already[$pair]=0
+  else
     echo "Attempting to load unauthorized env file!"
-    command ls -l $1
+    command ls -l $env_file
     echo ""
     echo "**********************************************"
     echo ""
-    cat $1
+    cat $env_file
     echo ""
     echo "**********************************************"
     echo ""
     echo -n "Would you like to authorize it? (type 'yes') "
 
-    if ! _autoenv_ask_for_yes; then
-      return 1
+    if _autoenv_ask_for_yes; then
+      _autoenv_authorize $env_file
+      _autoenv_asked_already[$pair]=0
+    else
+      _autoenv_asked_already[$pair]=1
     fi
 
-    _autoenv_authorize $1
   fi
-  return 0
+  return ${_autoenv_asked_already[$pair]}
 }
 
 # Get directory of this file (absolute, with resolved symlinks).
diff --git a/tests/autoenv.t b/tests/autoenv.t
index b0c520f..2859896 100644
--- a/tests/autoenv.t
+++ b/tests/autoenv.t
@@ -14,6 +14,7 @@ Now try to make it accept it
 
   $ _autoenv_stack_entered=()
   $ rm $AUTOENV_ENV_FILENAME
+  $ _autoenv_asked_already=()
   $ _autoenv_ask_for_yes() { echo "yes" }
   $ cd .
   Attempting to load unauthorized env file!
@@ -37,8 +38,9 @@ Now lets see that it actually checks the shasum value.
   $ cd .
   ENTERED
 
-  $ _autoenv_stack_entered=()
   $ rm $AUTOENV_ENV_FILENAME
+  $ _autoenv_stack_entered=()
+  $ _autoenv_asked_already=()
   $ test_autoenv_add_to_env $PWD/.env mischief
   $ cd .
   Attempting to load unauthorized env file!
@@ -56,8 +58,9 @@ Now lets see that it actually checks the shasum value.
 
 Now, will it take no for an answer?
 
-  $ _autoenv_stack_entered=()
   $ rm $AUTOENV_ENV_FILENAME
+  $ _autoenv_stack_entered=()
+  $ _autoenv_asked_already=()
   $ _autoenv_ask_for_yes() { echo "no"; return 1 }
   $ cd .
   Attempting to load unauthorized env file!
@@ -74,7 +77,8 @@ Now, will it take no for an answer?
 
 Lets also try one more time to ensure it didn't add it.
 
-  $ _autoenv_ask_for_yes() { echo "yes"; return 0 }
+  $ _autoenv_asked_already=()
+  $ _autoenv_ask_for_yes() { echo "no"; return 1 }
   $ cd .
   Attempting to load unauthorized env file!
   -* /tmp/cramtests-*/autoenv.t/.env (glob)
@@ -85,11 +89,35 @@ Lets also try one more time to ensure it didn't add it.
   
   **********************************************
   
+  Would you like to authorize it? (type 'yes') no
+
+And now see if we're not being asked again after not allowing it.
+
+  $ _autoenv_ask_for_yes() { echo "should_not_be_used"; return 1 }
+  $ cd .
+
+
+Reloading the script should keep the current state, e.g. when reloading your
+~/.zshrc.
+But it should re-ask for unauthorized files.
+
+  $ cd ..
+  $ $TEST_SOURCE_AUTOENV
+  $ _autoenv_ask_for_yes() { echo "yes"; return 0 }
+  $ cd -
+  Attempting to load unauthorized env file!
+  -* /tmp/cramtests-*/autoenv.t/.env (glob)
+  
+  **********************************************
+  
+  echo ENTERED
+  
+  **********************************************
+  
   Would you like to authorize it? (type 'yes') yes
   ENTERED
 
-Reloading the script should keep the current state, e.g. when reloading your
-~/.zshrc.
+With an authorized file, it should not re-ask you.
 
   $ $TEST_SOURCE_AUTOENV
   $ cd .
diff --git a/tests/leave.t b/tests/leave.t
index 7b80466..1e55a5c 100644
--- a/tests/leave.t
+++ b/tests/leave.t
@@ -42,6 +42,7 @@ Leave the directory and answer "no".
 
   $ cd sub
   ENTERED
+  $ _autoenv_asked_already=()
   $ _autoenv_ask_for_yes() { echo "yes"; return 0 }
   $ cd ..
   Attempting to load unauthorized env file!
diff --git a/tests/recurse-upwards.t b/tests/recurse-upwards.t
index ff4fce6..d57bc32 100644
--- a/tests/recurse-upwards.t
+++ b/tests/recurse-upwards.t
@@ -110,6 +110,7 @@ Changing the root .env should trigger re-authentication via autoenv_source_paren
 First, let's answer "no".
 
   $ echo "echo NEW" >| .env
+  $ _autoenv_asked_already=()
   $ _autoenv_ask_for_yes() { echo "no"; return 1 }
   $ cd sub
   autoenv_source_parent_from_sub:
@@ -135,6 +136,7 @@ This currently does not trigger re-execution of the .env file.
 
 Touching the .env file will now source the parent env file.
 
+  $ _autoenv_asked_already=()
   $ touch -t 201401010104 .env
   $ cd .
   autoenv_source_parent_from_sub: