diff --git a/autoload/fugitive.vim b/autoload/fugitive.vim
index 9bd089c..8f25832 100644
--- a/autoload/fugitive.vim
+++ b/autoload/fugitive.vim
@@ -4525,14 +4525,11 @@ endfunction
 function! s:diffoff_all(dir) abort
   let curwin = winnr()
   for nr in range(1,winnr('$'))
-    if getwinvar(nr,'&diff')
+    if getwinvar(nr, '&diff') && !empty(getwinvar(nr, 'fugitive_diff_restore'))
       if nr != winnr()
         execute nr.'wincmd w'
-        let restorewinnr = 1
-      endif
-      if s:Dir() ==# a:dir
-        call s:diffoff()
       endif
+      call s:diffoff()
     endif
   endfor
   execute curwin.'wincmd w'