diff --git a/autoload/gitgutter.vim b/autoload/gitgutter.vim
index f336eec..fe2f810 100644
--- a/autoload/gitgutter.vim
+++ b/autoload/gitgutter.vim
@@ -45,6 +45,8 @@ function! gitgutter#process_buffer(bufnr, force) abort
       endif
 
     endif
+  else
+    call s:clear(a:bufnr)
   endif
 endfunction
 
@@ -59,9 +61,7 @@ function! gitgutter#disable() abort
   for bufnr in buflist
     let file = expand('#'.bufnr.':p')
     if !empty(file)
-      call gitgutter#sign#clear_signs(bufnr)
-      call gitgutter#sign#remove_dummy_sign(bufnr, 1)
-      call gitgutter#hunk#reset(bufnr)
+      call s:clear(bufnr)
     endif
   endfor
 
@@ -87,3 +87,8 @@ function! s:has_fresh_changes(bufnr) abort
   return getbufvar(a:bufnr, 'changedtick') != gitgutter#utility#getbufvar(a:bufnr, 'tick')
 endfunction
 
+function! s:clear(bufnr)
+  call gitgutter#sign#clear_signs(a:bufnr)
+  call gitgutter#sign#remove_dummy_sign(a:bufnr, 1)
+  call gitgutter#hunk#reset(a:bufnr)
+endfunction
diff --git a/autoload/gitgutter/utility.vim b/autoload/gitgutter/utility.vim
index 7cdbb7c..6885489 100644
--- a/autoload/gitgutter/utility.vim
+++ b/autoload/gitgutter/utility.vim
@@ -39,7 +39,8 @@ function! gitgutter#utility#is_active(bufnr) abort
         \ !pumvisible() &&
         \ s:is_file_buffer(a:bufnr) &&
         \ s:exists_file(a:bufnr) &&
-        \ s:not_git_dir(a:bufnr)
+        \ s:not_git_dir(a:bufnr) &&
+        \ !s:vimdiff(a:bufnr)
 endfunction
 
 function! s:not_git_dir(bufnr) abort
@@ -175,3 +176,23 @@ endfunction
 function! s:strip_trailing_new_line(line) abort
   return substitute(a:line, '\n$', '', '')
 endfunction
+
+" Returns 1 if any of the given buffer's windows has the `&diff` option set,
+" or 0 otherwise.
+function! s:vimdiff(bufnr)
+  if exists('*win_findbuf')
+    for winid in win_findbuf(a:bufnr)
+      if getwinvar(winid, '&diff')
+        return 1
+      endif
+    endfor
+    return 0
+  else
+    for winnr in range(1, winnr('$'))
+      if winbufnr(winnr) == a:bufnr && getwinvar(winnr, '&diff')
+        return 1
+      endif
+    endfor
+    return 0
+  endif
+endfunction