diff --git a/autoload/gitgutter/sign.vim b/autoload/gitgutter/sign.vim
index e9a91f3..4fc0296 100644
--- a/autoload/gitgutter/sign.vim
+++ b/autoload/gitgutter/sign.vim
@@ -20,7 +20,7 @@ function! gitgutter#sign#disable() abort
   let g:gitgutter_signs = 0
   call gitgutter#highlight#define_sign_text_highlights()
 
-  if !g:gitgutter_highlight_lines
+  if !g:gitgutter_highlight_lines && !g:gitgutter_highlight_linenrs
     call gitgutter#sign#clear_signs(bufnr(''))
   endif
 endfunction