diff --git a/autoload/gitgutter/sign.vim b/autoload/gitgutter/sign.vim
index 1ee305a..e9a91f3 100644
--- a/autoload/gitgutter/sign.vim
+++ b/autoload/gitgutter/sign.vim
@@ -11,7 +11,7 @@ function! gitgutter#sign#enable() abort
   let g:gitgutter_signs = 1
   call gitgutter#highlight#define_sign_text_highlights()
 
-  if !old_signs && !g:gitgutter_highlight_lines
+  if !old_signs && !g:gitgutter_highlight_lines && !g:gitgutter_highlight_linenrs
     call gitgutter#all(1)
   endif
 endfunction