diff --git a/autoload/gitgutter.vim b/autoload/gitgutter.vim
index 06dcdeb..5d393f9 100644
--- a/autoload/gitgutter.vim
+++ b/autoload/gitgutter.vim
@@ -23,7 +23,7 @@ function! gitgutter#process_buffer(file, realtime)
         call hunk#set_hunks(diff#parse_diff(diff))
         let modified_lines = diff#process_hunks(hunk#hunks())
 
-        if g:gitgutter_signs
+        if g:gitgutter_signs || g:gitgutter_highlight_lines
           call sign#update_signs(a:file, modified_lines)
         endif
 
@@ -77,19 +77,34 @@ endfunction
 function! gitgutter#line_highlights_disable()
   let g:gitgutter_highlight_lines = 0
   call highlight#define_sign_line_highlights()
+
+  if !g:gitgutter_signs
+    call sign#clear_signs(utility#file())
+    call sign#remove_dummy_sign(0)
+  endif
+
   redraw!
 endfunction
 
 function! gitgutter#line_highlights_enable()
+  let old_highlight_lines = g:gitgutter_highlight_lines
+
   let g:gitgutter_highlight_lines = 1
   call highlight#define_sign_line_highlights()
+
+  if !old_highlight_lines && !g:gitgutter_signs
+    call gitgutter#all()
+  endif
+
   redraw!
 endfunction
 
 function! gitgutter#line_highlights_toggle()
-  let g:gitgutter_highlight_lines = !g:gitgutter_highlight_lines
-  call highlight#define_sign_line_highlights()
-  redraw!
+  if g:gitgutter_highlight_lines
+    call gitgutter#line_highlights_disable()
+  else
+    call gitgutter#line_highlights_enable()
+  endif
 endfunction
 
 " }}}
@@ -97,14 +112,24 @@ endfunction
 " Signs {{{
 
 function! gitgutter#signs_enable()
+  let old_signs = g:gitgutter_signs
+
   let g:gitgutter_signs = 1
-  call gitgutter#all()
+  call highlight#define_sign_text_highlights()
+
+  if !old_signs && !g:gitgutter_highlight_lines
+    call gitgutter#all()
+  endif
 endfunction
 
 function! gitgutter#signs_disable()
   let g:gitgutter_signs = 0
-  call sign#clear_signs(utility#file())
-  call sign#remove_dummy_sign(0)
+  call highlight#define_sign_text_highlights()
+
+  if !g:gitgutter_highlight_lines
+    call sign#clear_signs(utility#file())
+    call sign#remove_dummy_sign(0)
+  endif
 endfunction
 
 function! gitgutter#signs_toggle()
diff --git a/autoload/highlight.vim b/autoload/highlight.vim
index d19b227..44218ca 100644
--- a/autoload/highlight.vim
+++ b/autoload/highlight.vim
@@ -6,17 +6,24 @@ function! highlight#define_highlights()
   let [guibg, ctermbg] = highlight#get_background_colors('SignColumn')
 
   " Highlights used by the signs.
+
   execute "highlight GitGutterAddDefault    guifg=#009900 guibg=" . guibg . " ctermfg=2 ctermbg=" . ctermbg
   execute "highlight GitGutterChangeDefault guifg=#bbbb00 guibg=" . guibg . " ctermfg=3 ctermbg=" . ctermbg
   execute "highlight GitGutterDeleteDefault guifg=#ff2222 guibg=" . guibg . " ctermfg=1 ctermbg=" . ctermbg
   highlight default link GitGutterChangeDeleteDefault GitGutterChangeDefault
 
+  execute "highlight GitGutterAddInvisible    guifg=bg" . " guibg=" . guibg . " ctermfg=" . ctermbg . " ctermbg=" . ctermbg
+  execute "highlight GitGutterChangeInvisible guifg=bg" . " guibg=" . guibg . " ctermfg=" . ctermbg . " ctermbg=" . ctermbg
+  execute "highlight GitGutterDeleteInvisible guifg=bg" . " guibg=" . guibg . " ctermfg=" . ctermbg . " ctermbg=" . ctermbg
+  highlight default link GitGutterChangeDeleteInvisible GitGutterChangeInvisble
+
   highlight default link GitGutterAdd          GitGutterAddDefault
   highlight default link GitGutterChange       GitGutterChangeDefault
   highlight default link GitGutterDelete       GitGutterDeleteDefault
   highlight default link GitGutterChangeDelete GitGutterChangeDeleteDefault
 
   " Highlights used for the whole line.
+
   highlight default link GitGutterAddLine          DiffAdd
   highlight default link GitGutterChangeLine       DiffChange
   highlight default link GitGutterDeleteLine       DiffDelete
@@ -43,6 +50,21 @@ function! highlight#define_sign_text()
 endfunction
 
 function! highlight#define_sign_text_highlights()
+  " Once a sign's text attribute has been defined, it cannot be undefined or
+  " set to an empty value.  So to make signs' text disappear we make it
+  " invisible.
+  if g:gitgutter_signs
+    highlight link GitGutterAdd          GitGutterAddDefault
+    highlight link GitGutterChange       GitGutterChangeDefault
+    highlight link GitGutterDelete       GitGutterDeleteDefault
+    highlight link GitGutterChangeDelete GitGutterChangeDeleteDefault
+  else
+    highlight link GitGutterAdd          GitGutterAddInvisible
+    highlight link GitGutterChange       GitGutterChangeInvisible
+    highlight link GitGutterDelete       GitGutterDeleteInvisible
+    highlight link GitGutterChangeDelete GitGutterChangeDeleteInvisible
+  endif
+
   sign define GitGutterLineAdded           texthl=GitGutterAdd
   sign define GitGutterLineModified        texthl=GitGutterChange
   sign define GitGutterLineRemoved         texthl=GitGutterDelete