diff --git a/autoload/sign.vim b/autoload/sign.vim
index 0d5833c..acf8827 100644
--- a/autoload/sign.vim
+++ b/autoload/sign.vim
@@ -8,7 +8,7 @@ let s:next_sign_id  = s:first_sign_id
 let s:dummy_sign_id = s:first_sign_id - 1
 
 
-" Removes gitgutter's signs from the given file.
+" Removes gitgutter's signs (excluding dummy sign) from the given file.
 function! sign#clear_signs(file_name)
   call sign#find_current_signs(a:file_name)
   for sign in values(getbufvar(a:file_name, 'gitgutter_gitgutter_signs'))
@@ -26,19 +26,33 @@ function! sign#update_signs(file_name, modified_lines)
   call sign#find_current_signs(a:file_name)
 
   let new_gitgutter_signs_line_numbers = map(copy(a:modified_lines), 'v:val[0]')
-  call sign#remove_obsolete_gitgutter_signs(a:file_name, new_gitgutter_signs_line_numbers)
+  let obsolete_signs = sign#obsolete_gitgutter_signs_to_remove(a:file_name, new_gitgutter_signs_line_numbers)
 
+  let flicker_possible = s:remove_all_old_signs && !empty(a:modified_lines)
+  if flicker_possible
+    call sign#add_dummy_sign()
+  endif
+
+  call sign#remove_signs(obsolete_signs)
   call sign#upsert_new_gitgutter_signs(a:file_name, a:modified_lines)
+
+  if flicker_possible
+    call sign#remove_dummy_sign(0)
+  endif
 endfunction
 
 
 function! sign#add_dummy_sign()
-  execute "sign place" s:dummy_sign_id "line=" . 9999 "name=GitGutterDummy file=" . utility#file()
+  if !getbufvar(utility#file(), 'gitgutter_dummy_sign')
+    execute "sign place" s:dummy_sign_id "line=" . 9999 "name=GitGutterDummy file=" . utility#file()
+    call setbufvar(utility#file(), 'gitgutter_dummy_sign', 1)
+  endif
 endfunction
 
-function! sign#remove_dummy_sign()
-  if exists('s:dummy_sign_id')
+function! sign#remove_dummy_sign(force)
+  if getbufvar(utility#file(), 'gitgutter_dummy_sign') && (a:force || !g:gitgutter_sign_column_always)
     execute "sign unplace" s:dummy_sign_id "file=" . utility#file()
+    call setbufvar(utility#file(), 'gitgutter_dummy_sign', 0)
   endif
 endfunction
 
@@ -51,6 +65,7 @@ endfunction
 function! sign#find_current_signs(file_name)
   let gitgutter_signs = {}  " <line_number>: {'id': <id>, 'name': <name>}
   let other_signs = []
+  let dummy_sign_placed = 0
 
   redir => signs
     silent execute "sign place file=" . a:file_name
@@ -61,7 +76,9 @@ function! sign#find_current_signs(file_name)
     " We assume splitting is faster than a regexp.
     let components  = split(sign_line)
     let name        = split(components[2], '=')[1]
-    if name !~# 'GitGutterDummy'
+    if name =~# 'GitGutterDummy'
+      let dummy_sign_placed = 1
+    else
       let line_number = str2nr(split(components[0], '=')[1])
       if name =~# 'GitGutter'
         let id = str2nr(split(components[1], '=')[1])
@@ -72,18 +89,31 @@ function! sign#find_current_signs(file_name)
     end
   endfor
 
+  call setbufvar(a:file_name, 'gitgutter_dummy_sign', dummy_sign_placed)
   call setbufvar(a:file_name, 'gitgutter_gitgutter_signs', gitgutter_signs)
   call setbufvar(a:file_name, 'gitgutter_other_signs', other_signs)
 endfunction
 
 
-function! sign#remove_obsolete_gitgutter_signs(file_name, new_gitgutter_signs_line_numbers)
+function! sign#obsolete_gitgutter_signs_to_remove(file_name, new_gitgutter_signs_line_numbers)
+  let signs_to_remove = []
+  let remove_all_signs = 1
   let old_gitgutter_signs = getbufvar(a:file_name, 'gitgutter_gitgutter_signs')
   for line_number in keys(old_gitgutter_signs)
     if index(a:new_gitgutter_signs_line_numbers, str2nr(line_number)) == -1
-      execute "sign unplace" old_gitgutter_signs[line_number].id
+      call add(signs_to_remove, old_gitgutter_signs[line_number].id)
+    else
+      let remove_all_signs = 0
     endif
   endfor
+  let s:remove_all_old_signs = remove_all_signs
+  return signs_to_remove
+endfunction
+
+function! sign#remove_signs(sign_ids)
+  for id in a:sign_ids
+    execute "sign unplace" id
+  endfor
 endfunction
 
 
diff --git a/autoload/utility.vim b/autoload/utility.vim
index 5ce00f4..8a67b4a 100644
--- a/autoload/utility.vim
+++ b/autoload/utility.vim
@@ -64,10 +64,6 @@ function! utility#command_in_directory_of_file(cmd)
   return substitute(utility#cmd_in_dir, "'", '"', 'g')
 endfunction
 
-function! utility#differences(hunks)
-  return len(a:hunks) != 0
-endfunction
-
 function! utility#highlight_name_for_change(text)
   if a:text ==# 'added'
     return 'GitGutterLineAdded'
diff --git a/plugin/gitgutter.vim b/plugin/gitgutter.vim
index 050b44f..f54f2af 100644
--- a/plugin/gitgutter.vim
+++ b/plugin/gitgutter.vim
@@ -55,6 +55,9 @@ command GitGutterAll call GitGutterAll()
 function! GitGutter(file, realtime)
   call utility#set_file(a:file)
   if utility#is_active()
+    if g:gitgutter_sign_column_always
+      call sign#add_dummy_sign()
+    endif
     try
       if !a:realtime || utility#has_fresh_changes(a:file)
         let diff           = diff#run_diff(a:realtime || utility#has_unsaved_changes(a:file), 1)
@@ -62,15 +65,6 @@ function! GitGutter(file, realtime)
         let modified_lines = diff#process_hunks(s:hunks)
 
         if g:gitgutter_signs
-          if g:gitgutter_sign_column_always
-            call sign#add_dummy_sign()
-          else
-            if utility#differences(s:hunks)
-              call sign#add_dummy_sign()  " prevent flicker
-            else
-              call sign#remove_dummy_sign()
-            endif
-          endif
           call sign#update_signs(a:file, modified_lines)
         endif
 
@@ -93,7 +87,7 @@ command GitGutter call GitGutter(utility#current_file(), 0)
 function! GitGutterDisable()
   let g:gitgutter_enabled = 0
   call sign#clear_signs(utility#file())
-  call sign#remove_dummy_sign()
+  call sign#remove_dummy_sign(1)
   call hunk#reset()
 endfunction
 command GitGutterDisable call GitGutterDisable()
@@ -153,7 +147,7 @@ command GitGutterSignsEnable call GitGutterSignsEnable()
 function! GitGutterSignsDisable()
   let g:gitgutter_signs = 0
   call sign#clear_signs(utility#file())
-  call sign#remove_dummy_sign()
+  call sign#remove_dummy_sign(0)
 endfunction
 command GitGutterSignsDisable call GitGutterSignsDisable()