diff --git a/bin/pyenv-virtualenv b/bin/pyenv-virtualenv
index 84cf37d..a250658 100755
--- a/bin/pyenv-virtualenv
+++ b/bin/pyenv-virtualenv
@@ -537,10 +537,11 @@ for hook in "${before_hooks[@]}"; do eval "$hook"; done
 
 # Plan cleanup on unsuccessful installation.
 cleanup() {
+  [[ -L "${COMPAT_VIRTUALENV_PATH}" ]] && rm "${COMPAT_VIRTUALENV_PATH}"
   [ -z "${PREFIX_EXISTS}" ] && rm -rf "$VIRTUALENV_PATH"
 }
 
-trap cleanup SIGINT
+trap cleanup SIGINT ERR
 
 # Invoke virtualenv and record exit status in $STATUS.
 STATUS=0