diff --git a/plugins/communicators/ssh/communicator.rb b/plugins/communicators/ssh/communicator.rb
index 681ce7fde..0a9845436 100644
--- a/plugins/communicators/ssh/communicator.rb
+++ b/plugins/communicators/ssh/communicator.rb
@@ -52,7 +52,7 @@ module VagrantPlugins
         # If we're already attempting to switch out the SSH key, then
         # just return that we're ready (for Machine#guest).
         @lock.synchronize do
-          return true if @inserted_key
+          return true if @inserted_key || !@machine.config.ssh.insert_key
           @inserted_key = true
         end