diff --git a/CHANGELOG.md b/CHANGELOG.md
index edc1bc78b..73ca82841 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,8 @@ IMPROVEMENTS:
 BUG FIXES:
 
   - core: Fix cases where sometimes SSH connection would hang.
+  - core: On a graceful halt, force halt if capability "insert public key"
+      is missing. [GH-4684]
   - commands/box: `--cert` flag works properly. [GH-4691]
   - command/docker-logs: Won't crash if container is removed. [GH-3990]
   - command/docker-run: Synced folders will be attached properly. [GH-3873]
diff --git a/lib/vagrant/action/builtin/graceful_halt.rb b/lib/vagrant/action/builtin/graceful_halt.rb
index 393c81398..5ccd8389d 100644
--- a/lib/vagrant/action/builtin/graceful_halt.rb
+++ b/lib/vagrant/action/builtin/graceful_halt.rb
@@ -59,6 +59,9 @@ module Vagrant
               rescue Timeout::Error
                 # Don't worry about it, we catch the case later.
               end
+            rescue Errors::GuestCapabilityNotFound
+              # This happens if insert_public_key is called on a guest that
+              # doesn't support it. This will block a destroy so we let it go.
             rescue Errors::MachineGuestNotReady
               env[:ui].detail(I18n.t("vagrant.actions.vm.halt.guest_not_ready"))
             end