diff --git a/lib/vagrant/bundler.rb b/lib/vagrant/bundler.rb
index dfe8a17c1..05867da15 100644
--- a/lib/vagrant/bundler.rb
+++ b/lib/vagrant/bundler.rb
@@ -1,4 +1,3 @@
-require "fileutils"
 require "monitor"
 require "pathname"
 require "set"
@@ -80,9 +79,9 @@ module Vagrant
 
     # Removes any temporary files created by init
     def deinit
-      %w{ BUNDLE_APP_CONFIG BUNDLE_CONFIG BUNDLE_GEMFILE }.each do |entry|
-        FileUtils.remove_entry_secure(ENV[entry], true)
-      end
+      File.unlink(ENV["BUNDLE_APP_CONFIG"]) rescue nil
+      File.unlink(ENV["BUNDLE_CONFIG"]) rescue nil
+      File.unlink(ENV["GEMFILE"]) rescue nil
     end
 
     # Installs the list of plugins.