8468 - make more virtualbox commands retryable

Issue: https://github.com/mitchellh/vagrant/issues/8468

A lot of vboxmanage commands are flakey and frequently cause
bringing multiple machines up at once to fail, especially when
the host system is under heavy load. Most commands are also safe
to retry and just result in a no-op, so we can simply add
'retryable' to a lot of existing calls. For the others we need to
do a little bit of cleanup or reevaluate the parameters before
trying again.
This commit is contained in:
Samuel Clark 2017-04-25 12:30:08 -07:00
parent 5fa23c42dd
commit 7d2f7dab97
1 changed files with 115 additions and 70 deletions

View File

@ -17,19 +17,23 @@ module VagrantPlugins
end end
def clear_forwarded_ports def clear_forwarded_ports
args = [] retryable(on: Vagrant::Errors::VBoxManageError, tries: 3, sleep: 1) do
read_forwarded_ports(@uuid).each do |nic, name, _, _| args = []
args.concat(["--natpf#{nic}", "delete", name]) read_forwarded_ports(@uuid).each do |nic, name, _, _|
end args.concat(["--natpf#{nic}", "delete", name])
end
execute("modifyvm", @uuid, *args) if !args.empty? execute("modifyvm", @uuid, *args) if !args.empty?
end
end end
def clear_shared_folders def clear_shared_folders
info = execute("showvminfo", @uuid, "--machinereadable", retryable: true) retryable(on: Vagrant::Errors::VBoxManageError, tries: 3, sleep: 1) do
info.split("\n").each do |line| info = execute("showvminfo", @uuid, "--machinereadable", retryable: true)
if line =~ /^SharedFolderNameMachineMapping\d+="(.+?)"$/ info.split("\n").each do |line|
execute("sharedfolder", "remove", @uuid, "--name", $1.to_s) if line =~ /^SharedFolderNameMachineMapping\d+="(.+?)"$/
execute("sharedfolder", "remove", @uuid, "--name", $1.to_s)
end
end end
end end
end end
@ -41,22 +45,29 @@ module VagrantPlugins
args += ["--snapshot", snapshot_name, "--options", "link"] args += ["--snapshot", snapshot_name, "--options", "link"]
end end
execute("clonevm", master_id, *args) execute("clonevm", master_id, *args, retryable: true)
return get_machine_id(machine_name) return get_machine_id(machine_name)
end end
def create_dhcp_server(network, options) def create_dhcp_server(network, options)
execute("dhcpserver", "add", "--ifname", network, retryable(on: Vagrant::Errors::VBoxManageError, tries: 3, sleep: 1) do
"--ip", options[:dhcp_ip], begin
"--netmask", options[:netmask], execute("dhcpserver", "add", "--ifname", network,
"--lowerip", options[:dhcp_lower], "--ip", options[:dhcp_ip],
"--upperip", options[:dhcp_upper], "--netmask", options[:netmask],
"--enable") "--lowerip", options[:dhcp_lower],
"--upperip", options[:dhcp_upper],
"--enable")
rescue Vagrant::Errors::VBoxManageError => e
return if e.extra_data[:stderr] == 'VBoxManage: error: DHCP server already exists'
raise
end
end
end end
def create_host_only_network(options) def create_host_only_network(options)
# Create the interface # Create the interface
execute("hostonlyif", "create") =~ /^Interface '(.+?)' was successfully created$/ execute("hostonlyif", "create", retryable: true) =~ /^Interface '(.+?)' was successfully created$/
name = $1.to_s name = $1.to_s
# Get the IP so we can determine v4 vs v6 # Get the IP so we can determine v4 vs v6
@ -66,11 +77,13 @@ module VagrantPlugins
if ip.ipv4? if ip.ipv4?
execute("hostonlyif", "ipconfig", name, execute("hostonlyif", "ipconfig", name,
"--ip", options[:adapter_ip], "--ip", options[:adapter_ip],
"--netmask", options[:netmask]) "--netmask", options[:netmask],
retryable: true)
elsif ip.ipv6? elsif ip.ipv6?
execute("hostonlyif", "ipconfig", name, execute("hostonlyif", "ipconfig", name,
"--ipv6", options[:adapter_ip], "--ipv6", options[:adapter_ip],
"--netmasklengthv6", options[:netmask].to_s) "--netmasklengthv6", options[:netmask].to_s,
retryable: true)
else else
raise "BUG: Unknown IP type: #{ip.inspect}" raise "BUG: Unknown IP type: #{ip.inspect}"
end end
@ -85,7 +98,26 @@ module VagrantPlugins
end end
def create_snapshot(machine_id, snapshot_name) def create_snapshot(machine_id, snapshot_name)
execute("snapshot", machine_id, "take", snapshot_name) execute("snapshot", machine_id, "take", snapshot_name, retryable: true) do |type, data|
if type == :stderr
# Append the data so we can see the full view
total << data.gsub("\r", "")
# Break up the lines. We can't get the progress until we see an "OK"
lines = total.split("\n")
# The progress of the import will be in the last line. Do a greedy
# regular expression to find what we're looking for.
match = /.+(\d{2})%/.match(lines.last)
if match
current = match[1].to_i
if current > last
last = current
yield current if block_given?
end
end
end
end
end end
def delete_snapshot(machine_id, snapshot_name) def delete_snapshot(machine_id, snapshot_name)
@ -95,7 +127,7 @@ module VagrantPlugins
yield 0 if block_given? yield 0 if block_given?
# Snapshot and report the % progress # Snapshot and report the % progress
execute("snapshot", machine_id, "delete", snapshot_name) do |type, data| execute("snapshot", machine_id, "delete", snapshot_name, retryable: true) do |type, data|
if type == :stderr if type == :stderr
# Append the data so we can see the full view # Append the data so we can see the full view
total << data.gsub("\r", "") total << data.gsub("\r", "")
@ -142,7 +174,7 @@ module VagrantPlugins
total = "" total = ""
yield 0 if block_given? yield 0 if block_given?
execute("snapshot", machine_id, "restore", snapshot_name) do |type, data| execute("snapshot", machine_id, "restore", snapshot_name, retryable: true) do |type, data|
if type == :stderr if type == :stderr
# Append the data so we can see the full view # Append the data so we can see the full view
total << data.gsub("\r", "") total << data.gsub("\r", "")
@ -165,7 +197,7 @@ module VagrantPlugins
end end
def delete def delete
execute("unregistervm", @uuid, "--delete") execute("unregistervm", @uuid, "--delete", retryable: true)
end end
def delete_unused_host_only_networks def delete_unused_host_only_networks
@ -192,12 +224,12 @@ module VagrantPlugins
raw("dhcpserver", "remove", "--ifname", name) raw("dhcpserver", "remove", "--ifname", name)
# Delete the actual host only network interface. # Delete the actual host only network interface.
execute("hostonlyif", "remove", name) execute("hostonlyif", "remove", name, retryable: true)
end end
end end
def discard_saved_state def discard_saved_state
execute("discardstate", @uuid) execute("discardstate", @uuid, retryable: true)
end end
def enable_adapters(adapters) def enable_adapters(adapters)
@ -230,7 +262,7 @@ module VagrantPlugins
end end
end end
execute("modifyvm", @uuid, *args) execute("modifyvm", @uuid, *args, retryable: true)
end end
def execute_command(command) def execute_command(command)
@ -238,11 +270,20 @@ module VagrantPlugins
end end
def export(path) def export(path)
execute("export", @uuid, "--output", path.to_s) retryable(on: Vagrant::Errors::VBoxManageError, tries: 3, sleep: 1) do
begin
execute("export", @uuid, "--output", path.to_s)
rescue Vagrant::Errors::VBoxManageError => e
raise if !e.extra_data[:stderr].include?("VERR_E_FILE_ERROR")
# If the file already exists we'll throw a custom error
raise Vagrant::Errors::VirtualBoxFileExists,
stderr: e.extra_data[:stderr]
end
end
end end
def forward_ports(ports) def forward_ports(ports)
args = []
ports.each do |options| ports.each do |options|
pf_builder = [options[:name], pf_builder = [options[:name],
options[:protocol] || "tcp", options[:protocol] || "tcp",
@ -251,11 +292,11 @@ module VagrantPlugins
options[:guestip] || "", options[:guestip] || "",
options[:guestport]] options[:guestport]]
args.concat(["--natpf#{options[:adapter] || 1}", args = ["--natpf#{options[:adapter] || 1}",
pf_builder.join(",")]) pf_builder.join(",")]
end
execute("modifyvm", @uuid, *args) if !args.empty? execute("modifyvm", @uuid, *args, retryable: true)
end
end end
def get_machine_id(machine_name) def get_machine_id(machine_name)
@ -266,7 +307,7 @@ module VagrantPlugins
end end
def halt def halt
execute("controlvm", @uuid, "poweroff") execute("controlvm", @uuid, "poweroff", retryable: true)
end end
def import(ovf) def import(ovf)
@ -315,7 +356,7 @@ module VagrantPlugins
end end
end end
execute("import", ovf , *name_params, *disk_params) do |type, data| execute("import", ovf , *name_params, *disk_params, retryable: true) do |type, data|
if type == :stdout if type == :stdout
# Keep track of the stdout so that we can get the VM name # Keep track of the stdout so that we can get the VM name
output << data output << data
@ -590,26 +631,30 @@ module VagrantPlugins
def reconfig_host_only(interface) def reconfig_host_only(interface)
execute("hostonlyif", "ipconfig", interface[:name], execute("hostonlyif", "ipconfig", interface[:name],
"--ipv6", interface[:ipv6]) "--ipv6", interface[:ipv6], retryable: true)
end end
def remove_dhcp_server(network_name) def remove_dhcp_server(network_name)
execute("dhcpserver", "remove", "--netname", network_name) execute("dhcpserver", "remove", "--netname", network_name, retryable: true)
end end
def set_mac_address(mac) def set_mac_address(mac)
execute("modifyvm", @uuid, "--macaddress1", mac) execute("modifyvm", @uuid, "--macaddress1", mac, retryable: true)
end end
def set_name(name) def set_name(name)
execute("modifyvm", @uuid, "--name", name, retryable: true) retryable(on: Vagrant::Errors::VBoxManageError, tries: 3, sleep: 1) do
rescue Vagrant::Errors::VBoxManageError => e begin
raise if !e.extra_data[:stderr].include?("VERR_ALREADY_EXISTS") execute("modifyvm", @uuid, "--name", name)
rescue Vagrant::Errors::VBoxManageError => e
raise if !e.extra_data[:stderr].include?("VERR_ALREADY_EXISTS")
# We got VERR_ALREADY_EXISTS. This means that we're renaming to # We got VERR_ALREADY_EXISTS. This means that we're renaming to
# a VM name that already exists. Raise a custom error. # a VM name that already exists. Raise a custom error.
raise Vagrant::Errors::VirtualBoxNameExists, raise Vagrant::Errors::VirtualBoxNameExists,
stderr: e.extra_data[:stderr] stderr: e.extra_data[:stderr]
end
end
end end
def share_folders(folders) def share_folders(folders)
@ -625,10 +670,10 @@ module VagrantPlugins
args << "--transient" if folder.key?(:transient) && folder[:transient] args << "--transient" if folder.key?(:transient) && folder[:transient]
# Enable symlinks on the shared folder # Enable symlinks on the shared folder
execute("setextradata", @uuid, "VBoxInternal2/SharedFoldersEnableSymlinksCreate/#{folder[:name]}", "1") execute("setextradata", @uuid, "VBoxInternal2/SharedFoldersEnableSymlinksCreate/#{folder[:name]}", "1", retryable: true)
# Add the shared folder # Add the shared folder
execute("sharedfolder", "add", @uuid, *args) execute("sharedfolder", "add", @uuid, *args, retryable: true)
end end
end end
@ -650,40 +695,40 @@ module VagrantPlugins
def start(mode) def start(mode)
command = ["startvm", @uuid, "--type", mode.to_s] command = ["startvm", @uuid, "--type", mode.to_s]
r = raw(*command) retryable(on: Vagrant::Errors::VBoxManageError, tries: 3, sleep: 1) do
r = raw(*command)
if r.exit_code == 0 || r.stdout =~ /VM ".+?" has been successfully started/ if r.exit_code == 0 || r.stdout =~ /VM ".+?" has been successfully started/
# Some systems return an exit code 1 for some reason. For that # Some systems return an exit code 1 for some reason. For that
# we depend on the output. # we depend on the output.
return true return true
end
# If we reached this point then it didn't work out.
raise Vagrant::Errors::VBoxManageError,
command: command.inspect,
stderr: r.stderr
end end
# If we reached this point then it didn't work out.
raise Vagrant::Errors::VBoxManageError,
command: command.inspect,
stderr: r.stderr
end end
def suspend def suspend
execute("controlvm", @uuid, "savestate") execute("controlvm", @uuid, "savestate", retryable: true)
end end
def unshare_folders(names) def unshare_folders(names)
names.each do |name| names.each do |name|
begin retryable(on: Vagrant::Errors::VBoxManageError, tries: 3, sleep: 1) do
execute( begin
"sharedfolder", "remove", @uuid, execute(
"--name", name, "sharedfolder", "remove", @uuid,
"--transient") "--name", name,
"--transient")
execute( execute(
"setextradata", @uuid, "setextradata", @uuid,
"VBoxInternal2/SharedFoldersEnableSymlinksCreate/#{name}") "VBoxInternal2/SharedFoldersEnableSymlinksCreate/#{name}")
rescue Vagrant::Errors::VBoxManageError => e rescue Vagrant::Errors::VBoxManageError => e
if e.extra_data[:stderr].include?("VBOX_E_FILE_ERROR") raise if !e.extra_data[:stderr].include?("VBOX_E_FILE_ERROR")
# The folder doesn't exist. ignore.
else
raise
end end
end end
end end