From c20624bfdc71e1d9bbc85ca8bfbbbac0fd0bf2c1 Mon Sep 17 00:00:00 2001 From: mpoeter Date: Tue, 9 Sep 2014 19:17:04 +0200 Subject: [PATCH 1/7] Add support for linked clones for VirtualBox. --- lib/vagrant/errors.rb | 8 +++ plugins/providers/virtualbox/action.rb | 10 ++- .../virtualbox/action/create_clone.rb | 51 +++++++++++++++ .../virtualbox/action/import_master.rb | 62 +++++++++++++++++++ plugins/providers/virtualbox/config.rb | 10 +++ plugins/providers/virtualbox/driver/meta.rb | 2 + .../virtualbox/driver/version_4_3.rb | 25 ++++++-- templates/locales/en.yml | 8 +++ 8 files changed, 171 insertions(+), 5 deletions(-) create mode 100644 plugins/providers/virtualbox/action/create_clone.rb create mode 100644 plugins/providers/virtualbox/action/import_master.rb diff --git a/lib/vagrant/errors.rb b/lib/vagrant/errors.rb index 8995f6505..b4a609aa1 100644 --- a/lib/vagrant/errors.rb +++ b/lib/vagrant/errors.rb @@ -740,6 +740,14 @@ module Vagrant error_key(:boot_timeout) end + class VMCloneFailure < VagrantError + error_key(:failure, "vagrant.actions.vm.clone") + end + + class VMCreateMasterFailure < VagrantError + error_key(:failure, "vagrant.actions.vm.clone.create_master") + end + class VMCustomizationFailed < VagrantError error_key(:failure, "vagrant.actions.vm.customize") end diff --git a/plugins/providers/virtualbox/action.rb b/plugins/providers/virtualbox/action.rb index 668cbe0ef..b0051ce3d 100644 --- a/plugins/providers/virtualbox/action.rb +++ b/plugins/providers/virtualbox/action.rb @@ -12,6 +12,7 @@ module VagrantPlugins autoload :CleanMachineFolder, File.expand_path("../action/clean_machine_folder", __FILE__) autoload :ClearForwardedPorts, File.expand_path("../action/clear_forwarded_ports", __FILE__) autoload :ClearNetworkInterfaces, File.expand_path("../action/clear_network_interfaces", __FILE__) + autoload :CreateClone, File.expand_path("../action/create_clone", __FILE__) autoload :Created, File.expand_path("../action/created", __FILE__) autoload :Customize, File.expand_path("../action/customize", __FILE__) autoload :Destroy, File.expand_path("../action/destroy", __FILE__) @@ -21,6 +22,7 @@ module VagrantPlugins autoload :ForcedHalt, File.expand_path("../action/forced_halt", __FILE__) autoload :ForwardPorts, File.expand_path("../action/forward_ports", __FILE__) autoload :Import, File.expand_path("../action/import", __FILE__) + autoload :ImportMaster, File.expand_path("../action/import_master", __FILE__) autoload :IsPaused, File.expand_path("../action/is_paused", __FILE__) autoload :IsRunning, File.expand_path("../action/is_running", __FILE__) autoload :IsSaved, File.expand_path("../action/is_saved", __FILE__) @@ -313,7 +315,13 @@ module VagrantPlugins if !env[:result] b2.use CheckAccessible b2.use Customize, "pre-import" - b2.use Import + + if env[:machine].provider_config.use_linked_clone + b2.use ImportMaster + b2.use CreateClone + else + b2.use Import + end b2.use MatchMACAddress end end diff --git a/plugins/providers/virtualbox/action/create_clone.rb b/plugins/providers/virtualbox/action/create_clone.rb new file mode 100644 index 000000000..e26b5721f --- /dev/null +++ b/plugins/providers/virtualbox/action/create_clone.rb @@ -0,0 +1,51 @@ +require "log4r" +#require "lockfile" + +module VagrantPlugins + module ProviderVirtualBox + module Action + class CreateClone + def initialize(app, env) + @app = app + @logger = Log4r::Logger.new("vagrant::action::vm::clone") + end + + def call(env) + @logger.info("Creating linked clone from master '#{env[:master_id]}'") + + env[:ui].info I18n.t("vagrant.actions.vm.clone.creating", name: env[:machine].box.name) + env[:machine].id = env[:machine].provider.driver.clonevm(env[:master_id], env[:machine].box.name, "base") do |progress| + env[:ui].clear_line + env[:ui].report_progress(progress, 100, false) + end + + # Clear the line one last time since the progress meter doesn't disappear immediately. + env[:ui].clear_line + + # Flag as erroneous and return if clone failed + raise Vagrant::Errors::VMCloneFailure if !env[:machine].id + + # Continue + @app.call(env) + end + + def recover(env) + if env[:machine].state.id != :not_created + return if env["vagrant.error"].is_a?(Vagrant::Errors::VagrantError) + + # If we're not supposed to destroy on error then just return + return if !env[:destroy_on_error] + + # Interrupted, destroy the VM. We note that we don't want to + # validate the configuration here, and we don't want to confirm + # we want to destroy. + destroy_env = env.clone + destroy_env[:config_validate] = false + destroy_env[:force_confirm_destroy] = true + env[:action_runner].run(Action.action_destroy, destroy_env) + end + end + end + end + end +end diff --git a/plugins/providers/virtualbox/action/import_master.rb b/plugins/providers/virtualbox/action/import_master.rb new file mode 100644 index 000000000..090b769c4 --- /dev/null +++ b/plugins/providers/virtualbox/action/import_master.rb @@ -0,0 +1,62 @@ +require "log4r" +#require "lockfile" + +module VagrantPlugins + module ProviderVirtualBox + module Action + class ImportMaster + def initialize(app, env) + @app = app + @logger = Log4r::Logger.new("vagrant::action::vm::create_master") + end + + def call(env) + master_id_file = env[:machine].box.directory.join("master_id") + + env[:master_id] = master_id_file.read.chomp if master_id_file.file? + if env[:master_id] && env[:machine].provider.driver.vm_exists?(env[:master_id]) + # Master VM already exists -> nothing to do - continue. + @app.call(env) + end + + env[:ui].info I18n.t("vagrant.actions.vm.clone.importing", name: env[:machine].box.name) + + #TODO - prevent concurrent creation of master vms for the same box. + + # Import the virtual machine + ovf_file = env[:machine].box.directory.join("box.ovf").to_s + env[:master_id] = env[:machine].provider.driver.import(ovf_file) do |progress| + env[:ui].clear_line + env[:ui].report_progress(progress, 100, false) + end + + # Clear the line one last time since the progress meter doesn't disappear immediately. + env[:ui].clear_line + + # Flag as erroneous and return if import failed + raise Vagrant::Errors::VMImportFailure if !env[:master_id] + + @logger.info("Imported box #{env[:machine].box.name} as master vm with id #{env[:master_id]}") + + @logger.info("Creating base snapshot for master VM.") + env[:machine].provider.driver.create_snapshot(env[:master_id], "base")do |progress| + env[:ui].clear_line + env[:ui].report_progress(progress, 100, false) + end + + @logger.debug("Writing id of master VM '#{env[:master_id]}' to #{master_id_file}") + master_id_file.open("w+") do |f| + f.write(env[:master_id]) + end + + # If we got interrupted, then the import could have been + # interrupted and its not a big deal. Just return out. + return if env[:interrupted] + + # Import completed successfully. Continue the chain + @app.call(env) + end + end + end + end +end diff --git a/plugins/providers/virtualbox/config.rb b/plugins/providers/virtualbox/config.rb index aed1c692c..7a0e08d08 100644 --- a/plugins/providers/virtualbox/config.rb +++ b/plugins/providers/virtualbox/config.rb @@ -32,6 +32,12 @@ module VagrantPlugins # @return [Boolean] attr_accessor :gui + # If set to `true`, then a linked clone is created from a master + # VM generated from the specified box. + # + # @return [Boolean] + attr_accessor :use_linked_clone + # This should be set to the name of the machine in the VirtualBox # GUI. # @@ -59,6 +65,7 @@ module VagrantPlugins @name = UNSET_VALUE @network_adapters = {} @gui = UNSET_VALUE + @use_linked_clone = UNSET_VALUE # We require that network adapter 1 is a NAT device. network_adapter(1, :nat) @@ -136,6 +143,9 @@ module VagrantPlugins # Default is to not show a GUI @gui = false if @gui == UNSET_VALUE + # Do not create linked clone by default + @use_linked_clone = false if @use_linked_clone == UNSET_VALUE + # The default name is just nothing, and we default it @name = nil if @name == UNSET_VALUE end diff --git a/plugins/providers/virtualbox/driver/meta.rb b/plugins/providers/virtualbox/driver/meta.rb index 1da74d5d3..5f0700c32 100644 --- a/plugins/providers/virtualbox/driver/meta.rb +++ b/plugins/providers/virtualbox/driver/meta.rb @@ -79,8 +79,10 @@ module VagrantPlugins def_delegators :@driver, :clear_forwarded_ports, :clear_shared_folders, + :clonevm, :create_dhcp_server, :create_host_only_network, + :create_snapshot, :delete, :delete_unused_host_only_networks, :discard_saved_state, diff --git a/plugins/providers/virtualbox/driver/version_4_3.rb b/plugins/providers/virtualbox/driver/version_4_3.rb index cc700c0fa..76ccdcd6a 100644 --- a/plugins/providers/virtualbox/driver/version_4_3.rb +++ b/plugins/providers/virtualbox/driver/version_4_3.rb @@ -34,6 +34,15 @@ module VagrantPlugins end end + def clonevm(master_id, box_name, snapshot_name) + @logger.debug("Creating linked clone from master vm with id #{master_id} from snapshot '#{snapshot_name}'") + + machine_name = "#{box_name}_#{snapshot_name}_clone_#{(Time.now.to_f * 1000.0).to_i}_#{rand(100000)}" + execute("clonevm", master_id, "--snapshot", snapshot_name, "--options", "link", "--register", "--name", machine_name) + + return get_machine_id machine_name + end + def create_dhcp_server(network, options) execute("dhcpserver", "add", "--ifname", network, "--ip", options[:dhcp_ip], @@ -62,6 +71,10 @@ module VagrantPlugins } end + def create_snapshot(machine_id, snapshot_name) + execute("snapshot", machine_id, "take", snapshot_name) + end + def delete execute("unregistervm", @uuid, "--delete") end @@ -156,6 +169,13 @@ module VagrantPlugins execute("modifyvm", @uuid, *args) if !args.empty? end + def get_machine_id(machine_name) + output = execute("list", "vms", retryable: true) + match = /^"#{Regexp.escape(machine_name)}" \{(.+?)\}$/.match(output) + return match[1].to_s if match + nil + end + def halt execute("controlvm", @uuid, "poweroff") end @@ -231,10 +251,7 @@ module VagrantPlugins end end - output = execute("list", "vms", retryable: true) - match = /^"#{Regexp.escape(specified_name)}" \{(.+?)\}$/.match(output) - return match[1].to_s if match - nil + return get_machine_id specified_name end def max_network_adapters diff --git a/templates/locales/en.yml b/templates/locales/en.yml index 4be7a92e7..4cbf5a0a3 100644 --- a/templates/locales/en.yml +++ b/templates/locales/en.yml @@ -1482,6 +1482,14 @@ en: deleting: Clearing any previously set network interfaces... clear_shared_folders: deleting: Cleaning previously set shared folders... + clone: + importing: Importing box '%{name}' as master vm... + creating: Creating linked clone... + failure: Creation of the linked clone failed. + + create_master: + failure: |- + Failed to create lock-file for master VM creation for box %{box}. customize: failure: |- A customization command failed: From c3151c0b8829582873dfb355f0ca5a2edcb75208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=B6ter?= Date: Wed, 3 Jun 2015 13:27:03 +0200 Subject: [PATCH 2/7] Ignore failure when trying to delete lock file. Deleting the lock file can fail when another process is currently trying to acquire it (-> race condition). It is safe to ignore this error since the other process will eventually acquire the lock and again try to delete the lock file. --- lib/vagrant/environment.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/vagrant/environment.rb b/lib/vagrant/environment.rb index 571ae75ab..455921485 100644 --- a/lib/vagrant/environment.rb +++ b/lib/vagrant/environment.rb @@ -484,7 +484,11 @@ module Vagrant if name != "dotlock" lock("dotlock", retry: true) do f.close - File.delete(lock_path) + begin + File.delete(lock_path) + rescue + @logger.debug("Failed to delete lock file #{lock_path} - some other thread might be trying to acquire it -> ignoring this error") + end end end From 9d63ca4dd237947756ea59b447013e78313f4c3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=B6ter?= Date: Wed, 3 Jun 2015 13:31:43 +0200 Subject: [PATCH 3/7] Acquire lock to prevent concurrent creation of master VM for the same box. --- .../virtualbox/action/import_master.rb | 68 ++++++++++--------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/plugins/providers/virtualbox/action/import_master.rb b/plugins/providers/virtualbox/action/import_master.rb index 090b769c4..cba6216fd 100644 --- a/plugins/providers/virtualbox/action/import_master.rb +++ b/plugins/providers/virtualbox/action/import_master.rb @@ -12,46 +12,50 @@ module VagrantPlugins def call(env) master_id_file = env[:machine].box.directory.join("master_id") - - env[:master_id] = master_id_file.read.chomp if master_id_file.file? - if env[:master_id] && env[:machine].provider.driver.vm_exists?(env[:master_id]) - # Master VM already exists -> nothing to do - continue. - @app.call(env) - end - env[:ui].info I18n.t("vagrant.actions.vm.clone.importing", name: env[:machine].box.name) + env[:machine].env.lock(env[:machine].box.name, retry: true) do + env[:master_id] = master_id_file.read.chomp if master_id_file.file? + if env[:master_id] && env[:machine].provider.driver.vm_exists?(env[:master_id]) + # Master VM already exists -> nothing to do - continue. + @logger.info("Master VM for '#{env[:machine].box.name}' already exists (id=#{env[:master_id]}) - skipping import step.") + return @app.call(env) + end - #TODO - prevent concurrent creation of master vms for the same box. - - # Import the virtual machine - ovf_file = env[:machine].box.directory.join("box.ovf").to_s - env[:master_id] = env[:machine].provider.driver.import(ovf_file) do |progress| + env[:ui].info I18n.t("vagrant.actions.vm.clone.importing", name: env[:machine].box.name) + + # Import the virtual machine + ovf_file = env[:machine].box.directory.join("box.ovf").to_s + env[:master_id] = env[:machine].provider.driver.import(ovf_file) do |progress| + env[:ui].clear_line + env[:ui].report_progress(progress, 100, false) + end + + # Clear the line one last time since the progress meter doesn't disappear immediately. env[:ui].clear_line - env[:ui].report_progress(progress, 100, false) - end - - # Clear the line one last time since the progress meter doesn't disappear immediately. - env[:ui].clear_line + + # Flag as erroneous and return if import failed + raise Vagrant::Errors::VMImportFailure if !env[:master_id] - # Flag as erroneous and return if import failed - raise Vagrant::Errors::VMImportFailure if !env[:master_id] - - @logger.info("Imported box #{env[:machine].box.name} as master vm with id #{env[:master_id]}") - - @logger.info("Creating base snapshot for master VM.") - env[:machine].provider.driver.create_snapshot(env[:master_id], "base")do |progress| - env[:ui].clear_line - env[:ui].report_progress(progress, 100, false) - end + @logger.info("Imported box #{env[:machine].box.name} as master vm with id #{env[:master_id]}") - @logger.debug("Writing id of master VM '#{env[:master_id]}' to #{master_id_file}") - master_id_file.open("w+") do |f| - f.write(env[:master_id]) - end + @logger.info("Creating base snapshot for master VM.") + env[:machine].provider.driver.create_snapshot(env[:master_id], "base") do |progress| + env[:ui].clear_line + env[:ui].report_progress(progress, 100, false) + end + + @logger.debug("Writing id of master VM '#{env[:master_id]}' to #{master_id_file}") + master_id_file.open("w+") do |f| + f.write(env[:master_id]) + end + end # If we got interrupted, then the import could have been # interrupted and its not a big deal. Just return out. - return if env[:interrupted] + if env[:interrupted] + @logger.info("Import of master VM was interrupted -> exiting.") + return + end # Import completed successfully. Continue the chain @app.call(env) From 2717f5605ae4372c93a3eb768242a4355c37f1de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=B6ter?= Date: Mon, 8 Jun 2015 18:03:03 +0200 Subject: [PATCH 4/7] Document use_linked_clone property for VirtualBox provider. --- .../v2/virtualbox/configuration.html.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/website/docs/source/v2/virtualbox/configuration.html.md b/website/docs/source/v2/virtualbox/configuration.html.md index 22cb2a6c5..c9ee51540 100644 --- a/website/docs/source/v2/virtualbox/configuration.html.md +++ b/website/docs/source/v2/virtualbox/configuration.html.md @@ -36,6 +36,30 @@ config.vm.provider "virtualbox" do |v| end ``` +## Linked Clones + +By default new machines are created by importing the base box. For large +boxes this produces a large overhead in terms of time (the import operation) +and space (the new machine contains a copy of the base box's image). +Using linked clones can drastically reduce this overhead. + +Linked clones are based on a master VM, which is generated by importing the +base box only once the first time it is required. For the linked clones only +differencing disk images are created where the parent disk image belongs to +the master VM. + +```ruby +config.vm.provider "virtualbox" do |v| + v.use_linked_clone = true +end +``` + +
+ Note: the generated master VMs are currently not removed + automatically by Vagrant. This has to be done manually. However, a master + VM can only be remove when there are no linked clones connected to it. +
+ ## VBoxManage Customizations [VBoxManage](http://www.virtualbox.org/manual/ch08.html) is a utility that can From dbcb513075a6117b867f4d2933f4715cbbd187aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=B6ter?= Date: Thu, 16 Jul 2015 13:23:57 +0200 Subject: [PATCH 5/7] Fix typo. --- website/docs/source/v2/virtualbox/configuration.html.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/source/v2/virtualbox/configuration.html.md b/website/docs/source/v2/virtualbox/configuration.html.md index c9ee51540..5c908ab68 100644 --- a/website/docs/source/v2/virtualbox/configuration.html.md +++ b/website/docs/source/v2/virtualbox/configuration.html.md @@ -57,7 +57,7 @@ end
Note: the generated master VMs are currently not removed automatically by Vagrant. This has to be done manually. However, a master - VM can only be remove when there are no linked clones connected to it. + VM can only be removed when there are no linked clones connected to it.
## VBoxManage Customizations From 772f276ee3a1cb6ea62b851bb41f45b49a168587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=B6ter?= Date: Thu, 16 Jul 2015 13:27:24 +0200 Subject: [PATCH 6/7] Port support for linked clones to VirtualBox 5.0 driver. --- .../virtualbox/driver/version_5_0.rb | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/plugins/providers/virtualbox/driver/version_5_0.rb b/plugins/providers/virtualbox/driver/version_5_0.rb index d7d3b58df..e2aeb35a1 100644 --- a/plugins/providers/virtualbox/driver/version_5_0.rb +++ b/plugins/providers/virtualbox/driver/version_5_0.rb @@ -34,6 +34,15 @@ module VagrantPlugins end end + def clonevm(master_id, box_name, snapshot_name) + @logger.debug("Creating linked clone from master vm with id #{master_id} from snapshot '#{snapshot_name}'") + + machine_name = "#{box_name}_#{snapshot_name}_clone_#{(Time.now.to_f * 1000.0).to_i}_#{rand(100000)}" + execute("clonevm", master_id, "--snapshot", snapshot_name, "--options", "link", "--register", "--name", machine_name) + + return get_machine_id machine_name + end + def create_dhcp_server(network, options) execute("dhcpserver", "add", "--ifname", network, "--ip", options[:dhcp_ip], @@ -62,6 +71,10 @@ module VagrantPlugins } end + def create_snapshot(machine_id, snapshot_name) + execute("snapshot", machine_id, "take", snapshot_name) + end + def delete execute("unregistervm", @uuid, "--delete") end @@ -156,6 +169,13 @@ module VagrantPlugins execute("modifyvm", @uuid, *args) if !args.empty? end + def get_machine_id(machine_name) + output = execute("list", "vms", retryable: true) + match = /^"#{Regexp.escape(machine_name)}" \{(.+?)\}$/.match(output) + return match[1].to_s if match + nil + end + def halt execute("controlvm", @uuid, "poweroff") end @@ -231,10 +251,7 @@ module VagrantPlugins end end - output = execute("list", "vms", retryable: true) - match = /^"#{Regexp.escape(specified_name)}" \{(.+?)\}$/.match(output) - return match[1].to_s if match - nil + return get_machine_id specified_name end def max_network_adapters From 2a2f0a4751a7ea40f7db9941904291746d9729ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=B6ter?= Date: Wed, 12 Aug 2015 14:25:54 +0200 Subject: [PATCH 7/7] Use hash of machine name for lock file to avoid problems with invalid characters for file names. --- plugins/providers/virtualbox/action/import_master.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/providers/virtualbox/action/import_master.rb b/plugins/providers/virtualbox/action/import_master.rb index cba6216fd..61b0064eb 100644 --- a/plugins/providers/virtualbox/action/import_master.rb +++ b/plugins/providers/virtualbox/action/import_master.rb @@ -13,7 +13,7 @@ module VagrantPlugins def call(env) master_id_file = env[:machine].box.directory.join("master_id") - env[:machine].env.lock(env[:machine].box.name, retry: true) do + env[:machine].env.lock(Digest::MD5.hexdigest(env[:machine].box.name), retry: true) do env[:master_id] = master_id_file.read.chomp if master_id_file.file? if env[:master_id] && env[:machine].provider.driver.vm_exists?(env[:master_id]) # Master VM already exists -> nothing to do - continue.