From deba93ce5c6f1c31fe17a432baa0886604a09ee0 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Tue, 9 May 2017 18:54:15 -0700 Subject: [PATCH 01/14] Add optional support for docker-compose Adds configuration switch to enable using docker-compose to create and manage docker containers. --- plugins/providers/docker/config.rb | 4 + plugins/providers/docker/driver.rb | 3 +- plugins/providers/docker/driver/compose.rb | 166 +++++++++++++++++++++ plugins/providers/docker/provider.rb | 12 +- 4 files changed, 179 insertions(+), 6 deletions(-) create mode 100644 plugins/providers/docker/driver/compose.rb diff --git a/plugins/providers/docker/config.rb b/plugins/providers/docker/config.rb index f47a906be..e9cae75b3 100644 --- a/plugins/providers/docker/config.rb +++ b/plugins/providers/docker/config.rb @@ -19,6 +19,8 @@ module VagrantPlugins # @return [String] attr_accessor :build_dir + attr_accessor :compose + # An optional file name of a Dockerfile to be used when building # the image. This requires Docker >1.5.0. # @@ -138,6 +140,7 @@ module VagrantPlugins @build_args = [] @build_dir = UNSET_VALUE @cmd = UNSET_VALUE + @compose = UNSET_VALUE @create_args = UNSET_VALUE @dockerfile = UNSET_VALUE @env = {} @@ -201,6 +204,7 @@ module VagrantPlugins @build_args = [] if @build_args == UNSET_VALUE @build_dir = nil if @build_dir == UNSET_VALUE @cmd = [] if @cmd == UNSET_VALUE + @compose = false if @compose == UNSET_VALUE @create_args = [] if @create_args == UNSET_VALUE @dockerfile = nil if @dockerfile == UNSET_VALUE @env ||= {} diff --git a/plugins/providers/docker/driver.rb b/plugins/providers/docker/driver.rb index c845ed2a9..6e7a136ad 100644 --- a/plugins/providers/docker/driver.rb +++ b/plugins/providers/docker/driver.rb @@ -1,7 +1,8 @@ require "json" - require "log4r" +require_relative "./driver/compose" + module VagrantPlugins module DockerProvider class Driver diff --git a/plugins/providers/docker/driver/compose.rb b/plugins/providers/docker/driver/compose.rb new file mode 100644 index 000000000..6cfd24cb2 --- /dev/null +++ b/plugins/providers/docker/driver/compose.rb @@ -0,0 +1,166 @@ +require "json" +require "log4r" + +module VagrantPlugins + module DockerProvider + class Driver + class Compose < Driver + + # @return [String] Compose file format version + COMPOSE_VERSION = "2".freeze + + # @return [Pathname] data directory to store composition + attr_reader :data_directory + # @return [Vagrant::Machine] + attr_reader :machine + + # Create a new driver instance + # + # @param [Vagrant::Machine] machine Machine instance for this driver + def initialize(machine) + super() + @machine = machine + @data_directory = Pathname.new(machine.env.local_data_path). + join("docker-compose") + @data_directory.mkpath + @logger = Log4r::Logger.new("vagrant::docker::driver::compose") + @compose_lock = Mutex.new + end + + def build(dir, **opts, &block) + update_composition do |composition| + composition["build"] = dir + end + end + + def create(params, **opts, &block) + # NOTE: Use the direct machine name as we don't + # need to worry about uniqueness with compose + name = machine.name.to_s + image = params.fetch(:image) + links = params.fetch(:links) + ports = Array(params[:ports]) + volumes = Array(params[:volumes]) + cmd = Array(params.fetch(:cmd)) + env = params.fetch(:env) + expose = Array(params[:expose]) + + begin + update_composition do |composition| + services = composition["services"] ||= {} + services[name] = { + "image" => image, + "environment" => env, + "expose" => expose, + "ports" => ports, + "volumes" => volumes, + "links" => links, + "command" => cmd + } + end + rescue + update_composition(false) do |composition| + composition["services"].delete(name) + end + raise + end + get_container_id(name) + end + + def rm(cid) + if created?(cid) + destroy = false + compose_execute("rm", "-f", machine.name.to_s) + update_composition do |composition| + if composition["services"] && composition["services"].key?(machine.name.to_s) + @logger.info("Removing container `#{machine.name}`") + composition["services"].delete(machine.name.to_s) + destroy = composition["services"].empty? + end + end + if destroy + @logger.info("No containers remain. Destroying full environment.") + compose_execute("down", "--remove-orphans") + end + end + end + + def created?(cid) + result = super + if !result + composition = get_current_composition + if composition["services"] && composition["services"].has_key?(machine.name.to_s) + result = true + end + end + result + end + + private + + # Lookup the ID for the container with the given name + # + # @param [String] name Name of container + # @return [String] Container ID + def get_container_id(name) + compose_execute("ps", "-q", name).chomp + end + + # Execute a `docker-compose` command + def compose_execute(*cmd, **opts) + @compose_lock.synchronize do + execute("docker-compose", "-f", composition_path.to_s, + "-p", machine.env.cwd.basename.to_s, *cmd, **opts) + end + end + + # Apply any changes made to the composition + def apply_composition! + machine.env.lock("compose", retry: true) do + compose_execute("up", "-d", "--remove-orphans") + end + end + + # Update the composition and apply changes if requested + # + # @param [Boolean] apply Apply composition changes + def update_composition(apply=true) + machine.env.lock("compose", retry: true) do + composition = get_current_composition + yield composition + write_composition(composition) + apply_composition! if apply + end + end + + # @return [Hash] current composition contents + def get_current_composition + composition = {"version" => COMPOSE_VERSION.dup} + if composition_path.exist? + composition.merge!( + YAML.load(composition_path.read) + ) + end + composition + end + + # Save the composition + # + # @param [Hash] composition New composition + def write_composition(composition) + tmp_file = Tempfile.new("vagrant-docker-compose") + tmp_file.write(composition.to_yaml) + tmp_file.close + @compose_lock.synchronize do + FileUtils.mv(tmp_file.path, composition_path.to_s) + end + end + + # @return [Pathname] path to the docker-compose.yml file + def composition_path + data_directory.join("docker-compose.yml") + end + end + end + end +end diff --git a/plugins/providers/docker/provider.rb b/plugins/providers/docker/provider.rb index 8f23eb3a6..b6d27524a 100644 --- a/plugins/providers/docker/provider.rb +++ b/plugins/providers/docker/provider.rb @@ -31,11 +31,13 @@ module VagrantPlugins # Returns the driver instance for this provider. def driver - return @driver if @driver - @driver = Driver.new - - # If we are running on a host machine, then we set the executor - # to execute remotely. + if !@driver + if @machine.provider_config.compose + @driver = Driver::Compose.new(@machine) + else + @driver = Driver.new + end + end if host_vm? @driver.executor = Executor::Vagrant.new(host_vm) end From 4a05e8561f35f2c6368f9607907c1247000ad8e0 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Wed, 10 May 2017 09:19:30 -0700 Subject: [PATCH 02/14] Update `force_host_vm` configuration documentation and add `compose`. --- website/source/docs/docker/configuration.html.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/website/source/docs/docker/configuration.html.md b/website/source/docs/docker/configuration.html.md index ce463b1c8..b756d9931 100644 --- a/website/source/docs/docker/configuration.html.md +++ b/website/source/docs/docker/configuration.html.md @@ -30,6 +30,10 @@ General settings: * `cmd` (array of strings) - Custom command to run on the container. Example: `["ls", "/app"]`. + * `compose` (boolean) - If true, Vagrant will use `docker-compose` to + manage the lifecycle and configuration of containers. This defaults + to false. + * `create_args` (array of strings) - Additional arguments to pass to `docker run` when the container is started. This can be used to set parameters that are not exposed via the Vagrantfile. @@ -51,8 +55,8 @@ General settings: * `force_host_vm` (boolean) - If true, then a host VM will be spun up even if the computer running Vagrant supports Linux containers. This is useful to enforce a consistent environment to run Docker. This value - defaults to "true" on Mac and Windows hosts and defaults to "false" on - Linux hosts. Mac/Windows users who choose to use a different Docker + defaults to "false" on Linux, Mac, and Windows hosts and defaults to "true" + on other hosts. Users on other hosts who choose to use a different Docker provider or opt-in to the native Docker builds can explicitly set this value to false to disable the behavior. From d4bfade19f405ab741aa29ee7ec518ca39d65607 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Wed, 10 May 2017 09:33:04 -0700 Subject: [PATCH 03/14] Include documentation on accessor in docker provider configuration --- plugins/providers/docker/config.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/providers/docker/config.rb b/plugins/providers/docker/config.rb index e9cae75b3..e733338c1 100644 --- a/plugins/providers/docker/config.rb +++ b/plugins/providers/docker/config.rb @@ -19,6 +19,10 @@ module VagrantPlugins # @return [String] attr_accessor :build_dir + # Use docker-compose to manage the lifecycle and environment for + # containers instead of using docker directly. + # + # @return [Boolean] attr_accessor :compose # An optional file name of a Dockerfile to be used when building From d1c1c175a07a44a7917f76afe1910e35482385a0 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Thu, 11 May 2017 10:13:31 -0700 Subject: [PATCH 04/14] Support modifications of composition outside services --- plugins/providers/docker/config.rb | 7 ++ plugins/providers/docker/driver/compose.rb | 82 +++++++++++++++---- plugins/providers/docker/errors.rb | 4 + templates/locales/providers_docker.yml | 5 ++ .../source/docs/docker/configuration.html.md | 5 ++ 5 files changed, 85 insertions(+), 18 deletions(-) diff --git a/plugins/providers/docker/config.rb b/plugins/providers/docker/config.rb index e733338c1..ca3d5a2ae 100644 --- a/plugins/providers/docker/config.rb +++ b/plugins/providers/docker/config.rb @@ -25,6 +25,12 @@ module VagrantPlugins # @return [Boolean] attr_accessor :compose + # Configuration Hash used for build the docker-compose composition + # file. This can be used for adding networks or volumes. + # + # @return [Hash] + attr_reader :compose_configuration + # An optional file name of a Dockerfile to be used when building # the image. This requires Docker >1.5.0. # @@ -145,6 +151,7 @@ module VagrantPlugins @build_dir = UNSET_VALUE @cmd = UNSET_VALUE @compose = UNSET_VALUE + @compose_configuration = {} @create_args = UNSET_VALUE @dockerfile = UNSET_VALUE @env = {} diff --git a/plugins/providers/docker/driver/compose.rb b/plugins/providers/docker/driver/compose.rb index 6cfd24cb2..1a2ff66a3 100644 --- a/plugins/providers/docker/driver/compose.rb +++ b/plugins/providers/docker/driver/compose.rb @@ -6,6 +6,8 @@ module VagrantPlugins class Driver class Compose < Driver + # @return [Integer] Maximum number of seconds to wait for lock + LOCK_TIMEOUT = 60 # @return [String] Compose file format version COMPOSE_VERSION = "2".freeze @@ -25,11 +27,22 @@ module VagrantPlugins @data_directory.mkpath @logger = Log4r::Logger.new("vagrant::docker::driver::compose") @compose_lock = Mutex.new + @logger.debug("Docker compose driver initialize for machine `#{@machine.name}` (`#{@machine.id}`)") + @logger.debug("Data directory for composition file `#{@data_directory}`") end def build(dir, **opts, &block) - update_composition do |composition| - composition["build"] = dir + @logger.debug("Applying build using `#{dir}` directory.") + begin + update_composition(:apply) do |composition| + composition["build"] = dir + end + rescue => error + @logger.error("Failed to apply build using `#{dir}` directory: #{error.class} - #{error}") + update_composition do |composition| + composition.delete("build") + end + raise end end @@ -44,9 +57,9 @@ module VagrantPlugins cmd = Array(params.fetch(:cmd)) env = params.fetch(:env) expose = Array(params[:expose]) - + @logger.debug("Creating container `#{name}`") begin - update_composition do |composition| + update_composition(:apply) do |composition| services = composition["services"] ||= {} services[name] = { "image" => image, @@ -58,8 +71,9 @@ module VagrantPlugins "command" => cmd } end - rescue - update_composition(false) do |composition| + rescue => error + @logger.error("Failed to create container `#{name}`: #{error.class} - #{error}") + update_composition do |composition| composition["services"].delete(name) end raise @@ -71,16 +85,19 @@ module VagrantPlugins if created?(cid) destroy = false compose_execute("rm", "-f", machine.name.to_s) - update_composition do |composition| + update_composition(:conditional_apply) do |composition| if composition["services"] && composition["services"].key?(machine.name.to_s) @logger.info("Removing container `#{machine.name}`") composition["services"].delete(machine.name.to_s) destroy = composition["services"].empty? end + !destroy end if destroy @logger.info("No containers remain. Destroying full environment.") - compose_execute("down", "--remove-orphans") + compose_execute("down", "--remove-orphans", "--volumes", "--rmi", "local") + @logger.info("Deleting composition path `#{composition_path}`") + composition_path.delete end end end @@ -88,7 +105,7 @@ module VagrantPlugins def created?(cid) result = super if !result - composition = get_current_composition + composition = get_composition if composition["services"] && composition["services"].has_key?(machine.name.to_s) result = true end @@ -108,7 +125,7 @@ module VagrantPlugins # Execute a `docker-compose` command def compose_execute(*cmd, **opts) - @compose_lock.synchronize do + synchronized do execute("docker-compose", "-f", composition_path.to_s, "-p", machine.env.cwd.basename.to_s, *cmd, **opts) end @@ -124,23 +141,29 @@ module VagrantPlugins # Update the composition and apply changes if requested # # @param [Boolean] apply Apply composition changes - def update_composition(apply=true) - machine.env.lock("compose", retry: true) do - composition = get_current_composition - yield composition - write_composition(composition) - apply_composition! if apply + def update_composition(*args) + synchronized do + machine.env.lock("compose", retry: true) do + composition = get_composition + result = yield composition + write_composition(composition) + if args.include?(:apply) || (args.include?(:conditional) && result) + apply_composition! + end + end end end # @return [Hash] current composition contents - def get_current_composition + def get_composition composition = {"version" => COMPOSE_VERSION.dup} if composition_path.exist? composition.merge!( YAML.load(composition_path.read) ) end + composition.merge!(machine.provider_config.compose_configuration.dup) + @logger.debug("Fetched composition with provider configuration applied: #{composition}") composition end @@ -148,10 +171,11 @@ module VagrantPlugins # # @param [Hash] composition New composition def write_composition(composition) + @logger.debug("Saving composition to `#{composition_path}`: #{composition}") tmp_file = Tempfile.new("vagrant-docker-compose") tmp_file.write(composition.to_yaml) tmp_file.close - @compose_lock.synchronize do + synchronized do FileUtils.mv(tmp_file.path, composition_path.to_s) end end @@ -160,6 +184,28 @@ module VagrantPlugins def composition_path data_directory.join("docker-compose.yml") end + + def synchronized + if !@compose_lock.owned? + timeout = LOCK_TIMEOUT.to_f + until @compose_lock.owned? + if @compose_lock.try_lock + if timeout > 0 + timeout -= sleep(1) + else + raise Errors::ComposeLockTimeoutError + end + end + end + got_lock = true + end + begin + result = yield + ensure + @compose_lock.unlock if got_lock + end + result + end end end end diff --git a/plugins/providers/docker/errors.rb b/plugins/providers/docker/errors.rb index 1387fbfec..096f6fa5b 100644 --- a/plugins/providers/docker/errors.rb +++ b/plugins/providers/docker/errors.rb @@ -9,6 +9,10 @@ module VagrantPlugins error_key(:communicator_non_docker) end + class ComposeLockTimeoutError < DockerError + error_key(:compose_lock_timeout) + end + class ContainerNotRunningError < DockerError error_key(:not_running) end diff --git a/templates/locales/providers_docker.yml b/templates/locales/providers_docker.yml index 946372396..0c10338ce 100644 --- a/templates/locales/providers_docker.yml +++ b/templates/locales/providers_docker.yml @@ -117,6 +117,11 @@ en: run exits and doesn't keep running. errors: + compose_lock_timeout: |- + Vagrant enountered a timeout waiting for the docker compose driver + to become available. Please try to run your command again. If you + continue to experience this error it may be resolved by disabling + parallel execution. not_created: |- The container hasn't been created yet. not_running: |- diff --git a/website/source/docs/docker/configuration.html.md b/website/source/docs/docker/configuration.html.md index b756d9931..a72d6d6e0 100644 --- a/website/source/docs/docker/configuration.html.md +++ b/website/source/docs/docker/configuration.html.md @@ -34,6 +34,11 @@ General settings: manage the lifecycle and configuration of containers. This defaults to false. + * `compose_configuration` (Hash) - Configuration values used for populating + the `docker-compose.yml` file. The value of this Hash is directly merged + and written to the `docker-compose.yml` file allowing customization of + non-services items like networks and volumes. + * `create_args` (array of strings) - Additional arguments to pass to `docker run` when the container is started. This can be used to set parameters that are not exposed via the Vagrantfile. From b333e5cd82e92fc944f615cfdc521e8fc000e70c Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Thu, 11 May 2017 14:00:53 -0700 Subject: [PATCH 05/14] Fix argument construction when adding dockerfile path Fixes #7914 --- plugins/providers/docker/action/build.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/providers/docker/action/build.rb b/plugins/providers/docker/action/build.rb index 0cd613045..f6c071fc0 100644 --- a/plugins/providers/docker/action/build.rb +++ b/plugins/providers/docker/action/build.rb @@ -43,7 +43,7 @@ module VagrantPlugins dockerfile = machine.provider_config.dockerfile dockerfile_path = File.join(build_dir, dockerfile) - args.push("--file=\"#{dockerfile_path}\"") + args.push("--file").push(dockerfile_path) machine.ui.output( I18n.t("docker_provider.building_named_dockerfile", file: machine.provider_config.dockerfile)) From 4673bbb90747179799696f4a30dc2f5f0eb27326 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Thu, 11 May 2017 14:01:42 -0700 Subject: [PATCH 06/14] Properly define service build within composition. Full cleanup on destroy. --- plugins/providers/docker/driver/compose.rb | 74 ++++++++++++++++------ 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/plugins/providers/docker/driver/compose.rb b/plugins/providers/docker/driver/compose.rb index 1a2ff66a3..bfe7a9d38 100644 --- a/plugins/providers/docker/driver/compose.rb +++ b/plugins/providers/docker/driver/compose.rb @@ -32,15 +32,44 @@ module VagrantPlugins end def build(dir, **opts, &block) - @logger.debug("Applying build using `#{dir}` directory.") + name = machine.name.to_s + @logger.debug("Applying build for `#{name}` using `#{dir}` directory.") begin update_composition(:apply) do |composition| - composition["build"] = dir + services = composition["services"] ||= {} + services[name] ||= {} + services[name]["build"] = {"context" => dir} + # Extract custom dockerfile location if set + if opts[:extra_args] && opts[:extra_args].include?("--file") + services[name]["build"]["dockerfile"] = opts[:extra_args][opts[:extra_args].index("--file") + 1] + end + # Extract any build args that can be found + case opts[:build_args] + when Array + if opts[:build_args].include?("--build-arg") + idx = 0 + build_args = {} + while(idx < opts[:build_args].size) + arg_value = opts[:build_args][idx] + idx += 1 + if arg_value.start_with?("--build-arg") + if !arg_value.include?("=") + arg_value = opts[:build_args][idx] + idx += 1 + end + key, val = arg_value.to_s.split("=", 2).to_s.split("=") + build_args[key] = val + end + end + end + when Hash + services[name]["build"]["args"] = opts[:build_args] + end end rescue => error @logger.error("Failed to apply build using `#{dir}` directory: #{error.class} - #{error}") update_composition do |composition| - composition.delete("build") + composition["services"].delete(name) end raise end @@ -61,7 +90,8 @@ module VagrantPlugins begin update_composition(:apply) do |composition| services = composition["services"] ||= {} - services[name] = { + services[name] ||= {} + services[name].merge( "image" => image, "environment" => env, "expose" => expose, @@ -69,7 +99,7 @@ module VagrantPlugins "volumes" => volumes, "links" => links, "command" => cmd - } + ) end rescue => error @logger.error("Failed to create container `#{name}`: #{error.class} - #{error}") @@ -84,24 +114,32 @@ module VagrantPlugins def rm(cid) if created?(cid) destroy = false - compose_execute("rm", "-f", machine.name.to_s) - update_composition(:conditional_apply) do |composition| - if composition["services"] && composition["services"].key?(machine.name.to_s) - @logger.info("Removing container `#{machine.name}`") - composition["services"].delete(machine.name.to_s) - destroy = composition["services"].empty? + synchronized do + compose_execute("rm", "-f", machine.name.to_s) + update_composition do |composition| + if composition["services"] && composition["services"].key?(machine.name.to_s) + @logger.info("Removing container `#{machine.name}`") + if composition["services"].size > 1 + composition["services"].delete(machine.name.to_s) + else + destroy = true + end + end + end + if destroy + @logger.info("No containers remain. Destroying full environment.") + compose_execute("down", "--volumes", "--rmi", "local") + @logger.info("Deleting composition path `#{composition_path}`") + composition_path.delete end - !destroy - end - if destroy - @logger.info("No containers remain. Destroying full environment.") - compose_execute("down", "--remove-orphans", "--volumes", "--rmi", "local") - @logger.info("Deleting composition path `#{composition_path}`") - composition_path.delete end end end + def rmi(*_) + true + end + def created?(cid) result = super if !result From 9242a695457a2f39ca9c7d9c31bc04e4c7e82f82 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Thu, 11 May 2017 14:25:57 -0700 Subject: [PATCH 07/14] Allow direct set of composition and ensure basic types are used --- plugins/providers/docker/config.rb | 11 ++++++++++- plugins/providers/docker/driver/compose.rb | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/plugins/providers/docker/config.rb b/plugins/providers/docker/config.rb index ca3d5a2ae..64d1ae620 100644 --- a/plugins/providers/docker/config.rb +++ b/plugins/providers/docker/config.rb @@ -29,7 +29,7 @@ module VagrantPlugins # file. This can be used for adding networks or volumes. # # @return [Hash] - attr_reader :compose_configuration + attr_accessor :compose_configuration # An optional file name of a Dockerfile to be used when building # the image. This requires Docker >1.5.0. @@ -252,6 +252,11 @@ module VagrantPlugins @vagrant_machine = @vagrant_machine.to_sym if @vagrant_machine @expose.uniq! + + if @compose_configuration.is_a?(Hash) + # Ensures configuration is using basic types + @compose_configuration = JSON.parse(@compose_configuration.to_json) + end end def validate(machine) @@ -272,6 +277,10 @@ module VagrantPlugins end end + if !@compose_configuration.is_a?(Hash) + errors << I18n.t("docker_provider.errors.config.compose_configuration_hash") + end + if !@create_args.is_a?(Array) errors << I18n.t("docker_provider.errors.config.create_args_array") end diff --git a/plugins/providers/docker/driver/compose.rb b/plugins/providers/docker/driver/compose.rb index bfe7a9d38..5ad3c8cd9 100644 --- a/plugins/providers/docker/driver/compose.rb +++ b/plugins/providers/docker/driver/compose.rb @@ -91,7 +91,7 @@ module VagrantPlugins update_composition(:apply) do |composition| services = composition["services"] ||= {} services[name] ||= {} - services[name].merge( + services[name].merge!( "image" => image, "environment" => env, "expose" => expose, From ed8378bcf5e1b6b6e120c652338eeb6551eb0c4b Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Thu, 11 May 2017 14:26:35 -0700 Subject: [PATCH 08/14] Add output for incorrect type on compose_configuration option --- templates/locales/providers_docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/locales/providers_docker.yml b/templates/locales/providers_docker.yml index 0c10338ce..50f9959ab 100644 --- a/templates/locales/providers_docker.yml +++ b/templates/locales/providers_docker.yml @@ -139,6 +139,8 @@ en: "build_dir" must exist and contain a Dockerfile build_dir_or_image: |- One of "build_dir" or "image" must be set + compose_configuration_hash: |- + "compose_configuration" must be a hash create_args_array: |- "create_args" must be an array invalid_link: |- From 42c90422219da4331fcc5135bcd3d329e3db7ab6 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Thu, 11 May 2017 17:28:04 -0700 Subject: [PATCH 09/14] Deep merge configuration settings and set any extra options --- plugins/providers/docker/driver/compose.rb | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/plugins/providers/docker/driver/compose.rb b/plugins/providers/docker/driver/compose.rb index 5ad3c8cd9..dbdf91567 100644 --- a/plugins/providers/docker/driver/compose.rb +++ b/plugins/providers/docker/driver/compose.rb @@ -1,3 +1,5 @@ +# NOTE: DETACHED + require "json" require "log4r" @@ -84,13 +86,22 @@ module VagrantPlugins ports = Array(params[:ports]) volumes = Array(params[:volumes]) cmd = Array(params.fetch(:cmd)) - env = params.fetch(:env) + env = Hash[*params.fetch(:env).flatten.map(&:to_s)] expose = Array(params[:expose]) @logger.debug("Creating container `#{name}`") begin update_composition(:apply) do |composition| services = composition["services"] ||= {} services[name] ||= {} + if params[:extra_args].is_a?(Hash) + services[name].merge!( + Hash[ + params[:extra_args].map{ |k, v| + [k.to_s, v] + } + ] + ) + end services[name].merge!( "image" => image, "environment" => env, @@ -100,6 +111,9 @@ module VagrantPlugins "links" => links, "command" => cmd ) + services[name]["hostname"] = params[:hostname] if params[:hostname] + services[name]["privileged"] = true if params[:privileged] + services[name]["pty"] = true if params[:pty] end rescue => error @logger.error("Failed to create container `#{name}`: #{error.class} - #{error}") @@ -196,11 +210,9 @@ module VagrantPlugins def get_composition composition = {"version" => COMPOSE_VERSION.dup} if composition_path.exist? - composition.merge!( - YAML.load(composition_path.read) - ) + composition = Vagrant::Util::DeepMerge.deep_merge(composition, YAML.load(composition_path.read)) end - composition.merge!(machine.provider_config.compose_configuration.dup) + composition = Vagrant::Util::DeepMerge.deep_merge(composition, machine.provider_config.compose_configuration.dup) @logger.debug("Fetched composition with provider configuration applied: #{composition}") composition end From ed1b25f1b266cc5889a9ed455d0638927ebc6bd0 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Thu, 11 May 2017 17:28:52 -0700 Subject: [PATCH 10/14] Include spec coverage on compose driver --- .../providers/docker/driver_compose_test.rb | 267 ++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 test/unit/plugins/providers/docker/driver_compose_test.rb diff --git a/test/unit/plugins/providers/docker/driver_compose_test.rb b/test/unit/plugins/providers/docker/driver_compose_test.rb new file mode 100644 index 000000000..2fdb5fd53 --- /dev/null +++ b/test/unit/plugins/providers/docker/driver_compose_test.rb @@ -0,0 +1,267 @@ +require_relative "../../../base" + +require Vagrant.source_root.join("lib/vagrant/util/deep_merge") +require Vagrant.source_root.join("plugins/providers/docker/driver") + +describe VagrantPlugins::DockerProvider::Driver::Compose do + let(:cmd_executed) { @cmd } + let(:cid) { 'side-1-song-10' } + let(:docker_yml){ double("docker-yml", path: "/tmp-file") } + let(:machine){ double("machine", env: env, name: :docker_1, id: :docker_id, provider_config: provider_config) } + let(:compose_configuration){ {} } + let(:provider_config) do + double("provider-config", + compose: true, + compose_configuration: compose_configuration + ) + end + let(:env) do + double("env", + cwd: Pathname.new("/compose/cwd"), + local_data_path: local_data_path + ) + end + let(:composition_content){ "--- {}\n" } + let(:composition_path) do + double("composition-path", + to_s: "docker-compose.yml", + exist?: true, + read: composition_content, + delete: true + ) + end + let(:data_directory){ double("data-directory", join: composition_path) } + let(:local_data_path){ double("local-data-path") } + let(:compose_execute_up){ ["docker-compose", "-f", "docker-compose.yml", "-p", "cwd", "up", "-d", "--remove-orphans", {}] } + + + subject{ described_class.new(machine) } + + before do + allow(env).to receive(:lock).and_yield + allow(Pathname).to receive(:new).with(local_data_path).and_return(local_data_path) + allow(local_data_path).to receive(:join).and_return(data_directory) + allow(data_directory).to receive(:mkpath) + allow(FileUtils).to receive(:mv) + allow(Tempfile).to receive(:new).with("vagrant-docker-compose").and_return(docker_yml) + allow(docker_yml).to receive(:write) + allow(docker_yml).to receive(:close) + subject.stub(:execute) do |*args| + args.delete_if{|i| i.is_a?(Hash) } + @cmd = args.join(' ') + end + end + + describe '#create' do + let(:params) { { + image: 'jimi/hendrix:eletric-ladyland', + cmd: ['play', 'voodoo-chile'], + ports: '8080:80', + volumes: '/host/path:guest/path', + detach: true, + links: [[:janis, 'joplin'], [:janis, 'janis']], + env: {key: 'value'}, + name: cid, + hostname: 'jimi-hendrix', + privileged: true + } } + + before { expect(subject).to receive(:execute).with(*compose_execute_up) } + after { subject.create(params) } + + it 'sets container name' do + expect(docker_yml).to receive(:write).with(/#{machine.name}/) + end + + it 'forwards ports' do + expect(docker_yml).to receive(:write).with(/#{params[:ports]}/) + end + + it 'shares folders' do + expect(docker_yml).to receive(:write).with(/#{params[:volumes]}/) + end + + it 'links containers' do + params[:links].each do |link| + expect(docker_yml).to receive(:write).with(/#{link}/) + subject.create(params) + end + end + + it 'sets environmental variables' do + expect(docker_yml).to receive(:write).with(/key.*value/) + end + + it 'is able to run a privileged container' do + expect(docker_yml).to receive(:write).with(/privileged/) + end + + it 'sets the hostname if specified' do + expect(docker_yml).to receive(:write).with(/#{params[:hostname]}/) + end + + it 'executes the provided command' do + expect(docker_yml).to receive(:write).with(/#{params[:image]}/) + end + end + + describe '#created?' do + let(:result) { subject.created?(cid) } + + it 'performs the check on all containers list' do + subject.created?(cid) + expect(cmd_executed).to match(/docker ps \-a \-q/) + end + + context 'when container exists' do + before { subject.stub(execute: "foo\n#{cid}\nbar") } + it { expect(result).to be_true } + end + + context 'when container does not exist' do + before { subject.stub(execute: "foo\n#{cid}extra\nbar") } + it { expect(result).to be_false } + end + end + + describe '#pull' do + it 'should pull images' do + subject.should_receive(:execute).with('docker', 'pull', 'foo') + subject.pull('foo') + end + end + + describe '#running?' do + let(:result) { subject.running?(cid) } + + it 'performs the check on the running containers list' do + subject.running?(cid) + expect(cmd_executed).to match(/docker ps \-q/) + expect(cmd_executed).to_not include('-a') + end + + context 'when container exists' do + before { subject.stub(execute: "foo\n#{cid}\nbar") } + it { expect(result).to be_true } + end + + context 'when container does not exist' do + before { subject.stub(execute: "foo\n#{cid}extra\nbar") } + it { expect(result).to be_false } + end + end + + describe '#privileged?' do + it 'identifies privileged containers' do + subject.stub(inspect_container: {'HostConfig' => {"Privileged" => true}}) + expect(subject).to be_privileged(cid) + end + + it 'identifies unprivileged containers' do + subject.stub(inspect_container: {'HostConfig' => {"Privileged" => false}}) + expect(subject).to_not be_privileged(cid) + end + end + + describe '#start' do + context 'when container is running' do + before { subject.stub(running?: true) } + + it 'does not start the container' do + subject.should_not_receive(:execute).with('docker', 'start', cid) + subject.start(cid) + end + end + + context 'when container is not running' do + before { subject.stub(running?: false) } + + it 'starts the container' do + subject.should_receive(:execute).with('docker', 'start', cid) + subject.start(cid) + end + end + end + + describe '#stop' do + context 'when container is running' do + before { subject.stub(running?: true) } + + it 'stops the container' do + subject.should_receive(:execute).with('docker', 'stop', '-t', '1', cid) + subject.stop(cid, 1) + end + + it "stops the container with the set timeout" do + subject.should_receive(:execute).with('docker', 'stop', '-t', '5', cid) + subject.stop(cid, 5) + end + end + + context 'when container is not running' do + before { subject.stub(running?: false) } + + it 'does not stop container' do + subject.should_not_receive(:execute).with('docker', 'stop', '-t', '1', cid) + subject.stop(cid, 1) + end + end + end + + describe '#rm' do + context 'when container has been created' do + before { subject.stub(created?: true) } + + it 'removes the container' do + expect(subject).to receive(:execute).with("docker-compose", "-f", "docker-compose.yml", "-p", "cwd", "rm", "-f", "docker_1", {}) + subject.rm(cid) + end + end + + context 'when container has not been created' do + before { subject.stub(created?: false) } + + it 'does not attempt to remove the container' do + expect(subject).not_to receive(:execute).with("docker-compose", "-f", "docker-compose.yml", "-p", "cwd", "rm", "-f", "docker_1", {}) + subject.rm(cid) + end + end + end + + describe '#inspect_container' do + let(:data) { '[{"json": "value"}]' } + + before { subject.stub(execute: data) } + + it 'inspects the container' do + subject.should_receive(:execute).with('docker', 'inspect', cid) + subject.inspect_container(cid) + end + + it 'parses the json output' do + expect(subject.inspect_container(cid)).to eq('json' => 'value') + end + end + + describe '#all_containers' do + let(:containers) { "container1\ncontainer2" } + + before { subject.stub(execute: containers) } + + it 'returns an array of all known containers' do + subject.should_receive(:execute).with('docker', 'ps', '-a', '-q', '--no-trunc') + expect(subject.all_containers).to eq(['container1', 'container2']) + end + end + + describe '#docker_bridge_ip' do + let(:containers) { " inet 123.456.789.012/16 " } + + before { subject.stub(execute: containers) } + + it 'returns an array of all known containers' do + subject.should_receive(:execute).with('/sbin/ip', '-4', 'addr', 'show', 'scope', 'global', 'docker0') + expect(subject.docker_bridge_ip).to eq('123.456.789.012') + end + end +end From 6096bb299b5ea1d80776111b46f24d6e082c9745 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Fri, 12 May 2017 06:46:31 -0700 Subject: [PATCH 11/14] Only set image if option given. Do not apply when setting build options. --- plugins/providers/docker/driver/compose.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/providers/docker/driver/compose.rb b/plugins/providers/docker/driver/compose.rb index dbdf91567..a2d833a23 100644 --- a/plugins/providers/docker/driver/compose.rb +++ b/plugins/providers/docker/driver/compose.rb @@ -1,5 +1,3 @@ -# NOTE: DETACHED - require "json" require "log4r" @@ -37,7 +35,7 @@ module VagrantPlugins name = machine.name.to_s @logger.debug("Applying build for `#{name}` using `#{dir}` directory.") begin - update_composition(:apply) do |composition| + update_composition do |composition| services = composition["services"] ||= {} services[name] ||= {} services[name]["build"] = {"context" => dir} @@ -103,7 +101,6 @@ module VagrantPlugins ) end services[name].merge!( - "image" => image, "environment" => env, "expose" => expose, "ports" => ports, @@ -111,6 +108,7 @@ module VagrantPlugins "links" => links, "command" => cmd ) + services[name]["image"] = image if image services[name]["hostname"] = params[:hostname] if params[:hostname] services[name]["privileged"] = true if params[:privileged] services[name]["pty"] = true if params[:pty] From 36ecd40c529424bcfd4135a15a3e00cab64b0915 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Fri, 12 May 2017 14:58:49 -0700 Subject: [PATCH 12/14] Support optional detach and pass blocks through to execution. --- plugins/providers/docker/driver/compose.rb | 27 +++++++++++++++++----- plugins/providers/docker/errors.rb | 4 ++++ templates/locales/providers_docker.yml | 4 ++++ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/plugins/providers/docker/driver/compose.rb b/plugins/providers/docker/driver/compose.rb index a2d833a23..92e4e2fd8 100644 --- a/plugins/providers/docker/driver/compose.rb +++ b/plugins/providers/docker/driver/compose.rb @@ -20,6 +20,9 @@ module VagrantPlugins # # @param [Vagrant::Machine] machine Machine instance for this driver def initialize(machine) + if !Vagrant::Util::Which.which("vagrant-compose") + raise Errors::DockerComposeNotInstalledError + end super() @machine = machine @data_directory = Pathname.new(machine.env.local_data_path). @@ -88,7 +91,10 @@ module VagrantPlugins expose = Array(params[:expose]) @logger.debug("Creating container `#{name}`") begin - update_composition(:apply) do |composition| + update_args = [:apply] + update_args.push(:detach) if params[:detach] + update_args << block + update_composition(*update_args) do |composition| services = composition["services"] ||= {} services[name] ||= {} if params[:extra_args].is_a?(Hash) @@ -174,17 +180,26 @@ module VagrantPlugins end # Execute a `docker-compose` command - def compose_execute(*cmd, **opts) + def compose_execute(*cmd, **opts, &block) synchronized do execute("docker-compose", "-f", composition_path.to_s, - "-p", machine.env.cwd.basename.to_s, *cmd, **opts) + "-p", machine.env.cwd.basename.to_s, *cmd, **opts, &block) end end # Apply any changes made to the composition - def apply_composition! + def apply_composition!(*args) + block = args.detect{|arg| arg.is_a?(Proc) } + execute_args = ["up", "--remove-orphans"] + if args.include?(:detach) + execute_args << "-d" + end machine.env.lock("compose", retry: true) do - compose_execute("up", "-d", "--remove-orphans") + if block + compose_execute(*execute_args, &block) + else + compose_execute(*execute_args) + end end end @@ -198,7 +213,7 @@ module VagrantPlugins result = yield composition write_composition(composition) if args.include?(:apply) || (args.include?(:conditional) && result) - apply_composition! + apply_composition!(*args) end end end diff --git a/plugins/providers/docker/errors.rb b/plugins/providers/docker/errors.rb index 096f6fa5b..36e1b9859 100644 --- a/plugins/providers/docker/errors.rb +++ b/plugins/providers/docker/errors.rb @@ -21,6 +21,10 @@ module VagrantPlugins error_key(:not_created) end + class DockerComposeNotInstalledError < DockerError + error_key(:docker_compose_not_installed) + end + class ExecuteError < DockerError error_key(:execute_error) end diff --git a/templates/locales/providers_docker.yml b/templates/locales/providers_docker.yml index 50f9959ab..e21b90d81 100644 --- a/templates/locales/providers_docker.yml +++ b/templates/locales/providers_docker.yml @@ -122,6 +122,10 @@ en: to become available. Please try to run your command again. If you continue to experience this error it may be resolved by disabling parallel execution. + docker_compose_not_installed: |- + Vagrant has been instructed to use to use the Compose driver for the + Docker plugin but was unable to locate the `docker-compose` executable. + Ensure that `docker-compose` is installed and available on the PATH. not_created: |- The container hasn't been created yet. not_running: |- From 27ca3ef831a22383e12903833f8cd77fab4bee4c Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Fri, 12 May 2017 15:11:10 -0700 Subject: [PATCH 13/14] Convert Windows paths in volumes if detected --- plugins/providers/docker/driver/compose.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/plugins/providers/docker/driver/compose.rb b/plugins/providers/docker/driver/compose.rb index 92e4e2fd8..804cdad32 100644 --- a/plugins/providers/docker/driver/compose.rb +++ b/plugins/providers/docker/driver/compose.rb @@ -85,7 +85,19 @@ module VagrantPlugins image = params.fetch(:image) links = params.fetch(:links) ports = Array(params[:ports]) - volumes = Array(params[:volumes]) + volumes = Array(params[:volumes]).map do |v| + v = v.to_s + if v.include?(":") && (Vagrant::Util::Platform.windows? || Vagrant::Util::Platform.wsl?) + host, guest = v.split(":", 2) + host = Vagrant::Util::Platform.windows_path(host) + # NOTE: Docker does not support UNC style paths (which also + # means that there's no long path support). Hopefully this + # will be fixed someday and the gsub below can be removed. + host.gsub!(/^[^A-Za-z]+/, "") + v = [host, guest].join(":") + end + v + end cmd = Array(params.fetch(:cmd)) env = Hash[*params.fetch(:env).flatten.map(&:to_s)] expose = Array(params[:expose]) From 3e05ac06393e934a6910a5d747022565ff1e618c Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Mon, 15 May 2017 09:09:25 -0700 Subject: [PATCH 14/14] Stub out the which check within compose tests --- test/unit/plugins/providers/docker/driver_compose_test.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit/plugins/providers/docker/driver_compose_test.rb b/test/unit/plugins/providers/docker/driver_compose_test.rb index 2fdb5fd53..fa1cada06 100644 --- a/test/unit/plugins/providers/docker/driver_compose_test.rb +++ b/test/unit/plugins/providers/docker/driver_compose_test.rb @@ -32,12 +32,13 @@ describe VagrantPlugins::DockerProvider::Driver::Compose do end let(:data_directory){ double("data-directory", join: composition_path) } let(:local_data_path){ double("local-data-path") } - let(:compose_execute_up){ ["docker-compose", "-f", "docker-compose.yml", "-p", "cwd", "up", "-d", "--remove-orphans", {}] } + let(:compose_execute_up){ ["docker-compose", "-f", "docker-compose.yml", "-p", "cwd", "up", "--remove-orphans", "-d", {}] } subject{ described_class.new(machine) } before do + allow(Vagrant::Util::Which).to receive(:which).and_return("/dev/null/docker-compose") allow(env).to receive(:lock).and_yield allow(Pathname).to receive(:new).with(local_data_path).and_return(local_data_path) allow(local_data_path).to receive(:join).and_return(data_directory)