Merge pull request #8576 from chrisroberts/enhancement/docker-compose

Add optional compose driver

plugins/providers/docker/action/build.rb

@@ -43,7 +43,7 @@ module VagrantPlugins
               dockerfile      = machine.provider_config.dockerfile
               dockerfile_path = File.join(build_dir, dockerfile)
 
-              args.push("--file=\"#{dockerfile_path}\"")
+              args.push("--file").push(dockerfile_path)
               machine.ui.output(
                 I18n.t("docker_provider.building_named_dockerfile",
                 file: machine.provider_config.dockerfile))

plugins/providers/docker/config.rb

@@ -19,6 +19,18 @@ module VagrantPlugins
       # @return [String]
       attr_accessor :build_dir
 
+      # Use docker-compose to manage the lifecycle and environment for
+      # containers instead of using docker directly.
+      #
+      # @return [Boolean]
+      attr_accessor :compose
+
+      # Configuration Hash used for build the docker-compose composition
+      # file. This can be used for adding networks or volumes.
+      #
+      # @return [Hash]
+      attr_accessor :compose_configuration
+
       # An optional file name of a Dockerfile to be used when building
       # the image. This requires Docker >1.5.0.
       #
@@ -138,6 +150,8 @@ module VagrantPlugins
         @build_args = []
         @build_dir  = UNSET_VALUE
         @cmd        = UNSET_VALUE
+        @compose    = UNSET_VALUE
+        @compose_configuration = {}
         @create_args = UNSET_VALUE
         @dockerfile = UNSET_VALUE
         @env        = {}
@@ -201,6 +215,7 @@ module VagrantPlugins
         @build_args = [] if @build_args == UNSET_VALUE
         @build_dir  = nil if @build_dir == UNSET_VALUE
         @cmd        = [] if @cmd == UNSET_VALUE
+        @compose    = false if @compose == UNSET_VALUE
         @create_args = [] if @create_args == UNSET_VALUE
         @dockerfile = nil if @dockerfile == UNSET_VALUE
         @env       ||= {}
@@ -237,6 +252,11 @@ module VagrantPlugins
         @vagrant_machine = @vagrant_machine.to_sym if @vagrant_machine
 
         @expose.uniq!
+
+        if @compose_configuration.is_a?(Hash)
+          # Ensures configuration is using basic types
+          @compose_configuration = JSON.parse(@compose_configuration.to_json)
+        end
       end
 
       def validate(machine)
@@ -257,6 +277,10 @@ module VagrantPlugins
           end
         end
 
+        if !@compose_configuration.is_a?(Hash)
+          errors << I18n.t("docker_provider.errors.config.compose_configuration_hash")
+        end
+
         if !@create_args.is_a?(Array)
           errors << I18n.t("docker_provider.errors.config.create_args_array")
         end

plugins/providers/docker/driver.rb

@@ -1,7 +1,8 @@
 require "json"
-
 require "log4r"
 
+require_relative "./driver/compose"
+
 module VagrantPlugins
   module DockerProvider
     class Driver

plugins/providers/docker/driver/compose.rb

@@ -0,0 +1,287 @@
+require "json"
+require "log4r"
+
+module VagrantPlugins
+  module DockerProvider
+    class Driver
+      class Compose < Driver
+
+        # @return [Integer] Maximum number of seconds to wait for lock
+        LOCK_TIMEOUT = 60
+        # @return [String] Compose file format version
+        COMPOSE_VERSION = "2".freeze
+
+        # @return [Pathname] data directory to store composition
+        attr_reader :data_directory
+        # @return [Vagrant::Machine]
+        attr_reader :machine
+
+        # Create a new driver instance
+        #
+        # @param [Vagrant::Machine] machine Machine instance for this driver
+        def initialize(machine)
+          if !Vagrant::Util::Which.which("vagrant-compose")
+            raise Errors::DockerComposeNotInstalledError
+          end
+          super()
+          @machine = machine
+          @data_directory = Pathname.new(machine.env.local_data_path).
+            join("docker-compose")
+          @data_directory.mkpath
+          @logger = Log4r::Logger.new("vagrant::docker::driver::compose")
+          @compose_lock = Mutex.new
+          @logger.debug("Docker compose driver initialize for machine `#{@machine.name}` (`#{@machine.id}`)")
+          @logger.debug("Data directory for composition file `#{@data_directory}`")
+        end
+
+        def build(dir, **opts, &block)
+          name = machine.name.to_s
+          @logger.debug("Applying build for `#{name}` using `#{dir}` directory.")
+          begin
+            update_composition do |composition|
+              services = composition["services"] ||= {}
+              services[name] ||= {}
+              services[name]["build"] = {"context" => dir}
+              # Extract custom dockerfile location if set
+              if opts[:extra_args] && opts[:extra_args].include?("--file")
+                services[name]["build"]["dockerfile"] = opts[:extra_args][opts[:extra_args].index("--file") + 1]
+              end
+              # Extract any build args that can be found
+              case opts[:build_args]
+              when Array
+                if opts[:build_args].include?("--build-arg")
+                  idx = 0
+                  build_args = {}
+                  while(idx < opts[:build_args].size)
+                    arg_value = opts[:build_args][idx]
+                    idx += 1
+                    if arg_value.start_with?("--build-arg")
+                      if !arg_value.include?("=")
+                        arg_value = opts[:build_args][idx]
+                        idx += 1
+                      end
+                      key, val = arg_value.to_s.split("=", 2).to_s.split("=")
+                      build_args[key] = val
+                    end
+                  end
+                end
+              when Hash
+                services[name]["build"]["args"] = opts[:build_args]
+              end
+            end
+          rescue => error
+            @logger.error("Failed to apply build using `#{dir}` directory: #{error.class} - #{error}")
+            update_composition do |composition|
+              composition["services"].delete(name)
+            end
+            raise
+          end
+        end
+
+        def create(params, **opts, &block)
+          # NOTE: Use the direct machine name as we don't
+          # need to worry about uniqueness with compose
+          name    = machine.name.to_s
+          image   = params.fetch(:image)
+          links   = params.fetch(:links)
+          ports   = Array(params[:ports])
+          volumes = Array(params[:volumes]).map do |v|
+            v = v.to_s
+            if v.include?(":") && (Vagrant::Util::Platform.windows? || Vagrant::Util::Platform.wsl?)
+              host, guest = v.split(":", 2)
+              host = Vagrant::Util::Platform.windows_path(host)
+              # NOTE: Docker does not support UNC style paths (which also
+              # means that there's no long path support). Hopefully this
+              # will be fixed someday and the gsub below can be removed.
+              host.gsub!(/^[^A-Za-z]+/, "")
+              v = [host, guest].join(":")
+            end
+            v
+          end
+          cmd     = Array(params.fetch(:cmd))
+          env     = Hash[*params.fetch(:env).flatten.map(&:to_s)]
+          expose  = Array(params[:expose])
+          @logger.debug("Creating container `#{name}`")
+          begin
+            update_args = [:apply]
+            update_args.push(:detach) if params[:detach]
+            update_args << block
+            update_composition(*update_args) do |composition|
+              services = composition["services"] ||= {}
+              services[name] ||= {}
+              if params[:extra_args].is_a?(Hash)
+                services[name].merge!(
+                  Hash[
+                    params[:extra_args].map{ |k, v|
+                      [k.to_s, v]
+                    }
+                  ]
+                )
+              end
+              services[name].merge!(
+                "environment" => env,
+                "expose" => expose,
+                "ports" => ports,
+                "volumes" => volumes,
+                "links" => links,
+                "command" => cmd
+              )
+              services[name]["image"] = image if image
+              services[name]["hostname"] = params[:hostname] if params[:hostname]
+              services[name]["privileged"] = true if params[:privileged]
+              services[name]["pty"] = true if params[:pty]
+            end
+          rescue => error
+            @logger.error("Failed to create container `#{name}`: #{error.class} - #{error}")
+            update_composition do |composition|
+              composition["services"].delete(name)
+            end
+            raise
+          end
+          get_container_id(name)
+        end
+
+        def rm(cid)
+          if created?(cid)
+            destroy = false
+            synchronized do
+              compose_execute("rm", "-f", machine.name.to_s)
+              update_composition do |composition|
+                if composition["services"] && composition["services"].key?(machine.name.to_s)
+                  @logger.info("Removing container `#{machine.name}`")
+                  if composition["services"].size > 1
+                    composition["services"].delete(machine.name.to_s)
+                  else
+                    destroy = true
+                  end
+                end
+              end
+              if destroy
+                @logger.info("No containers remain. Destroying full environment.")
+                compose_execute("down", "--volumes", "--rmi", "local")
+                @logger.info("Deleting composition path `#{composition_path}`")
+                composition_path.delete
+              end
+            end
+          end
+        end
+
+        def rmi(*_)
+          true
+        end
+
+        def created?(cid)
+          result = super
+          if !result
+            composition = get_composition
+            if composition["services"] && composition["services"].has_key?(machine.name.to_s)
+              result = true
+            end
+          end
+          result
+        end
+
+        private
+
+        # Lookup the ID for the container with the given name
+        #
+        # @param [String] name Name of container
+        # @return [String] Container ID
+        def get_container_id(name)
+          compose_execute("ps", "-q", name).chomp
+        end
+
+        # Execute a `docker-compose` command
+        def compose_execute(*cmd, **opts, &block)
+          synchronized do
+            execute("docker-compose", "-f", composition_path.to_s,
+              "-p", machine.env.cwd.basename.to_s, *cmd, **opts, &block)
+          end
+        end
+
+        # Apply any changes made to the composition
+        def apply_composition!(*args)
+          block = args.detect{|arg| arg.is_a?(Proc) }
+          execute_args = ["up", "--remove-orphans"]
+          if args.include?(:detach)
+            execute_args << "-d"
+          end
+          machine.env.lock("compose", retry: true) do
+            if block
+              compose_execute(*execute_args, &block)
+            else
+              compose_execute(*execute_args)
+            end
+          end
+        end
+
+        # Update the composition and apply changes if requested
+        #
+        # @param [Boolean] apply Apply composition changes
+        def update_composition(*args)
+          synchronized do
+            machine.env.lock("compose", retry: true) do
+              composition = get_composition
+              result = yield composition
+              write_composition(composition)
+              if args.include?(:apply) || (args.include?(:conditional) && result)
+                apply_composition!(*args)
+              end
+            end
+          end
+        end
+
+        # @return [Hash] current composition contents
+        def get_composition
+          composition = {"version" => COMPOSE_VERSION.dup}
+          if composition_path.exist?
+            composition = Vagrant::Util::DeepMerge.deep_merge(composition, YAML.load(composition_path.read))
+          end
+          composition = Vagrant::Util::DeepMerge.deep_merge(composition, machine.provider_config.compose_configuration.dup)
+          @logger.debug("Fetched composition with provider configuration applied: #{composition}")
+          composition
+        end
+
+        # Save the composition
+        #
+        # @param [Hash] composition New composition
+        def write_composition(composition)
+          @logger.debug("Saving composition to `#{composition_path}`: #{composition}")
+          tmp_file = Tempfile.new("vagrant-docker-compose")
+          tmp_file.write(composition.to_yaml)
+          tmp_file.close
+          synchronized do
+            FileUtils.mv(tmp_file.path, composition_path.to_s)
+          end
+        end
+
+        # @return [Pathname] path to the docker-compose.yml file
+        def composition_path
+          data_directory.join("docker-compose.yml")
+        end
+
+        def synchronized
+          if !@compose_lock.owned?
+            timeout = LOCK_TIMEOUT.to_f
+            until @compose_lock.owned?
+              if @compose_lock.try_lock
+                if timeout > 0
+                  timeout -= sleep(1)
+                else
+                  raise Errors::ComposeLockTimeoutError
+                end
+              end
+            end
+            got_lock = true
+          end
+          begin
+            result = yield
+          ensure
+            @compose_lock.unlock if got_lock
+          end
+          result
+        end
+      end
+    end
+  end
+end

plugins/providers/docker/errors.rb

@@ -9,6 +9,10 @@ module VagrantPlugins
         error_key(:communicator_non_docker)
       end
 
+      class ComposeLockTimeoutError < DockerError
+        error_key(:compose_lock_timeout)
+      end
+
       class ContainerNotRunningError < DockerError
         error_key(:not_running)
       end
@@ -17,6 +21,10 @@ module VagrantPlugins
         error_key(:not_created)
       end
 
+      class DockerComposeNotInstalledError < DockerError
+        error_key(:docker_compose_not_installed)
+      end
+
       class ExecuteError < DockerError
         error_key(:execute_error)
       end

plugins/providers/docker/provider.rb

@@ -31,11 +31,13 @@ module VagrantPlugins
 
       # Returns the driver instance for this provider.
       def driver
-        return @driver if @driver
+        if !@driver
+          if @machine.provider_config.compose
+            @driver = Driver::Compose.new(@machine)
+          else
             @driver = Driver.new
-
+          end
-        # If we are running on a host machine, then we set the executor
+        end
-        # to execute remotely.
         if host_vm?
           @driver.executor = Executor::Vagrant.new(host_vm)
         end

templates/locales/providers_docker.yml

@@ -117,6 +117,15 @@ en:
         run exits and doesn't keep running.
 
     errors:
+      compose_lock_timeout: |-
+        Vagrant enountered a timeout waiting for the docker compose driver
+        to become available. Please try to run your command again. If you
+        continue to experience this error it may be resolved by disabling
+        parallel execution.
+      docker_compose_not_installed: |-
+        Vagrant has been instructed to use to use the Compose driver for the
+        Docker plugin but was unable to locate the `docker-compose` executable.
+        Ensure that `docker-compose` is installed and available on the PATH.
       not_created: |-
         The container hasn't been created yet.
       not_running: |-
@@ -134,6 +143,8 @@ en:
           "build_dir" must exist and contain a Dockerfile
         build_dir_or_image: |-
           One of "build_dir" or "image" must be set
+        compose_configuration_hash: |-
+          "compose_configuration" must be a hash
         create_args_array: |-
           "create_args" must be an array
         invalid_link: |-

test/unit/plugins/providers/docker/driver_compose_test.rb

@@ -0,0 +1,268 @@
+require_relative "../../../base"
+
+require Vagrant.source_root.join("lib/vagrant/util/deep_merge")
+require Vagrant.source_root.join("plugins/providers/docker/driver")
+
+describe VagrantPlugins::DockerProvider::Driver::Compose do
+  let(:cmd_executed) { @cmd }
+  let(:cid)          { 'side-1-song-10' }
+  let(:docker_yml){ double("docker-yml", path: "/tmp-file") }
+  let(:machine){ double("machine", env: env, name: :docker_1, id: :docker_id, provider_config: provider_config) }
+  let(:compose_configuration){ {} }
+  let(:provider_config) do
+    double("provider-config",
+      compose: true,
+      compose_configuration: compose_configuration
+    )
+  end
+  let(:env) do
+    double("env",
+      cwd: Pathname.new("/compose/cwd"),
+      local_data_path: local_data_path
+    )
+  end
+  let(:composition_content){ "--- {}\n" }
+  let(:composition_path) do
+    double("composition-path",
+      to_s: "docker-compose.yml",
+      exist?: true,
+      read: composition_content,
+      delete: true
+    )
+  end
+  let(:data_directory){ double("data-directory", join: composition_path) }
+  let(:local_data_path){ double("local-data-path") }
+  let(:compose_execute_up){ ["docker-compose", "-f", "docker-compose.yml", "-p", "cwd", "up", "--remove-orphans", "-d", {}] }
+
+
+  subject{ described_class.new(machine) }
+
+  before do
+    allow(Vagrant::Util::Which).to receive(:which).and_return("/dev/null/docker-compose")
+    allow(env).to receive(:lock).and_yield
+    allow(Pathname).to receive(:new).with(local_data_path).and_return(local_data_path)
+    allow(local_data_path).to receive(:join).and_return(data_directory)
+    allow(data_directory).to receive(:mkpath)
+    allow(FileUtils).to receive(:mv)
+    allow(Tempfile).to receive(:new).with("vagrant-docker-compose").and_return(docker_yml)
+    allow(docker_yml).to receive(:write)
+    allow(docker_yml).to receive(:close)
+    subject.stub(:execute) do |*args|
+      args.delete_if{|i| i.is_a?(Hash) }
+      @cmd = args.join(' ')
+    end
+  end
+
+  describe '#create' do
+    let(:params) { {
+      image:      'jimi/hendrix:eletric-ladyland',
+      cmd:        ['play', 'voodoo-chile'],
+      ports:      '8080:80',
+      volumes:    '/host/path:guest/path',
+      detach:     true,
+      links:      [[:janis, 'joplin'], [:janis, 'janis']],
+      env:        {key: 'value'},
+      name:       cid,
+      hostname:   'jimi-hendrix',
+      privileged: true
+    } }
+
+    before { expect(subject).to receive(:execute).with(*compose_execute_up) }
+    after { subject.create(params) }
+
+    it 'sets container name' do
+      expect(docker_yml).to receive(:write).with(/#{machine.name}/)
+    end
+
+    it 'forwards ports' do
+      expect(docker_yml).to receive(:write).with(/#{params[:ports]}/)
+    end
+
+    it 'shares folders' do
+      expect(docker_yml).to receive(:write).with(/#{params[:volumes]}/)
+    end
+
+    it 'links containers' do
+      params[:links].each do |link|
+        expect(docker_yml).to receive(:write).with(/#{link}/)
+        subject.create(params)
+      end
+    end
+
+    it 'sets environmental variables' do
+      expect(docker_yml).to receive(:write).with(/key.*value/)
+    end
+
+    it 'is able to run a privileged container' do
+      expect(docker_yml).to receive(:write).with(/privileged/)
+    end
+
+    it 'sets the hostname if specified' do
+      expect(docker_yml).to receive(:write).with(/#{params[:hostname]}/)
+    end
+
+    it 'executes the provided command' do
+      expect(docker_yml).to receive(:write).with(/#{params[:image]}/)
+    end
+  end
+
+  describe '#created?' do
+    let(:result) { subject.created?(cid) }
+
+    it 'performs the check on all containers list' do
+      subject.created?(cid)
+      expect(cmd_executed).to match(/docker ps \-a \-q/)
+    end
+
+    context 'when container exists' do
+      before { subject.stub(execute: "foo\n#{cid}\nbar") }
+      it { expect(result).to be_true }
+    end
+
+    context 'when container does not exist' do
+      before { subject.stub(execute: "foo\n#{cid}extra\nbar") }
+      it { expect(result).to be_false }
+    end
+  end
+
+  describe '#pull' do
+    it 'should pull images' do
+      subject.should_receive(:execute).with('docker', 'pull', 'foo')
+      subject.pull('foo')
+    end
+  end
+
+  describe '#running?' do
+    let(:result) { subject.running?(cid) }
+
+    it 'performs the check on the running containers list' do
+      subject.running?(cid)
+      expect(cmd_executed).to match(/docker ps \-q/)
+      expect(cmd_executed).to_not include('-a')
+    end
+
+    context 'when container exists' do
+      before { subject.stub(execute: "foo\n#{cid}\nbar") }
+      it { expect(result).to be_true }
+    end
+
+    context 'when container does not exist' do
+      before { subject.stub(execute: "foo\n#{cid}extra\nbar") }
+      it { expect(result).to be_false }
+    end
+  end
+
+  describe '#privileged?' do
+    it 'identifies privileged containers' do
+      subject.stub(inspect_container: {'HostConfig' => {"Privileged" => true}})
+      expect(subject).to be_privileged(cid)
+    end
+
+    it 'identifies unprivileged containers' do
+      subject.stub(inspect_container: {'HostConfig' => {"Privileged" => false}})
+      expect(subject).to_not be_privileged(cid)
+    end
+  end
+
+  describe '#start' do
+    context 'when container is running' do
+      before { subject.stub(running?: true) }
+
+      it 'does not start the container' do
+        subject.should_not_receive(:execute).with('docker', 'start', cid)
+        subject.start(cid)
+      end
+    end
+
+    context 'when container is not running' do
+      before { subject.stub(running?: false) }
+
+      it 'starts the container' do
+        subject.should_receive(:execute).with('docker', 'start', cid)
+        subject.start(cid)
+      end
+    end
+  end
+
+  describe '#stop' do
+    context 'when container is running' do
+      before { subject.stub(running?: true) }
+
+      it 'stops the container' do
+        subject.should_receive(:execute).with('docker', 'stop', '-t', '1', cid)
+        subject.stop(cid, 1)
+      end
+
+      it "stops the container with the set timeout" do
+        subject.should_receive(:execute).with('docker', 'stop', '-t', '5', cid)
+        subject.stop(cid, 5)
+      end
+    end
+
+    context 'when container is not running' do
+      before { subject.stub(running?: false) }
+
+      it 'does not stop container' do
+        subject.should_not_receive(:execute).with('docker', 'stop', '-t', '1', cid)
+        subject.stop(cid, 1)
+      end
+    end
+  end
+
+  describe '#rm' do
+    context 'when container has been created' do
+      before { subject.stub(created?: true) }
+
+      it 'removes the container' do
+        expect(subject).to receive(:execute).with("docker-compose", "-f", "docker-compose.yml", "-p", "cwd", "rm", "-f", "docker_1", {})
+        subject.rm(cid)
+      end
+    end
+
+    context 'when container has not been created' do
+      before { subject.stub(created?: false) }
+
+      it 'does not attempt to remove the container' do
+        expect(subject).not_to receive(:execute).with("docker-compose", "-f", "docker-compose.yml", "-p", "cwd", "rm", "-f", "docker_1", {})
+        subject.rm(cid)
+      end
+    end
+  end
+
+  describe '#inspect_container' do
+    let(:data) { '[{"json": "value"}]' }
+
+    before { subject.stub(execute: data) }
+
+    it 'inspects the container' do
+      subject.should_receive(:execute).with('docker', 'inspect', cid)
+      subject.inspect_container(cid)
+    end
+
+    it 'parses the json output' do
+      expect(subject.inspect_container(cid)).to eq('json' => 'value')
+    end
+  end
+
+  describe '#all_containers' do
+    let(:containers) { "container1\ncontainer2" }
+
+    before { subject.stub(execute: containers) }
+
+    it 'returns an array of all known containers' do
+      subject.should_receive(:execute).with('docker', 'ps', '-a', '-q', '--no-trunc')
+      expect(subject.all_containers).to eq(['container1', 'container2'])
+    end
+  end
+
+  describe '#docker_bridge_ip' do
+    let(:containers) { " inet 123.456.789.012/16 " }
+
+    before { subject.stub(execute: containers) }
+
+    it 'returns an array of all known containers' do
+      subject.should_receive(:execute).with('/sbin/ip', '-4', 'addr', 'show', 'scope', 'global', 'docker0')
+      expect(subject.docker_bridge_ip).to eq('123.456.789.012')
+    end
+  end
+end

website/source/docs/docker/configuration.html.md

@@ -30,6 +30,15 @@ General settings:
   * `cmd` (array of strings) - Custom command to run on the container.
     Example: `["ls", "/app"]`.
 
+  * `compose` (boolean) - If true, Vagrant will use `docker-compose` to
+    manage the lifecycle and configuration of containers. This defaults
+    to false.
+
+  * `compose_configuration` (Hash) - Configuration values used for populating
+    the `docker-compose.yml` file. The value of this Hash is directly merged
+    and written to the `docker-compose.yml` file allowing customization of
+    non-services items like networks and volumes.
+
   * `create_args` (array of strings) - Additional arguments to pass to
     `docker run` when the container is started. This can be used to set
     parameters that are not exposed via the Vagrantfile.
@@ -51,8 +60,8 @@ General settings:
   * `force_host_vm` (boolean) - If true, then a host VM will be spun up
     even if the computer running Vagrant supports Linux containers. This
     is useful to enforce a consistent environment to run Docker. This value
-    defaults to "true" on Mac and Windows hosts and defaults to "false" on
+    defaults to "false" on Linux, Mac, and Windows hosts and defaults to "true"
-    Linux hosts. Mac/Windows users who choose to use a different Docker
+    on other hosts. Users on other hosts who choose to use a different Docker
     provider or opt-in to the native Docker builds can explicitly set this
     value to false to disable the behavior.