diff --git a/plugins/providers/docker/config.rb b/plugins/providers/docker/config.rb
index 84d195d1a..d156b2143 100644
--- a/plugins/providers/docker/config.rb
+++ b/plugins/providers/docker/config.rb
@@ -90,6 +90,19 @@ module VagrantPlugins
 
       def merge(other)
         super.tap do |result|
+          # This is a bit confusing. The tests explain the purpose of this
+          # better than the code lets on, I believe.
+          if (other.image != UNSET_VALUE || other.build_dir != UNSET_VALUE) &&
+            (other.image == UNSET_VALUE || other.build_dir == UNSET_VALUE)
+            if other.image != UNSET_VALUE && @build_dir != UNSET_VALUE
+              result.build_dir = nil
+            end
+
+            if other.build_dir != UNSET_VALUE && @image != UNSET_VALUE
+              result.image = nil
+            end
+          end
+
           env = {}
           env.merge!(@env) if @env
           env.merge!(other.env) if other.env
diff --git a/test/unit/plugins/providers/docker/config_spec.rb b/test/unit/plugins/providers/docker/config_spec.rb
index ceb435b81..cb316238d 100644
--- a/test/unit/plugins/providers/docker/config_spec.rb
+++ b/test/unit/plugins/providers/docker/config_spec.rb
@@ -127,6 +127,33 @@ describe VagrantPlugins::DockerProvider::Config do
 
     subject { one.merge(two) }
 
+    context "#build_dir and #image" do
+      it "overrides image if build_dir is set previously" do
+        one.build_dir = "foo"
+        two.image = "bar"
+
+        expect(subject.build_dir).to be_nil
+        expect(subject.image).to eq("bar")
+      end
+
+      it "overrides image if build_dir is set previously" do
+        one.image = "foo"
+        two.build_dir = "bar"
+
+        expect(subject.image).to be_nil
+        expect(subject.build_dir).to eq("bar")
+      end
+
+      it "preserves if both set" do
+        one.image = "foo"
+        two.image = "baz"
+        two.build_dir = "bar"
+
+        expect(subject.image).to eq("baz")
+        expect(subject.build_dir).to eq("bar")
+      end
+    end
+
     context "env vars" do
       it "should merge the values" do
         one.env["foo"] = "bar"