From da45ca707c72c822ba8697a02ae2e0b22e59f507 Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Wed, 9 Nov 2016 16:00:09 -0800 Subject: [PATCH] Add md5 and sha1 checksum support to Downloader. Allows checksum validation on downloaded files via Util::Downloader using MD5 and/or SHA1 checksums. This also integrates checksum validation support with the shell provisioner for downloaded remote files. --- lib/vagrant/errors.rb | 4 + lib/vagrant/util/downloader.rb | 56 +++++++++- plugins/provisioners/shell/config.rb | 6 ++ plugins/provisioners/shell/provisioner.rb | 7 +- templates/locales/en.yml | 7 ++ .../provisioners/shell/provisioner_test.rb | 76 ++++++++++++- test/unit/vagrant/util/downloader_test.rb | 101 ++++++++++++++++++ .../source/docs/provisioning/shell.html.md | 4 + 8 files changed, 258 insertions(+), 3 deletions(-) diff --git a/lib/vagrant/errors.rb b/lib/vagrant/errors.rb index 9bff60a47..2139f994b 100644 --- a/lib/vagrant/errors.rb +++ b/lib/vagrant/errors.rb @@ -340,6 +340,10 @@ module Vagrant error_key(:downloader_interrupted) end + class DownloaderChecksumError < VagrantError + error_key(:downloader_checksum_error) + end + class EnvInval < VagrantError error_key(:env_inval) end diff --git a/lib/vagrant/util/downloader.rb b/lib/vagrant/util/downloader.rb index 03e623897..25a545b0d 100644 --- a/lib/vagrant/util/downloader.rb +++ b/lib/vagrant/util/downloader.rb @@ -1,7 +1,8 @@ require "uri" require "log4r" - +require "digest/md5" +require "digest/sha1" require "vagrant/util/busy" require "vagrant/util/platform" require "vagrant/util/subprocess" @@ -18,6 +19,12 @@ module Vagrant # Vagrant/1.7.4 (+https://www.vagrantup.com; ruby2.1.0) USER_AGENT = "Vagrant/#{VERSION} (+https://www.vagrantup.com; #{RUBY_ENGINE}#{RUBY_VERSION})".freeze + # Supported file checksum + CHECKSUM_MAP = { + :md5 => Digest::MD5, + :sha1 => Digest::SHA1 + }.freeze + attr_reader :source attr_reader :destination @@ -52,6 +59,10 @@ module Vagrant @ui = options[:ui] @client_cert = options[:client_cert] @location_trusted = options[:location_trusted] + @checksums = { + :md5 => options[:md5], + :sha1 => options[:sha1] + } end # This executes the actual download, downloading the source file @@ -161,6 +172,8 @@ module Vagrant end end + validate_download!(@source, @destination, @checksums) + # Everything succeeded true end @@ -178,6 +191,47 @@ module Vagrant protected + # Apply any checksum validations based on provided + # options content + # + # @param source [String] Source of file + # @param path [String, Pathname] local file path + # @param checksums [Hash] User provided options + # @option checksums [String] :md5 Compare MD5 checksum + # @option checksums [String] :sha1 Compare SHA1 checksum + # @return [Boolean] + def validate_download!(source, path, checksums) + CHECKSUM_MAP.each do |type, klass| + if checksums[type] + result = checksum_file(klass, path) + if checksums[type] != result + raise Errors::DownloaderChecksumError.new( + source: source, + path: path, + expected_checksum: checksums[type], + actual_checksum: result + ) + end + end + end + true + end + + # Generate checksum on given file + # + # @param digest_class [Class] Digest class to use for generating checksum + # @param path [String, Pathname] Path to file + # @return [String] hexdigest result + def checksum_file(digest_class, path) + digester = digest_class.new + File.open(path.to_s, 'rb') do |file| + while(data = file.read(1024)) + digester << data + end + end + digester.hexdigest + end + def execute_curl(options, subprocess_options, &data_proc) options = options.dup options << subprocess_options diff --git a/plugins/provisioners/shell/config.rb b/plugins/provisioners/shell/config.rb index d5f260b91..049eb15d0 100644 --- a/plugins/provisioners/shell/config.rb +++ b/plugins/provisioners/shell/config.rb @@ -5,6 +5,8 @@ module VagrantPlugins class Config < Vagrant.plugin("2", :config) attr_accessor :inline attr_accessor :path + attr_accessor :md5 + attr_accessor :sha1 attr_accessor :env attr_accessor :upload_path attr_accessor :args @@ -19,6 +21,8 @@ module VagrantPlugins @args = UNSET_VALUE @inline = UNSET_VALUE @path = UNSET_VALUE + @md5 = UNSET_VALUE + @sha1 = UNSET_VALUE @env = UNSET_VALUE @upload_path = UNSET_VALUE @privileged = UNSET_VALUE @@ -33,6 +37,8 @@ module VagrantPlugins @args = nil if @args == UNSET_VALUE @inline = nil if @inline == UNSET_VALUE @path = nil if @path == UNSET_VALUE + @md5 = nil if @md5 == UNSET_VALUE + @sha1 = nil if @sha1 == UNSET_VALUE @env = {} if @env == UNSET_VALUE @upload_path = "/tmp/vagrant-shell" if @upload_path == UNSET_VALUE @privileged = true if @privileged == UNSET_VALUE diff --git a/plugins/provisioners/shell/provisioner.rb b/plugins/provisioners/shell/provisioner.rb index 13d4d112e..7c06d19df 100644 --- a/plugins/provisioners/shell/provisioner.rb +++ b/plugins/provisioners/shell/provisioner.rb @@ -177,7 +177,12 @@ module VagrantPlugins download_path.delete if download_path.file? begin - Vagrant::Util::Downloader.new(config.path, download_path).download! + Vagrant::Util::Downloader.new( + config.path, + download_path, + md5: config.md5, + sha1: config.sha1 + ).download! ext = File.extname(config.path) script = download_path.read ensure diff --git a/templates/locales/en.yml b/templates/locales/en.yml index d3114d845..5006beedf 100644 --- a/templates/locales/en.yml +++ b/templates/locales/en.yml @@ -734,6 +734,13 @@ en: downloader_interrupted: |- The download was interrupted by an external signal. It did not complete. + downloader_checksum_error: |- + The calculated checksum of the requested file does not match the expected + checksum! + + File source: %{source} + Expected checksum: %{expected_checksum} + Calculated checksum: %{actual_checksum} env_inval: |- Vagrant received an "EINVAL" error while attempting to set some environment variables. This is usually caused by the total size of your diff --git a/test/unit/plugins/provisioners/shell/provisioner_test.rb b/test/unit/plugins/provisioners/shell/provisioner_test.rb index 8f83f2554..6ba995e04 100644 --- a/test/unit/plugins/provisioners/shell/provisioner_test.rb +++ b/test/unit/plugins/provisioners/shell/provisioner_test.rb @@ -2,14 +2,19 @@ require File.expand_path("../../../../base", __FILE__) require Vagrant.source_root.join("plugins/provisioners/shell/provisioner") describe "Vagrant::Shell::Provisioner" do + include_context "unit" + let(:env){ isolated_environment } let(:machine) { - double(:machine).tap { |machine| + double(:machine, env: env, id: "ID").tap { |machine| machine.stub_chain(:config, :vm, :communicator).and_return(:not_winrm) machine.stub_chain(:communicate, :tap) {} } } + before do + allow(env).to receive(:tmp_path).and_return(Pathname.new("/dev/null")) + end context "with a script that contains invalid us-ascii byte sequences" do let(:config) { @@ -37,4 +42,73 @@ describe "Vagrant::Shell::Provisioner" do }.not_to raise_error end end + + context "with remote script" do + + context "that does not have matching sha1 checksum" do + let(:config) { + double( + :config, + :args => "doesn't matter", + :env => {}, + :upload_path => "arbitrary", + :remote? => true, + :path => "http://example.com/script.sh", + :binary => false, + :md5 => nil, + :sha1 => 'EXPECTED_VALUE' + ) + } + + let(:file){ double("file") } + let(:digest){ double("digest") } + before do + Vagrant::Util::Downloader.any_instance.should_receive(:execute_curl).and_return(true) + expect(File).to receive(:open).with(%r{/dev/null/.+}, "rb").and_yield(file).once + allow(File).to receive(:open).and_call_original + expect(file).to receive(:read).and_return(nil) + expect(Digest::SHA1).to receive(:new).and_return(digest) + expect(digest).to receive(:hexdigest).and_return('INVALID_VALUE') + end + + it "should raise an exception" do + vsp = VagrantPlugins::Shell::Provisioner.new(machine, config) + + expect{ vsp.provision }.to raise_error(Vagrant::Errors::DownloaderChecksumError) + end + end + + context "that does not have matching md5 checksum" do + let(:config) { + double( + :config, + :args => "doesn't matter", + :env => {}, + :upload_path => "arbitrary", + :remote? => true, + :path => "http://example.com/script.sh", + :binary => false, + :md5 => 'EXPECTED_VALUE', + :sha1 => nil + ) + } + + let(:file){ double("file") } + let(:digest){ double("digest") } + before do + Vagrant::Util::Downloader.any_instance.should_receive(:execute_curl).and_return(true) + expect(File).to receive(:open).with(%r{/dev/null/.+}, "rb").and_yield(file).once + allow(File).to receive(:open).and_call_original + expect(file).to receive(:read).and_return(nil) + expect(Digest::MD5).to receive(:new).and_return(digest) + expect(digest).to receive(:hexdigest).and_return('INVALID_VALUE') + end + + it "should raise an exception" do + vsp = VagrantPlugins::Shell::Provisioner.new(machine, config) + + expect{ vsp.provision }.to raise_error(Vagrant::Errors::DownloaderChecksumError) + end + end + end end diff --git a/test/unit/vagrant/util/downloader_test.rb b/test/unit/vagrant/util/downloader_test.rb index f89c6cbe9..df88eed6d 100644 --- a/test/unit/vagrant/util/downloader_test.rb +++ b/test/unit/vagrant/util/downloader_test.rb @@ -93,6 +93,107 @@ describe Vagrant::Util::Downloader do expect(subject.download!).to be_true end end + + context "with checksum" do + let(:checksum_expected_value){ 'MD5_CHECKSUM_VALUE' } + let(:checksum_invalid_value){ 'INVALID_VALUE' } + let(:digest){ double("digest") } + let(:file){ double("file") } + + before do + allow(digest).to receive(:<<) + allow(File).to receive(:open).and_yield(file) + allow(file).to receive(:read).and_return(nil) + end + + [Digest::MD5, Digest::SHA1].each do |klass| + short_name = klass.to_s.split("::").last.downcase + + context "using #{short_name} digest" do + subject { described_class.new(source, destination, short_name.to_sym => checksum_expected_value) } + + context "that matches expected value" do + before do + expect(klass).to receive(:new).and_return(digest) + expect(digest).to receive(:hexdigest).and_return(checksum_expected_value) + end + + it "should not raise an exception" do + expect(subject.download!).to be_true + end + end + + context "that does not match expected value" do + before do + expect(klass).to receive(:new).and_return(digest) + expect(digest).to receive(:hexdigest).and_return(checksum_invalid_value) + end + + it "should raise an exception" do + expect{ subject.download! }.to raise_error(Vagrant::Errors::DownloaderChecksumError) + end + end + end + end + + context "using both md5 and sha1 digests" do + context "that both match expected values" do + subject { described_class.new(source, destination, md5: checksum_expected_value, sha1: checksum_expected_value) } + + before do + expect(Digest::MD5).to receive(:new).and_return(digest) + expect(Digest::SHA1).to receive(:new).and_return(digest) + expect(digest).to receive(:hexdigest).and_return(checksum_expected_value).exactly(2).times + end + + it "should not raise an exception" do + expect(subject.download!).to be_true + end + end + + context "that only sha1 matches expected value" do + subject { described_class.new(source, destination, md5: checksum_invalid_value, sha1: checksum_expected_value) } + + before do + allow(Digest::MD5).to receive(:new).and_return(digest) + allow(Digest::SHA1).to receive(:new).and_return(digest) + expect(digest).to receive(:hexdigest).and_return(checksum_expected_value).at_least(:once) + end + + it "should raise an exception" do + expect{ subject.download! }.to raise_error(Vagrant::Errors::DownloaderChecksumError) + end + end + + context "that only md5 matches expected value" do + subject { described_class.new(source, destination, md5: checksum_expected_value, sha1: checksum_invalid_value) } + + before do + allow(Digest::MD5).to receive(:new).and_return(digest) + allow(Digest::SHA1).to receive(:new).and_return(digest) + expect(digest).to receive(:hexdigest).and_return(checksum_expected_value).at_least(:once) + end + + it "should raise an exception" do + expect{ subject.download! }.to raise_error(Vagrant::Errors::DownloaderChecksumError) + end + end + + context "that none match expected value" do + subject { described_class.new(source, destination, md5: checksum_invalid_value, sha1: checksum_invalid_value) } + + before do + allow(Digest::MD5).to receive(:new).and_return(digest) + allow(Digest::SHA1).to receive(:new).and_return(digest) + expect(digest).to receive(:hexdigest).and_return(checksum_expected_value).at_least(:once) + end + + it "should raise an exception" do + expect{ subject.download! }.to raise_error(Vagrant::Errors::DownloaderChecksumError) + end + end + end + end end describe "#head" do diff --git a/website/source/docs/provisioning/shell.html.md b/website/source/docs/provisioning/shell.html.md index 8a272f4e5..3530c181d 100644 --- a/website/source/docs/provisioning/shell.html.md +++ b/website/source/docs/provisioning/shell.html.md @@ -79,6 +79,10 @@ The remainder of the available options are optional: enable auto-login for Windows as the user must be logged in for interactive mode to work. +* `md5` (string) - MD5 checksum used to validate remotely downloaded shell files. + +* `sha1` (string) - SHA1 checksum used to validate remotely downloaded shell files. + ## Inline Scripts