Commit f67139ae authored by David Cole's avatar David Cole
Browse files

Improve FILE(DOWNLOAD) and ExternalProject.

Improve FILE(DOWNLOAD ...):

- Add percent complete progress output to the FILE DOWNLOAD
  command. This progress output is off by default to
  preserve existing behavior. To turn it on, pass
  SHOW_PROGRESS as an argument.

- Add EXPECTED_MD5 argument. Verify that the downloaded
  file has the expected md5 sum after download is complete.

- Add documentation for SHOW_PROGRESS and EXPECTED_MD5.

  When the destination file exists already and has the
  expected md5 sum, then do not bother re-downloading
  the file. ("Short circuit" return.)

  Also, add a test that checks for the status output
  indicating that the short circuit behavior is actually
  occurring. Use a binary file for the test so that the
  md5 sum is guaranteed to be the same on all platforms
  regardless of "shifting text file line ending" issues.

Improve ExternalProject:

- Add argument URL_MD5.

- Add verify step that compares md5 sum of .tar.gz file
  before extracting it.

- Add md5 check t...
parent 282a119e
......@@ -17,6 +17,7 @@
# [SVN_REPOSITORY url] # URL of Subversion repo
# [SVN_REVISION rev] # Revision to checkout from Subversion repo
# [URL /.../src.tgz] # Full path or URL of source
# [URL_MD5 md5] # MD5 checksum of file at URL
# [TIMEOUT seconds] # Time allowed for file download operations
# #--Update/Patch step----------
# [UPDATE_COMMAND cmd...] # Source work-tree update command
......@@ -111,19 +112,19 @@
# License text for the above reference.)
# Pre-compute a regex to match documented keywords for each command.
file(STRINGS "${CMAKE_CURRENT_LIST_FILE}" lines LIMIT_COUNT 100
REGEX "^# ( \\[[A-Z_]+ [^]]*\\] +#.*$|[A-Za-z_]+\\()")
file(STRINGS "${CMAKE_CURRENT_LIST_FILE}" lines LIMIT_COUNT 103
REGEX "^# ( \\[[A-Z0-9_]+ [^]]*\\] +#.*$|[A-Za-z0-9_]+\\()")
foreach(line IN LISTS lines)
if("${line}" MATCHES "^# [A-Za-z_]+\\(")
if("${line}" MATCHES "^# [A-Za-z0-9_]+\\(")
if(_ep_func)
set(_ep_keywords_${_ep_func} "${_ep_keywords_${_ep_func}})$")
endif()
string(REGEX REPLACE "^# ([A-Za-z_]+)\\(.*" "\\1" _ep_func "${line}")
string(REGEX REPLACE "^# ([A-Za-z0-9_]+)\\(.*" "\\1" _ep_func "${line}")
#message("function [${_ep_func}]")
set(_ep_keywords_${_ep_func} "^(")
set(_ep_keyword_sep)
else()
string(REGEX REPLACE "^# \\[([A-Z_]+) .*" "\\1" _ep_key "${line}")
string(REGEX REPLACE "^# \\[([A-Z0-9_]+) .*" "\\1" _ep_key "${line}")
#message(" keyword [${_ep_key}]")
set(_ep_keywords_${_ep_func}
"${_ep_keywords_${_ep_func}}${_ep_keyword_sep}${_ep_key}")
......@@ -148,7 +149,7 @@ function(_ep_parse_arguments f name ns args)
foreach(arg IN LISTS args)
set(is_value 1)
if(arg MATCHES "^[A-Z][A-Z_][A-Z_]+$" AND
if(arg MATCHES "^[A-Z][A-Z0-9_][A-Z0-9_]+$" AND
NOT ((arg STREQUAL "${key}") AND (key STREQUAL "COMMAND")) AND
NOT arg MATCHES "^(TRUE|FALSE)$")
if(_ep_keywords_${f} AND arg MATCHES "${_ep_keywords_${f}}")
......@@ -203,7 +204,7 @@ define_property(DIRECTORY PROPERTY "EP_PREFIX" INHERITED
)
function(_ep_write_downloadfile_script script_filename remote local timeout)
function(_ep_write_downloadfile_script script_filename remote local timeout md5)
if(timeout)
set(timeout_args TIMEOUT ${timeout})
set(timeout_msg "${timeout} seconds")
......@@ -212,6 +213,12 @@ function(_ep_write_downloadfile_script script_filename remote local timeout)
set(timeout_msg "none")
endif()
if(md5)
set(md5_args EXPECTED_MD5 ${md5})
else()
set(md5_args "# no EXPECTED_MD5")
endif()
file(WRITE ${script_filename}
"message(STATUS \"downloading...
src='${remote}'
......@@ -221,6 +228,8 @@ function(_ep_write_downloadfile_script script_filename remote local timeout)
file(DOWNLOAD
\"${remote}\"
\"${local}\"
SHOW_PROGRESS
${md5_args}
${timeout_args}
STATUS status
LOG log)
......@@ -243,6 +252,51 @@ message(STATUS \"downloading... done\")
endfunction(_ep_write_downloadfile_script)
function(_ep_write_verifyfile_script script_filename local md5)
file(WRITE ${script_filename}
"message(STATUS \"verifying file...
file='${local}'\")
set(verified 0)
# If an expected md5 checksum exists, compare against it:
#
if(NOT \"${md5}\" STREQUAL \"\")
execute_process(COMMAND \${CMAKE_COMMAND} -E md5sum \"${local}\"
OUTPUT_VARIABLE ov
OUTPUT_STRIP_TRAILING_WHITESPACE
RESULT_VARIABLE rv)
if(NOT rv EQUAL 0)
message(FATAL_ERROR \"error: computing md5sum of '${local}' failed\")
endif()
string(REGEX MATCH \"^([0-9A-Fa-f]+)\" md5_actual \"\${ov}\")
string(TOLOWER \"\${md5_actual}\" md5_actual)
string(TOLOWER \"${md5}\" md5)
if(NOT \"\${md5}\" STREQUAL \"\${md5_actual}\")
message(FATAL_ERROR \"error: md5sum of '${local}' does not match expected value
md5_expected: \${md5}
md5_actual: \${md5_actual}
\")
endif()
set(verified 1)
endif()
if(verified)
message(STATUS \"verifying file... done\")
else()
message(STATUS \"verifying file... warning: did not verify file - no URL_MD5 checksum argument? corrupt file?\")
endif()
"
)
endfunction(_ep_write_verifyfile_script)
function(_ep_write_extractfile_script script_filename filename tmp directory)
set(args "")
......@@ -678,9 +732,10 @@ function(_ep_add_download_command name)
list(APPEND depends ${stamp_dir}/${name}-svninfo.txt)
elseif(url)
get_filename_component(work_dir "${source_dir}" PATH)
get_property(md5 TARGET ${name} PROPERTY _EP_URL_MD5)
set(repository "external project URL")
set(module "${url}")
set(tag "")
set(tag "${md5}")
configure_file(
"${CMAKE_ROOT}/Modules/RepositoryInfo.txt.in"
"${stamp_dir}/${name}-urlinfo.txt"
......@@ -701,14 +756,16 @@ function(_ep_add_download_command name)
endif()
set(file ${download_dir}/${fname})
get_property(timeout TARGET ${name} PROPERTY _EP_TIMEOUT)
_ep_write_downloadfile_script("${stamp_dir}/download-${name}.cmake" "${url}" "${file}" "${timeout}")
_ep_write_downloadfile_script("${stamp_dir}/download-${name}.cmake" "${url}" "${file}" "${timeout}" "${md5}")
set(cmd ${CMAKE_COMMAND} -P ${stamp_dir}/download-${name}.cmake
COMMAND)
set(comment "Performing download step (download and extract) for '${name}'")
set(comment "Performing download step (download, verify and extract) for '${name}'")
else()
set(file "${url}")
set(comment "Performing download step (extract) for '${name}'")
set(comment "Performing download step (verify and extract) for '${name}'")
endif()
_ep_write_verifyfile_script("${stamp_dir}/verify-${name}.cmake" "${file}" "${md5}")
list(APPEND cmd ${CMAKE_COMMAND} -P ${stamp_dir}/verify-${name}.cmake)
# TODO: Support other archive formats.
_ep_write_extractfile_script("${stamp_dir}/extract-${name}.cmake" "${file}" "${tmp_dir}" "${source_dir}")
list(APPEND cmd ${CMAKE_COMMAND} -P ${stamp_dir}/extract-${name}.cmake)
......
......@@ -2443,7 +2443,8 @@ namespace{
fout->write(chPtr, realsize);
return realsize;
}
static size_t
cmFileCommandCurlDebugCallback(CURL *, curl_infotype, char *chPtr,
size_t size, void *data)
......@@ -2456,6 +2457,69 @@ namespace{
}
class cURLProgressHelper
{
public:
cURLProgressHelper(cmFileCommand *fc)
{
this->CurrentPercentage = -1;
this->FileCommand = fc;
}
bool UpdatePercentage(double value, double total, std::string &status)
{
int OldPercentage = this->CurrentPercentage;
if (0.0 == total)
{
this->CurrentPercentage = 100;
}
else
{
this->CurrentPercentage = static_cast<int>(value/total*100.0 + 0.5);
}
bool updated = (OldPercentage != this->CurrentPercentage);
if (updated)
{
cmOStringStream oss;
oss << "[download " << this->CurrentPercentage << "% complete]";
status = oss.str();
}
return updated;
}
cmFileCommand *GetFileCommand()
{
return this->FileCommand;
}
private:
int CurrentPercentage;
cmFileCommand *FileCommand;
};
static int
cmFileCommandCurlProgressCallback(void *clientp,
double dltotal, double dlnow,
double ultotal, double ulnow)
{
cURLProgressHelper *helper =
reinterpret_cast<cURLProgressHelper *>(clientp);
std::string status;
if (helper->UpdatePercentage(dlnow, dltotal, status))
{
cmFileCommand *fc = helper->GetFileCommand();
cmMakefile *mf = fc->GetMakefile();
mf->DisplayStatus(status.c_str(), -1);
}
return 0;
}
}
#endif
......@@ -2469,8 +2533,8 @@ namespace {
cURLEasyGuard(CURL * easy)
: Easy(easy)
{}
~cURLEasyGuard(void)
~cURLEasyGuard(void)
{
if (this->Easy)
{
......@@ -2491,6 +2555,7 @@ namespace {
}
#endif
bool
cmFileCommand::HandleDownloadCommand(std::vector<std::string>
const& args)
......@@ -2508,9 +2573,13 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
++i;
std::string file = *i;
++i;
long timeout = 0;
std::string verboseLog;
std::string statusVar;
std::string expectedMD5sum;
bool showProgress = false;
while(i != args.end())
{
if(*i == "TIMEOUT")
......@@ -2549,9 +2618,65 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
}
statusVar = *i;
}
else if(*i == "EXPECTED_MD5")
{
++i;
if( i == args.end())
{
this->SetError("FILE(DOWNLOAD url file EXPECTED_MD5 sum) missing "
"sum value for EXPECTED_MD5.");
return false;
}
expectedMD5sum = cmSystemTools::LowerCase(*i);
}
else if(*i == "SHOW_PROGRESS")
{
showProgress = true;
}
++i;
}
// If file exists already, and caller specified an expected md5 sum,
// and the existing file already has the expected md5 sum, then simply
// return.
//
if(cmSystemTools::FileExists(file.c_str()) &&
!expectedMD5sum.empty())
{
char computedMD5[32];
if (!cmSystemTools::ComputeFileMD5(file.c_str(), computedMD5))
{
this->SetError("FILE(DOWNLOAD ) error; cannot compute MD5 sum on "
"pre-existing file");
return false;
}
std::string actualMD5sum = cmSystemTools::LowerCase(
std::string(computedMD5, 32));
if (expectedMD5sum == actualMD5sum)
{
this->Makefile->DisplayStatus(
"FILE(DOWNLOAD ) returning early: file already exists with "
"expected MD5 sum", -1);
if(statusVar.size())
{
cmOStringStream result;
result << (int)0 << ";\""
"returning early: file already exists with expected MD5 sum\"";
this->Makefile->AddDefinition(statusVar.c_str(),
result.str().c_str());
}
return true;
}
}
// Make sure parent directory exists so we can write to the file
// as we receive downloaded bits from curl...
//
std::string dir = cmSystemTools::GetFilenamePath(file.c_str());
if(!cmSystemTools::FileExists(dir.c_str()) &&
!cmSystemTools::MakeDirectory(dir.c_str()))
......@@ -2570,6 +2695,7 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
"file for write.");
return false;
}
::CURL *curl;
::curl_global_init(CURL_GLOBAL_DEFAULT);
curl = ::curl_easy_init();
......@@ -2585,28 +2711,31 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
::CURLcode res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
if (res != CURLE_OK)
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set url: ";
errstring += ::curl_easy_strerror(res);
std::string errstring = "FILE(DOWNLOAD ) error; cannot set url: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
res = ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
cmFileCommandWriteMemoryCallback);
cmFileCommandWriteMemoryCallback);
if (res != CURLE_OK)
{
std::string errstring =
"FILE(DOWNLOAD ) error; cannot set write function: ";
errstring += ::curl_easy_strerror(res);
{
std::string errstring =
"FILE(DOWNLOAD ) error; cannot set write function: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
res = ::curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION,
cmFileCommandCurlDebugCallback);
cmFileCommandCurlDebugCallback);
if (res != CURLE_OK)
{
std::string errstring =
"FILE(DOWNLOAD ) error; cannot set debug function: ";
errstring += ::curl_easy_strerror(res);
std::string errstring =
"FILE(DOWNLOAD ) error; cannot set debug function: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
......@@ -2618,14 +2747,16 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set write data: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
res = ::curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void *)&chunkDebug);
if (res != CURLE_OK)
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set write data: ";
std::string errstring = "FILE(DOWNLOAD ) error; cannot set debug data: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
......@@ -2637,24 +2768,70 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set verbose: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
}
if(timeout > 0)
{
res = ::curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout );
if (res != CURLE_OK)
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set verbose: ";
std::string errstring = "FILE(DOWNLOAD ) error; cannot set timeout: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
}
// Need the progress helper's scope to last through the duration of
// the curl_easy_perform call... so this object is declared at function
// scope intentionally, rather than inside the "if(showProgress)"
// block...
//
cURLProgressHelper helper(this);
if(showProgress)
{
res = ::curl_easy_setopt(curl,
CURLOPT_NOPROGRESS, 0);
if (res != CURLE_OK)
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set noprogress value: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
res = ::curl_easy_setopt(curl,
CURLOPT_PROGRESSFUNCTION, cmFileCommandCurlProgressCallback);
if (res != CURLE_OK)
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set progress function: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
res = ::curl_easy_setopt(curl,
CURLOPT_PROGRESSDATA, reinterpret_cast<void*>(&helper));
if (res != CURLE_OK)
{
std::string errstring = "FILE(DOWNLOAD ) error; cannot set progress data: ";
errstring += ::curl_easy_strerror(res);
this->SetError(errstring.c_str());
return false;
}
}
res = ::curl_easy_perform(curl);
/* always cleanup */
g_curl.release();
::curl_easy_cleanup(curl);
if(statusVar.size())
{
cmOStringStream result;
......@@ -2662,7 +2839,44 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
this->Makefile->AddDefinition(statusVar.c_str(),
result.str().c_str());
}
::curl_global_cleanup();
// Explicitly flush/close so we can measure the md5 accurately.
//
fout.flush();
fout.close();
// Verify MD5 sum if requested:
//
if (!expectedMD5sum.empty())
{
char computedMD5[32];
if (!cmSystemTools::ComputeFileMD5(file.c_str(), computedMD5))
{
this->SetError("FILE(DOWNLOAD ) error; cannot compute MD5 sum on "
"downloaded file");
return false;
}
std::string actualMD5sum = cmSystemTools::LowerCase(
std::string(computedMD5, 32));
if (expectedMD5sum != actualMD5sum)
{
cmOStringStream oss;
oss << "FILE(DOWNLOAD ) error; expected and actual MD5 sums differ"
<< std::endl
<< " for file: [" << file << "]" << std::endl
<< " expected MD5 sum: [" << expectedMD5sum << "]" << std::endl
<< " actual MD5 sum: [" << actualMD5sum << "]" << std::endl
;
this->SetError(oss.str().c_str());
return false;
}
}
if(chunkDebug.size())
{
chunkDebug.push_back(0);
......@@ -2680,6 +2894,7 @@ cmFileCommand::HandleDownloadCommand(std::vector<std::string>
this->Makefile->AddDefinition(verboseLog.c_str(),
&*chunkDebug.begin());
}
return true;
#else
this->SetError("FILE(DOWNLOAD ) "
......
......@@ -80,7 +80,8 @@ public:
" file(RELATIVE_PATH variable directory file)\n"
" file(TO_CMAKE_PATH path result)\n"
" file(TO_NATIVE_PATH path result)\n"
" file(DOWNLOAD url file [TIMEOUT timeout] [STATUS status] [LOG log])\n"
" file(DOWNLOAD url file [TIMEOUT timeout] [STATUS status] [LOG log]\n"
" [EXPECTED_MD5 sum] [SHOW_PROGRESS])\n"
"WRITE will write a message into a file called 'filename'. It "
"overwrites the file if it already exists, and creates the file "
"if it does not exist.\n"
......@@ -152,7 +153,12 @@ public:
"and the second element is a string value for the error. A 0 "
"numeric error means no error in the operation. "
"If TIMEOUT time is specified, the operation will "
"timeout after time seconds, time should be specified as an integer."
"timeout after time seconds, time should be specified as an integer. "
"If EXPECTED_MD5 sum is specified, the operation will verify that the "
"downloaded file's actual md5 sum matches the expected value. If it "
"does not match, the operation fails with an error. "
"If SHOW_PROGRESS is specified, progress information will be printed "
"as status messages until the operation is complete."
"\n"
"The file() command also provides COPY and INSTALL signatures:\n"
" file(<COPY|INSTALL> files... DESTINATION <dir>\n"
......
......@@ -28,6 +28,11 @@ AddCMakeTest(Math "")
AddCMakeTest(CMakeMinimumRequired "")
AddCMakeTest(CompilerIdVendor "")
AddCMakeTest(FileDownload "")
set_property(TEST CMake.FileDownload PROPERTY
PASS_REGULAR_EXPRESSION "file already exists with expected MD5 sum"
)
if(HAVE_ELF_H)
AddCMakeTest(ELF "")
endif()
......
set(url "file://@CMAKE_CURRENT_SOURCE_DIR@/FileDownloadInput.png")
set(dir "@CMAKE_CURRENT_BINARY_DIR@/downloads")
message(STATUS "FileDownload:1")
file(DOWNLOAD
${url}
${dir}/file1.png
TIMEOUT 2
)
message(STATUS "FileDownload:2")
file(DOWNLOAD
${url}
${dir}/file2.png
TIMEOUT 2
SHOW_PROGRESS
)
# Two calls in a row, exactly the same arguments.
# Since downloaded file should exist already for 2nd call,
# the 2nd call will short-circuit and return early...
#
if(EXISTS ${dir}/file3.png)
file(REMOVE ${dir}/file3.png)
endif()
message(STATUS "FileDownload:3")
file(DOWNLOAD
${url}
${dir}/file3.png
TIMEOUT 2
EXPECTED_MD5 d16778650db435bda3a8c3435c3ff5d1
)
message(STATUS "FileDownload:4")
file(DOWNLOAD
${url}
${dir}/file3.png
TIMEOUT 2
EXPECTED_MD5 d16778650db435bda3a8c3435c3ff5d1
)
......@@ -65,7 +65,9 @@ ExternalProject_Add(${proj}
SVN_REPOSITORY ""
SVN_REVISION ""
TEST_COMMAND ""
TIMEOUT ""
URL ""
URL_MD5 ""
UPDATE_COMMAND ""
)
......@@ -96,6 +98,7 @@ endif()
set(proj TutorialStep1-LocalTAR)
ExternalProject_Add(${proj}
URL "${CMAKE_CURRENT_SOURCE_DIR}/Step1.tar"
URL_MD5 a87c5b47c0201c09ddfe1d5738fdb1e3
LIST_SEPARATOR ::
PATCH_COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/Step1Patch.cmake