Commit d308e944 authored by Brad King's avatar Brad King Committed by Kitware Robot
Browse files

Merge topic 'glob_configure_depends'

6c4f8b45 Adjust help documentation for file(GLOB), add topic notes
20612978 Add tests for `file(GLOB)` CONFIGURE_DEPENDS flag
3f4b81f5 Add glob verify support to XCode, VS, Ninja, and Makefile generators
ca0befc2 Add `CONFIGURE_DEPENDS` flag support to cmFileCommand::HandleGlobCommand
599c93e2

 Add cmGlobVerificationManager class, integrate with cmake and cmState
Acked-by: Kitware Robot's avatarKitware Robot <kwrobot@kitware.com>
Merge-request: !1767
parents 2e49bb64 6c4f8b45
Pipeline #97834 passed with stage
in 0 seconds
......@@ -98,10 +98,10 @@ command.
::
file(GLOB <variable>
[LIST_DIRECTORIES true|false] [RELATIVE <path>]
[LIST_DIRECTORIES true|false] [RELATIVE <path>] [CONFIGURE_DEPENDS]
[<globbing-expressions>...])
file(GLOB_RECURSE <variable> [FOLLOW_SYMLINKS]
[LIST_DIRECTORIES true|false] [RELATIVE <path>]
[LIST_DIRECTORIES true|false] [RELATIVE <path>] [CONFIGURE_DEPENDS]
[<globbing-expressions>...])
Generate a list of files that match the ``<globbing-expressions>`` and
......@@ -110,6 +110,11 @@ regular expressions, but much simpler. If ``RELATIVE`` flag is
specified, the results will be returned as relative paths to the given
path. The results will be ordered lexicographically.
If the ``CONFIGURE_DEPENDS`` flag is specified, CMake will add logic
to the main build system check target to rerun the flagged ``GLOB`` commands
at build time. If any of the outputs change, CMake will regenerate the build
system.
By default ``GLOB`` lists directories - directories are omitted in result if
``LIST_DIRECTORIES`` is set to false.
......@@ -118,6 +123,10 @@ By default ``GLOB`` lists directories - directories are omitted in result if
your source tree. If no CMakeLists.txt file changes when a source is
added or removed then the generated build system cannot know when to
ask CMake to regenerate.
The ``CONFIGURE_DEPENDS`` flag may not work reliably on all generators, or if
a new generator is added in the future that cannot support it, projects using
it will be stuck. Even if ``CONFIGURE_DEPENDS`` works reliably, there is
still a cost to perform the check on every rebuild.
Examples of globbing expressions include::
......
glob_configure_depends
----------------------
* The :command:`file(GLOB)` and :command:`file(GLOB_RECURSE)` commands
learned a new flag ``CONFIGURE_DEPENDS`` which enables expression of
build system dependency on globbed directory's contents.
......@@ -246,6 +246,8 @@ set(SRCS
cmGlobalGeneratorFactory.h
cmGlobalUnixMakefileGenerator3.cxx
cmGlobalUnixMakefileGenerator3.h
cmGlobVerificationManager.cxx
cmGlobVerificationManager.h
cmGraphAdjacencyList.h
cmGraphVizWriter.cxx
cmGraphVizWriter.h
......
......@@ -758,7 +758,11 @@ bool cmFileCommand::HandleGlobCommand(std::vector<std::string> const& args,
}
std::vector<std::string> files;
bool configureDepends = false;
bool warnConfigureLate = false;
bool warnFollowedSymlinks = false;
const cmake::WorkingMode workingMode =
this->Makefile->GetCMakeInstance()->GetWorkingMode();
while (i != args.end()) {
if (*i == "LIST_DIRECTORIES") {
++i;
......@@ -807,6 +811,27 @@ bool cmFileCommand::HandleGlobCommand(std::vector<std::string> const& args,
this->SetError("GLOB requires a glob expression after the directory.");
return false;
}
} else if (*i == "CONFIGURE_DEPENDS") {
// Generated build system depends on glob results
if (!configureDepends && warnConfigureLate) {
this->Makefile->IssueMessage(
cmake::AUTHOR_WARNING,
"CONFIGURE_DEPENDS flag was given after a glob expression was "
"already evaluated.");
}
if (workingMode != cmake::NORMAL_MODE) {
this->Makefile->IssueMessage(
cmake::FATAL_ERROR,
"CONFIGURE_DEPENDS is invalid for script and find package modes.");
return false;
}
configureDepends = true;
++i;
if (i == args.end()) {
this->SetError(
"GLOB requires a glob expression after CONFIGURE_DEPENDS.");
return false;
}
} else {
std::string expr = *i;
if (!cmsys::SystemTools::FileIsFullPath(*i)) {
......@@ -849,6 +874,19 @@ bool cmFileCommand::HandleGlobCommand(std::vector<std::string> const& args,
std::vector<std::string>& foundFiles = g.GetFiles();
files.insert(files.end(), foundFiles.begin(), foundFiles.end());
if (configureDepends) {
std::sort(foundFiles.begin(), foundFiles.end());
foundFiles.erase(std::unique(foundFiles.begin(), foundFiles.end()),
foundFiles.end());
this->Makefile->GetCMakeInstance()->AddGlobCacheEntry(
recurse, (recurse ? g.GetRecurseListDirs() : g.GetListDirs()),
(recurse ? g.GetRecurseThroughSymlinks() : false),
(g.GetRelative() ? g.GetRelative() : ""), expr, foundFiles, variable,
this->Makefile->GetBacktrace());
} else {
warnConfigureLate = true;
}
++i;
}
}
......
/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying
file Copyright.txt or https://cmake.org/licensing for details. */
#include "cmGlobVerificationManager.h"
#include "cmsys/FStream.hxx"
#include <sstream>
#include "cmGeneratedFileStream.h"
#include "cmListFileCache.h"
#include "cmSystemTools.h"
#include "cmVersion.h"
#include "cmake.h"
bool cmGlobVerificationManager::SaveVerificationScript(const std::string& path)
{
if (this->Cache.empty()) {
return true;
}
std::string scriptFile = path;
scriptFile += cmake::GetCMakeFilesDirectory();
std::string stampFile = scriptFile;
cmSystemTools::MakeDirectory(scriptFile);
scriptFile += "/VerifyGlobs.cmake";
stampFile += "/cmake.verify_globs";
cmGeneratedFileStream verifyScriptFile(scriptFile.c_str());
verifyScriptFile.SetCopyIfDifferent(true);
if (!verifyScriptFile) {
cmSystemTools::Error("Unable to open verification script file for save. ",
scriptFile.c_str());
cmSystemTools::ReportLastSystemError("");
return false;
}
verifyScriptFile << std::boolalpha;
verifyScriptFile << "# CMAKE generated file: DO NOT EDIT!\n"
<< "# Generated by CMake Version "
<< cmVersion::GetMajorVersion() << "."
<< cmVersion::GetMinorVersion() << "\n";
for (auto const& i : this->Cache) {
CacheEntryKey k = std::get<0>(i);
CacheEntryValue v = std::get<1>(i);
if (!v.Initialized) {
continue;
}
verifyScriptFile << "\n";
for (auto const& bt : v.Backtraces) {
verifyScriptFile << "# " << std::get<0>(bt);
std::get<1>(bt).PrintTitle(verifyScriptFile);
verifyScriptFile << "\n";
}
k.PrintGlobCommand(verifyScriptFile, "NEW_GLOB");
verifyScriptFile << "\n";
verifyScriptFile << "set(OLD_GLOB\n";
for (const std::string& file : v.Files) {
verifyScriptFile << " \"" << file << "\"\n";
}
verifyScriptFile << " )\n";
verifyScriptFile << "if(NOT \"${NEW_GLOB}\" STREQUAL \"${OLD_GLOB}\")\n"
<< " message(\"-- GLOB mismatch!\")\n"
<< " file(TOUCH_NOCREATE \"" << stampFile << "\")\n"
<< "endif()\n";
}
verifyScriptFile.Close();
cmsys::ofstream verifyStampFile(stampFile.c_str());
if (!verifyStampFile) {
cmSystemTools::Error("Unable to open verification stamp file for write. ",
stampFile.c_str());
return false;
}
verifyStampFile << "# This file is generated by CMake for checking of the "
"VerifyGlobs.cmake file\n";
this->VerifyScript = scriptFile;
this->VerifyStamp = stampFile;
return true;
}
bool cmGlobVerificationManager::DoWriteVerifyTarget() const
{
return !this->VerifyScript.empty() && !this->VerifyStamp.empty();
}
bool cmGlobVerificationManager::CacheEntryKey::operator<(
const CacheEntryKey& r) const
{
if (this->Recurse < r.Recurse) {
return true;
}
if (this->Recurse > r.Recurse) {
return false;
}
if (this->ListDirectories < r.ListDirectories) {
return true;
}
if (this->ListDirectories > r.ListDirectories) {
return false;
}
if (this->FollowSymlinks < r.FollowSymlinks) {
return true;
}
if (this->FollowSymlinks > r.FollowSymlinks) {
return false;
}
if (this->Relative < r.Relative) {
return true;
}
if (this->Relative > r.Relative) {
return false;
}
if (this->Expression < r.Expression) {
return true;
}
if (this->Expression > r.Expression) {
return false;
}
return false;
}
void cmGlobVerificationManager::CacheEntryKey::PrintGlobCommand(
std::ostream& out, const std::string& cmdVar)
{
out << "file(GLOB" << (this->Recurse ? "_RECURSE " : " ");
out << cmdVar << " ";
if (this->Recurse && this->FollowSymlinks) {
out << "FOLLOW_SYMLINKS ";
}
out << "LIST_DIRECTORIES " << this->ListDirectories << " ";
if (!this->Relative.empty()) {
out << "RELATIVE \"" << this->Relative << "\" ";
}
out << "\"" << this->Expression << "\")";
}
void cmGlobVerificationManager::AddCacheEntry(
const bool recurse, const bool listDirectories, const bool followSymlinks,
const std::string& relative, const std::string& expression,
const std::vector<std::string>& files, const std::string& variable,
const cmListFileBacktrace& backtrace)
{
CacheEntryKey key = CacheEntryKey(recurse, listDirectories, followSymlinks,
relative, expression);
CacheEntryValue& value = this->Cache[key];
if (!value.Initialized) {
value.Files = files;
value.Initialized = true;
value.Backtraces.emplace_back(variable, backtrace);
} else if (value.Initialized && value.Files != files) {
std::ostringstream message;
message << std::boolalpha;
message << "The glob expression\n";
key.PrintGlobCommand(message, variable);
backtrace.PrintTitle(message);
message << "\nwas already present in the glob cache but the directory\n"
"contents have changed during the configuration run.\n";
message << "Matching glob expressions:";
for (auto const& bt : value.Backtraces) {
message << "\n " << std::get<0>(bt);
std::get<1>(bt).PrintTitle(message);
}
cmSystemTools::Error(message.str().c_str());
} else {
value.Backtraces.emplace_back(variable, backtrace);
}
}
/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying
file Copyright.txt or https://cmake.org/licensing for details. */
#ifndef cmGlobVerificationManager_h
#define cmGlobVerificationManager_h
#include "cmConfigure.h" // IWYU pragma: keep
#include "cmListFileCache.h"
#include <iosfwd>
#include <map>
#include <string>
#include <utility>
#include <vector>
/** \class cmGlobVerificationManager
* \brief Class for expressing build-time dependencies on glob expressions.
*
* Generates a CMake script which verifies glob outputs during prebuild.
*
*/
class cmGlobVerificationManager
{
public:
cmGlobVerificationManager() {}
protected:
///! Save verification script for given makefile.
///! Saves to output <path>/<CMakeFilesDirectory>/VerifyGlobs.cmake
bool SaveVerificationScript(const std::string& path);
///! Add an entry into the glob cache
void AddCacheEntry(bool recurse, bool listDirectories, bool followSymlinks,
const std::string& relative,
const std::string& expression,
const std::vector<std::string>& files,
const std::string& variable,
const cmListFileBacktrace& bt);
///! Check targets should be written in generated build system.
bool DoWriteVerifyTarget() const;
///! Get the paths to the generated script and stamp files
std::string const& GetVerifyScript() const { return this->VerifyScript; }
std::string const& GetVerifyStamp() const { return this->VerifyStamp; }
private:
struct CacheEntryKey
{
const bool Recurse;
const bool ListDirectories;
const bool FollowSymlinks;
const std::string Relative;
const std::string Expression;
CacheEntryKey(const bool rec, const bool l, const bool s,
const std::string& rel, const std::string& e)
: Recurse(rec)
, ListDirectories(l)
, FollowSymlinks(s)
, Relative(rel)
, Expression(e)
{
}
bool operator<(const CacheEntryKey& r) const;
void PrintGlobCommand(std::ostream& out, const std::string& cmdVar);
};
struct CacheEntryValue
{
bool Initialized;
std::vector<std::string> Files;
std::vector<std::pair<std::string, cmListFileBacktrace>> Backtraces;
CacheEntryValue()
: Initialized(false)
{
}
};
typedef std::map<CacheEntryKey, CacheEntryValue> CacheEntryMap;
CacheEntryMap Cache;
std::string VerifyScript;
std::string VerifyStamp;
// Only cmState should be able to add cache values.
// cmGlobVerificationManager should never be used directly.
friend class cmState; // allow access to add cache values
};
#endif
......@@ -475,6 +475,7 @@ cmGlobalNinjaGenerator::cmGlobalNinjaGenerator(cmake* cm)
, PolicyCMP0058(cmPolicies::WARN)
, NinjaSupportsConsolePool(false)
, NinjaSupportsImplicitOuts(false)
, NinjaSupportsManifestRestat(false)
, NinjaSupportsDyndeps(0)
{
#ifdef _WIN32
......@@ -597,6 +598,9 @@ void cmGlobalNinjaGenerator::CheckNinjaFeatures()
this->NinjaSupportsImplicitOuts = !cmSystemTools::VersionCompare(
cmSystemTools::OP_LESS, this->NinjaVersion.c_str(),
this->RequiredNinjaVersionForImplicitOuts().c_str());
this->NinjaSupportsManifestRestat = !cmSystemTools::VersionCompare(
cmSystemTools::OP_LESS, this->NinjaVersion.c_str(),
RequiredNinjaVersionForManifestRestat().c_str());
{
// Our ninja branch adds ".dyndep-#" to its version number,
// where '#' is a feature-specific version number. Extract it.
......@@ -1361,6 +1365,7 @@ void cmGlobalNinjaGenerator::WriteTargetRebuildManifest(std::ostream& os)
/*generator=*/true);
cmNinjaDeps implicitDeps;
cmNinjaDeps explicitDeps;
for (cmLocalGenerator* localGen : this->LocalGenerators) {
std::vector<std::string> const& lf =
localGen->GetMakefile()->GetListFiles();
......@@ -1370,10 +1375,6 @@ void cmGlobalNinjaGenerator::WriteTargetRebuildManifest(std::ostream& os)
}
implicitDeps.push_back(this->CMakeCacheFile);
std::sort(implicitDeps.begin(), implicitDeps.end());
implicitDeps.erase(std::unique(implicitDeps.begin(), implicitDeps.end()),
implicitDeps.end());
cmNinjaVars variables;
// Use 'console' pool to get non buffered output of the CMake re-run call
// Available since Ninja 1.5
......@@ -1381,12 +1382,71 @@ void cmGlobalNinjaGenerator::WriteTargetRebuildManifest(std::ostream& os)
variables["pool"] = "console";
}
cmake* cm = this->GetCMakeInstance();
if (this->SupportsManifestRestat() && cm->DoWriteGlobVerifyTarget()) {
std::ostringstream verify_cmd;
verify_cmd << lg->ConvertToOutputFormat(cmSystemTools::GetCMakeCommand(),
cmOutputConverter::SHELL)
<< " -P "
<< lg->ConvertToOutputFormat(cm->GetGlobVerifyScript(),
cmOutputConverter::SHELL);
WriteRule(*this->RulesFileStream, "VERIFY_GLOBS", verify_cmd.str(),
"Re-checking globbed directories...",
"Rule for re-checking globbed directories.",
/*depfile=*/"",
/*deptype=*/"",
/*rspfile=*/"",
/*rspcontent*/ "",
/*restat=*/"",
/*generator=*/true);
std::string verifyForce = cm->GetGlobVerifyScript() + "_force";
cmNinjaDeps verifyForceDeps(1, this->NinjaOutputPath(verifyForce));
this->WritePhonyBuild(os, "Phony target to force glob verification run.",
verifyForceDeps, cmNinjaDeps());
variables["restat"] = "1";
std::string const verifyScriptFile =
this->NinjaOutputPath(cm->GetGlobVerifyScript());
std::string const verifyStampFile =
this->NinjaOutputPath(cm->GetGlobVerifyStamp());
this->WriteBuild(os,
"Re-run CMake to check if globbed directories changed.",
"VERIFY_GLOBS",
/*outputs=*/cmNinjaDeps(1, verifyStampFile),
/*implicitOuts=*/cmNinjaDeps(),
/*explicitDeps=*/cmNinjaDeps(),
/*implicitDeps=*/verifyForceDeps,
/*orderOnlyDeps=*/cmNinjaDeps(), variables);
variables.erase("restat");
implicitDeps.push_back(verifyScriptFile);
explicitDeps.push_back(verifyStampFile);
} else if (!this->SupportsManifestRestat() &&
cm->DoWriteGlobVerifyTarget()) {
std::ostringstream msg;
msg << "The detected version of Ninja:\n"
<< " " << this->NinjaVersion << "\n"
<< "is less than the version of Ninja required by CMake for adding "
"restat dependencies to the build.ninja manifest regeneration "
"target:\n"
<< " " << this->RequiredNinjaVersionForManifestRestat() << "\n";
msg << "Any pre-check scripts, such as those generated for file(GLOB "
"CONFIGURE_DEPENDS), will not be run by Ninja.";
this->GetCMakeInstance()->IssueMessage(cmake::AUTHOR_WARNING, msg.str());
}
std::sort(implicitDeps.begin(), implicitDeps.end());
implicitDeps.erase(std::unique(implicitDeps.begin(), implicitDeps.end()),
implicitDeps.end());
std::string const ninjaBuildFile = this->NinjaOutputPath(NINJA_BUILD_FILE);
this->WriteBuild(os, "Re-run CMake if any of its inputs changed.",
"RERUN_CMAKE",
/*outputs=*/cmNinjaDeps(1, ninjaBuildFile),
/*implicitOuts=*/cmNinjaDeps(),
/*explicitDeps=*/cmNinjaDeps(), implicitDeps,
/*implicitOuts=*/cmNinjaDeps(), explicitDeps, implicitDeps,
/*orderOnlyDeps=*/cmNinjaDeps(), variables);
cmNinjaDeps missingInputs;
......@@ -1419,6 +1479,11 @@ bool cmGlobalNinjaGenerator::SupportsImplicitOuts() const
return this->NinjaSupportsImplicitOuts;
}
bool cmGlobalNinjaGenerator::SupportsManifestRestat() const
{
return this->NinjaSupportsManifestRestat;
}
void cmGlobalNinjaGenerator::WriteTargetClean(std::ostream& os)
{
WriteRule(*this->RulesFileStream, "CLEAN", ninjaCmd() + " -t clean",
......
......@@ -346,8 +346,10 @@ public:
static std::string RequiredNinjaVersion() { return "1.3"; }
static std::string RequiredNinjaVersionForConsolePool() { return "1.5"; }
static std::string RequiredNinjaVersionForImplicitOuts() { return "1.7"; }
static std::string RequiredNinjaVersionForManifestRestat() { return "1.8"; }
bool SupportsConsolePool() const;
bool SupportsImplicitOuts() const;
bool SupportsManifestRestat() const;
std::string NinjaOutputPath(std::string const& path) const;
bool HasOutputPathPrefix() const { return !this->OutputPathPrefix.empty(); }
......@@ -460,6 +462,7 @@ private:
std::string NinjaVersion;
bool NinjaSupportsConsolePool;
bool NinjaSupportsImplicitOuts;
bool NinjaSupportsManifestRestat;
unsigned long NinjaSupportsDyndeps;
private:
......
......@@ -301,6 +301,13 @@ void cmGlobalUnixMakefileGenerator3::WriteMainCMakefile()
lfiles.insert(lfiles.end(), lg->GetMakefile()->GetListFiles().begin(),
lg->GetMakefile()->GetListFiles().end());
}
cmake* cm = this->GetCMakeInstance();
if (cm->DoWriteGlobVerifyTarget()) {
lfiles.push_back(cm->GetGlobVerifyScript());
lfiles.push_back(cm->GetGlobVerifyStamp());
}
// Sort the list and remove duplicates.
std::sort(lfiles.begin(), lfiles.end(), std::less<std::string>());
#if !defined(__VMS) // The Compaq STL on VMS crashes, so accept duplicates.
......
......@@ -87,18 +87,18 @@ bool cmGlobalVisualStudio8Generator::AddCheckTarget()
{
// Add a special target on which all other targets depend that
// checks the build system and optionally re-runs CMake.
const char* no_working_directory = 0;
// Skip the target if no regeneration is to be done.
if (this->GlobalSettingIsOn("CMAKE_SUPPRESS_REGENERATION")) {
return false;
}
const char* no_working_directory = nullptr;
std::vector<std::string> no_depends;
std::vector<cmLocalGenerator*> const& generators = this->LocalGenerators;
cmLocalVisualStudio7Generator* lg =
static_cast<cmLocalVisualStudio7Generator*>(generators[0]);
cmMakefile* mf = lg->GetMakefile();
// Skip the target if no regeneration is to be done.
if (this->GlobalSettingIsOn("CMAKE_SUPPRESS_REGENERATION")) {
return false;
}
cmCustomCommandLines noCommandLines;
cmTarget* tgt = mf->AddUtilityCommand(
CMAKE_CHECK_BUILD_SYSTEM_TARGET, cmMakefile::TargetOrigin::Generator,
......@@ -144,6 +144,30 @@ bool cmGlobalVisualStudio8Generator::AddCheckTarget()
listFiles.insert(listFiles.end(), lmf->GetListFiles().begin(),
lmf->GetListFiles().end());
}
// Add a custom prebuild target to run the VerifyGlobs script.
cmake* cm = this->GetCMakeInstance();
if (cm->DoWriteGlobVerifyTarget()) {
cmCustomCommandLine verifyCommandLine;
verifyCommandLine.push_back(cmSystemTools::GetCMakeCommand());
verifyCommandLine.push_back("-P");
verifyCommandLine.push_back(cm->GetGlobVerifyScript());
cmCustomCommandLines verifyCommandLines;
verifyCommandLines.push_back(verifyCommandLine);
std::vector<std::string> byproducts;
byproducts.push_back(cm->GetGlobVerifyStamp());
mf->AddCustomCommandToTarget(CMAKE_CHECK_BUILD_SYSTEM_TARGET, byproducts,
no_depends, verifyCommandLines,
cmTarget::PRE_BUILD, "Checking File Globs",
no_working_directory, false);
// Ensure ZERO_CHECK always runs in Visual Studio using MSBuild,
// otherwise the prebuild command will not be run.
tgt->SetProperty("VS_GLOBAL_DisableFastUpToDateCheck", "true");
listFiles.push_back(cm->GetGlobVerifyStamp());
}
// Sort the list of input files and remove duplicates.
std::sort(listFiles.begin(), listFiles.end(), std::less<std::string>());
std::vector<std::string>::iterator new_end =
......@@ -151,8 +175,6 @@ bool cmGlobalVisualStudio8Generator::AddCheckTarget()
listFiles.erase(new_end, listFiles.end());
// Create a rule to re-run CMake.
std::string stampName = cmake::GetCMakeFilesDirectoryPostSlash();
stampName += "generate.stamp";
cmCustomCommandLine commandLine;
commandLine.push_back(cmSystemTools::GetCMakeCommand());
std::string argH = "-H";
......
......@@ -537,6 +537,12 @@ void cmGlobalXCodeGenerator::CreateReRunCMakeFile(
std::vector<std::string>::iterator new_end =
std::unique(lfiles.begin(), lfiles.end());
lfiles.erase(new_end, lfiles.end());