Commit 7d50e1c6 authored by Sebastian Holtermann's avatar Sebastian Holtermann
Browse files

Autogen: Refactor AUTOMOC and AUTOUIC and add source file parse data caching

New features
------------

CMake's `AUTOMOC` and `AUTOUIC` now cache information extracted when parsing
source files in `CMakeFiles/<ORIGIN>_autogen.dir/ParseCache.txt`.
This leads to faster `<ORIGIN>_autogen` target rebuilds, because source files
will be parsed again only if they're newer than the `ParseCache.txt` file.
The parse cache will be recomputed if it is older than the CMake executable.

`AUTOMOC` and `AUTOUIC` now check if `moc` or `uic` output files are older
than the `moc` or `uic` executable.  If an output file is older than the
compiler, it will be regenerated.  Therefore if a new `moc` or `uic` version
is installed, all output files will be regenerated.

`AUTOMOC` and `AUTOUIC` error and warning messages are more detailed.

Internal changes
----------------

`moc` and `uic` output file names are not computed in the `_autogen`
target anymore but in `cmQtAutoGenInitializer`.  This makes the available at
the configuration stage for improved dependency computations (to be done).

In `AutogenInfo.cmake`, equally sized lists for "source file names",
"source file flags" and "compiler output file names" are passed to the
`_autogen` target.  This replaces the separate file lists for
`AUTOMOC` and `AUTOUIC`.

Files times are read from the file system only once by using `cmFileTime`
instances instead of `cmQtAutoGenerator::FileSystem::FileIsOlderThan` calls.

All calls to not thread safe file system functions are moved to non concurrent
fence jobs (see `cmWorkerPool::JobT::IsFence()`).  This renders the
`cmQtAutoGenerator::FileSystem` wrapper class obsolete and it is removed.

Instead of composing a single large settings string that is fed to the
`cmCryptoHash`, now all setting sub strings are fed one by one to the
`cmCryptoHash` and the finalized result is stored.

The `std::mutex` in `cmQtAutoGenerator::Logger` is tagged `mutable` and most
`cmQtAutoGenerator::Logger` methods become `const`.

Outlook
-------

This patch provides the framework required to

- extract dependencies from `.ui` files in `AUTOUIC`.
  These will help to address issue
  #15420 "AUTOUIC: Track uic external inputs".

- generate adaptive `make` and `ninja` files in the `_autogen` target.
  These will help to address issue
  #16776 "AUTOUIC: Ninja needs two passes to correctly build Qt project".

- generate (possibly empty) `moc` and `uic` files for all headers instead of a
  `mocs_compilation.cpp` file.
  This will help to address issue
  #17277 "AUTOMOC: Provide a option to allow AUTOMOC to compile individual "
         "moc_x.cxx instead of including all in mocs_compilation.cxx"
parent c6f6e2b3
......@@ -194,11 +194,11 @@ std::string cmQtAutoGen::QuotedCommand(std::vector<std::string> const& command)
std::string cmQtAutoGen::SubDirPrefix(std::string const& filename)
{
std::string res(cmSystemTools::GetFilenamePath(filename));
if (!res.empty()) {
res += '/';
std::string::size_type slash_pos = filename.rfind('/');
if (slash_pos == std::string::npos) {
return std::string();
}
return res;
return filename.substr(0, slash_pos + 1);
}
std::string cmQtAutoGen::AppendFilenameSuffix(std::string const& filename,
......
......@@ -34,6 +34,7 @@
#include <map>
#include <set>
#include <string>
#include <unordered_set>
#include <utility>
#include <vector>
......@@ -77,7 +78,8 @@ static std::string FileProjectRelativePath(cmMakefile* makefile,
return res;
}
/* @brief Tests if targetDepend is a STATIC_LIBRARY and if any of its
/**
* Tests if targetDepend is a STATIC_LIBRARY and if any of its
* recursive STATIC_LIBRARY dependencies depends on targetOrigin
* (STATIC_LIBRARY cycle).
*/
......@@ -384,6 +386,10 @@ bool cmQtAutoGenInitializer::InitCustomTargets()
} else {
AddCleanFile(makefile, this->AutogenTarget.SettingsFile);
}
this->AutogenTarget.ParseCacheFile = this->Dir.Info;
this->AutogenTarget.ParseCacheFile += "/ParseCache.txt";
AddCleanFile(makefile, this->AutogenTarget.ParseCacheFile);
}
// Autogen target: Compute user defined dependencies
......@@ -1255,53 +1261,107 @@ bool cmQtAutoGenInitializer::SetupWriteAutogenInfo()
ofs.Write("AM_INCLUDE_DIR", this->Dir.Include);
ofs.WriteConfig("AM_INCLUDE_DIR", this->Dir.ConfigInclude);
// Use sorted sets
std::set<std::string> headers;
std::set<std::string> sources;
std::set<std::string> moc_headers;
std::set<std::string> moc_sources;
std::vector<std::string> headers;
std::vector<std::string> headersFlags;
std::vector<std::string> headersBuildPaths;
std::vector<std::string> sources;
std::vector<std::string> sourcesFlags;
std::set<std::string> moc_skip;
std::set<std::string> uic_headers;
std::set<std::string> uic_sources;
std::set<std::string> uic_skip;
// Filter headers
{
auto headerCount = this->AutogenTarget.Headers.size();
headers.reserve(headerCount);
headersFlags.reserve(headerCount);
std::vector<MUFile const*> sortedHeaders;
{
sortedHeaders.reserve(headerCount);
for (auto const& pair : this->AutogenTarget.Headers) {
MUFile const& muf = *pair.second;
if (muf.Generated && !this->CMP0071Accept) {
sortedHeaders.emplace_back(pair.second.get());
}
std::sort(sortedHeaders.begin(), sortedHeaders.end(),
[](MUFile const* a, MUFile const* b) {
return (a->RealPath < b->RealPath);
});
}
for (MUFile const* const muf : sortedHeaders) {
if (muf->Generated && !this->CMP0071Accept) {
continue;
}
if (muf.SkipMoc) {
moc_skip.insert(muf.RealPath);
if (muf->SkipMoc) {
moc_skip.insert(muf->RealPath);
}
if (muf->SkipUic) {
uic_skip.insert(muf->RealPath);
}
if (muf->MocIt || muf->UicIt) {
headers.emplace_back(muf->RealPath);
std::string flags;
flags += muf->MocIt ? 'M' : 'm';
flags += muf->UicIt ? 'U' : 'u';
headersFlags.emplace_back(std::move(flags));
}
}
if (muf.SkipUic) {
uic_skip.insert(muf.RealPath);
}
if (muf.MocIt && muf.UicIt) {
headers.insert(muf.RealPath);
} else if (muf.MocIt) {
moc_headers.insert(muf.RealPath);
} else if (muf.UicIt) {
uic_headers.insert(muf.RealPath);
// Header build paths
{
cmFilePathChecksum const fpathCheckSum(makefile);
std::unordered_set<std::string> emitted;
for (std::string const& hdr : headers) {
std::string basePath = fpathCheckSum.getPart(hdr);
basePath += "/moc_";
basePath += cmSystemTools::GetFilenameWithoutLastExtension(hdr);
for (unsigned int ii = 1; ii != 1024; ++ii) {
std::string path = basePath;
if (ii > 1) {
path += '_';
path += std::to_string(ii);
}
path += ".cpp";
if (emitted.emplace(path).second) {
headersBuildPaths.emplace_back(std::move(path));
break;
}
}
}
}
// Filter sources
{
auto sourcesCount = this->AutogenTarget.Sources.size();
sources.reserve(sourcesCount);
sourcesFlags.reserve(sourcesCount);
std::vector<MUFile const*> sorted;
sorted.reserve(sourcesCount);
for (auto const& pair : this->AutogenTarget.Sources) {
MUFile const& muf = *pair.second;
if (muf.Generated && !this->CMP0071Accept) {
sorted.emplace_back(pair.second.get());
}
std::sort(sorted.begin(), sorted.end(),
[](MUFile const* a, MUFile const* b) {
return (a->RealPath < b->RealPath);
});
for (MUFile const* const muf : sorted) {
if (muf->Generated && !this->CMP0071Accept) {
continue;
}
if (muf.SkipMoc) {
moc_skip.insert(muf.RealPath);
if (muf->SkipMoc) {
moc_skip.insert(muf->RealPath);
}
if (muf->SkipUic) {
uic_skip.insert(muf->RealPath);
}
if (muf.SkipUic) {
uic_skip.insert(muf.RealPath);
if (muf->MocIt || muf->UicIt) {
sources.emplace_back(muf->RealPath);
std::string flags;
flags += muf->MocIt ? 'M' : 'm';
flags += muf->UicIt ? 'U' : 'u';
sourcesFlags.emplace_back(std::move(flags));
}
if (muf.MocIt && muf.UicIt) {
sources.insert(muf.RealPath);
} else if (muf.MocIt) {
moc_sources.insert(muf.RealPath);
} else if (muf.UicIt) {
uic_sources.insert(muf.RealPath);
}
}
......@@ -1311,17 +1371,20 @@ bool cmQtAutoGenInitializer::SetupWriteAutogenInfo()
ofs.Write("AM_QT_UIC_EXECUTABLE", this->Uic.Executable);
ofs.Write("# Files\n");
ofs.Write("AM_CMAKE_EXECUTABLE", cmSystemTools::GetCMakeCommand());
ofs.Write("AM_SETTINGS_FILE", this->AutogenTarget.SettingsFile);
ofs.WriteConfig("AM_SETTINGS_FILE",
this->AutogenTarget.ConfigSettingsFile);
ofs.Write("AM_PARSE_CACHE_FILE", this->AutogenTarget.ParseCacheFile);
ofs.WriteStrings("AM_HEADERS", headers);
ofs.WriteStrings("AM_HEADERS_FLAGS", headersFlags);
ofs.WriteStrings("AM_HEADERS_BUILD_PATHS", headersBuildPaths);
ofs.WriteStrings("AM_SOURCES", sources);
ofs.WriteStrings("AM_SOURCES_FLAGS", sourcesFlags);
// Write moc settings
if (this->Moc.Enabled) {
ofs.Write("# MOC settings\n");
ofs.WriteStrings("AM_MOC_HEADERS", moc_headers);
ofs.WriteStrings("AM_MOC_SOURCES", moc_sources);
ofs.WriteStrings("AM_MOC_SKIP", moc_skip);
ofs.WriteStrings("AM_MOC_DEFINITIONS", this->Moc.Defines);
ofs.WriteConfigStrings("AM_MOC_DEFINITIONS", this->Moc.ConfigDefines);
......@@ -1343,8 +1406,6 @@ bool cmQtAutoGenInitializer::SetupWriteAutogenInfo()
uic_skip.insert(this->Uic.SkipUi.begin(), this->Uic.SkipUi.end());
ofs.Write("# UIC settings\n");
ofs.WriteStrings("AM_UIC_HEADERS", uic_headers);
ofs.WriteStrings("AM_UIC_SOURCES", uic_sources);
ofs.WriteStrings("AM_UIC_SKIP", uic_skip);
ofs.WriteStrings("AM_UIC_TARGET_OPTIONS", this->Uic.Options);
ofs.WriteConfigStrings("AM_UIC_TARGET_OPTIONS", this->Uic.ConfigOptions);
......
......@@ -183,6 +183,7 @@ private:
// Configuration files
std::string InfoFile;
std::string SettingsFile;
std::string ParseCacheFile;
std::map<std::string, std::string> ConfigSettingsFile;
// Dependencies
bool DependOrigin = false;
......
......@@ -66,7 +66,8 @@ std::string cmQtAutoGenerator::Logger::HeadLine(std::string const& title)
return head;
}
void cmQtAutoGenerator::Logger::Info(GenT genType, std::string const& message)
void cmQtAutoGenerator::Logger::Info(GenT genType,
std::string const& message) const
{
std::string msg = GeneratorName(genType);
msg += ": ";
......@@ -81,7 +82,7 @@ void cmQtAutoGenerator::Logger::Info(GenT genType, std::string const& message)
}
void cmQtAutoGenerator::Logger::Warning(GenT genType,
std::string const& message)
std::string const& message) const
{
std::string msg;
if (message.find('\n') == std::string::npos) {
......@@ -106,7 +107,7 @@ void cmQtAutoGenerator::Logger::Warning(GenT genType,
void cmQtAutoGenerator::Logger::WarningFile(GenT genType,
std::string const& filename,
std::string const& message)
std::string const& message) const
{
std::string msg = " ";
msg += Quoted(filename);
......@@ -116,7 +117,8 @@ void cmQtAutoGenerator::Logger::WarningFile(GenT genType,
Warning(genType, msg);
}
void cmQtAutoGenerator::Logger::Error(GenT genType, std::string const& message)
void cmQtAutoGenerator::Logger::Error(GenT genType,
std::string const& message) const
{
std::string msg;
msg += HeadLine(GeneratorName(genType) + " error");
......@@ -134,7 +136,7 @@ void cmQtAutoGenerator::Logger::Error(GenT genType, std::string const& message)
void cmQtAutoGenerator::Logger::ErrorFile(GenT genType,
std::string const& filename,
std::string const& message)
std::string const& message) const
{
std::string emsg = " ";
emsg += Quoted(filename);
......@@ -146,7 +148,7 @@ void cmQtAutoGenerator::Logger::ErrorFile(GenT genType,
void cmQtAutoGenerator::Logger::ErrorCommand(
GenT genType, std::string const& message,
std::vector<std::string> const& command, std::string const& output)
std::vector<std::string> const& command, std::string const& output) const
{
std::string msg;
msg.push_back('\n');
......@@ -191,7 +193,7 @@ bool cmQtAutoGenerator::FileRead(std::string& content,
content.clear();
if (!cmSystemTools::FileExists(filename, true)) {
if (error != nullptr) {
error->append("Not a file.");
*error = "Not a file.";
}
return false;
}
......@@ -203,7 +205,7 @@ bool cmQtAutoGenerator::FileRead(std::string& content,
return [&ifs, length, &content, error]() -> bool {
if (!ifs) {
if (error != nullptr) {
error->append("Opening the file for reading failed.");
*error = "Opening the file for reading failed.";
}
return false;
}
......@@ -213,7 +215,7 @@ bool cmQtAutoGenerator::FileRead(std::string& content,
if (!ifs) {
content.clear();
if (error != nullptr) {
error->append("Reading from the file failed.");
*error = "Reading from the file failed.";
}
return false;
}
......@@ -228,7 +230,7 @@ bool cmQtAutoGenerator::FileWrite(std::string const& filename,
// Make sure the parent directory exists
if (!cmQtAutoGenerator::MakeParentDirectory(filename)) {
if (error != nullptr) {
error->assign("Could not create parent directory.");
*error = "Could not create parent directory.";
}
return false;
}
......@@ -240,14 +242,14 @@ bool cmQtAutoGenerator::FileWrite(std::string const& filename,
return [&ofs, &content, error]() -> bool {
if (!ofs) {
if (error != nullptr) {
error->assign("Opening file for writing failed.");
*error = "Opening file for writing failed.";
}
return false;
}
ofs << content;
if (!ofs.good()) {
if (error != nullptr) {
error->assign("File writing failed.");
*error = "File writing failed.";
}
return false;
}
......@@ -255,176 +257,17 @@ bool cmQtAutoGenerator::FileWrite(std::string const& filename,
}();
}
cmQtAutoGenerator::FileSystem::FileSystem() = default;
cmQtAutoGenerator::FileSystem::~FileSystem() = default;
std::string cmQtAutoGenerator::FileSystem::GetRealPath(
std::string const& filename)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmSystemTools::GetRealPath(filename);
}
std::string cmQtAutoGenerator::FileSystem::CollapseFullPath(
std::string const& file, std::string const& dir)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmSystemTools::CollapseFullPath(file, dir);
}
void cmQtAutoGenerator::FileSystem::SplitPath(
const std::string& p, std::vector<std::string>& components,
bool expand_home_dir)
{
std::lock_guard<std::mutex> lock(Mutex_);
cmSystemTools::SplitPath(p, components, expand_home_dir);
}
std::string cmQtAutoGenerator::FileSystem::JoinPath(
const std::vector<std::string>& components)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmSystemTools::JoinPath(components);
}
std::string cmQtAutoGenerator::FileSystem::JoinPath(
std::vector<std::string>::const_iterator first,
std::vector<std::string>::const_iterator last)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmSystemTools::JoinPath(first, last);
}
std::string cmQtAutoGenerator::FileSystem::GetFilenameWithoutLastExtension(
const std::string& filename)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmSystemTools::GetFilenameWithoutLastExtension(filename);
}
std::string cmQtAutoGenerator::FileSystem::SubDirPrefix(
std::string const& filename)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmQtAutoGen::SubDirPrefix(filename);
}
void cmQtAutoGenerator::FileSystem::setupFilePathChecksum(
std::string const& currentSrcDir, std::string const& currentBinDir,
std::string const& projectSrcDir, std::string const& projectBinDir)
{
std::lock_guard<std::mutex> lock(Mutex_);
FilePathChecksum_.setupParentDirs(currentSrcDir, currentBinDir,
projectSrcDir, projectBinDir);
}
std::string cmQtAutoGenerator::FileSystem::GetFilePathChecksum(
std::string const& filename)
{
std::lock_guard<std::mutex> lock(Mutex_);
return FilePathChecksum_.getPart(filename);
}
bool cmQtAutoGenerator::FileSystem::FileExists(std::string const& filename)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmSystemTools::FileExists(filename);
}
bool cmQtAutoGenerator::FileSystem::FileExists(std::string const& filename,
bool isFile)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmSystemTools::FileExists(filename, isFile);
}
unsigned long cmQtAutoGenerator::FileSystem::FileLength(
std::string const& filename)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmSystemTools::FileLength(filename);
}
bool cmQtAutoGenerator::FileSystem::FileIsOlderThan(
std::string const& buildFile, std::string const& sourceFile,
std::string* error)
{
bool res(false);
int result = 0;
{
std::lock_guard<std::mutex> lock(Mutex_);
res = cmSystemTools::FileTimeCompare(buildFile, sourceFile, &result);
}
if (res) {
res = (result < 0);
} else {
if (error != nullptr) {
error->append(
"File modification time comparison failed for the files\n ");
error->append(Quoted(buildFile));
error->append("\nand\n ");
error->append(Quoted(sourceFile));
}
}
return res;
}
bool cmQtAutoGenerator::FileSystem::FileRead(std::string& content,
std::string const& filename,
std::string* error)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmQtAutoGenerator::FileRead(content, filename, error);
}
bool cmQtAutoGenerator::FileSystem::FileWrite(std::string const& filename,
std::string const& content,
std::string* error)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmQtAutoGenerator::FileWrite(filename, content, error);
}
bool cmQtAutoGenerator::FileSystem::FileDiffers(std::string const& filename,
bool cmQtAutoGenerator::FileDiffers(std::string const& filename,
std::string const& content)
{
bool differs = true;
{
std::string oldContents;
if (FileRead(oldContents, filename)) {
differs = (oldContents != content);
}
if (FileRead(oldContents, filename) && (oldContents == content)) {
differs = false;
}
return differs;
}
bool cmQtAutoGenerator::FileSystem::FileRemove(std::string const& filename)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmSystemTools::RemoveFile(filename);
}
bool cmQtAutoGenerator::FileSystem::Touch(std::string const& filename,
bool create)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmSystemTools::Touch(filename, create);
}
bool cmQtAutoGenerator::FileSystem::MakeDirectory(std::string const& dirname)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmSystemTools::MakeDirectory(dirname);
}
bool cmQtAutoGenerator::FileSystem::MakeParentDirectory(
std::string const& filename)
{
std::lock_guard<std::mutex> lock(Mutex_);
return cmQtAutoGenerator::MakeParentDirectory(filename);
}
cmQtAutoGenerator::cmQtAutoGenerator() = default;
cmQtAutoGenerator::~cmQtAutoGenerator() = default;
......@@ -435,6 +278,12 @@ bool cmQtAutoGenerator::Run(std::string const& infoFile,
// Info settings
InfoFile_ = infoFile;
cmSystemTools::ConvertToUnixSlashes(InfoFile_);
if (!InfoFileTime_.Load(InfoFile_)) {
std::string msg = "Autogen: The info file ";
msg += Quoted(InfoFile_);
msg += " is not readable\n";
cmSystemTools::Stderr(msg);
}
InfoDir_ = cmSystemTools::GetFilenamePath(infoFile);
InfoConfig_ = config;
......
......@@ -5,7 +5,7 @@
#include "cmConfigure.h" // IWYU pragma: keep
#include "cmFilePathChecksum.h"
#include "cmFileTime.h"
#include "cmQtAutoGen.h"
#include <mutex>
......@@ -14,13 +14,17 @@
class cmMakefile;
/// @brief Base class for QtAutoGen gernerators
/** \class cmQtAutoGenerator
* \brief Base class for QtAutoGen generators
*/
class cmQtAutoGenerator : public cmQtAutoGen
{
public:
// -- Types
/// @brief Thread safe logging
/**
* Thread safe logger
*/
class Logger
{
public:
......@@ -37,24 +41,24 @@ public:
bool ColorOutput() const { return this->ColorOutput_; }
void SetColorOutput(bool value);
// -- Log info
void Info(GenT genType, std::string const& message);
void Info(GenT genType, std::string const& message) const;
// -- Log warning
void Warning(GenT genType, std::string const& message);
void Warning(GenT genType, std::string const& message) const;
void WarningFile(GenT genType, std::string const& filename,
std::string const& message);
std::string const& message) const;
// -- Log error
void Error(GenT genType, std::string const& message);
void Error(GenT genType, std::string const& message) const;
void ErrorFile(GenT genType, std::string const& filename,
std::string const& message);
std::string const& message) const;
void ErrorCommand(GenT genType, std::string const& message,
std::vector<std::string> const& command,
std::string const& output);
std::string const& output) const;
private:
static std::string HeadLine(std::string const& title);
private:
std::mutex Mutex_;
mutable std::mutex Mutex_;
unsigned int Verbosity_ = 0;
bool ColorOutput_ = false;
};
......@@ -66,70 +70,8 @@ public:
static bool FileWrite(std::string const& filename,
std::string const& content,
std::string* error = nullptr);
/// @brief Thread safe file system interface
class FileSystem
{
public:
FileSystem();
~FileSystem();
// -- Paths
/// @brief Wrapper for cmSystemTools::GetRealPath
std::string GetRealPath(std::string const& filename);