Commit a008578d authored by Sebastian Holtermann's avatar Sebastian Holtermann
Browse files

Autogen: Process files concurrently in AUTOMOC and AUTOUIC

This introduces concurrent thread processing in the `_autogen`
target wich processes AUTOMOC and AUTOUIC.
Source file parsing is distributed among the threads by
using a job queue from which the threads pull new parse jobs.
Each thread might start an independent ``moc`` or ``uic`` process.
Altogether this roughly speeds up the AUTOMOC and AUTOUIC build
process by the number of physical CPUs on the host system.

The exact number of threads to start in  the `_autogen` target
is controlled by the new AUTOGEN_PARALLEL target property which
is initialized by the new CMAKE_AUTOGEN_PARALLEL variable.
If AUTOGEN_PARALLEL is empty or unset (which is the default)
the thread count is set to the number of physical CPUs on
the host system.

The AUTOMOC/AUTOUIC generator and the AUTORCC generator are
refactored to use a libuv loop internally.

Closes #17422.
parent 488baaf0
# Meta
set(ARCC_MULTI_CONFIG @_multi_config@)
# Directories and files
set(ARCC_CMAKE_BINARY_DIR "@CMAKE_BINARY_DIR@/")
set(ARCC_CMAKE_SOURCE_DIR "@CMAKE_SOURCE_DIR@/")
set(ARCC_CMAKE_CURRENT_SOURCE_DIR "@CMAKE_CURRENT_SOURCE_DIR@/")
set(ARCC_CMAKE_CURRENT_BINARY_DIR "@CMAKE_CURRENT_BINARY_DIR@/")
set(ARCC_BUILD_DIR @_build_dir@)
# Qt environment
set(ARCC_RCC_EXECUTABLE @_qt_rcc_executable@)
......
# Meta
set(AM_MULTI_CONFIG @_multi_config@)
set(AM_PARALLEL @_parallel@)
# Directories and files
set(AM_CMAKE_BINARY_DIR "@CMAKE_BINARY_DIR@/")
set(AM_CMAKE_SOURCE_DIR "@CMAKE_SOURCE_DIR@/")
......
......@@ -2,16 +2,13 @@
file Copyright.txt or https://cmake.org/licensing for details. */
#include "cmQtAutoGen.h"
#include "cmAlgorithms.h"
#include "cmProcessOutput.h"
#include "cmSystemTools.h"
#include "cmsys/FStream.hxx"
#include "cmsys/RegularExpression.hxx"
#include <algorithm>
#include <iterator>
#include <sstream>
#include <stddef.h>
// - Static variables
......@@ -21,8 +18,8 @@ std::string const genNameUic = "AutoUic";
std::string const genNameRcc = "AutoRcc";
std::string const mcNameSingle = "SINGLE";
std::string const mcNameWrap = "WRAP";
std::string const mcNameFull = "FULL";
std::string const mcNameWrapper = "WRAPPER";
std::string const mcNameMulti = "MULTI";
// - Static functions
......@@ -80,203 +77,53 @@ void MergeOptions(std::vector<std::string>& baseOpts,
baseOpts.insert(baseOpts.end(), extraOpts.begin(), extraOpts.end());
}
/// @brief Reads the resource files list from from a .qrc file - Qt4 version
/// @return True if the .qrc file was successfully parsed
static bool RccListInputsQt4(std::string const& fileName,
std::vector<std::string>& files,
std::string* errorMessage)
{
bool allGood = true;
// Read qrc file content into string
std::string qrcContents;
{
cmsys::ifstream ifs(fileName.c_str());
if (ifs) {
std::ostringstream osst;
osst << ifs.rdbuf();
qrcContents = osst.str();
} else {
if (errorMessage != nullptr) {
std::string& err = *errorMessage;
err = "rcc file not readable:\n ";
err += cmQtAutoGen::Quoted(fileName);
err += "\n";
}
allGood = false;
}
}
if (allGood) {
// qrc file directory
std::string qrcDir(cmSystemTools::GetFilenamePath(fileName));
if (!qrcDir.empty()) {
qrcDir += '/';
}
cmsys::RegularExpression fileMatchRegex("(<file[^<]+)");
cmsys::RegularExpression fileReplaceRegex("(^<file[^>]*>)");
size_t offset = 0;
while (fileMatchRegex.find(qrcContents.c_str() + offset)) {
std::string qrcEntry = fileMatchRegex.match(1);
offset += qrcEntry.size();
{
fileReplaceRegex.find(qrcEntry);
std::string tag = fileReplaceRegex.match(1);
qrcEntry = qrcEntry.substr(tag.size());
}
if (!cmSystemTools::FileIsFullPath(qrcEntry.c_str())) {
qrcEntry = qrcDir + qrcEntry;
}
files.push_back(qrcEntry);
}
}
return allGood;
}
/// @brief Reads the resource files list from from a .qrc file - Qt5 version
/// @return True if the .qrc file was successfully parsed
static bool RccListInputsQt5(std::string const& rccCommand,
std::vector<std::string> const& rccListOptions,
std::string const& fileName,
std::vector<std::string>& files,
std::string* errorMessage)
{
if (rccCommand.empty()) {
cmSystemTools::Error("rcc executable not available");
return false;
}
std::string const fileDir = cmSystemTools::GetFilenamePath(fileName);
std::string const fileNameName = cmSystemTools::GetFilenameName(fileName);
// Run rcc list command
bool result = false;
int retVal = 0;
std::string rccStdOut;
std::string rccStdErr;
{
std::vector<std::string> command;
command.push_back(rccCommand);
command.insert(command.end(), rccListOptions.begin(),
rccListOptions.end());
command.push_back(fileNameName);
result = cmSystemTools::RunSingleCommand(
command, &rccStdOut, &rccStdErr, &retVal, fileDir.c_str(),
cmSystemTools::OUTPUT_NONE, 0.0, cmProcessOutput::Auto);
}
if (!result || retVal) {
if (errorMessage != nullptr) {
std::string& err = *errorMessage;
err = "rcc list process failed for:\n ";
err += cmQtAutoGen::Quoted(fileName);
err += "\n";
err += rccStdOut;
err += "\n";
err += rccStdErr;
err += "\n";
}
return false;
}
// Lambda to strip CR characters
auto StripCR = [](std::string& line) {
std::string::size_type cr = line.find('\r');
if (cr != std::string::npos) {
line = line.substr(0, cr);
}
};
// Parse rcc std output
{
std::istringstream ostr(rccStdOut);
std::string oline;
while (std::getline(ostr, oline)) {
StripCR(oline);
if (!oline.empty()) {
files.push_back(oline);
}
}
}
// Parse rcc error output
{
std::istringstream estr(rccStdErr);
std::string eline;
while (std::getline(estr, eline)) {
StripCR(eline);
if (cmHasLiteralPrefix(eline, "RCC: Error in")) {
static std::string searchString = "Cannot find file '";
std::string::size_type pos = eline.find(searchString);
if (pos == std::string::npos) {
if (errorMessage != nullptr) {
std::string& err = *errorMessage;
err = "rcc lists unparsable output:\n";
err += cmQtAutoGen::Quoted(eline);
err += "\n";
}
return false;
}
pos += searchString.length();
std::string::size_type sz = eline.size() - pos - 1;
files.push_back(eline.substr(pos, sz));
}
}
}
// Convert relative paths to absolute paths
for (std::string& resFile : files) {
resFile = cmSystemTools::CollapseCombinedPath(fileDir, resFile);
}
return true;
}
// - Class definitions
std::string const cmQtAutoGen::listSep = "<<<S>>>";
std::string const cmQtAutoGen::ListSep = "<<<S>>>";
unsigned int const cmQtAutoGen::ParallelMax = 64;
std::string const& cmQtAutoGen::GeneratorName(Generator type)
std::string const& cmQtAutoGen::GeneratorName(GeneratorT type)
{
switch (type) {
case Generator::GEN:
case GeneratorT::GEN:
return genNameGen;
case Generator::MOC:
case GeneratorT::MOC:
return genNameMoc;
case Generator::UIC:
case GeneratorT::UIC:
return genNameUic;
case Generator::RCC:
case GeneratorT::RCC:
return genNameRcc;
}
return genNameGen;
}
std::string cmQtAutoGen::GeneratorNameUpper(Generator genType)
std::string cmQtAutoGen::GeneratorNameUpper(GeneratorT genType)
{
return cmSystemTools::UpperCase(cmQtAutoGen::GeneratorName(genType));
}
std::string const& cmQtAutoGen::MultiConfigName(MultiConfig config)
std::string const& cmQtAutoGen::MultiConfigName(MultiConfigT config)
{
switch (config) {
case MultiConfig::SINGLE:
case MultiConfigT::SINGLE:
return mcNameSingle;
case MultiConfig::WRAP:
return mcNameWrap;
case MultiConfig::FULL:
return mcNameFull;
case MultiConfigT::WRAPPER:
return mcNameWrapper;
case MultiConfigT::MULTI:
return mcNameMulti;
}
return mcNameWrap;
return mcNameWrapper;
}
cmQtAutoGen::MultiConfig cmQtAutoGen::MultiConfigType(std::string const& name)
cmQtAutoGen::MultiConfigT cmQtAutoGen::MultiConfigType(std::string const& name)
{
if (name == mcNameSingle) {
return MultiConfig::SINGLE;
return MultiConfigT::SINGLE;
}
if (name == mcNameFull) {
return MultiConfig::FULL;
if (name == mcNameMulti) {
return MultiConfigT::MULTI;
}
return MultiConfig::WRAP;
return MultiConfigT::WRAPPER;
}
std::string cmQtAutoGen::Quoted(std::string const& text)
......@@ -294,6 +141,33 @@ std::string cmQtAutoGen::Quoted(std::string const& text)
return res;
}
std::string cmQtAutoGen::QuotedCommand(std::vector<std::string> const& command)
{
std::string res;
for (std::string const& item : command) {
if (!res.empty()) {
res.push_back(' ');
}
std::string const cesc = cmQtAutoGen::Quoted(item);
if (item.empty() || (cesc.size() > (item.size() + 2)) ||
(cesc.find(' ') != std::string::npos)) {
res += cesc;
} else {
res += item;
}
}
return res;
}
std::string cmQtAutoGen::SubDirPrefix(std::string const& filename)
{
std::string res(cmSystemTools::GetFilenamePath(filename));
if (!res.empty()) {
res += '/';
}
return res;
}
std::string cmQtAutoGen::AppendFilenameSuffix(std::string const& filename,
std::string const& suffix)
{
......@@ -333,27 +207,79 @@ void cmQtAutoGen::RccMergeOptions(std::vector<std::string>& baseOpts,
MergeOptions(baseOpts, newOpts, valueOpts, isQt5);
}
bool cmQtAutoGen::RccListInputs(std::string const& rccCommand,
std::vector<std::string> const& rccListOptions,
std::string const& fileName,
std::vector<std::string>& files,
std::string* errorMessage)
void cmQtAutoGen::RccListParseContent(std::string const& content,
std::vector<std::string>& files)
{
bool allGood = false;
if (cmSystemTools::FileExists(fileName.c_str())) {
if (rccListOptions.empty()) {
allGood = RccListInputsQt4(fileName, files, errorMessage);
} else {
allGood = RccListInputsQt5(rccCommand, rccListOptions, fileName, files,
errorMessage);
cmsys::RegularExpression fileMatchRegex("(<file[^<]+)");
cmsys::RegularExpression fileReplaceRegex("(^<file[^>]*>)");
const char* contentChars = content.c_str();
while (fileMatchRegex.find(contentChars)) {
std::string const qrcEntry = fileMatchRegex.match(1);
contentChars += qrcEntry.size();
{
fileReplaceRegex.find(qrcEntry);
std::string const tag = fileReplaceRegex.match(1);
files.push_back(qrcEntry.substr(tag.size()));
}
} else {
if (errorMessage != nullptr) {
std::string& err = *errorMessage;
err = "rcc resource file does not exist:\n ";
err += cmQtAutoGen::Quoted(fileName);
err += "\n";
}
}
bool cmQtAutoGen::RccListParseOutput(std::string const& rccStdOut,
std::string const& rccStdErr,
std::vector<std::string>& files,
std::string& error)
{
// Lambda to strip CR characters
auto StripCR = [](std::string& line) {
std::string::size_type cr = line.find('\r');
if (cr != std::string::npos) {
line = line.substr(0, cr);
}
};
// Parse rcc std output
{
std::istringstream ostr(rccStdOut);
std::string oline;
while (std::getline(ostr, oline)) {
StripCR(oline);
if (!oline.empty()) {
files.push_back(oline);
}
}
}
// Parse rcc error output
{
std::istringstream estr(rccStdErr);
std::string eline;
while (std::getline(estr, eline)) {
StripCR(eline);
if (cmHasLiteralPrefix(eline, "RCC: Error in")) {
static std::string const searchString = "Cannot find file '";
std::string::size_type pos = eline.find(searchString);
if (pos == std::string::npos) {
error = "rcc lists unparsable output:\n";
error += cmQtAutoGen::Quoted(eline);
error += "\n";
return false;
}
pos += searchString.length();
std::string::size_type sz = eline.size() - pos - 1;
files.push_back(eline.substr(pos, sz));
}
}
}
return true;
}
void cmQtAutoGen::RccListConvertFullPath(std::string const& qrcFileDir,
std::vector<std::string>& files)
{
for (std::string& entry : files) {
std::string tmp = cmSystemTools::CollapseCombinedPath(qrcFileDir, entry);
entry = std::move(tmp);
}
return allGood;
}
......@@ -9,14 +9,18 @@
#include <vector>
/** \class cmQtAutoGen
* \brief Class used as namespace for QtAutogen related types and functions
* \brief Common base class for QtAutoGen classes
*/
class cmQtAutoGen
{
public:
static std::string const listSep;
/// @brief Nested lists separator
static std::string const ListSep;
/// @brief Maximum number of parallel threads/processes in a generator
static unsigned int const ParallelMax;
enum Generator
/// @brief AutoGen generator type
enum class GeneratorT
{
GEN, // General
MOC,
......@@ -24,27 +28,33 @@ public:
RCC
};
enum MultiConfig
/// @brief Multiconfiguration type
enum class MultiConfigT
{
SINGLE, // Single configuration
WRAP, // Multi configuration using wrapper files
FULL // Full multi configuration using per config sources
SINGLE, // Single configuration
WRAPPER, // Multi configuration using wrapper files
MULTI // Multi configuration using per config sources
};
public:
/// @brief Returns the generator name
static std::string const& GeneratorName(Generator genType);
static std::string const& GeneratorName(GeneratorT genType);
/// @brief Returns the generator name in upper case
static std::string GeneratorNameUpper(Generator genType);
static std::string GeneratorNameUpper(GeneratorT genType);
/// @brief Returns the multi configuration name string
static std::string const& MultiConfigName(MultiConfig config);
static std::string const& MultiConfigName(MultiConfigT config);
/// @brief Returns the multi configuration type
static MultiConfig MultiConfigType(std::string const& name);
static MultiConfigT MultiConfigType(std::string const& name);
/// @brief Returns a the string escaped and enclosed in quotes
static std::string Quoted(std::string const& text);
static std::string QuotedCommand(std::vector<std::string> const& command);
/// @brief Returns the parent directory of the file with a "/" suffix
static std::string SubDirPrefix(std::string const& filename);
/// @brief Appends the suffix to the filename before the last dot
static std::string AppendFilenameSuffix(std::string const& filename,
std::string const& suffix);
......@@ -59,14 +69,21 @@ public:
std::vector<std::string> const& newOpts,
bool isQt5);
/// @brief Reads the resource files list from from a .qrc file
/// @arg fileName Must be the absolute path of the .qrc file
/// @return True if the rcc file was successfully read
static bool RccListInputs(std::string const& rccCommand,
std::vector<std::string> const& rccListOptions,
std::string const& fileName,
std::vector<std::string>& files,
std::string* errorMessage = nullptr);
/// @brief Parses the content of a qrc file
///
/// Use when rcc does not support the "--list" option
static void RccListParseContent(std::string const& content,
std::vector<std::string>& files);
/// @brief Parses the output of the "rcc --list ..." command
static bool RccListParseOutput(std::string const& rccStdOut,
std::string const& rccStdErr,
std::vector<std::string>& files,
std::string& error);
/// @brief Converts relative qrc entry paths to full paths
static void RccListConvertFullPath(std::string const& qrcFileDir,
std::vector<std::string>& files);
};
#endif
This diff is collapsed.
......@@ -6,71 +6,283 @@
#include "cmConfigure.h" // IWYU pragma: keep
#include "cmQtAutoGen.h"
#include "cmUVHandlePtr.h"
#include "cm_uv.h"
#include <array>
#include <functional>
#include <mutex>
#include <stddef.h>
#include <stdint.h>
#include <string>
#include <vector>
class cmMakefile;
class cmQtAutoGenerator
/// @brief Base class for QtAutoGen gernerators
class cmQtAutoGenerator : public cmQtAutoGen
{
CM_DISABLE_COPY(cmQtAutoGenerator)
public:
// -- Types
/// @brief Thread safe logging
class Logger
{
public:
// -- Verbosity
bool Verbose() const { return this->Verbose_; }
void SetVerbose(bool value);
bool ColorOutput() const { return this->ColorOutput_; }
void SetColorOutput(bool value);
// -- Log info
void Info(GeneratorT genType, std::string const& message);
// -- Log warning
void Warning(GeneratorT genType, std::string const& message);
void WarningFile(GeneratorT genType, std::string const& filename,
std::string const& message);
// -- Log error
void Error(GeneratorT genType, std::string const& message);
void ErrorFile(GeneratorT genType, std::string const& filename,
std::string const& message);
void ErrorCommand(GeneratorT genType, std::string const& message,
std::vector<std::string> const& command,
std::string const& output);
private:
static std::string HeadLine(std::string const& title);
private:
std::mutex Mutex_;
bool volatile Verbose_ = false;
bool volatile ColorOutput_ = false;
};
/// @brief Thread safe file system interface
class FileSystem
{
public:
FileSystem(Logger* log)
: Log_(log)
{
}
Logger* Log() const { return Log_; }
std::string RealPath(std::string const& filename);
bool FileExists(std::string const& filename);
bool FileIsOlderThan(std::string const& buildFile,
std::string const& sourceFile,
std::string* error = nullptr);
bool FileRead(std::string& content, std::string const& filename,
std::string* error = nullptr);
/// @brief Error logging version
bool FileRead(GeneratorT genType, std::string& content,
std::string const& filename);
bool FileWrite(std::string const& filename, std::string const& content,
std::string* error = nullptr);
/// @brief Error logging version
bool FileWrite(GeneratorT genType, std::string const& filename,
std::string const& content);
bool FileDiffers(std::string const& filename, std::string const& content);
bool FileRemove(std::string const& filename);
bool Touch(std::string const& filename);
bool MakeDirectory(std::string const& dirname);
/// @brief Error logging version
bool MakeDirectory(GeneratorT genType, std::string const& dirname);
bool MakeParentDirectory(std::string const& filename);
/// @brief Error logging version
bool MakeParentDirectory(GeneratorT genType, std::string const& filename);
private:
std::mutex Mutex_;
Logger* Log_;
};
/// @brief Return value and output of an external process
struct ProcessResultT
{
void reset();
bool error() const
{
return (ExitStatus != 0) || (TermSignal != 0) || !ErrorMessage.empty();
}
std::int64_t ExitStatus = 0;