Commit 34975815 authored by Brad King's avatar Brad King Committed by Kitware Robot

Merge topic 'autogen_moc_uic_single_job_queue'

58f04b6e Autogen: Add ManySources test
a3f06209 Autogen: Rename `cmQtAutoGeneratorMocUic` class to `cmQtAutoMocUic`
8cb26a0a Autogen: Factor out concurrency framework to cmWorkerPool class
Acked-by: Kitware Robot's avatarKitware Robot <kwrobot@kitware.com>
Merge-request: !3224
parents 68a0b51e 58f04b6e
......@@ -350,8 +350,8 @@ set(SRCS
cmQtAutoGenGlobalInitializer.h
cmQtAutoGenInitializer.cxx
cmQtAutoGenInitializer.h
cmQtAutoGeneratorMocUic.cxx
cmQtAutoGeneratorMocUic.h
cmQtAutoMocUic.cxx
cmQtAutoMocUic.h
cmQtAutoRcc.cxx
cmQtAutoRcc.h
cmRST.cxx
......@@ -391,6 +391,8 @@ set(SRCS
cmVariableWatch.h
cmVersion.cxx
cmVersion.h
cmWorkerPool.cxx
cmWorkerPool.h
cmWorkingDirectory.cxx
cmWorkingDirectory.h
cmXMLParser.cxx
......
......@@ -14,12 +14,6 @@
#include "cmSystemTools.h"
#include "cmake.h"
#include <algorithm>
#include <sstream>
#include <utility>
// -- Class methods
cmQtAutoGenerator::Logger::Logger()
{
// Initialize logger
......@@ -431,232 +425,6 @@ bool cmQtAutoGenerator::FileSystem::MakeParentDirectory(
return cmQtAutoGenerator::MakeParentDirectory(filename);
}
int cmQtAutoGenerator::ReadOnlyProcessT::PipeT::init(uv_loop_t* uv_loop,
ReadOnlyProcessT* process)
{
Process_ = process;
Target_ = nullptr;
return UVPipe_.init(*uv_loop, 0, this);
}
int cmQtAutoGenerator::ReadOnlyProcessT::PipeT::startRead(std::string* target)
{
Target_ = target;
return uv_read_start(uv_stream(), &PipeT::UVAlloc, &PipeT::UVData);
}
void cmQtAutoGenerator::ReadOnlyProcessT::PipeT::reset()
{
Process_ = nullptr;
Target_ = nullptr;
UVPipe_.reset();
Buffer_.clear();
Buffer_.shrink_to_fit();
}
void cmQtAutoGenerator::ReadOnlyProcessT::PipeT::UVAlloc(uv_handle_t* handle,
size_t suggestedSize,
uv_buf_t* buf)
{
auto& pipe = *reinterpret_cast<PipeT*>(handle->data);
pipe.Buffer_.resize(suggestedSize);
buf->base = pipe.Buffer_.data();
buf->len = pipe.Buffer_.size();
}
void cmQtAutoGenerator::ReadOnlyProcessT::PipeT::UVData(uv_stream_t* stream,
ssize_t nread,
const uv_buf_t* buf)
{
auto& pipe = *reinterpret_cast<PipeT*>(stream->data);
if (nread > 0) {
// Append data to merged output
if ((buf->base != nullptr) && (pipe.Target_ != nullptr)) {
pipe.Target_->append(buf->base, nread);
}
} else if (nread < 0) {
// EOF or error
auto* proc = pipe.Process_;
// Check it this an unusual error
if (nread != UV_EOF) {
if (!proc->Result()->error()) {
proc->Result()->ErrorMessage =
"libuv reading from pipe failed with error code ";
proc->Result()->ErrorMessage += std::to_string(nread);
}
}
// Clear libuv pipe handle and try to finish
pipe.reset();
proc->UVTryFinish();
}
}
void cmQtAutoGenerator::ProcessResultT::reset()
{
ExitStatus = 0;
TermSignal = 0;
if (!StdOut.empty()) {
StdOut.clear();
StdOut.shrink_to_fit();
}
if (!StdErr.empty()) {
StdErr.clear();
StdErr.shrink_to_fit();
}
if (!ErrorMessage.empty()) {
ErrorMessage.clear();
ErrorMessage.shrink_to_fit();
}
}
void cmQtAutoGenerator::ReadOnlyProcessT::setup(
ProcessResultT* result, bool mergedOutput,
std::vector<std::string> const& command, std::string const& workingDirectory)
{
Setup_.WorkingDirectory = workingDirectory;
Setup_.Command = command;
Setup_.Result = result;
Setup_.MergedOutput = mergedOutput;
}
static std::string getUVError(const char* prefixString, int uvErrorCode)
{
std::ostringstream ost;
ost << prefixString << ": " << uv_strerror(uvErrorCode);
return ost.str();
}
bool cmQtAutoGenerator::ReadOnlyProcessT::start(
uv_loop_t* uv_loop, std::function<void()>&& finishedCallback)
{
if (IsStarted() || (Result() == nullptr)) {
return false;
}
// Reset result before the start
Result()->reset();
// Fill command string pointers
if (!Setup().Command.empty()) {
CommandPtr_.reserve(Setup().Command.size() + 1);
for (std::string const& arg : Setup().Command) {
CommandPtr_.push_back(arg.c_str());
}
CommandPtr_.push_back(nullptr);
} else {
Result()->ErrorMessage = "Empty command";
}
if (!Result()->error()) {
if (UVPipeOut_.init(uv_loop, this) != 0) {
Result()->ErrorMessage = "libuv stdout pipe initialization failed";
}
}
if (!Result()->error()) {
if (UVPipeErr_.init(uv_loop, this) != 0) {
Result()->ErrorMessage = "libuv stderr pipe initialization failed";
}
}
if (!Result()->error()) {
// -- Setup process stdio options
// stdin
UVOptionsStdIO_[0].flags = UV_IGNORE;
UVOptionsStdIO_[0].data.stream = nullptr;
// stdout
UVOptionsStdIO_[1].flags =
static_cast<uv_stdio_flags>(UV_CREATE_PIPE | UV_WRITABLE_PIPE);
UVOptionsStdIO_[1].data.stream = UVPipeOut_.uv_stream();
// stderr
UVOptionsStdIO_[2].flags =
static_cast<uv_stdio_flags>(UV_CREATE_PIPE | UV_WRITABLE_PIPE);
UVOptionsStdIO_[2].data.stream = UVPipeErr_.uv_stream();
// -- Setup process options
std::fill_n(reinterpret_cast<char*>(&UVOptions_), sizeof(UVOptions_), 0);
UVOptions_.exit_cb = &ReadOnlyProcessT::UVExit;
UVOptions_.file = CommandPtr_[0];
UVOptions_.args = const_cast<char**>(CommandPtr_.data());
UVOptions_.cwd = Setup_.WorkingDirectory.c_str();
UVOptions_.flags = UV_PROCESS_WINDOWS_HIDE;
UVOptions_.stdio_count = static_cast<int>(UVOptionsStdIO_.size());
UVOptions_.stdio = UVOptionsStdIO_.data();
// -- Spawn process
int uvErrorCode = UVProcess_.spawn(*uv_loop, UVOptions_, this);
if (uvErrorCode != 0) {
Result()->ErrorMessage =
getUVError("libuv process spawn failed ", uvErrorCode);
}
}
// -- Start reading from stdio streams
if (!Result()->error()) {
if (UVPipeOut_.startRead(&Result()->StdOut) != 0) {
Result()->ErrorMessage = "libuv start reading from stdout pipe failed";
}
}
if (!Result()->error()) {
if (UVPipeErr_.startRead(Setup_.MergedOutput ? &Result()->StdOut
: &Result()->StdErr) != 0) {
Result()->ErrorMessage = "libuv start reading from stderr pipe failed";
}
}
if (!Result()->error()) {
IsStarted_ = true;
FinishedCallback_ = std::move(finishedCallback);
} else {
// Clear libuv handles and finish
UVProcess_.reset();
UVPipeOut_.reset();
UVPipeErr_.reset();
CommandPtr_.clear();
}
return IsStarted();
}
void cmQtAutoGenerator::ReadOnlyProcessT::UVExit(uv_process_t* handle,
int64_t exitStatus,
int termSignal)
{
auto& proc = *reinterpret_cast<ReadOnlyProcessT*>(handle->data);
if (proc.IsStarted() && !proc.IsFinished()) {
// Set error message on demand
proc.Result()->ExitStatus = exitStatus;
proc.Result()->TermSignal = termSignal;
if (!proc.Result()->error()) {
if (termSignal != 0) {
proc.Result()->ErrorMessage = "Process was terminated by signal ";
proc.Result()->ErrorMessage +=
std::to_string(proc.Result()->TermSignal);
} else if (exitStatus != 0) {
proc.Result()->ErrorMessage = "Process failed with return value ";
proc.Result()->ErrorMessage +=
std::to_string(proc.Result()->ExitStatus);
}
}
// Reset process handle and try to finish
proc.UVProcess_.reset();
proc.UVTryFinish();
}
}
void cmQtAutoGenerator::ReadOnlyProcessT::UVTryFinish()
{
// There still might be data in the pipes after the process has finished.
// Therefore check if the process is finished AND all pipes are closed
// before signaling the worker thread to continue.
if (UVProcess_.get() == nullptr) {
if (UVPipeOut_.uv_pipe() == nullptr) {
if (UVPipeErr_.uv_pipe() == nullptr) {
IsFinished_ = true;
FinishedCallback_();
}
}
}
}
cmQtAutoGenerator::cmQtAutoGenerator() = default;
cmQtAutoGenerator::~cmQtAutoGenerator() = default;
......
......@@ -7,14 +7,8 @@
#include "cmFilePathChecksum.h"
#include "cmQtAutoGen.h"
#include "cmUVHandlePtr.h"
#include "cm_uv.h"
#include <array>
#include <functional>
#include <mutex>
#include <stddef.h>
#include <stdint.h>
#include <string>
#include <vector>
......@@ -137,102 +131,6 @@ public:
cmFilePathChecksum FilePathChecksum_;
};
/// @brief Return value and output of an external process
struct ProcessResultT
{
void reset();
bool error() const
{
return (ExitStatus != 0) || (TermSignal != 0) || !ErrorMessage.empty();
}
std::int64_t ExitStatus = 0;
int TermSignal = 0;
std::string StdOut;
std::string StdErr;
std::string ErrorMessage;
};
/// @brief External process management class
struct ReadOnlyProcessT
{
// -- Types
/// @brief libuv pipe buffer class
class PipeT
{
public:
int init(uv_loop_t* uv_loop, ReadOnlyProcessT* process);
int startRead(std::string* target);
void reset();
// -- Libuv casts
uv_pipe_t* uv_pipe() { return UVPipe_.get(); }
uv_stream_t* uv_stream()
{
return reinterpret_cast<uv_stream_t*>(uv_pipe());
}
uv_handle_t* uv_handle()
{
return reinterpret_cast<uv_handle_t*>(uv_pipe());
}
// -- Libuv callbacks
static void UVAlloc(uv_handle_t* handle, size_t suggestedSize,
uv_buf_t* buf);
static void UVData(uv_stream_t* stream, ssize_t nread,
const uv_buf_t* buf);
private:
ReadOnlyProcessT* Process_ = nullptr;
std::string* Target_ = nullptr;
std::vector<char> Buffer_;
cm::uv_pipe_ptr UVPipe_;
};
/// @brief Process settings
struct SetupT
{
std::string WorkingDirectory;
std::vector<std::string> Command;
ProcessResultT* Result = nullptr;
bool MergedOutput = false;
};
// -- Const accessors
const SetupT& Setup() const { return Setup_; }
ProcessResultT* Result() const { return Setup_.Result; }
bool IsStarted() const { return IsStarted_; }
bool IsFinished() const { return IsFinished_; }
// -- Runtime
void setup(ProcessResultT* result, bool mergedOutput,
std::vector<std::string> const& command,
std::string const& workingDirectory = std::string());
bool start(uv_loop_t* uv_loop, std::function<void()>&& finishedCallback);
private:
// -- Friends
friend class PipeT;
// -- Libuv callbacks
static void UVExit(uv_process_t* handle, int64_t exitStatus,
int termSignal);
void UVTryFinish();
// -- Setup
SetupT Setup_;
// -- Runtime
bool IsStarted_ = false;
bool IsFinished_ = false;
std::function<void()> FinishedCallback_;
std::vector<const char*> CommandPtr_;
std::array<uv_stdio_container_t, 3> UVOptionsStdIO_;
uv_process_options_t UVOptions_;
cm::uv_process_ptr UVProcess_;
PipeT UVPipeOut_;
PipeT UVPipeErr_;
};
public:
// -- Constructors
cmQtAutoGenerator();
......
/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying
file Copyright.txt or https://cmake.org/licensing for details. */
#ifndef cmQtAutoGeneratorMocUic_h
#define cmQtAutoGeneratorMocUic_h
#ifndef cmQtAutoMocUic_h
#define cmQtAutoMocUic_h
#include "cmConfigure.h" // IWYU pragma: keep
#include "cmQtAutoGen.h"
#include "cmQtAutoGenerator.h"
#include "cmUVHandlePtr.h"
#include "cmUVSignalHackRAII.h" // IWYU pragma: keep
#include "cm_uv.h"
#include "cmWorkerPool.h"
#include "cmsys/RegularExpression.hxx"
#include <condition_variable>
#include <cstddef>
#include <deque>
#include <array>
#include <atomic>
#include <map>
#include <memory> // IWYU pragma: keep
#include <mutex>
#include <set>
#include <string>
#include <thread>
#include <unordered_set>
#include <utility>
#include <vector>
......@@ -28,18 +24,18 @@
class cmMakefile;
// @brief AUTOMOC and AUTOUIC generator
class cmQtAutoGeneratorMocUic : public cmQtAutoGenerator
class cmQtAutoMocUic : public cmQtAutoGenerator
{
public:
cmQtAutoGeneratorMocUic();
~cmQtAutoGeneratorMocUic() override;
cmQtAutoMocUic();
~cmQtAutoMocUic() override;
cmQtAutoGeneratorMocUic(cmQtAutoGeneratorMocUic const&) = delete;
cmQtAutoGeneratorMocUic& operator=(cmQtAutoGeneratorMocUic const&) = delete;
cmQtAutoMocUic(cmQtAutoMocUic const&) = delete;
cmQtAutoMocUic& operator=(cmQtAutoMocUic const&) = delete;
public:
// -- Types
class WorkerT;
typedef std::multimap<std::string, std::array<std::string, 2>> IncludesMap;
/// @brief Search key plus regular expression pair
///
......@@ -173,31 +169,71 @@ public:
cmsys::RegularExpression RegExpInclude;
};
/// @brief Abstract job class for threaded processing
/// @brief Abstract job class for concurrent job processing
///
class JobT
class JobT : public cmWorkerPool::JobT
{
public:
JobT() = default;
virtual ~JobT() = default;
protected:
/**
* @brief Protected default constructor
*/
JobT(bool fence = false)
: cmWorkerPool::JobT(fence)
{
}
//! Get the generator. Only valid during Process() call!
cmQtAutoMocUic* Gen() const
{
return static_cast<cmQtAutoMocUic*>(UserData());
};
//! Get the file system interface. Only valid during Process() call!
FileSystem& FileSys() { return Gen()->FileSys(); }
//! Get the logger. Only valid during Process() call!
Logger& Log() { return Gen()->Log(); }
// -- Error logging with automatic abort
void LogError(GenT genType, std::string const& message) const;
void LogFileError(GenT genType, std::string const& filename,
std::string const& message) const;
void LogCommandError(GenT genType, std::string const& message,
std::vector<std::string> const& command,
std::string const& output) const;
JobT(JobT const&) = delete;
JobT& operator=(JobT const&) = delete;
/**
* @brief Run an external process. Use only during Process() call!
*/
bool RunProcess(GenT genType, cmWorkerPool::ProcessResultT& result,
std::vector<std::string> const& command);
};
// -- Abstract processing interface
virtual void Process(WorkerT& wrk) = 0;
/// @brief Fence job utility class
///
class JobFenceT : public JobT
{
public:
JobFenceT()
: JobT(true)
{
}
void Process() override{};
};
// Job management types
typedef std::unique_ptr<JobT> JobHandleT;
typedef std::deque<JobHandleT> JobQueueT;
/// @brief Generate moc_predefs.h
///
class JobMocPredefsT : public JobT
{
private:
void Process() override;
};
/// @brief Parse source job
/// @brief Parses a source file
///
class JobParseT : public JobT
{
public:
JobParseT(std::string&& fileName, bool moc, bool uic, bool header = false)
JobParseT(std::string fileName, bool moc, bool uic, bool header = false)
: FileName(std::move(fileName))
, AutoMoc(moc)
, AutoUic(uic)
......@@ -213,18 +249,15 @@ public:
std::string FileBase;
};
void Process(WorkerT& wrk) override;
bool ParseMocSource(WorkerT& wrk, MetaT const& meta);
bool ParseMocHeader(WorkerT& wrk, MetaT const& meta);
std::string MocStringHeaders(WorkerT& wrk,
std::string const& fileBase) const;
std::string MocFindIncludedHeader(WorkerT& wrk,
std::string const& includerDir,
void Process() override;
bool ParseMocSource(MetaT const& meta);
bool ParseMocHeader(MetaT const& meta);
std::string MocStringHeaders(std::string const& fileBase) const;
std::string MocFindIncludedHeader(std::string const& includerDir,
std::string const& includeBase);
bool ParseUic(WorkerT& wrk, MetaT const& meta);
bool ParseUicInclude(WorkerT& wrk, MetaT const& meta,
std::string&& includeString);
std::string UicFindIncludedFile(WorkerT& wrk, MetaT const& meta,
bool ParseUic(MetaT const& meta);
bool ParseUicInclude(MetaT const& meta, std::string&& includeString);
std::string UicFindIncludedFile(MetaT const& meta,
std::string const& includeString);
private:
......@@ -234,12 +267,20 @@ public:
bool Header = false;
};
/// @brief Generate moc_predefs
/// @brief Generates additional jobs after all files have been parsed
///
class JobMocPredefsT : public JobT
class JobPostParseT : public JobFenceT
{
private:
void Process(WorkerT& wrk) override;
void Process() override;
};
/// @brief Generate mocs_compilation.cpp
///
class JobMocsCompilationT : public JobFenceT
{
private:
void Process() override;
};
/// @brief Moc a file job
......@@ -247,20 +288,20 @@ public:
class JobMocT : public JobT
{
public:
JobMocT(std::string&& sourceFile, std::string includerFile,
std::string&& includeString)
JobMocT(std::string sourceFile, std::string includerFile,
std::string includeString)
: SourceFile(std::move(sourceFile))
, IncluderFile(std::move(includerFile))
, IncludeString(std::move(includeString))
{
}
void FindDependencies(WorkerT& wrk, std::string const& content);
void FindDependencies(std::string const& content);
private:
void Process(WorkerT& wrk) override;
bool UpdateRequired(WorkerT& wrk);
void GenerateMoc(WorkerT& wrk);
void Process() override;
bool UpdateRequired();
void GenerateMoc();
public:
std::string SourceFile;
......@@ -276,8 +317,8 @@ public:
class JobUicT : public JobT
{
public:
JobUicT(std::string&& sourceFile, std::string includerFile,