Commit 7a18e4be authored by Rich Chiodo's avatar Rich Chiodo
Browse files

Simplify and reenable backtraces

Add tests to verify support for backtraces is available.

Example: https://github.com/Kitware/kwiver
Old (3.10) codemodel json : 14 MBs
New codemodel json with traces : 4MB
New codemodel json without traces : 2MB
parent 5656ebc8
Pipeline #91729 passed with stage
......@@ -5,3 +5,5 @@
*.pyc
Testing
/.vs
/CMakeSettings.json
......@@ -440,6 +440,22 @@ The "codemodel" request can be used after a project was "compute"d successfully.
It will list the complete project structure as it is known to cmake.
If the optional parameter "includeTraces" is set to true, traces will be added
to responses and a "referenceTraces" key will also be included in the reply.
Reference traces contain an id to file/line/name location mapping for converting
embedded backtraces in other data to actual file/line/name locations.
Example:
"referencedTraces": [
{
"id": 1
"line": 3,
"name": "add_library",
"path": "D:/src/CMakeLists.txt"
}
]
The reply will contain a key "configurations", which will contain a list of
configuration objects. Configuration objects are used to destinquish between
different configurations the build directory might have enabled. While most
......@@ -517,6 +533,9 @@ Each target object can have the following keys:
with the sysroot path.
"fileGroups"
contains the source files making up the target.
"crossReferences" (protocol v2 and includeTraces=true)
contains the location of the target in the corresponding CMakeLists.txt
file and the locations of the related statements like "target_link_libraries"
FileGroups are used to group sources using similar settings together.
......@@ -542,10 +561,20 @@ Each fileGroup object may contain the following keys:
All file paths in the fileGroup are either absolute or relative to the
sourceDirectory of the target.
CrossReferences object is used to report the location of the target (including
the entire call stack if the target is defined in a function) and the related
"target_link_libraries", "target_include_directories", "target_compile_definitions"
and "target_compile_options" statements.
See the example below for details on the internal format of the "crossReferences" object.
Line numbers stated in the "backtrace" entries are 1-based. The last entry of a backtrace
is a special entry with missing "line" and "name" fields that specifies the initial
CMakeLists.txt file.
Example::
[== "CMake Server" ==[
{"type":"codemodel"}
{"type":"codemodel", "includeTraces" : true}
]== "CMake Server" ==]
CMake will reply::
......@@ -578,7 +607,22 @@ CMake will reply::
"linkerLanguage": "C",
"name": "cmForm",
"sourceDirectory": "/home/code/src/cmake/Source/CursesDialog/form",
"type": "STATIC_LIBRARY"
"type": "STATIC_LIBRARY",
"crossReferences": {
"backtrace": [
1,
2
],
"relatedStatements": [
{
"backtrace": [
3,
4,
],
"type": "target_link_libraries"
}
]
}
}
]
},
......@@ -586,6 +630,18 @@ CMake will reply::
]
}
],
"referencedTraces": [
{
"id" : 1
"line": 3,
"name": "add_library",
"path": "F:/CMake/HelloWorldWithLib/Lib/CMakeLists.txt"
},
{
"id" : 2
"path": "F:/CMake/HelloWorldWithLib/Lib/CMakeLists.txt"
},
]
"cookie": "",
"inReplyTo": "codemodel",
"type": "reply"
......@@ -600,6 +656,10 @@ The "ctestInfo" request can be used after a project was "compute"d successfully.
It will list the complete project test structure as it is known to cmake.
The reply will contain a key "referenceTraces" if on protocol v2.
Reference traces contain an id to file/line/name location mapping for converting
embedded backtraces to actual file/line/name locations.
The reply will contain a key "configurations", which will contain a list of
configuration objects. Configuration objects are used to destinquish between
different configurations the build directory might have enabled. While most
......@@ -629,6 +689,17 @@ Each test object can have the following keys:
contains the test command.
"properties"
contains a list of test property objects.
"backtrace" (protocol v2)
contains a list of backtrace objects that specify where the test was defined.
Each backtrace object can have the following keys:
"path"
contains the full path to the file containing the statement.
"line"
contains the line number in the file where the statement was defined.
"name"
contains the name of the statement that added the test.
Each test property object can have the following keys:
......
......@@ -277,6 +277,20 @@ bool cmListFileParser::AddArgument(cmListFileLexer_Token* token,
return true;
}
std::map<size_t, cmListFileContext> s_idToFrameMap;
std::map<cmListFileContext, size_t> s_frameToIdMap;
size_t ComputeFrameId(cmListFileContext const& frame)
{
auto it = s_frameToIdMap.find(frame);
if (it == s_frameToIdMap.end()) {
// Zero is a special id indicating not found. Always start at 1
it = s_frameToIdMap.emplace(frame, s_frameToIdMap.size() + 1).first;
s_idToFrameMap.emplace(it->second, it->first);
}
return it->second;
}
struct cmListFileBacktrace::Entry : public cmListFileContext
{
Entry(cmListFileContext const& lfc, Entry* up)
......@@ -403,9 +417,10 @@ void cmListFileBacktrace::PrintTitle(std::ostream& out) const
}
cmOutputConverter converter(this->Bottom);
cmListFileContext lfc = *this->Cur;
if (!this->Bottom.GetState()->GetIsInTryCompile()) {
lfc.FilePath = converter.ConvertToRelativePath(
this->Bottom.GetState()->GetSourceDirectory(), lfc.FilePath);
cmState* state = this->Bottom.GetState();
if (state && !state->GetIsInTryCompile()) {
lfc.FilePath = converter.ConvertToRelativePath(state->GetSourceDirectory(),
lfc.FilePath);
}
out << (lfc.Line ? " at " : " in ") << lfc;
}
......@@ -418,6 +433,7 @@ void cmListFileBacktrace::PrintCallStack(std::ostream& out) const
bool first = true;
cmOutputConverter converter(this->Bottom);
cmState* state = this->Bottom.GetState();
for (Entry* i = this->Cur->Up; i; i = i->Up) {
if (i->Name.empty()) {
// Skip this whole-file scope. When we get here we already will
......@@ -429,9 +445,9 @@ void cmListFileBacktrace::PrintCallStack(std::ostream& out) const
out << "Call Stack (most recent call first):\n";
}
cmListFileContext lfc = *i;
if (!this->Bottom.GetState()->GetIsInTryCompile()) {
if (state && !state->GetIsInTryCompile()) {
lfc.FilePath = converter.ConvertToRelativePath(
this->Bottom.GetState()->GetSourceDirectory(), lfc.FilePath);
state->GetSourceDirectory(), lfc.FilePath);
}
out << " " << lfc << "\n";
}
......@@ -450,6 +466,40 @@ size_t cmListFileBacktrace::Depth() const
return depth;
}
std::vector<size_t> const& cmListFileBacktrace::GetFrameIds() const
{
cmState* state = this->Bottom.GetState();
bool inTryCompile = state && state->GetIsInTryCompile();
auto& frameIds =
inTryCompile ? this->CompilingFrameIds : this->NonCompilingFrameIds;
if (this->Cur != nullptr && frameIds.empty()) {
cmOutputConverter converter(this->Bottom);
for (Entry* i = this->Cur; i; i = i->Up) {
cmListFileContext lfc = *i;
if (!inTryCompile && state) {
lfc.FilePath = converter.ConvertToRelativePath(
state->GetSourceDirectory(), lfc.FilePath);
}
frameIds.emplace_back(ComputeFrameId(lfc));
}
}
return frameIds;
}
std::vector<std::pair<size_t, cmListFileContext>>
cmListFileBacktrace::ConvertFrameIds(
std::unordered_set<size_t> const& frameIds)
{
std::vector<std::pair<size_t, cmListFileContext>> results;
for (auto id : frameIds) {
auto it = s_idToFrameMap.find(id);
if (it != s_idToFrameMap.end()) {
results.push_back(std::make_pair(it->first, it->second));
}
}
return std::move(results);
}
std::ostream& operator<<(std::ostream& os, cmListFileContext const& lfc)
{
os << lfc.FilePath;
......
......@@ -8,6 +8,7 @@
#include <iosfwd>
#include <stddef.h>
#include <string>
#include <unordered_set>
#include <vector>
#include "cmStateSnapshot.h"
......@@ -142,6 +143,13 @@ public:
// Get the number of 'frames' in this backtrace
size_t Depth() const;
// Return a list of ids that can be used to query for traces later
std::vector<size_t> const& GetFrameIds() const;
// Convert a list of frame ids into their actual representation
static std::vector<std::pair<size_t, cmListFileContext>> ConvertFrameIds(
std::unordered_set<size_t> const& frameIds);
private:
struct Entry;
......@@ -150,6 +158,9 @@ private:
cmListFileBacktrace(cmStateSnapshot const& bottom, Entry* up,
cmListFileContext const& lfc);
cmListFileBacktrace(cmStateSnapshot const& bottom, Entry* cur);
std::vector<size_t> mutable NonCompilingFrameIds;
std::vector<size_t> mutable CompilingFrameIds;
};
struct cmListFile
......
......@@ -56,6 +56,7 @@ cmServer::cmServer(cmConnection* conn, bool supportExperimental)
{
// Register supported protocols:
this->RegisterProtocol(new cmServerProtocol1);
this->RegisterProtocol(new cmServerProtocol2);
}
cmServer::~cmServer()
......
......@@ -96,6 +96,13 @@ static const std::string kCTEST_COMMAND = "ctestCommand";
static const std::string kCTEST_INFO = "ctestInfo";
static const std::string kMINIMUM_CMAKE_VERSION = "minimumCMakeVersion";
static const std::string kIS_GENERATOR_PROVIDED_KEY = "isGeneratorProvided";
static const std::string kTARGET_CROSS_REFERENCES_KEY = "crossReferences";
static const std::string kLINE_NUMBER_KEY = "line";
static const std::string kBACKTRACE_KEY = "backtrace";
static const std::string kRELATED_STATEMENTS_KEY = "relatedStatements";
static const std::string KREFERENCED_TRACES_KEY = "referencedTraces";
static const std::string kID_KEY = "id";
static const std::string kINCLUDE_TRACES_KEY = "includeTraces";
static const std::string kSTART_MAGIC = "[== \"CMake Server\" ==[";
static const std::string kEND_MAGIC = "]== \"CMake Server\" ==]";
......
......@@ -11,6 +11,7 @@
#include "cmInstallGenerator.h"
#include "cmInstallTargetGenerator.h"
#include "cmLinkLineComputer.h"
#include "cmListFileCache.h"
#include "cmLocalGenerator.h"
#include "cmMakefile.h"
#include "cmProperty.h"
......@@ -259,6 +260,11 @@ std::pair<int, int> cmServerProtocol1::ProtocolVersion() const
return std::make_pair(1, 2);
}
std::pair<int, int> cmServerProtocol2::ProtocolVersion() const
{
return std::make_pair(2, 0);
}
static void setErrorMessage(std::string* errorMessage, const std::string& text)
{
if (errorMessage) {
......@@ -770,8 +776,66 @@ static Json::Value DumpSourceFilesList(
return result;
}
static Json::Value DumpBacktrace_Protocol1(
const cmListFileBacktrace& backtrace)
{
Json::Value result = Json::arrayValue;
cmListFileBacktrace backtraceCopy = backtrace;
while (!backtraceCopy.Top().FilePath.empty()) {
Json::Value entry = Json::objectValue;
entry[kPATH_KEY] = backtraceCopy.Top().FilePath;
if (backtraceCopy.Top().Line) {
entry[kLINE_NUMBER_KEY] = static_cast<int>(backtraceCopy.Top().Line);
}
if (!backtraceCopy.Top().Name.empty()) {
entry[kNAME_KEY] = backtraceCopy.Top().Name;
}
result.append(entry);
backtraceCopy = backtraceCopy.Pop();
}
return result;
}
static Json::Value DumpBacktrace_Protocol2(
const cmListFileBacktrace& backtrace,
std::unordered_set<size_t>* seenTraceIds)
{
Json::Value result = Json::arrayValue;
auto ids = backtrace.GetFrameIds();
for (auto& id : ids) {
result.append(id);
seenTraceIds->emplace(id);
}
return std::move(result);
}
static Json::Value DumpBacktrace(const cmListFileBacktrace& backtrace,
std::unordered_set<size_t>* seenTraceIds)
{
if (seenTraceIds == nullptr) {
return DumpBacktrace_Protocol1(backtrace);
}
return DumpBacktrace_Protocol2(backtrace, seenTraceIds);
}
static void DumpBacktraceRange(Json::Value& result, const std::string& type,
cmBacktraceRange range,
std::unordered_set<size_t>* seenTraceIds)
{
for (auto const& bt : range) {
Json::Value obj = Json::objectValue;
obj[kTYPE_KEY] = type;
obj[kBACKTRACE_KEY] = DumpBacktrace(bt, seenTraceIds);
result.append(obj);
}
}
static Json::Value DumpCTestInfo(cmLocalGenerator* lg, cmTest* testInfo,
const std::string& config)
const std::string& config,
std::unordered_set<size_t>* seenTraceIds)
{
Json::Value result = Json::objectValue;
result[kCTEST_NAME] = testInfo->GetName();
......@@ -805,25 +869,34 @@ static Json::Value DumpCTestInfo(cmLocalGenerator* lg, cmTest* testInfo,
}
result[kPROPERTIES_KEY] = properties;
return result;
if (seenTraceIds != nullptr) {
// Need backtrace to figure out where this test was originally added
result[kBACKTRACE_KEY] =
DumpBacktrace(testInfo->GetBacktrace(), seenTraceIds);
}
return std::move(result);
}
static void DumpMakefileTests(cmLocalGenerator* lg, const std::string& config,
Json::Value* result)
Json::Value* result,
std::unordered_set<size_t>* seenTraceIds)
{
auto mf = lg->GetMakefile();
std::vector<cmTest*> tests;
mf->GetTests(config, tests);
for (auto test : tests) {
Json::Value tmp = DumpCTestInfo(lg, test, config);
Json::Value tmp = DumpCTestInfo(lg, test, config, seenTraceIds);
if (!tmp.isNull()) {
result->append(tmp);
}
}
}
static Json::Value DumpCTestProjectList(const cmake* cm,
std::string const& config)
static Json::Value DumpCTestProjectList(
const cmake* cm, std::string const& config,
std::unordered_set<size_t>* seenTraceIds)
{
Json::Value result = Json::arrayValue;
......@@ -839,7 +912,8 @@ static Json::Value DumpCTestProjectList(const cmake* cm,
for (const auto& lg : projectIt.second) {
// Make sure they're generated.
lg->GenerateTestFiles();
DumpMakefileTests(lg, config, &tests);
DumpMakefileTests(lg, config, &tests, seenTraceIds);
}
pObj[kCTEST_INFO] = tests;
......@@ -850,30 +924,33 @@ static Json::Value DumpCTestProjectList(const cmake* cm,
return result;
}
static Json::Value DumpCTestConfiguration(const cmake* cm,
const std::string& config)
static Json::Value DumpCTestConfiguration(
const cmake* cm, const std::string& config,
std::unordered_set<size_t>* seenTraceIds)
{
Json::Value result = Json::objectValue;
result[kNAME_KEY] = config;
result[kPROJECTS_KEY] = DumpCTestProjectList(cm, config);
result[kPROJECTS_KEY] = DumpCTestProjectList(cm, config, seenTraceIds);
return result;
}
static Json::Value DumpCTestConfigurationsList(const cmake* cm)
static Json::Value DumpCTestConfigurationsList(
const cmake* cm, std::unordered_set<size_t>* seenTraceIds)
{
Json::Value result = Json::arrayValue;
for (const std::string& c : getConfigurations(cm)) {
result.append(DumpCTestConfiguration(cm, c));
result.append(DumpCTestConfiguration(cm, c, seenTraceIds));
}
return result;
}
static Json::Value DumpTarget(cmGeneratorTarget* target,
const std::string& config)
static Json::Value DumpTarget(
cmGeneratorTarget* target, const std::string& config,
std::unordered_set<size_t>* seenTraceIds = nullptr)
{
cmLocalGenerator* lg = target->GetLocalGenerator();
const cmState* state = lg->GetState();
......@@ -936,6 +1013,29 @@ static Json::Value DumpTarget(cmGeneratorTarget* target,
result[kINSTALL_PATHS] = installPaths;
}
if (seenTraceIds != nullptr) {
Json::Value crossRefs = Json::objectValue;
crossRefs[kBACKTRACE_KEY] =
DumpBacktrace(target->Target->GetBacktrace(), seenTraceIds);
Json::Value statements = Json::arrayValue;
DumpBacktraceRange(statements, "target_compile_definitions",
target->Target->GetCompileDefinitionsBacktraces(),
seenTraceIds);
DumpBacktraceRange(statements, "target_include_directories",
target->Target->GetIncludeDirectoriesBacktraces(),
seenTraceIds);
DumpBacktraceRange(statements, "target_compile_options",
target->Target->GetCompileOptionsBacktraces(),
seenTraceIds);
DumpBacktraceRange(statements, "target_link_libraries",
target->Target->GetLinkImplementationBacktraces(),
seenTraceIds);
crossRefs[kRELATED_STATEMENTS_KEY] = std::move(statements);
result[kTARGET_CROSS_REFERENCES_KEY] = std::move(crossRefs);
}
if (target->HaveWellDefinedOutputFiles()) {
Json::Value artifacts = Json::arrayValue;
artifacts.append(
......@@ -1019,7 +1119,8 @@ static Json::Value DumpTarget(cmGeneratorTarget* target,
}
static Json::Value DumpTargetsList(
const std::vector<cmLocalGenerator*>& generators, const std::string& config)
const std::vector<cmLocalGenerator*>& generators, const std::string& config,
std::unordered_set<size_t>* seenTraceIds)
{
Json::Value result = Json::arrayValue;
......@@ -1031,7 +1132,7 @@ static Json::Value DumpTargetsList(
std::sort(targetList.begin(), targetList.end());
for (cmGeneratorTarget* target : targetList) {
Json::Value tmp = DumpTarget(target, config);
Json::Value tmp = DumpTarget(target, config, seenTraceIds);
if (!tmp.isNull()) {
result.append(tmp);
}
......@@ -1040,7 +1141,8 @@ static Json::Value DumpTargetsList(
return result;
}
static Json::Value DumpProjectList(const cmake* cm, std::string const& config)
static Json::Value DumpProjectList(const cmake* cm, std::string const& config,
std::unordered_set<size_t>* seenTraceIds)
{
Json::Value result = Json::arrayValue;
......@@ -1060,7 +1162,8 @@ static Json::Value DumpProjectList(const cmake* cm, std::string const& config)
pObj[kMINIMUM_CMAKE_VERSION] = minVersion ? minVersion : "";
pObj[kSOURCE_DIRECTORY_KEY] = mf->GetCurrentSourceDirectory();
pObj[kBUILD_DIRECTORY_KEY] = mf->GetCurrentBinaryDirectory();
pObj[kTARGETS_KEY] = DumpTargetsList(projectIt.second, config);
pObj[kTARGETS_KEY] =
DumpTargetsList(projectIt.second, config, seenTraceIds);
// For a project-level install rule it might be defined in any of its
// associated generators.
......@@ -1083,22 +1186,54 @@ static Json::Value DumpProjectList(const cmake* cm, std::string const& config)
}
static Json::Value DumpConfiguration(const cmake* cm,
const std::string& config)
const std::string& config,
std::unordered_set<size_t>* seenTraceIds)
{
Json::Value result = Json::objectValue;
result[kNAME_KEY] = config;
result[kPROJECTS_KEY] = DumpProjectList(cm, config);
result[kPROJECTS_KEY] = DumpProjectList(cm, config, seenTraceIds);
return result;
}
static Json::Value DumpConfigurationsList(const cmake* cm)
static Json::Value DumpConfigurationsList(
const cmake* cm, std::unordered_set<size_t>* seenTraceIds)
{
Json::Value result = Json::arrayValue;
for (std::string const& c : getConfigurations(cm)) {
result.append(DumpConfiguration(cm, c));
result.append(DumpConfiguration(cm, c, seenTraceIds));
}
return result;
}
static Json::Value DumpFrame(size_t id, const cmListFileContext& frame)
{
Json::Value entry = Json::objectValue;
entry[kID_KEY] = id;
entry[kPATH_KEY] = frame.FilePath;
if (frame.Line) {
entry[kLINE_NUMBER_KEY] = static_cast<int>(frame.Line);
}
if (!frame.Name.empty()) {
entry[kNAME_KEY] = frame.Name;
}
return entry;
}
static Json::Value DumpReferencedTraces(
std::unordered_set<size_t>& seenTraceIds)
{
Json::Value result = Json::arrayValue;
auto frames = cmListFileBacktrace::ConvertFrameIds(seenTraceIds);
for (const auto& pair : frames) {
result.append(DumpFrame(pair.first, pair.second));
}
return result;
......@@ -1112,7 +1247,26 @@ cmServerResponse cmServerProtocol1::ProcessCodeModel(
}
Json::Value result = Json::objectValue;
result[kCONFIGURATIONS_KEY] = DumpConfigurationsList(this->CMakeInstance());
result[kCONFIGURATIONS_KEY] =
DumpConfigurationsList(this->CMakeInstance(), nullptr);
return request.Reply(result);
}
cmServerResponse cmServerProtocol2::ProcessCodeModel(
const cmServerRequest& request)
{
if (this->m_State != STATE_COMPUTED) {
return request.ReportError("No build system was generated yet.");
}
auto includeTraces = request.Data[kINCLUDE_TRACES_KEY].asBool();
std::unordered_set<size_t> seenTraceIds;
Json::Value result = Json::objectValue;
result[kCONFIGURATIONS_KEY] = DumpConfigurationsList(
this->CMakeInstance(), includeTraces ? &seenTraceIds : nullptr);
if (includeTraces) {
result[KREFERENCED_TRACES_KEY] = DumpReferencedTraces(seenTraceIds);
}
return request.Reply(result);
}
......@@ -1338,7 +1492,26 @@ cmServerResponse cmServerProtocol1::ProcessCTests(
Json::Value result = Json::objectValue;
result[kCONFIGURATIONS_KEY] =
DumpCTestConfigurationsList(this->CMakeInstance());
DumpCTestConfigurationsList(this->CMakeInstance(), nullptr);