cmCTestLaunch.cxx 17.4 KB
Newer Older
1
2
/* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
   file Copyright.txt or https://cmake.org/licensing for details.  */
3
4
#include "cmCTestLaunch.h"

5
6
7
#include "cmsys/FStream.hxx"
#include "cmsys/Process.h"
#include "cmsys/RegularExpression.hxx"
Daniel Pfeifer's avatar
Daniel Pfeifer committed
8
#include <iostream>
9
#include <memory> // IWYU pragma: keep
Daniel Pfeifer's avatar
Daniel Pfeifer committed
10
11
12
#include <stdlib.h>
#include <string.h>

13
#include "cmCryptoHash.h"
14
#include "cmGeneratedFileStream.h"
15
16
#include "cmGlobalGenerator.h"
#include "cmMakefile.h"
17
#include "cmProcessOutput.h"
Daniel Pfeifer's avatar
Daniel Pfeifer committed
18
#include "cmStateSnapshot.h"
19
#include "cmSystemTools.h"
20
#include "cmXMLWriter.h"
21
22
#include "cmake.h"

23
24
#ifdef _WIN32
#include <fcntl.h> // for _O_BINARY
25
#include <io.h>    // for _setmode
26
27
28
#include <stdio.h> // for std{out,err} and fileno
#endif

29
30
31
cmCTestLaunch::cmCTestLaunch(int argc, const char* const* argv)
{
  this->Passthru = true;
Daniel Pfeifer's avatar
Daniel Pfeifer committed
32
  this->Process = nullptr;
33
34
35
  this->ExitCode = 1;
  this->CWD = cmSystemTools::GetCurrentWorkingDirectory();

36
  if (!this->ParseArguments(argc, argv)) {
37
    return;
38
  }
39
40
41
42
43
44
45
46
47
48
49
50

  this->ComputeFileNames();

  this->ScrapeRulesLoaded = false;
  this->HaveOut = false;
  this->HaveErr = false;
  this->Process = cmsysProcess_New();
}

cmCTestLaunch::~cmCTestLaunch()
{
  cmsysProcess_Delete(this->Process);
51
  if (!this->Passthru) {
52
53
    cmSystemTools::RemoveFile(this->LogOut);
    cmSystemTools::RemoveFile(this->LogErr);
54
  }
55
56
57
58
59
60
}

bool cmCTestLaunch::ParseArguments(int argc, const char* const* argv)
{
  // Launcher options occur first and are separated from the real
  // command line by a '--' option.
61
62
63
64
65
66
67
68
69
70
71
72
  enum Doing
  {
    DoingNone,
    DoingOutput,
    DoingSource,
    DoingLanguage,
    DoingTargetName,
    DoingTargetType,
    DoingBuildDir,
    DoingCount,
    DoingFilterPrefix
  };
73
74
  Doing doing = DoingNone;
  int arg0 = 0;
75
  for (int i = 1; !arg0 && i < argc; ++i) {
76
    const char* arg = argv[i];
77
78
79
    if (strcmp(arg, "--") == 0) {
      arg0 = i + 1;
    } else if (strcmp(arg, "--output") == 0) {
80
      doing = DoingOutput;
81
    } else if (strcmp(arg, "--source") == 0) {
82
      doing = DoingSource;
83
    } else if (strcmp(arg, "--language") == 0) {
84
      doing = DoingLanguage;
85
    } else if (strcmp(arg, "--target-name") == 0) {
86
      doing = DoingTargetName;
87
    } else if (strcmp(arg, "--target-type") == 0) {
88
      doing = DoingTargetType;
89
    } else if (strcmp(arg, "--build-dir") == 0) {
90
      doing = DoingBuildDir;
91
    } else if (strcmp(arg, "--filter-prefix") == 0) {
92
      doing = DoingFilterPrefix;
93
    } else if (doing == DoingOutput) {
94
95
      this->OptionOutput = arg;
      doing = DoingNone;
96
    } else if (doing == DoingSource) {
97
98
      this->OptionSource = arg;
      doing = DoingNone;
99
    } else if (doing == DoingLanguage) {
100
      this->OptionLanguage = arg;
101
      if (this->OptionLanguage == "CXX") {
102
103
        this->OptionLanguage = "C++";
      }
104
105
      doing = DoingNone;
    } else if (doing == DoingTargetName) {
106
107
      this->OptionTargetName = arg;
      doing = DoingNone;
108
    } else if (doing == DoingTargetType) {
109
110
      this->OptionTargetType = arg;
      doing = DoingNone;
111
    } else if (doing == DoingBuildDir) {
112
113
      this->OptionBuildDir = arg;
      doing = DoingNone;
114
    } else if (doing == DoingFilterPrefix) {
115
116
      this->OptionFilterPrefix = arg;
      doing = DoingNone;
117
    }
118
  }
119
120

  // Extract the real command line.
121
  if (arg0) {
122
123
    this->RealArgC = argc - arg0;
    this->RealArgV = argv + arg0;
124
    for (int i = 0; i < this->RealArgC; ++i) {
125
126
      this->HandleRealArg(this->RealArgV[i]);
    }
127
128
    return true;
  }
129
  this->RealArgC = 0;
Daniel Pfeifer's avatar
Daniel Pfeifer committed
130
  this->RealArgV = nullptr;
131
132
  std::cerr << "No launch/command separator ('--') found!\n";
  return false;
133
134
135
136
137
138
}

void cmCTestLaunch::HandleRealArg(const char* arg)
{
#ifdef _WIN32
  // Expand response file arguments.
139
140
  if (arg[0] == '@' && cmSystemTools::FileExists(arg + 1)) {
    cmsys::ifstream fin(arg + 1);
141
    std::string line;
142
    while (cmSystemTools::GetLineFromStream(fin, line)) {
143
144
      cmSystemTools::ParseWindowsCommandLine(line.c_str(), this->RealArgs);
    }
145
146
    return;
  }
147
148
149
150
151
152
153
154
155
#endif
  this->RealArgs.push_back(arg);
}

void cmCTestLaunch::ComputeFileNames()
{
  // We just passthru the behavior of the real command unless the
  // CTEST_LAUNCH_LOGS environment variable is set.
  const char* d = getenv("CTEST_LAUNCH_LOGS");
156
  if (!(d && *d)) {
157
    return;
158
  }
159
160
161
162
163
164
165
166
167
168
  this->Passthru = false;

  // The environment variable specifies the directory into which we
  // generate build logs.
  this->LogDir = d;
  cmSystemTools::ConvertToUnixSlashes(this->LogDir);
  this->LogDir += "/";

  // We hash the input command working dir and command line to obtain
  // a repeatable and (probably) unique name for log files.
169
170
171
  cmCryptoHash md5(cmCryptoHash::AlgoMD5);
  md5.Initialize();
  md5.Append(this->CWD);
172
173
  for (std::string const& realArg : this->RealArgs) {
    md5.Append(realArg);
174
  }
175
  this->LogHash = md5.FinalizeHex();
176
177
178
179
180
181
182
183
184
185
186
187
188
189

  // We store stdout and stderr in temporary log files.
  this->LogOut = this->LogDir;
  this->LogOut += "launch-";
  this->LogOut += this->LogHash;
  this->LogOut += "-out.txt";
  this->LogErr = this->LogDir;
  this->LogErr += "launch-";
  this->LogErr += this->LogHash;
  this->LogErr += "-err.txt";
}

void cmCTestLaunch::RunChild()
{
190
  // Ignore noopt make rules
191
  if (this->RealArgs.empty() || this->RealArgs[0] == ":") {
192
193
    this->ExitCode = 0;
    return;
194
  }
195

196
197
198
199
  // Prepare to run the real command.
  cmsysProcess* cp = this->Process;
  cmsysProcess_SetCommand(cp, this->RealArgV);

200
201
  cmsys::ofstream fout;
  cmsys::ofstream ferr;
202
  if (this->Passthru) {
203
204
205
    // In passthru mode we just share the output pipes.
    cmsysProcess_SetPipeShared(cp, cmsysProcess_Pipe_STDOUT, 1);
    cmsysProcess_SetPipeShared(cp, cmsysProcess_Pipe_STDERR, 1);
206
  } else {
207
    // In full mode we record the child output pipes to log files.
208
209
210
    fout.open(this->LogOut.c_str(), std::ios::out | std::ios::binary);
    ferr.open(this->LogErr.c_str(), std::ios::out | std::ios::binary);
  }
211

212
213
214
215
216
217
218
#ifdef _WIN32
  // Do this so that newline transformation is not done when writing to cout
  // and cerr below.
  _setmode(fileno(stdout), _O_BINARY);
  _setmode(fileno(stderr), _O_BINARY);
#endif

219
220
221
222
  // Run the real command.
  cmsysProcess_Execute(cp);

  // Record child stdout and stderr if necessary.
223
  if (!this->Passthru) {
Daniel Pfeifer's avatar
Daniel Pfeifer committed
224
    char* data = nullptr;
225
    int length = 0;
226
227
    cmProcessOutput processOutput;
    std::string strdata;
Daniel Pfeifer's avatar
Daniel Pfeifer committed
228
    while (int p = cmsysProcess_WaitForData(cp, &data, &length, nullptr)) {
229
      if (p == cmsysProcess_Pipe_STDOUT) {
230
231
232
        processOutput.DecodeText(data, length, strdata, 1);
        fout.write(strdata.c_str(), strdata.size());
        std::cout.write(strdata.c_str(), strdata.size());
233
        this->HaveOut = true;
234
      } else if (p == cmsysProcess_Pipe_STDERR) {
235
236
237
        processOutput.DecodeText(data, length, strdata, 2);
        ferr.write(strdata.c_str(), strdata.size());
        std::cerr.write(strdata.c_str(), strdata.size());
238
239
240
        this->HaveErr = true;
      }
    }
241
242
243
244
245
246
247
248
249
250
    processOutput.DecodeText(std::string(), strdata, 1);
    if (!strdata.empty()) {
      fout.write(strdata.c_str(), strdata.size());
      std::cout.write(strdata.c_str(), strdata.size());
    }
    processOutput.DecodeText(std::string(), strdata, 2);
    if (!strdata.empty()) {
      ferr.write(strdata.c_str(), strdata.size());
      std::cerr.write(strdata.c_str(), strdata.size());
    }
251
  }
252
253

  // Wait for the real command to finish.
Daniel Pfeifer's avatar
Daniel Pfeifer committed
254
  cmsysProcess_WaitForExit(cp, nullptr);
255
256
257
258
259
  this->ExitCode = cmsysProcess_GetExitValue(cp);
}

int cmCTestLaunch::Run()
{
260
  if (!this->Process) {
261
262
    std::cerr << "Could not allocate cmsysProcess instance!\n";
    return -1;
263
  }
264
265
266

  this->RunChild();

267
  if (this->CheckResults()) {
268
    return this->ExitCode;
269
  }
270

271
  this->LoadConfig();
272
273
274
275
276
277
278
  this->WriteXML();

  return this->ExitCode;
}

void cmCTestLaunch::LoadLabels()
{
279
  if (this->OptionBuildDir.empty() || this->OptionTargetName.empty()) {
280
    return;
281
  }
282
283
284
285
286
287
288
289
290
291
292
293
294

  // Labels are listed in per-target files.
  std::string fname = this->OptionBuildDir;
  fname += cmake::GetCMakeFilesDirectory();
  fname += "/";
  fname += this->OptionTargetName;
  fname += ".dir/Labels.txt";

  // We are interested in per-target labels for this source file.
  std::string source = this->OptionSource;
  cmSystemTools::ConvertToUnixSlashes(source);

  // Load the labels file.
295
  cmsys::ifstream fin(fname.c_str(), std::ios::in | std::ios::binary);
296
297
298
  if (!fin) {
    return;
  }
299
300
301
  bool inTarget = true;
  bool inSource = false;
  std::string line;
302
303
  while (cmSystemTools::GetLineFromStream(fin, line)) {
    if (line.empty() || line[0] == '#') {
304
305
      // Ignore blank and comment lines.
      continue;
306
307
    }
    if (line[0] == ' ') {
308
      // Label lines appear indented by one space.
309
310
      if (inTarget || inSource) {
        this->Labels.insert(line.c_str() + 1);
311
      }
312
    } else if (!this->OptionSource.empty() && !inSource) {
313
314
315
316
317
      // Non-indented lines specify a source file name.  The first one
      // is the end of the target-wide labels.  Use labels following a
      // matching source.
      inTarget = false;
      inSource = this->SourceMatches(line, source);
318
    } else {
319
320
      return;
    }
321
  }
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
}

bool cmCTestLaunch::SourceMatches(std::string const& lhs,
                                  std::string const& rhs)
{
  // TODO: Case sensitivity, UseRelativePaths, etc.  Note that both
  // paths in the comparison get generated by CMake.  This is done for
  // every source in the target, so it should be efficient (cannot use
  // cmSystemTools::IsSameFile).
  return lhs == rhs;
}

bool cmCTestLaunch::IsError() const
{
  return this->ExitCode != 0;
}

void cmCTestLaunch::WriteXML()
{
  // Name the xml file.
  std::string logXML = this->LogDir;
343
  logXML += this->IsError() ? "error-" : "warning-";
344
345
346
347
348
  logXML += this->LogHash;
  logXML += ".xml";

  // Use cmGeneratedFileStream to atomically create the report file.
  cmGeneratedFileStream fxml(logXML.c_str());
349
350
351
352
353
354
355
356
  cmXMLWriter xml(fxml, 2);
  xml.StartElement("Failure");
  xml.Attribute("type", this->IsError() ? "Error" : "Warning");
  this->WriteXMLAction(xml);
  this->WriteXMLCommand(xml);
  this->WriteXMLResult(xml);
  this->WriteXMLLabels(xml);
  xml.EndElement(); // Failure
357
358
}

359
void cmCTestLaunch::WriteXMLAction(cmXMLWriter& xml)
360
{
361
362
  xml.Comment("Meta-information about the build action");
  xml.StartElement("Action");
363
364

  // TargetName
365
  if (!this->OptionTargetName.empty()) {
366
    xml.Element("TargetName", this->OptionTargetName);
367
  }
368
369

  // Language
370
  if (!this->OptionLanguage.empty()) {
371
    xml.Element("Language", this->OptionLanguage);
372
  }
373
374

  // SourceFile
375
  if (!this->OptionSource.empty()) {
376
377
378
379
    std::string source = this->OptionSource;
    cmSystemTools::ConvertToUnixSlashes(source);

    // If file is in source tree use its relative location.
380
381
    if (cmSystemTools::FileIsFullPath(this->SourceDir) &&
        cmSystemTools::FileIsFullPath(source) &&
382
        cmSystemTools::IsSubDirectory(source, this->SourceDir)) {
383
      source = cmSystemTools::RelativePath(this->SourceDir, source);
384
    }
385

386
    xml.Element("SourceFile", source);
387
  }
388
389

  // OutputFile
390
  if (!this->OptionOutput.empty()) {
391
    xml.Element("OutputFile", this->OptionOutput);
392
  }
393
394

  // OutputType
Daniel Pfeifer's avatar
Daniel Pfeifer committed
395
  const char* outputType = nullptr;
396
397
  if (!this->OptionTargetType.empty()) {
    if (this->OptionTargetType == "EXECUTABLE") {
398
      outputType = "executable";
399
    } else if (this->OptionTargetType == "SHARED_LIBRARY") {
400
      outputType = "shared library";
401
    } else if (this->OptionTargetType == "MODULE_LIBRARY") {
402
      outputType = "module library";
403
    } else if (this->OptionTargetType == "STATIC_LIBRARY") {
404
405
      outputType = "static library";
    }
406
  } else if (!this->OptionSource.empty()) {
407
    outputType = "object file";
408
409
  }
  if (outputType) {
410
    xml.Element("OutputType", outputType);
411
  }
412

413
  xml.EndElement(); // Action
414
415
}

416
void cmCTestLaunch::WriteXMLCommand(cmXMLWriter& xml)
417
{
418
419
  xml.Comment("Details of command");
  xml.StartElement("Command");
420
  if (!this->CWD.empty()) {
421
    xml.Element("WorkingDirectory", this->CWD);
422
  }
423
424
  for (std::string const& realArg : this->RealArgs) {
    xml.Element("Argument", realArg);
425
  }
426
  xml.EndElement(); // Command
427
428
}

429
void cmCTestLaunch::WriteXMLResult(cmXMLWriter& xml)
430
{
431
432
  xml.Comment("Result of command");
  xml.StartElement("Result");
433
434

  // StdOut
435
436
437
  xml.StartElement("StdOut");
  this->DumpFileToXML(xml, this->LogOut);
  xml.EndElement(); // StdOut
438
439

  // StdErr
440
441
442
  xml.StartElement("StdErr");
  this->DumpFileToXML(xml, this->LogErr);
  xml.EndElement(); // StdErr
443
444

  // ExitCondition
445
  xml.StartElement("ExitCondition");
446
  cmsysProcess* cp = this->Process;
447
  switch (cmsysProcess_GetState(cp)) {
448
    case cmsysProcess_State_Starting:
449
450
      xml.Content("No process has been executed");
      break;
451
    case cmsysProcess_State_Executing:
452
453
      xml.Content("The process is still executing");
      break;
454
    case cmsysProcess_State_Disowned:
455
456
      xml.Content("Disowned");
      break;
457
    case cmsysProcess_State_Killed:
458
459
      xml.Content("Killed by parent");
      break;
460
461

    case cmsysProcess_State_Expired:
462
463
      xml.Content("Killed when timeout expired");
      break;
464
    case cmsysProcess_State_Exited:
465
466
      xml.Content(this->ExitCode);
      break;
467
    case cmsysProcess_State_Exception:
468
      xml.Content("Terminated abnormally: ");
469
470
      xml.Content(cmsysProcess_GetExceptionString(cp));
      break;
471
    case cmsysProcess_State_Error:
472
      xml.Content("Error administrating child process: ");
473
474
475
      xml.Content(cmsysProcess_GetErrorString(cp));
      break;
  };
476
  xml.EndElement(); // ExitCondition
477

478
  xml.EndElement(); // Result
479
480
}

481
void cmCTestLaunch::WriteXMLLabels(cmXMLWriter& xml)
482
483
{
  this->LoadLabels();
484
  if (!this->Labels.empty()) {
485
486
    xml.Comment("Interested parties");
    xml.StartElement("Labels");
487
488
    for (std::string const& label : this->Labels) {
      xml.Element("Label", label);
489
    }
490
491
    xml.EndElement(); // Labels
  }
492
493
}

494
void cmCTestLaunch::DumpFileToXML(cmXMLWriter& xml, std::string const& fname)
495
{
496
  cmsys::ifstream fin(fname.c_str(), std::ios::in | std::ios::binary);
497
498
499

  std::string line;
  const char* sep = "";
500

501
502
  while (cmSystemTools::GetLineFromStream(fin, line)) {
    if (MatchesFilterPrefix(line)) {
503
      continue;
504
    }
505
506
507
508
509
    if (this->Match(line, this->RegexWarningSuppress)) {
      line = "[CTest: warning suppressed] " + line;
    } else if (this->Match(line, this->RegexWarning)) {
      line = "[CTest: warning matched] " + line;
    }
510
511
    xml.Content(sep);
    xml.Content(line);
512
    sep = "\n";
513
  }
514
515
516
517
518
}

bool cmCTestLaunch::CheckResults()
{
  // Skip XML in passthru mode.
519
  if (this->Passthru) {
520
    return true;
521
  }
522
523

  // We always report failure for error conditions.
524
  if (this->IsError()) {
525
    return false;
526
  }
527
528

  // Scrape the output logs to look for warnings.
529
530
  if ((this->HaveErr && this->ScrapeLog(this->LogErr)) ||
      (this->HaveOut && this->ScrapeLog(this->LogOut))) {
531
    return false;
532
  }
533
534
535
536
537
  return true;
}

void cmCTestLaunch::LoadScrapeRules()
{
538
  if (this->ScrapeRulesLoaded) {
539
    return;
540
  }
541
542
543
544
545
546
547
548
549
550
551
552
553
554
  this->ScrapeRulesLoaded = true;

  // Common compiler warning formats.  These are much simpler than the
  // full log-scraping expressions because we do not need to extract
  // file and line information.
  this->RegexWarning.push_back("(^|[ :])[Ww][Aa][Rr][Nn][Ii][Nn][Gg]");
  this->RegexWarning.push_back("(^|[ :])[Rr][Ee][Mm][Aa][Rr][Kk]");
  this->RegexWarning.push_back("(^|[ :])[Nn][Oo][Tt][Ee]");

  // Load custom match rules given to us by CTest.
  this->LoadScrapeRules("Warning", this->RegexWarning);
  this->LoadScrapeRules("WarningSuppress", this->RegexWarningSuppress);
}

555
556
void cmCTestLaunch::LoadScrapeRules(
  const char* purpose, std::vector<cmsys::RegularExpression>& regexps)
557
558
559
560
561
{
  std::string fname = this->LogDir;
  fname += "Custom";
  fname += purpose;
  fname += ".txt";
562
  cmsys::ifstream fin(fname.c_str(), std::ios::in | std::ios::binary);
563
564
  std::string line;
  cmsys::RegularExpression rex;
565
  while (cmSystemTools::GetLineFromStream(fin, line)) {
566
    if (rex.compile(line)) {
567
568
      regexps.push_back(rex);
    }
569
  }
570
571
572
573
574
575
576
577
}

bool cmCTestLaunch::ScrapeLog(std::string const& fname)
{
  this->LoadScrapeRules();

  // Look for log file lines matching warning expressions but not
  // suppression expressions.
578
  cmsys::ifstream fin(fname.c_str(), std::ios::in | std::ios::binary);
579
  std::string line;
580
581
  while (cmSystemTools::GetLineFromStream(fin, line)) {
    if (MatchesFilterPrefix(line)) {
582
      continue;
583
    }
584

585
586
    if (this->Match(line, this->RegexWarning) &&
        !this->Match(line, this->RegexWarningSuppress)) {
587
588
      return true;
    }
589
  }
590
591
592
593
594
595
  return false;
}

bool cmCTestLaunch::Match(std::string const& line,
                          std::vector<cmsys::RegularExpression>& regexps)
{
596
597
  for (cmsys::RegularExpression& r : regexps) {
    if (r.find(line.c_str())) {
598
599
      return true;
    }
600
  }
601
602
603
  return false;
}

604
605
bool cmCTestLaunch::MatchesFilterPrefix(std::string const& line) const
{
Daniel Pfeifer's avatar
Daniel Pfeifer committed
606
607
  return !this->OptionFilterPrefix.empty() &&
    cmSystemTools::StringStartsWith(line, this->OptionFilterPrefix.c_str());
608
609
}

610
611
int cmCTestLaunch::Main(int argc, const char* const argv[])
{
612
  if (argc == 2) {
613
614
615
    std::cerr << "ctest --launch: this mode is for internal CTest use only"
              << std::endl;
    return 1;
616
  }
617
618
619
  cmCTestLaunch self(argc, argv);
  return self.Run();
}
620
621
622

void cmCTestLaunch::LoadConfig()
{
623
  cmake cm(cmake::RoleScript);
624
625
  cm.SetHomeDirectory("");
  cm.SetHomeOutputDirectory("");
626
  cm.GetCurrentSnapshot().SetDefaultDefinitions();
627
  cmGlobalGenerator gg(&cm);
628
  cmMakefile mf(&gg, cm.GetCurrentSnapshot());
629
630
  std::string fname = this->LogDir;
  fname += "CTestLaunchConfig.cmake";
631
  if (cmSystemTools::FileExists(fname) && mf.ReadListFile(fname.c_str())) {
632
    this->SourceDir = mf.GetSafeDefinition("CTEST_SOURCE_DIRECTORY");
633
    cmSystemTools::ConvertToUnixSlashes(this->SourceDir);
634
  }
635
}