cmCTestTestHandler.h 10 KB
Newer Older
1
2
/* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
   file Copyright.txt or https://cmake.org/licensing for details.  */
Ken Martin's avatar
Ken Martin committed
3
4
5
#ifndef cmCTestTestHandler_h
#define cmCTestTestHandler_h

6
#include "cmConfigure.h" // IWYU pragma: keep
7

8
#include "cmCTestGenericHandler.h"
Wouter Klouwen's avatar
Wouter Klouwen committed
9
#include "cmDuration.h"
10
#include "cmListFileCache.h"
Brad King's avatar
Brad King committed
11

12
#include "cmsys/RegularExpression.hxx"
13
#include <chrono>
14
#include <cstdint>
15
16
17
18
19
20
21
#include <iosfwd>
#include <map>
#include <set>
#include <stddef.h>
#include <string>
#include <utility>
#include <vector>
Ken Martin's avatar
Ken Martin committed
22

23
class cmCTest;
Ken Martin's avatar
Ken Martin committed
24
class cmMakefile;
25
class cmXMLWriter;
Ken Martin's avatar
Ken Martin committed
26
27
28
29
30

/** \class cmCTestTestHandler
 * \brief A class that handles ctest -S invocations
 *
 */
31
class cmCTestTestHandler : public cmCTestGenericHandler
Ken Martin's avatar
Ken Martin committed
32
{
33
  friend class cmCTestRunTest;
34
  friend class cmCTestMultiProcessHandler;
35

Ken Martin's avatar
Ken Martin committed
36
public:
37
  typedef cmCTestGenericHandler Superclass;
Ken Martin's avatar
Ken Martin committed
38

39
  /**
Ken Martin's avatar
Ken Martin committed
40
41
   * The main entry point for this class
   */
42
  int ProcessHandler() override;
Andy Cedilnik's avatar
Andy Cedilnik committed
43

44
  /**
luz.paz's avatar
luz.paz committed
45
   * When both -R and -I are used should the resulting test list be the
46
47
48
   * intersection or the union of the lists. By default it is the
   * intersection.
   */
Andy Cedilnik's avatar
Andy Cedilnik committed
49
  void SetUseUnion(bool val) { this->UseUnion = val; }
50

51
52
53
54
55
56
  /**
   * Set whether or not CTest should only execute the tests that failed
   * on the previous run.  By default this is false.
   */
  void SetRerunFailed(bool val) { this->RerunFailed = val; }

57
58
59
  /**
   * This method is called when reading CTest custom file
   */
60
  void PopulateCustomVectors(cmMakefile* mf) override;
Andy Cedilnik's avatar
Andy Cedilnik committed
61

62
  //! Control the use of the regular expresisons, call these methods to turn
63
  /// them on
Ken Martin's avatar
Ken Martin committed
64
65
  void UseIncludeRegExp();
  void UseExcludeRegExp();
66
67
  void SetIncludeRegExp(const char*);
  void SetExcludeRegExp(const char*);
Andy Cedilnik's avatar
Andy Cedilnik committed
68

69
70
  void SetMaxIndex(int n) { this->MaxIndex = n; }
  int GetMaxIndex() { return this->MaxIndex; }
Ken Martin's avatar
Ken Martin committed
71

72
  void SetTestOutputSizePassed(int n)
73
74
75
  {
    this->CustomMaximumPassedTestOutputSize = n;
  }
76
  void SetTestOutputSizeFailed(int n)
77
78
79
  {
    this->CustomMaximumFailedTestOutputSize = n;
  }
80

81
  //! pass the -I argument down
Ken Martin's avatar
Ken Martin committed
82
83
  void SetTestsToRunInformation(const char*);

84
  cmCTestTestHandler();
Ken Martin's avatar
Ken Martin committed
85

86
87
88
  /*
   * Add the test to the list of tests to be executed
   */
89
  bool AddTest(const std::vector<std::string>& args);
90

91
92
93
94
95
  /*
   * Set tests properties
   */
  bool SetTestsProperties(const std::vector<std::string>& args);

96
97
98
99
100
  /**
   * Set directory properties
   */
  bool SetDirectoryProperties(const std::vector<std::string>& args);

101
  void Initialize() override;
102

Bill Hoffman's avatar
Bill Hoffman committed
103
104
105
106
  // NOTE: This struct is Saved/Restored
  // in cmCTestTestHandler, if you add to this class
  // then you must add the new members to that code or
  // ctest -j N will break for that feature
Andy Cedilnik's avatar
Andy Cedilnik committed
107
108
  struct cmCTestTestProperties
  {
109
110
    std::string Name;
    std::string Directory;
Andy Cedilnik's avatar
Andy Cedilnik committed
111
    std::vector<std::string> Args;
112
    std::vector<std::string> RequiredFiles;
113
    std::vector<std::string> Depends;
114
    std::vector<std::string> AttachedFiles;
115
    std::vector<std::string> AttachOnFail;
116
    std::vector<std::pair<cmsys::RegularExpression, std::string>>
117
      ErrorRegularExpressions;
118
    std::vector<std::pair<cmsys::RegularExpression, std::string>>
119
      RequiredRegularExpressions;
120
121
    std::vector<std::pair<cmsys::RegularExpression, std::string>>
      SkipRegularExpressions;
122
    std::vector<std::pair<cmsys::RegularExpression, std::string>>
123
      TimeoutRegularExpressions;
124
    std::map<std::string, std::string> Measurements;
Andy Cedilnik's avatar
Andy Cedilnik committed
125
126
    bool IsInBasedOnREOptions;
    bool WillFail;
Betsy McPhail's avatar
Betsy McPhail committed
127
    bool Disabled;
128
    float Cost;
129
    int PreviousRuns;
130
    bool RunSerial;
Wouter Klouwen's avatar
Wouter Klouwen committed
131
    cmDuration Timeout;
132
    bool ExplicitTimeout;
Wouter Klouwen's avatar
Wouter Klouwen committed
133
    cmDuration AlternateTimeout;
134
    int Index;
135
    // Requested number of process slots
136
    int Processors;
137
138
    bool WantAffinity;
    std::vector<size_t> Affinity;
139
140
    // return code of test which will mark test as "not run"
    int SkipReturnCode;
141
    std::vector<std::string> Environment;
142
    std::vector<std::string> Labels;
Zach's avatar
Zach committed
143
    std::set<std::string> LockedResources;
144
145
146
147
    std::set<std::string> FixturesSetup;
    std::set<std::string> FixturesCleanup;
    std::set<std::string> FixturesRequired;
    std::set<std::string> RequireSuccessDepends;
148
149
    // Private test generator properties used to track backtraces
    cmListFileBacktrace Backtrace;
Andy Cedilnik's avatar
Andy Cedilnik committed
150
151
  };

152
153
154
155
  struct cmCTestTestResult
  {
    std::string Name;
    std::string Path;
156
    std::string Reason;
157
    std::string FullCommandLine;
Wouter Klouwen's avatar
Wouter Klouwen committed
158
    cmDuration ExecutionTime;
159
    std::int64_t ReturnValue;
160
    int Status;
161
    std::string ExceptionStatus;
162
    bool CompressOutput;
163
164
    std::string CompletionStatus;
    std::string Output;
165
    std::string DartString;
166
    int TestCount;
167
168
169
    cmCTestTestProperties* Properties;
  };

170
171
  struct cmCTestTestResultLess
  {
172
173
    bool operator()(const cmCTestTestResult& lhs,
                    const cmCTestTestResult& rhs) const
174
    {
175
      return lhs.TestCount < rhs.TestCount;
176
177
178
179
    }
  };

  // add configurations to a search path for an executable
180
181
182
183
  static void AddConfigurations(cmCTest* ctest,
                                std::vector<std::string>& attempted,
                                std::vector<std::string>& attemptedConfigs,
                                std::string filepath, std::string& filename);
184
185

  // full signature static method to find an executable
186
187
188
189
  static std::string FindExecutable(cmCTest* ctest, const char* testCommand,
                                    std::string& resultingConfig,
                                    std::vector<std::string>& extraPaths,
                                    std::vector<std::string>& failed);
190

191
  typedef std::vector<cmCTestTestProperties> ListOfTests;
192

Andy Cedilnik's avatar
Andy Cedilnik committed
193
protected:
194
195
196
197
  typedef std::set<cmCTestTestHandler::cmCTestTestResult,
                   cmCTestTestResultLess>
    SetOfTests;

198
  // compute a final test list
199
200
  virtual int PreProcessHandler();
  virtual int PostProcessHandler();
201
  virtual void GenerateTestCommand(std::vector<std::string>& args, int test);
202
  int ExecuteCommands(std::vector<std::string>& vec);
203

204
205
206
207
208
209
210
211
212
  bool ProcessOptions();
  void LogTestSummary(const std::vector<std::string>& passed,
                      const std::vector<std::string>& failed,
                      const cmDuration& durationInSecs);
  void LogDisabledTests(const std::vector<cmCTestTestResult>& disabledTests);
  void LogFailedTests(const std::vector<std::string>& failed,
                      const SetOfTests& resultsSet);
  bool GenerateXML();

213
214
215
216
  void WriteTestResultHeader(cmXMLWriter& xml,
                             cmCTestTestResult const& result);
  void WriteTestResultFooter(cmXMLWriter& xml,
                             cmCTestTestResult const& result);
217
  // Write attached test files into the xml
218
  void AttachFiles(cmXMLWriter& xml, cmCTestTestResult& result);
219

220
221
222
  //! Clean test output to specified length
  bool CleanTestOutput(std::string& output, size_t length);

Wouter Klouwen's avatar
Wouter Klouwen committed
223
  cmDuration ElapsedTestingTime;
224

Andy Cedilnik's avatar
Andy Cedilnik committed
225
  typedef std::vector<cmCTestTestResult> TestResultsVector;
226
  TestResultsVector TestResults;
Ken Martin's avatar
Ken Martin committed
227

228
  std::vector<std::string> CustomTestsIgnore;
229
230
  std::string StartTest;
  std::string EndTest;
231
232
  std::chrono::system_clock::time_point StartTestTime;
  std::chrono::system_clock::time_point EndTestTime;
Andy Cedilnik's avatar
Andy Cedilnik committed
233
  bool MemCheck;
234
235
  int CustomMaximumPassedTestOutputSize;
  int CustomMaximumFailedTestOutputSize;
236
  int MaxIndex;
237

238
public:
239
240
  enum
  { // Program statuses
241
242
243
244
245
246
247
248
249
250
251
    NOT_RUN = 0,
    TIMEOUT,
    SEGFAULT,
    ILLEGAL,
    INTERRUPT,
    NUMERICAL,
    OTHER_FAULT,
    FAILED,
    BAD_COMMAND,
    COMPLETED
  };
Ken Martin's avatar
Ken Martin committed
252

253
private:
Ken Martin's avatar
Ken Martin committed
254
255
256
  /**
   * Generate the Dart compatible output
   */
257
  virtual void GenerateDartOutput(cmXMLWriter& xml);
Ken Martin's avatar
Ken Martin committed
258

259
  void PrintLabelOrSubprojectSummary(bool isSubProject);
260

Ken Martin's avatar
Ken Martin committed
261
  /**
262
   * Run the tests for a directory and any subdirectories
Ken Martin's avatar
Ken Martin committed
263
   */
264
265
  void ProcessDirectory(std::vector<std::string>& passed,
                        std::vector<std::string>& failed);
Andy Cedilnik's avatar
Andy Cedilnik committed
266

Ken Martin's avatar
Ken Martin committed
267
268
269
  /**
   * Get the list of tests in directory and subdirectories.
   */
270
  void GetListOfTests();
271
272
273
  // compute the lists of tests that will actually run
  // based on union regex and -I stuff
  void ComputeTestList();
Brad King's avatar
Brad King committed
274

275
276
277
278
  // compute the lists of tests that will actually run
  // based on LastTestFailed.log
  void ComputeTestListForRerunFailed();

279
280
281
282
283
  // add required setup/cleanup tests not already in the
  // list of tests to be run and update dependencies between
  // tests to account for fixture setup/cleanup
  void UpdateForFixtures(ListOfTests& tests) const;

284
285
  void UpdateMaxTestNameWidth();

286
287
288
289
290
  bool GetValue(const char* tag, std::string& value, std::istream& fin);
  bool GetValue(const char* tag, int& value, std::istream& fin);
  bool GetValue(const char* tag, size_t& value, std::istream& fin);
  bool GetValue(const char* tag, bool& value, std::istream& fin);
  bool GetValue(const char* tag, double& value, std::istream& fin);
Ken Martin's avatar
Ken Martin committed
291
292
293
  /**
   * Find the executable for a test
   */
294
  std::string FindTheExecutable(const char* exe);
Ken Martin's avatar
Ken Martin committed
295

296
  std::string GetTestStatus(cmCTestTestResult const&);
Bill Hoffman's avatar
Bill Hoffman committed
297
  void ExpandTestsToRunInformation(size_t numPossibleTests);
298
  void ExpandTestsToRunInformationForRerunFailed();
Ken Martin's avatar
Ken Martin committed
299

300
301
  std::vector<std::string> CustomPreTest;
  std::vector<std::string> CustomPostTest;
Ken Martin's avatar
Ken Martin committed
302

303
  std::vector<int> TestsToRun;
Ken Martin's avatar
Ken Martin committed
304

305
306
  bool UseIncludeLabelRegExpFlag;
  bool UseExcludeLabelRegExpFlag;
Andy Cedilnik's avatar
Andy Cedilnik committed
307
308
309
  bool UseIncludeRegExpFlag;
  bool UseExcludeRegExpFlag;
  bool UseExcludeRegExpFirst;
310
311
  std::string IncludeLabelRegExp;
  std::string ExcludeLabelRegExp;
Andy Cedilnik's avatar
Andy Cedilnik committed
312
313
  std::string IncludeRegExp;
  std::string ExcludeRegExp;
314
315
316
  std::string ExcludeFixtureRegExp;
  std::string ExcludeFixtureSetupRegExp;
  std::string ExcludeFixtureCleanupRegExp;
317
318
  cmsys::RegularExpression IncludeLabelRegularExpression;
  cmsys::RegularExpression ExcludeLabelRegularExpression;
Andy Cedilnik's avatar
Andy Cedilnik committed
319
320
  cmsys::RegularExpression IncludeTestsRegularExpression;
  cmsys::RegularExpression ExcludeTestsRegularExpression;
Ken Martin's avatar
Ken Martin committed
321

322
  void GenerateRegressionImages(cmXMLWriter& xml, const std::string& dart);
Bill Hoffman's avatar
Bill Hoffman committed
323
  cmsys::RegularExpression DartStuff1;
324
325
326
  void CheckLabelFilter(cmCTestTestProperties& it);
  void CheckLabelFilterExclude(cmCTestTestProperties& it);
  void CheckLabelFilterInclude(cmCTestTestProperties& it);
Ken Martin's avatar
Ken Martin committed
327
328

  std::string TestsToRunString;
Andy Cedilnik's avatar
Andy Cedilnik committed
329
330
  bool UseUnion;
  ListOfTests TestList;
331
  size_t TotalNumberOfTests;
Andy Cedilnik's avatar
Andy Cedilnik committed
332
  cmsys::RegularExpression DartStuff;
Andy Cedilnik's avatar
Andy Cedilnik committed
333

Andy Cedilnik's avatar
Andy Cedilnik committed
334
  std::ostream* LogFile;
335
336

  bool RerunFailed;
Ken Martin's avatar
Ken Martin committed
337
338
339
};

#endif