cmCTestTestHandler.h 10.6 KB
Newer Older
1 2
/* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
   file Copyright.txt or https://cmake.org/licensing for details.  */
Ken Martin's avatar
Ken Martin committed
3 4 5
#ifndef cmCTestTestHandler_h
#define cmCTestTestHandler_h

6
#include "cmConfigure.h" // IWYU pragma: keep
7

8
#include <chrono>
9
#include <cstdint>
10 11 12 13 14 15
#include <iosfwd>
#include <map>
#include <set>
#include <string>
#include <utility>
#include <vector>
Ken Martin's avatar
Ken Martin committed
16

17 18 19 20 21
#include <stddef.h>

#include "cmsys/RegularExpression.hxx"

#include "cmCTestGenericHandler.h"
22
#include "cmCTestHardwareSpec.h"
23 24 25
#include "cmDuration.h"
#include "cmListFileCache.h"

26
class cmCTest;
Ken Martin's avatar
Ken Martin committed
27
class cmMakefile;
28
class cmXMLWriter;
Ken Martin's avatar
Ken Martin committed
29 30 31 32 33

/** \class cmCTestTestHandler
 * \brief A class that handles ctest -S invocations
 *
 */
34
class cmCTestTestHandler : public cmCTestGenericHandler
Ken Martin's avatar
Ken Martin committed
35
{
36
  friend class cmCTestRunTest;
37
  friend class cmCTestMultiProcessHandler;
38

Ken Martin's avatar
Ken Martin committed
39
public:
40
  using Superclass = cmCTestGenericHandler;
Ken Martin's avatar
Ken Martin committed
41

42
  /**
Ken Martin's avatar
Ken Martin committed
43 44
   * The main entry point for this class
   */
45
  int ProcessHandler() override;
Andy Cedilnik's avatar
Andy Cedilnik committed
46

47
  /**
luz.paz's avatar
luz.paz committed
48
   * When both -R and -I are used should the resulting test list be the
49 50 51
   * intersection or the union of the lists. By default it is the
   * intersection.
   */
Andy Cedilnik's avatar
Andy Cedilnik committed
52
  void SetUseUnion(bool val) { this->UseUnion = val; }
53

54 55 56 57 58 59
  /**
   * Set whether or not CTest should only execute the tests that failed
   * on the previous run.  By default this is false.
   */
  void SetRerunFailed(bool val) { this->RerunFailed = val; }

60 61 62
  /**
   * This method is called when reading CTest custom file
   */
63
  void PopulateCustomVectors(cmMakefile* mf) override;
Andy Cedilnik's avatar
Andy Cedilnik committed
64

65
  //! Control the use of the regular expresisons, call these methods to turn
66
  /// them on
Ken Martin's avatar
Ken Martin committed
67 68
  void UseIncludeRegExp();
  void UseExcludeRegExp();
69 70
  void SetIncludeRegExp(const char*);
  void SetExcludeRegExp(const char*);
Andy Cedilnik's avatar
Andy Cedilnik committed
71

72 73
  void SetMaxIndex(int n) { this->MaxIndex = n; }
  int GetMaxIndex() { return this->MaxIndex; }
Ken Martin's avatar
Ken Martin committed
74

75
  void SetTestOutputSizePassed(int n)
76 77 78
  {
    this->CustomMaximumPassedTestOutputSize = n;
  }
79
  void SetTestOutputSizeFailed(int n)
80 81 82
  {
    this->CustomMaximumFailedTestOutputSize = n;
  }
83

84
  //! pass the -I argument down
Ken Martin's avatar
Ken Martin committed
85 86
  void SetTestsToRunInformation(const char*);

87
  cmCTestTestHandler();
Ken Martin's avatar
Ken Martin committed
88

89 90 91
  /*
   * Add the test to the list of tests to be executed
   */
92
  bool AddTest(const std::vector<std::string>& args);
93

94 95 96 97 98
  /*
   * Set tests properties
   */
  bool SetTestsProperties(const std::vector<std::string>& args);

99 100 101 102 103
  /**
   * Set directory properties
   */
  bool SetDirectoryProperties(const std::vector<std::string>& args);

104
  void Initialize() override;
105

106 107 108 109 110 111 112 113 114 115
  struct cmCTestTestResourceRequirement
  {
    std::string ResourceType;
    int SlotsNeeded;
    int UnitsNeeded;

    bool operator==(const cmCTestTestResourceRequirement& other) const;
    bool operator!=(const cmCTestTestResourceRequirement& other) const;
  };

Bill Hoffman's avatar
Bill Hoffman committed
116 117 118 119
  // NOTE: This struct is Saved/Restored
  // in cmCTestTestHandler, if you add to this class
  // then you must add the new members to that code or
  // ctest -j N will break for that feature
Andy Cedilnik's avatar
Andy Cedilnik committed
120 121
  struct cmCTestTestProperties
  {
122 123
    std::string Name;
    std::string Directory;
Andy Cedilnik's avatar
Andy Cedilnik committed
124
    std::vector<std::string> Args;
125
    std::vector<std::string> RequiredFiles;
126
    std::vector<std::string> Depends;
127
    std::vector<std::string> AttachedFiles;
128
    std::vector<std::string> AttachOnFail;
129
    std::vector<std::pair<cmsys::RegularExpression, std::string>>
130
      ErrorRegularExpressions;
131
    std::vector<std::pair<cmsys::RegularExpression, std::string>>
132
      RequiredRegularExpressions;
133 134
    std::vector<std::pair<cmsys::RegularExpression, std::string>>
      SkipRegularExpressions;
135
    std::vector<std::pair<cmsys::RegularExpression, std::string>>
136
      TimeoutRegularExpressions;
137
    std::map<std::string, std::string> Measurements;
Andy Cedilnik's avatar
Andy Cedilnik committed
138 139
    bool IsInBasedOnREOptions;
    bool WillFail;
Betsy McPhail's avatar
Betsy McPhail committed
140
    bool Disabled;
141
    float Cost;
142
    int PreviousRuns;
143
    bool RunSerial;
Wouter Klouwen's avatar
Wouter Klouwen committed
144
    cmDuration Timeout;
145
    bool ExplicitTimeout;
Wouter Klouwen's avatar
Wouter Klouwen committed
146
    cmDuration AlternateTimeout;
147
    int Index;
148
    // Requested number of process slots
149
    int Processors;
150 151
    bool WantAffinity;
    std::vector<size_t> Affinity;
152 153
    // return code of test which will mark test as "not run"
    int SkipReturnCode;
154
    std::vector<std::string> Environment;
155
    std::vector<std::string> Labels;
Zach's avatar
Zach committed
156
    std::set<std::string> LockedResources;
157 158 159 160
    std::set<std::string> FixturesSetup;
    std::set<std::string> FixturesCleanup;
    std::set<std::string> FixturesRequired;
    std::set<std::string> RequireSuccessDepends;
161
    std::vector<std::vector<cmCTestTestResourceRequirement>> ResourceGroups;
162 163
    // Private test generator properties used to track backtraces
    cmListFileBacktrace Backtrace;
Andy Cedilnik's avatar
Andy Cedilnik committed
164 165
  };

166 167 168 169
  struct cmCTestTestResult
  {
    std::string Name;
    std::string Path;
170
    std::string Reason;
171
    std::string FullCommandLine;
Wouter Klouwen's avatar
Wouter Klouwen committed
172
    cmDuration ExecutionTime;
173
    std::int64_t ReturnValue;
174
    int Status;
175
    std::string ExceptionStatus;
176
    bool CompressOutput;
177 178
    std::string CompletionStatus;
    std::string Output;
179
    std::string DartString;
180
    int TestCount;
181 182 183
    cmCTestTestProperties* Properties;
  };

184 185
  struct cmCTestTestResultLess
  {
186 187
    bool operator()(const cmCTestTestResult& lhs,
                    const cmCTestTestResult& rhs) const
188
    {
189
      return lhs.TestCount < rhs.TestCount;
190 191 192 193
    }
  };

  // add configurations to a search path for an executable
194 195 196 197
  static void AddConfigurations(cmCTest* ctest,
                                std::vector<std::string>& attempted,
                                std::vector<std::string>& attemptedConfigs,
                                std::string filepath, std::string& filename);
198 199

  // full signature static method to find an executable
200 201 202 203
  static std::string FindExecutable(cmCTest* ctest, const char* testCommand,
                                    std::string& resultingConfig,
                                    std::vector<std::string>& extraPaths,
                                    std::vector<std::string>& failed);
204

205
  static bool ParseResourceGroupsProperty(
206
    const std::string& val,
207
    std::vector<std::vector<cmCTestTestResourceRequirement>>& resourceGroups);
208

209
  using ListOfTests = std::vector<cmCTestTestProperties>;
210

Andy Cedilnik's avatar
Andy Cedilnik committed
211
protected:
212 213
  using SetOfTests =
    std::set<cmCTestTestHandler::cmCTestTestResult, cmCTestTestResultLess>;
214

215
  // compute a final test list
216 217
  virtual int PreProcessHandler();
  virtual int PostProcessHandler();
218
  virtual void GenerateTestCommand(std::vector<std::string>& args, int test);
219
  int ExecuteCommands(std::vector<std::string>& vec);
220

221 222 223 224 225 226 227 228 229
  bool ProcessOptions();
  void LogTestSummary(const std::vector<std::string>& passed,
                      const std::vector<std::string>& failed,
                      const cmDuration& durationInSecs);
  void LogDisabledTests(const std::vector<cmCTestTestResult>& disabledTests);
  void LogFailedTests(const std::vector<std::string>& failed,
                      const SetOfTests& resultsSet);
  bool GenerateXML();

230 231 232 233
  void WriteTestResultHeader(cmXMLWriter& xml,
                             cmCTestTestResult const& result);
  void WriteTestResultFooter(cmXMLWriter& xml,
                             cmCTestTestResult const& result);
234
  // Write attached test files into the xml
235
  void AttachFiles(cmXMLWriter& xml, cmCTestTestResult& result);
236

237 238 239
  //! Clean test output to specified length
  bool CleanTestOutput(std::string& output, size_t length);

Wouter Klouwen's avatar
Wouter Klouwen committed
240
  cmDuration ElapsedTestingTime;
241

242
  using TestResultsVector = std::vector<cmCTestTestResult>;
243
  TestResultsVector TestResults;
Ken Martin's avatar
Ken Martin committed
244

245
  std::vector<std::string> CustomTestsIgnore;
246 247
  std::string StartTest;
  std::string EndTest;
248 249
  std::chrono::system_clock::time_point StartTestTime;
  std::chrono::system_clock::time_point EndTestTime;
Andy Cedilnik's avatar
Andy Cedilnik committed
250
  bool MemCheck;
251 252
  int CustomMaximumPassedTestOutputSize;
  int CustomMaximumFailedTestOutputSize;
253
  int MaxIndex;
254

255
public:
256 257
  enum
  { // Program statuses
258 259 260 261 262 263 264 265 266 267 268
    NOT_RUN = 0,
    TIMEOUT,
    SEGFAULT,
    ILLEGAL,
    INTERRUPT,
    NUMERICAL,
    OTHER_FAULT,
    FAILED,
    BAD_COMMAND,
    COMPLETED
  };
Ken Martin's avatar
Ken Martin committed
269

270
private:
Ken Martin's avatar
Ken Martin committed
271 272 273
  /**
   * Generate the Dart compatible output
   */
274
  virtual void GenerateDartOutput(cmXMLWriter& xml);
Ken Martin's avatar
Ken Martin committed
275

276
  void PrintLabelOrSubprojectSummary(bool isSubProject);
277

Ken Martin's avatar
Ken Martin committed
278
  /**
279
   * Run the tests for a directory and any subdirectories
Ken Martin's avatar
Ken Martin committed
280
   */
281 282
  void ProcessDirectory(std::vector<std::string>& passed,
                        std::vector<std::string>& failed);
Andy Cedilnik's avatar
Andy Cedilnik committed
283

Ken Martin's avatar
Ken Martin committed
284 285 286
  /**
   * Get the list of tests in directory and subdirectories.
   */
287
  void GetListOfTests();
288 289 290
  // compute the lists of tests that will actually run
  // based on union regex and -I stuff
  void ComputeTestList();
Brad King's avatar
Brad King committed
291

292 293 294 295
  // compute the lists of tests that will actually run
  // based on LastTestFailed.log
  void ComputeTestListForRerunFailed();

296 297 298 299 300
  // add required setup/cleanup tests not already in the
  // list of tests to be run and update dependencies between
  // tests to account for fixture setup/cleanup
  void UpdateForFixtures(ListOfTests& tests) const;

301 302
  void UpdateMaxTestNameWidth();

303 304 305 306 307
  bool GetValue(const char* tag, std::string& value, std::istream& fin);
  bool GetValue(const char* tag, int& value, std::istream& fin);
  bool GetValue(const char* tag, size_t& value, std::istream& fin);
  bool GetValue(const char* tag, bool& value, std::istream& fin);
  bool GetValue(const char* tag, double& value, std::istream& fin);
Ken Martin's avatar
Ken Martin committed
308 309 310
  /**
   * Find the executable for a test
   */
311
  std::string FindTheExecutable(const char* exe);
Ken Martin's avatar
Ken Martin committed
312

313
  std::string GetTestStatus(cmCTestTestResult const&);
Bill Hoffman's avatar
Bill Hoffman committed
314
  void ExpandTestsToRunInformation(size_t numPossibleTests);
315
  void ExpandTestsToRunInformationForRerunFailed();
Ken Martin's avatar
Ken Martin committed
316

317 318
  std::vector<std::string> CustomPreTest;
  std::vector<std::string> CustomPostTest;
Ken Martin's avatar
Ken Martin committed
319

320
  std::vector<int> TestsToRun;
Ken Martin's avatar
Ken Martin committed
321

322 323
  bool UseIncludeLabelRegExpFlag;
  bool UseExcludeLabelRegExpFlag;
Andy Cedilnik's avatar
Andy Cedilnik committed
324 325 326
  bool UseIncludeRegExpFlag;
  bool UseExcludeRegExpFlag;
  bool UseExcludeRegExpFirst;
327 328
  std::string IncludeLabelRegExp;
  std::string ExcludeLabelRegExp;
Andy Cedilnik's avatar
Andy Cedilnik committed
329 330
  std::string IncludeRegExp;
  std::string ExcludeRegExp;
331 332 333
  std::string ExcludeFixtureRegExp;
  std::string ExcludeFixtureSetupRegExp;
  std::string ExcludeFixtureCleanupRegExp;
334 335
  cmsys::RegularExpression IncludeLabelRegularExpression;
  cmsys::RegularExpression ExcludeLabelRegularExpression;
Andy Cedilnik's avatar
Andy Cedilnik committed
336 337
  cmsys::RegularExpression IncludeTestsRegularExpression;
  cmsys::RegularExpression ExcludeTestsRegularExpression;
Ken Martin's avatar
Ken Martin committed
338

339 340 341
  bool UseHardwareSpec;
  cmCTestHardwareSpec HardwareSpec;

342
  void GenerateRegressionImages(cmXMLWriter& xml, const std::string& dart);
Bill Hoffman's avatar
Bill Hoffman committed
343
  cmsys::RegularExpression DartStuff1;
344 345 346
  void CheckLabelFilter(cmCTestTestProperties& it);
  void CheckLabelFilterExclude(cmCTestTestProperties& it);
  void CheckLabelFilterInclude(cmCTestTestProperties& it);
Ken Martin's avatar
Ken Martin committed
347 348

  std::string TestsToRunString;
Andy Cedilnik's avatar
Andy Cedilnik committed
349 350
  bool UseUnion;
  ListOfTests TestList;
351
  size_t TotalNumberOfTests;
Andy Cedilnik's avatar
Andy Cedilnik committed
352
  cmsys::RegularExpression DartStuff;
Andy Cedilnik's avatar
Andy Cedilnik committed
353

Andy Cedilnik's avatar
Andy Cedilnik committed
354
  std::ostream* LogFile;
355 356

  bool RerunFailed;
Ken Martin's avatar
Ken Martin committed
357 358 359
};

#endif