cmCTestTestHandler.h 8.77 KB
Newer Older
1
2
3
/*============================================================================
  CMake - Cross Platform Makefile Generator
  Copyright 2000-2009 Kitware, Inc.
Ken Martin's avatar
Ken Martin committed
4

5
6
  Distributed under the OSI-approved BSD License (the "License");
  see accompanying file Copyright.txt for details.
Ken Martin's avatar
Ken Martin committed
7

8
9
10
11
  This software is distributed WITHOUT ANY WARRANTY; without even the
  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  See the License for more information.
============================================================================*/
Ken Martin's avatar
Ken Martin committed
12
13
14
15

#ifndef cmCTestTestHandler_h
#define cmCTestTestHandler_h

16
17
#include <cmConfigure.h>

18
#include "cmCTestGenericHandler.h"
19
#include "cmTypeMacro.h"
Brad King's avatar
Brad King committed
20

21
#include <cmsys/RegularExpression.hxx>
22
23
24
25
26
27
28
#include <iosfwd>
#include <map>
#include <set>
#include <stddef.h>
#include <string>
#include <utility>
#include <vector>
Ken Martin's avatar
Ken Martin committed
29

30
class cmCTest;
Ken Martin's avatar
Ken Martin committed
31
class cmMakefile;
32
class cmXMLWriter;
Ken Martin's avatar
Ken Martin committed
33
34
35
36
37

/** \class cmCTestTestHandler
 * \brief A class that handles ctest -S invocations
 *
 */
38
class cmCTestTestHandler : public cmCTestGenericHandler
Ken Martin's avatar
Ken Martin committed
39
{
40
  friend class cmCTestRunTest;
41
  friend class cmCTestMultiProcessHandler;
42
  friend class cmCTestBatchTestHandler;
43

Ken Martin's avatar
Ken Martin committed
44
public:
45
  cmTypeMacro(cmCTestTestHandler, cmCTestGenericHandler);
Ken Martin's avatar
Ken Martin committed
46

47
  /**
Ken Martin's avatar
Ken Martin committed
48
49
   * The main entry point for this class
   */
50
  int ProcessHandler() CM_OVERRIDE;
Andy Cedilnik's avatar
Andy Cedilnik committed
51

52
  /**
53
54
55
56
   * When both -R and -I are used should te resulting test list be the
   * intersection or the union of the lists. By default it is the
   * intersection.
   */
Andy Cedilnik's avatar
Andy Cedilnik committed
57
  void SetUseUnion(bool val) { this->UseUnion = val; }
58

59
60
61
62
63
64
  /**
   * Set whether or not CTest should only execute the tests that failed
   * on the previous run.  By default this is false.
   */
  void SetRerunFailed(bool val) { this->RerunFailed = val; }

65
66
67
  /**
   * This method is called when reading CTest custom file
   */
68
  void PopulateCustomVectors(cmMakefile* mf) CM_OVERRIDE;
Andy Cedilnik's avatar
Andy Cedilnik committed
69

Ken Martin's avatar
Ken Martin committed
70
  ///! Control the use of the regular expresisons, call these methods to turn
71
  /// them on
Ken Martin's avatar
Ken Martin committed
72
73
  void UseIncludeRegExp();
  void UseExcludeRegExp();
74
75
  void SetIncludeRegExp(const char*);
  void SetExcludeRegExp(const char*);
Andy Cedilnik's avatar
Andy Cedilnik committed
76

77
78
  void SetMaxIndex(int n) { this->MaxIndex = n; }
  int GetMaxIndex() { return this->MaxIndex; }
Ken Martin's avatar
Ken Martin committed
79

80
  void SetTestOutputSizePassed(int n)
81
82
83
  {
    this->CustomMaximumPassedTestOutputSize = n;
  }
84
  void SetTestOutputSizeFailed(int n)
85
86
87
  {
    this->CustomMaximumFailedTestOutputSize = n;
  }
88

Ken Martin's avatar
Ken Martin committed
89
90
91
  ///! pass the -I argument down
  void SetTestsToRunInformation(const char*);

92
  cmCTestTestHandler();
Ken Martin's avatar
Ken Martin committed
93

94
95
96
  /*
   * Add the test to the list of tests to be executed
   */
97
  bool AddTest(const std::vector<std::string>& args);
98

99
100
101
102
103
  /*
   * Set tests properties
   */
  bool SetTestsProperties(const std::vector<std::string>& args);

104
  void Initialize() CM_OVERRIDE;
105

Bill Hoffman's avatar
Bill Hoffman committed
106
107
108
109
  // NOTE: This struct is Saved/Restored
  // in cmCTestTestHandler, if you add to this class
  // then you must add the new members to that code or
  // ctest -j N will break for that feature
Andy Cedilnik's avatar
Andy Cedilnik committed
110
111
  struct cmCTestTestProperties
  {
112
113
    std::string Name;
    std::string Directory;
Andy Cedilnik's avatar
Andy Cedilnik committed
114
    std::vector<std::string> Args;
115
    std::vector<std::string> RequiredFiles;
116
    std::vector<std::string> Depends;
117
    std::vector<std::string> AttachedFiles;
118
    std::vector<std::string> AttachOnFail;
119
120
121
122
123
124
    std::vector<std::pair<cmsys::RegularExpression, std::string> >
      ErrorRegularExpressions;
    std::vector<std::pair<cmsys::RegularExpression, std::string> >
      RequiredRegularExpressions;
    std::vector<std::pair<cmsys::RegularExpression, std::string> >
      TimeoutRegularExpressions;
125
    std::map<std::string, std::string> Measurements;
Andy Cedilnik's avatar
Andy Cedilnik committed
126
127
    bool IsInBasedOnREOptions;
    bool WillFail;
128
    float Cost;
129
    int PreviousRuns;
130
    bool RunSerial;
131
    double Timeout;
132
    bool ExplicitTimeout;
133
    double AlternateTimeout;
134
    int Index;
135
    // Requested number of process slots
136
    int Processors;
137
138
    // return code of test which will mark test as "not run"
    int SkipReturnCode;
139
    std::vector<std::string> Environment;
140
    std::vector<std::string> Labels;
Zach's avatar
Zach committed
141
    std::set<std::string> LockedResources;
Andy Cedilnik's avatar
Andy Cedilnik committed
142
143
  };

144
145
146
147
  struct cmCTestTestResult
  {
    std::string Name;
    std::string Path;
148
    std::string Reason;
149
    std::string FullCommandLine;
150
151
152
153
    double ExecutionTime;
    int ReturnValue;
    int Status;
    bool CompressOutput;
154
155
    std::string CompletionStatus;
    std::string Output;
156
    std::string DartString;
157
    int TestCount;
158
159
160
    cmCTestTestProperties* Properties;
  };

161
162
  struct cmCTestTestResultLess
  {
163
164
    bool operator()(const cmCTestTestResult& lhs,
                    const cmCTestTestResult& rhs) const
165
    {
166
      return lhs.TestCount < rhs.TestCount;
167
168
169
170
    }
  };

  // add configurations to a search path for an executable
171
172
173
174
  static void AddConfigurations(cmCTest* ctest,
                                std::vector<std::string>& attempted,
                                std::vector<std::string>& attemptedConfigs,
                                std::string filepath, std::string& filename);
175
176

  // full signature static method to find an executable
177
178
179
180
  static std::string FindExecutable(cmCTest* ctest, const char* testCommand,
                                    std::string& resultingConfig,
                                    std::vector<std::string>& extraPaths,
                                    std::vector<std::string>& failed);
181

182
  typedef std::vector<cmCTestTestProperties> ListOfTests;
183

Andy Cedilnik's avatar
Andy Cedilnik committed
184
protected:
185
  // compute a final test list
186
187
  virtual int PreProcessHandler();
  virtual int PostProcessHandler();
188
  virtual void GenerateTestCommand(std::vector<std::string>& args, int test);
189
  int ExecuteCommands(std::vector<std::string>& vec);
190

191
192
  void WriteTestResultHeader(cmXMLWriter& xml, cmCTestTestResult* result);
  void WriteTestResultFooter(cmXMLWriter& xml, cmCTestTestResult* result);
193
  // Write attached test files into the xml
194
  void AttachFiles(cmXMLWriter& xml, cmCTestTestResult* result);
195

196
197
198
  //! Clean test output to specified length
  bool CleanTestOutput(std::string& output, size_t length);

199
  double ElapsedTestingTime;
200

Andy Cedilnik's avatar
Andy Cedilnik committed
201
  typedef std::vector<cmCTestTestResult> TestResultsVector;
202
  TestResultsVector TestResults;
Ken Martin's avatar
Ken Martin committed
203

204
  std::vector<std::string> CustomTestsIgnore;
205
206
207
208
  std::string StartTest;
  std::string EndTest;
  unsigned int StartTestTime;
  unsigned int EndTestTime;
Andy Cedilnik's avatar
Andy Cedilnik committed
209
  bool MemCheck;
210
211
  int CustomMaximumPassedTestOutputSize;
  int CustomMaximumFailedTestOutputSize;
212
  int MaxIndex;
213

214
public:
215
216
  enum
  { // Program statuses
217
218
219
220
221
222
223
224
225
226
227
    NOT_RUN = 0,
    TIMEOUT,
    SEGFAULT,
    ILLEGAL,
    INTERRUPT,
    NUMERICAL,
    OTHER_FAULT,
    FAILED,
    BAD_COMMAND,
    COMPLETED
  };
Ken Martin's avatar
Ken Martin committed
228

229
private:
Ken Martin's avatar
Ken Martin committed
230
231
232
  /**
   * Generate the Dart compatible output
   */
233
  virtual void GenerateDartOutput(cmXMLWriter& xml);
Ken Martin's avatar
Ken Martin committed
234

235
  void PrintLabelSummary();
Ken Martin's avatar
Ken Martin committed
236
  /**
237
   * Run the tests for a directory and any subdirectories
Ken Martin's avatar
Ken Martin committed
238
   */
239
240
  void ProcessDirectory(std::vector<std::string>& passed,
                        std::vector<std::string>& failed);
Andy Cedilnik's avatar
Andy Cedilnik committed
241

Ken Martin's avatar
Ken Martin committed
242
243
244
  /**
   * Get the list of tests in directory and subdirectories.
   */
245
  void GetListOfTests();
246
247
248
  // compute the lists of tests that will actually run
  // based on union regex and -I stuff
  void ComputeTestList();
Brad King's avatar
Brad King committed
249

250
251
252
253
254
255
  // compute the lists of tests that will actually run
  // based on LastTestFailed.log
  void ComputeTestListForRerunFailed();

  void UpdateMaxTestNameWidth();

256
257
258
259
260
  bool GetValue(const char* tag, std::string& value, std::istream& fin);
  bool GetValue(const char* tag, int& value, std::istream& fin);
  bool GetValue(const char* tag, size_t& value, std::istream& fin);
  bool GetValue(const char* tag, bool& value, std::istream& fin);
  bool GetValue(const char* tag, double& value, std::istream& fin);
Ken Martin's avatar
Ken Martin committed
261
262
263
  /**
   * Find the executable for a test
   */
264
  std::string FindTheExecutable(const char* exe);
Ken Martin's avatar
Ken Martin committed
265
266

  const char* GetTestStatus(int status);
Bill Hoffman's avatar
Bill Hoffman committed
267
  void ExpandTestsToRunInformation(size_t numPossibleTests);
268
  void ExpandTestsToRunInformationForRerunFailed();
Ken Martin's avatar
Ken Martin committed
269

270
271
  std::vector<std::string> CustomPreTest;
  std::vector<std::string> CustomPostTest;
Ken Martin's avatar
Ken Martin committed
272

273
  std::vector<int> TestsToRun;
Ken Martin's avatar
Ken Martin committed
274

275
276
  bool UseIncludeLabelRegExpFlag;
  bool UseExcludeLabelRegExpFlag;
Andy Cedilnik's avatar
Andy Cedilnik committed
277
278
279
  bool UseIncludeRegExpFlag;
  bool UseExcludeRegExpFlag;
  bool UseExcludeRegExpFirst;
280
281
  std::string IncludeLabelRegExp;
  std::string ExcludeLabelRegExp;
Andy Cedilnik's avatar
Andy Cedilnik committed
282
283
  std::string IncludeRegExp;
  std::string ExcludeRegExp;
284
285
  cmsys::RegularExpression IncludeLabelRegularExpression;
  cmsys::RegularExpression ExcludeLabelRegularExpression;
Andy Cedilnik's avatar
Andy Cedilnik committed
286
287
  cmsys::RegularExpression IncludeTestsRegularExpression;
  cmsys::RegularExpression ExcludeTestsRegularExpression;
Ken Martin's avatar
Ken Martin committed
288

289
  void GenerateRegressionImages(cmXMLWriter& xml, const std::string& dart);
Bill Hoffman's avatar
Bill Hoffman committed
290
  cmsys::RegularExpression DartStuff1;
291
292
293
  void CheckLabelFilter(cmCTestTestProperties& it);
  void CheckLabelFilterExclude(cmCTestTestProperties& it);
  void CheckLabelFilterInclude(cmCTestTestProperties& it);
Ken Martin's avatar
Ken Martin committed
294
295

  std::string TestsToRunString;
Andy Cedilnik's avatar
Andy Cedilnik committed
296
297
  bool UseUnion;
  ListOfTests TestList;
298
  size_t TotalNumberOfTests;
Andy Cedilnik's avatar
Andy Cedilnik committed
299
  cmsys::RegularExpression DartStuff;
Andy Cedilnik's avatar
Andy Cedilnik committed
300

Andy Cedilnik's avatar
Andy Cedilnik committed
301
  std::ostream* LogFile;
302
303

  bool RerunFailed;
Ken Martin's avatar
Ken Martin committed
304
305
306
};

#endif