cmCTestTestHandler.h 8.92 KB
Newer Older
1
2
3
/*============================================================================
  CMake - Cross Platform Makefile Generator
  Copyright 2000-2009 Kitware, Inc.
Ken Martin's avatar
Ken Martin committed
4

5
6
  Distributed under the OSI-approved BSD License (the "License");
  see accompanying file Copyright.txt for details.
Ken Martin's avatar
Ken Martin committed
7

8
9
10
11
  This software is distributed WITHOUT ANY WARRANTY; without even the
  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  See the License for more information.
============================================================================*/
Ken Martin's avatar
Ken Martin committed
12
13
14
15

#ifndef cmCTestTestHandler_h
#define cmCTestTestHandler_h

16
#include "cmCTestGenericHandler.h"
Brad King's avatar
Brad King committed
17

18
#include <cmsys/RegularExpression.hxx>
Ken Martin's avatar
Ken Martin committed
19
20

class cmMakefile;
21
class cmXMLWriter;
Ken Martin's avatar
Ken Martin committed
22
23
24
25
26

/** \class cmCTestTestHandler
 * \brief A class that handles ctest -S invocations
 *
 */
27
class cmCTestTestHandler : public cmCTestGenericHandler
Ken Martin's avatar
Ken Martin committed
28
{
29
  friend class cmCTestRunTest;
30
  friend class cmCTestMultiProcessHandler;
31
  friend class cmCTestBatchTestHandler;
Ken Martin's avatar
Ken Martin committed
32
public:
33
  cmTypeMacro(cmCTestTestHandler, cmCTestGenericHandler);
Ken Martin's avatar
Ken Martin committed
34

35
  /**
Ken Martin's avatar
Ken Martin committed
36
37
   * The main entry point for this class
   */
38
  int ProcessHandler();
Andy Cedilnik's avatar
Andy Cedilnik committed
39

40
  /**
41
42
43
44
   * When both -R and -I are used should te resulting test list be the
   * intersection or the union of the lists. By default it is the
   * intersection.
   */
Andy Cedilnik's avatar
Andy Cedilnik committed
45
  void SetUseUnion(bool val) { this->UseUnion = val; }
46

47
48
49
50
51
52
  /**
   * Set whether or not CTest should only execute the tests that failed
   * on the previous run.  By default this is false.
   */
  void SetRerunFailed(bool val) { this->RerunFailed = val; }

53
54
55
  /**
   * This method is called when reading CTest custom file
   */
56
  void PopulateCustomVectors(cmMakefile *mf);
Andy Cedilnik's avatar
Andy Cedilnik committed
57

Ken Martin's avatar
Ken Martin committed
58
59
60
61
62
63
  ///! Control the use of the regular expresisons, call these methods to turn
  ///them on
  void UseIncludeRegExp();
  void UseExcludeRegExp();
  void SetIncludeRegExp(const char *);
  void SetExcludeRegExp(const char *);
Andy Cedilnik's avatar
Andy Cedilnik committed
64

65
66
  void SetMaxIndex(int n) {this->MaxIndex = n;}
  int GetMaxIndex() {return this->MaxIndex;}
Ken Martin's avatar
Ken Martin committed
67

68
69
70
71
72
  void SetTestOutputSizePassed(int n)
    { this->CustomMaximumPassedTestOutputSize = n; }
  void SetTestOutputSizeFailed(int n)
    { this->CustomMaximumFailedTestOutputSize = n; }

Ken Martin's avatar
Ken Martin committed
73
74
75
  ///! pass the -I argument down
  void SetTestsToRunInformation(const char*);

76
  cmCTestTestHandler();
Ken Martin's avatar
Ken Martin committed
77

78
79
80
  /*
   * Add the test to the list of tests to be executed
   */
81
  bool AddTest(const std::vector<std::string>& args);
82

83
84
85
86
87
  /*
   * Set tests properties
   */
  bool SetTestsProperties(const std::vector<std::string>& args);

88
89
  void Initialize();

Bill Hoffman's avatar
Bill Hoffman committed
90
91
92
93
  // NOTE: This struct is Saved/Restored
  // in cmCTestTestHandler, if you add to this class
  // then you must add the new members to that code or
  // ctest -j N will break for that feature
Andy Cedilnik's avatar
Andy Cedilnik committed
94
95
  struct cmCTestTestProperties
  {
96
97
    std::string Name;
    std::string Directory;
Andy Cedilnik's avatar
Andy Cedilnik committed
98
    std::vector<std::string> Args;
99
    std::vector<std::string> RequiredFiles;
100
    std::vector<std::string> Depends;
101
    std::vector<std::string> AttachedFiles;
102
    std::vector<std::string> AttachOnFail;
Bill Hoffman's avatar
Bill Hoffman committed
103
104
105
106
    std::vector<std::pair<cmsys::RegularExpression,
                          std::string> > ErrorRegularExpressions;
    std::vector<std::pair<cmsys::RegularExpression,
                          std::string> > RequiredRegularExpressions;
107
108
    std::vector<std::pair<cmsys::RegularExpression,
                          std::string> > TimeoutRegularExpressions;
109
    std::map<std::string, std::string> Measurements;
Andy Cedilnik's avatar
Andy Cedilnik committed
110
111
    bool IsInBasedOnREOptions;
    bool WillFail;
112
    float Cost;
113
    int PreviousRuns;
114
    bool RunSerial;
115
    double Timeout;
116
    bool ExplicitTimeout;
117
    double AlternateTimeout;
118
    int Index;
119
120
    //Requested number of process slots
    int Processors;
121
122
    // return code of test which will mark test as "not run"
    int SkipReturnCode;
123
    std::vector<std::string> Environment;
124
    std::vector<std::string> Labels;
Zach's avatar
Zach committed
125
    std::set<std::string> LockedResources;
Andy Cedilnik's avatar
Andy Cedilnik committed
126
127
  };

128
129
130
131
  struct cmCTestTestResult
  {
    std::string Name;
    std::string Path;
132
    std::string Reason;
133
134
135
136
    std::string FullCommandLine;
    double      ExecutionTime;
    int         ReturnValue;
    int         Status;
137
    bool        CompressOutput;
138
139
    std::string CompletionStatus;
    std::string Output;
140
    std::string DartString;
141
142
143
144
    int         TestCount;
    cmCTestTestProperties* Properties;
  };

145
146
147
148
149
150
151
152
153
154
  struct cmCTestTestResultLess
  {
    bool operator() (const cmCTestTestResult &lhs,
                     const cmCTestTestResult &rhs) const
    {
    return lhs.TestCount < rhs.TestCount;
    }
  };

  // add configurations to a search path for an executable
Brad King's avatar
Brad King committed
155
  static void AddConfigurations(cmCTest *ctest,
156
157
158
159
160
161
162
163
164
165
166
167
                                std::vector<std::string> &attempted,
                                std::vector<std::string> &attemptedConfigs,
                                std::string filepath,
                                std::string &filename);

  // full signature static method to find an executable
  static std::string FindExecutable(cmCTest *ctest,
                                    const char *testCommand,
                                    std::string &resultingConfig,
                                    std::vector<std::string> &extraPaths,
                                    std::vector<std::string> &failed);

168
  typedef std::vector<cmCTestTestProperties> ListOfTests;
Andy Cedilnik's avatar
Andy Cedilnik committed
169
protected:
170
  // compute a final test list
171
172
  virtual int PreProcessHandler();
  virtual int PostProcessHandler();
173
  virtual void GenerateTestCommand(std::vector<std::string>& args, int test);
174
  int ExecuteCommands(std::vector<std::string>& vec);
175

176
177
  void WriteTestResultHeader(cmXMLWriter& xml, cmCTestTestResult* result);
  void WriteTestResultFooter(cmXMLWriter& xml, cmCTestTestResult* result);
178
  // Write attached test files into the xml
179
  void AttachFiles(cmXMLWriter& xml, cmCTestTestResult* result);
180

181
182
183
  //! Clean test output to specified length
  bool CleanTestOutput(std::string& output, size_t length);

Andy Cedilnik's avatar
Andy Cedilnik committed
184
  double                  ElapsedTestingTime;
185

Andy Cedilnik's avatar
Andy Cedilnik committed
186
187
  typedef std::vector<cmCTestTestResult> TestResultsVector;
  TestResultsVector    TestResults;
Ken Martin's avatar
Ken Martin committed
188

189
  std::vector<std::string> CustomTestsIgnore;
Andy Cedilnik's avatar
Andy Cedilnik committed
190
191
  std::string             StartTest;
  std::string             EndTest;
192
193
  unsigned int            StartTestTime;
  unsigned int            EndTestTime;
Andy Cedilnik's avatar
Andy Cedilnik committed
194
  bool MemCheck;
195
196
  int CustomMaximumPassedTestOutputSize;
  int CustomMaximumFailedTestOutputSize;
197
  int MaxIndex;
198
public:
199
200
201
202
203
204
205
206
207
208
209
210
  enum { // Program statuses
    NOT_RUN = 0,
    TIMEOUT,
    SEGFAULT,
    ILLEGAL,
    INTERRUPT,
    NUMERICAL,
    OTHER_FAULT,
    FAILED,
    BAD_COMMAND,
    COMPLETED
  };
Ken Martin's avatar
Ken Martin committed
211

212
private:
Ken Martin's avatar
Ken Martin committed
213
214
215
  /**
   * Generate the Dart compatible output
   */
216
  virtual void GenerateDartOutput(cmXMLWriter& xml);
Ken Martin's avatar
Ken Martin committed
217

218
  void PrintLabelSummary();
Ken Martin's avatar
Ken Martin committed
219
  /**
220
   * Run the tests for a directory and any subdirectories
Ken Martin's avatar
Ken Martin committed
221
   */
222
223
  void ProcessDirectory(std::vector<std::string> &passed,
                        std::vector<std::string> &failed);
Andy Cedilnik's avatar
Andy Cedilnik committed
224

Ken Martin's avatar
Ken Martin committed
225
226
227
  /**
   * Get the list of tests in directory and subdirectories.
   */
228
  void GetListOfTests();
229
230
231
  // compute the lists of tests that will actually run
  // based on union regex and -I stuff
  void ComputeTestList();
Brad King's avatar
Brad King committed
232

233
234
235
236
237
238
  // compute the lists of tests that will actually run
  // based on LastTestFailed.log
  void ComputeTestListForRerunFailed();

  void UpdateMaxTestNameWidth();

239
240
  bool GetValue(const char* tag,
                std::string& value,
241
                std::istream& fin);
242
243
  bool GetValue(const char* tag,
                int& value,
244
                std::istream& fin);
245
246
  bool GetValue(const char* tag,
                size_t& value,
247
                std::istream& fin);
248
249
  bool GetValue(const char* tag,
                bool& value,
250
                std::istream& fin);
251
252
  bool GetValue(const char* tag,
                double& value,
253
                std::istream& fin);
Ken Martin's avatar
Ken Martin committed
254
255
256
257
258
259
  /**
   * Find the executable for a test
   */
  std::string FindTheExecutable(const char *exe);

  const char* GetTestStatus(int status);
Bill Hoffman's avatar
Bill Hoffman committed
260
  void ExpandTestsToRunInformation(size_t numPossibleTests);
261
  void ExpandTestsToRunInformationForRerunFailed();
Ken Martin's avatar
Ken Martin committed
262

263
264
  std::vector<std::string> CustomPreTest;
  std::vector<std::string> CustomPostTest;
Ken Martin's avatar
Ken Martin committed
265

Andy Cedilnik's avatar
Andy Cedilnik committed
266
  std::vector<int>        TestsToRun;
Ken Martin's avatar
Ken Martin committed
267

268
269
  bool UseIncludeLabelRegExpFlag;
  bool UseExcludeLabelRegExpFlag;
Andy Cedilnik's avatar
Andy Cedilnik committed
270
271
272
  bool UseIncludeRegExpFlag;
  bool UseExcludeRegExpFlag;
  bool UseExcludeRegExpFirst;
273
274
  std::string IncludeLabelRegExp;
  std::string ExcludeLabelRegExp;
Andy Cedilnik's avatar
Andy Cedilnik committed
275
276
  std::string IncludeRegExp;
  std::string ExcludeRegExp;
277
278
  cmsys::RegularExpression IncludeLabelRegularExpression;
  cmsys::RegularExpression ExcludeLabelRegularExpression;
Andy Cedilnik's avatar
Andy Cedilnik committed
279
280
  cmsys::RegularExpression IncludeTestsRegularExpression;
  cmsys::RegularExpression ExcludeTestsRegularExpression;
Ken Martin's avatar
Ken Martin committed
281

282
  void GenerateRegressionImages(cmXMLWriter& xml, const std::string& dart);
Bill Hoffman's avatar
Bill Hoffman committed
283
  cmsys::RegularExpression DartStuff1;
284
285
286
  void CheckLabelFilter(cmCTestTestProperties& it);
  void CheckLabelFilterExclude(cmCTestTestProperties& it);
  void CheckLabelFilterInclude(cmCTestTestProperties& it);
Ken Martin's avatar
Ken Martin committed
287
288

  std::string TestsToRunString;
Andy Cedilnik's avatar
Andy Cedilnik committed
289
290
  bool UseUnion;
  ListOfTests TestList;
291
  size_t TotalNumberOfTests;
Andy Cedilnik's avatar
Andy Cedilnik committed
292
  cmsys::RegularExpression DartStuff;
Andy Cedilnik's avatar
Andy Cedilnik committed
293

Andy Cedilnik's avatar
Andy Cedilnik committed
294
  std::ostream* LogFile;
295
296

  bool RerunFailed;
Ken Martin's avatar
Ken Martin committed
297
298
299
};

#endif