Testing.py 16.9 KB
Newer Older
1
2
3
4
5
6
""" This module attempts to make it easy to create VTK-Python
unittests.  The module uses unittest for the test interface.  For more
documentation on what unittests are and how to use them, please read
these:

   http://www.python.org/doc/current/lib/module-unittest.html
7

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
   http://www.diveintopython.org/roman_divein.html


This VTK-Python test module supports image based tests with multiple
images per test suite and multiple images per individual test as well.
It also prints information appropriate for Dart
(http://public.kitware.com/Dart/).

This module defines several useful classes and functions to make
writing tests easy.  The most important of these are:

class vtkTest:
   Subclass this for your tests.  It also has a few useful internal
   functions that can be used to do some simple blackbox testing.

compareImage(renwin, img_fname, threshold=10):
   Compares renwin with image and generates image if it does not
   exist.  The threshold determines how closely the images must match.
   The function also handles multiple images and finds the best
   matching image.

29
30
31
32
33
34
35
compareImageWithSavedImage(src_img, img_fname, threshold=10):
   Compares given source image (in the form of a vtkImageData) with
   saved image and generates the image if it does not exist.  The
   threshold determines how closely the images must match.  The
   function also handles multiple images and finds the best matching
   image.

36
37
38
39
40
41
42
43
getAbsImagePath(img_basename):
   Returns the full path to the image given the basic image name.

main(cases):
   Does the testing given a list of tuples containing test classes and
   the starting string of the functions used for testing.

interact():
44
    Interacts with the user if necessary.  The behavior of this is
45
46
47
48
    rather trivial and works best when using Tkinter.  It does not do
    anything by default and stops to interact with the user when given
    the appropriate command line arguments.

49
50
51
52
53
isInteractive():
    If interact() is not good enough, use this to find if the mode is
    interactive or not and do whatever is necessary to generate an
    interactive view.

54
55
56
57
58
59
60
61
62
63
64
Examples:

  The best way to learn on how to use this module is to look at a few
  examples.  The end of this file contains a trivial example.  Please
  also look at the following examples:

    Rendering/Testing/Python/TestTkRenderWidget.py,
    Rendering/Testing/Python/TestTkRenderWindowInteractor.py

Created: September, 2002

65
Prabhu Ramachandran <prabhu@aero.iitb.ac.in>
66
67
68
"""

import sys, os, time
69
import os.path
70
71
72
73
74
75
76
77
78
79
80
81
import unittest, getopt
import vtk
import BlackBox

# location of the VTK data files.  Set via command line args or
# environment variable.
VTK_DATA_ROOT = ""

# location of the VTK baseline images.  Set via command line args or
# environment variable.
VTK_BASELINE_ROOT = ""

82
83
84
85
# location of the VTK difference images for failed tests.  Set via
# command line args or environment variable.
VTK_TEMP_DIR = ""

86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
# Verbosity of the test messages (used by unittest)
_VERBOSE = 0

# Determines if it is necessary to interact with the user.  If zero
# dont interact if 1 interact.  Set via command line args
_INTERACT = 0

# This will be set to 1 when the image test will not be performed.
# This option is used internally by the script and set via command
# line arguments.
_NO_IMAGE = 0


class vtkTest(unittest.TestCase):
    """A simple default VTK test class that defines a few useful
    blackbox tests that can be readily used.  Derive your test cases
    from this class and use the following if you'd like to.

    Note: Unittest instantiates this class (or your subclass) each
    time it tests a method.  So if you do not want that to happen when
    generating VTK pipelines you should create the pipeline in the
    class definition as done below for _blackbox.
    """
109

110
111
    _blackbox = BlackBox.Tester(debug=0)

112
    # Due to what seems to be a bug in python some objects leak.
113
    # Avoid the exit-with-error in vtkDebugLeaks.
114
    dl = vtk.vtkDebugLeaks()
115
    dl.SetExitError(0)
116
117
    dl = None

118
119
120
121
122
123
124
    def _testParse(self, obj):
        """Does a blackbox test by attempting to parse the class for
        its various methods using vtkMethodParser.  This is a useful
        test because it gets all the methods of the vtkObject, parses
        them and sorts them into different classes of objects."""
        self._blackbox.testParse(obj)

125
    def _testGetSet(self, obj, excluded_methods=[]):
126
127
128
129
        """Checks the Get/Set method pairs by setting the value using
        the current state and making sure that it equals the value it
        was originally.  This effectively calls _testParse
        internally. """
130
        self._blackbox.testGetSet(obj, excluded_methods)
131

132
    def _testBoolean(self, obj, excluded_methods=[]):
133
134
135
        """Checks the Boolean methods by setting the value on and off
        and making sure that the GetMethod returns the the set value.
        This effectively calls _testParse internally. """
136
        self._blackbox.testBoolean(obj, excluded_methods)
137

138
139
140
141
142
143
144
145


def interact():
    """Interacts with the user if necessary. """
    global _INTERACT
    if _INTERACT:
        raw_input("\nPress Enter/Return to continue with the testing. --> ")

146
147
148
149
def isInteractive():
    """Returns if the currently chosen mode is interactive or not
    based on command line options."""
    return _INTERACT
150

151
def getAbsImagePath(img_basename):
152
153
154
155
156
    """Returns the full path to the image given the basic image
    name."""
    global VTK_BASELINE_ROOT
    return os.path.join(VTK_BASELINE_ROOT, img_basename)

157
158
159
160
def _getTempImagePath(img_fname):
    x = os.path.join(VTK_TEMP_DIR, os.path.split(img_fname)[1])
    return os.path.abspath(x)

161
162
163
164
165
166
167
168
def compareImageWithSavedImage(src_img, img_fname, threshold=10):
    """Compares a source image (src_img, which is a vtkImageData) with
    the saved image file whose name is given in the second argument.
    If the image file does not exist the image is generated and
    stored.  If not the source image is compared to that of the
    figure.  This function also handles multiple images and finds the
    best matching image.
    """
169
170
171
    global _NO_IMAGE
    if _NO_IMAGE:
        return
172

173
174
175
176
177
    f_base, f_ext = os.path.splitext(img_fname)

    if not os.path.isfile(img_fname):
        # generate the image
        pngw = vtk.vtkPNGWriter()
178
        pngw.SetFileName(_getTempImagePath(img_fname))
179
        pngw.SetInputData(src_img)
180
        pngw.Write()
181
182
        return

183
184
185
186
    pngr = vtk.vtkPNGReader()
    pngr.SetFileName(img_fname)

    idiff = vtk.vtkImageDifference()
187
    idiff.SetInputData(src_img)
188
    idiff.SetImageConnection(pngr.GetOutputPort())
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238

    min_err = idiff.GetThresholdedError()
    img_err = min_err
    best_img = img_fname

    err_index = 0
    count = 0
    if min_err > threshold:
        count = 1
        test_failed = 1
        err_index = -1
        while 1: # keep trying images till we get the best match.
            new_fname = f_base + "_%d.png"%count
            if not os.path.exists(new_fname):
                # no other image exists.
                break
            # since file exists check if it matches.
            pngr.SetFileName(new_fname)
            pngr.Update()
            idiff.Update()
            alt_err = idiff.GetThresholdedError()
            if alt_err < threshold:
                # matched,
                err_index = count
                test_failed = 0
                min_err = alt_err
                img_err = alt_err
                best_img = new_fname
                break
            else:
                if alt_err < min_err:
                    # image is a better match.
                    err_index = count
                    min_err = alt_err
                    img_err = alt_err
                    best_img = new_fname

            count = count + 1
        # closes while loop.

        if test_failed:
            _handleFailedImage(idiff, pngr, best_img)
            # Print for Dart.
            _printDartImageError(img_err, err_index, f_base)
            msg = "Failed image test: %f\n"%idiff.GetThresholdedError()
            raise AssertionError, msg
    # output the image error even if a test passed
    _printDartImageSuccess(img_err, err_index)


239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
def compareImage(renwin, img_fname, threshold=10):
    """Compares renwin's (a vtkRenderWindow) contents with the image
    file whose name is given in the second argument.  If the image
    file does not exist the image is generated and stored.  If not the
    image in the render window is compared to that of the figure.
    This function also handles multiple images and finds the best
    matching image.  """

    global _NO_IMAGE
    if _NO_IMAGE:
        return

    w2if = vtk.vtkWindowToImageFilter()
    w2if.ReadFrontBufferOff()
    w2if.SetInput(renwin)
    return compareImageWithSavedImage(w2if.GetOutput(), img_fname, threshold)


257
258
def _printDartImageError(img_err, err_index, img_base):
    """Prints the XML data necessary for Dart."""
259
    img_base = _getTempImagePath(img_base)
260
261
262
263
264
265
266
267
    print "Failed image test with error: %f"%img_err
    print "<DartMeasurement name=\"ImageError\" type=\"numeric/double\">",
    print "%f </DartMeasurement>"%img_err
    if err_index <= 0:
        print "<DartMeasurement name=\"BaselineImage\" type=\"text/string\">Standard</DartMeasurement>",
    else:
        print "<DartMeasurement name=\"BaselineImage\" type=\"numeric/integer\">",
        print "%d </DartMeasurement>"%err_index
268

269
270
    print "<DartMeasurementFile name=\"TestImage\" type=\"image/png\">",
    print "%s </DartMeasurementFile>"%(img_base + '.png')
271

272
273
274
275
    print "<DartMeasurementFile name=\"DifferenceImage\" type=\"image/png\">",
    print "%s </DartMeasurementFile>"%(img_base + '.diff.png')
    print "<DartMeasurementFile name=\"ValidImage\" type=\"image/png\">",
    print "%s </DartMeasurementFile>"%(img_base + '.valid.png')
276
277
278
279
280
281
282
283
284
285
286


def _printDartImageSuccess(img_err, err_index):
    "Prints XML data for Dart when image test succeeded."
    print "<DartMeasurement name=\"ImageError\" type=\"numeric/double\">",
    print "%f </DartMeasurement>"%img_err
    if err_index <= 0:
        print "<DartMeasurement name=\"BaselineImage\" type=\"text/string\">Standard</DartMeasurement>",
    else:
       print "<DartMeasurement name=\"BaselineImage\" type=\"numeric/integer\">",
       print "%d </DartMeasurement>"%err_index
287

288
289
290
291
292

def _handleFailedImage(idiff, pngr, img_fname):
    """Writes all the necessary images when an image comparison
    failed."""
    f_base, f_ext = os.path.splitext(img_fname)
293

294
    # write the difference image gamma adjusted for the dashboard.
295
    gamma = vtk.vtkImageShiftScale()
296
    gamma.SetInputConnection(idiff.GetOutputPort())
297
298
299
    gamma.SetShift(0)
    gamma.SetScale(10)

300
301
302
303
    pngw = vtk.vtkPNGWriter()
    pngw.SetFileName(_getTempImagePath(f_base + ".diff.png"))
    pngw.SetInputConnection(gamma.GetOutputPort())
    pngw.Write()
304

305
306
    # Write out the image that was generated.  Write it out as full so that
    # it may be used as a baseline image if the tester deems it valid.
307
    pngw.SetInputConnection(idiff.GetInputConnection(0,0))
308
309
    pngw.SetFileName(_getTempImagePath(f_base + ".png"))
    pngw.Write()
310
311

    # write out the valid image that matched.
312
    pngw.SetInputConnection(idiff.GetInputConnection(1,0))
313
314
    pngw.SetFileName(_getTempImagePath(f_base + ".valid.png"))
    pngw.Write()
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329


def main(cases):
    """ Pass a list of tuples containing test classes and the starting
    string of the functions used for testing.

    Example:

    main ([(vtkTestClass, 'test'), (vtkTestClass1, 'test')])
    """

    processCmdLine()

    timer = vtk.vtkTimerLog()
    s_time = timer.GetCPUTime()
330
    s_wall_time = time.time()
331
332
333
334
335
336
337
338
339
340
341
342

    # run the tests
    result = test(cases)

    tot_time = timer.GetCPUTime() - s_time
    tot_wall_time = float(time.time() - s_wall_time)

    # output measurements for Dart
    print "<DartMeasurement name=\"WallTime\" type=\"numeric/double\">",
    print " %f </DartMeasurement>"%tot_wall_time
    print "<DartMeasurement name=\"CPUTime\" type=\"numeric/double\">",
    print " %f </DartMeasurement>"%tot_time
343
344
345

    # Delete these to eliminate debug leaks warnings.
    del cases, timer
346

347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
    if result.wasSuccessful():
        sys.exit(0)
    else:
        sys.exit(1)


def test(cases):
    """ Pass a list of tuples containing test classes and the
    functions used for testing.

    It returns a unittest._TextTestResult object.

    Example:

      test = test_suite([(vtkTestClass, 'test'),
                        (vtkTestClass1, 'test')])
    """
    # Make the test suites from the arguments.
    suites = []
    for case in cases:
        suites.append(unittest.makeSuite(case[0], case[1]))
    test_suite = unittest.TestSuite(suites)

    # Now run the tests.
    runner = unittest.TextTestRunner(verbosity=_VERBOSE)
    result = runner.run(test_suite)

374
    return result
375
376
377
378
379
380
381
382


def usage():
    msg="""Usage:\nTestScript.py [options]\nWhere options are:\n

    -D /path/to/VTKData
    --data-dir /path/to/VTKData

383
384
385
386
          Directory containing VTK Data use for tests.  If this option
          is not set via the command line the environment variable
          VTK_DATA_ROOT is used.  If the environment variable is not
          set the value defaults to '../../../../VTKData'.
387

388
389
390
391
    -B /path/to/valid/image_dir/
    --baseline-root /path/to/valid/image_dir/

          This is a path to the directory containing the valid images
392
393
394
395
          for comparison.  If this option is not set via the command
          line the environment variable VTK_BASELINE_ROOT is used.  If
          the environment variable is not set the value defaults to
          the same value set for -D (--data-dir).
396

397
398
399
400
401
402
403
404
405
    -T /path/to/valid/temporary_dir/
    --temp-dir /path/to/valid/temporary_dir/

          This is a path to the directory where the image differences
          are written.  If this option is not set via the command line
          the environment variable VTK_TEMP_DIR is used.  If the
          environment variable is not set the value defaults to
          '../../../Testing/Temporary'.

406
407
    -v level
    --verbose level
408

409
410
411
412
413
414
          Sets the verbosity of the test runner.  Valid values are 0,
          1, and 2 in increasing order of verbosity.

    -I
    --interact

415
416
417
418
          Interacts with the user when chosen.  If this is not chosen
          the test will run and exit as soon as it is finished.  When
          enabled, the behavior of this is rather trivial and works
          best when the test uses Tkinter.
419
420
421
422
423
424
425
426
427
428
429
430

    -n
    --no-image

          Does not do any image comparisons.  This is useful if you
          want to run the test and not worry about test images or
          image failures etc.

    -h
    --help

                 Prints this message.
431

432
433
"""
    return msg
434

435
436
437
438

def parseCmdLine():
    arguments = sys.argv[1:]

439
440
441
    options = "B:D:T:v:hnI"
    long_options = ['baseline-root=', 'data-dir=', 'temp-dir=',
                    'verbose=', 'help', 'no-image', 'interact']
442
443
444
445
446
447
448
449

    try:
        opts, args = getopt.getopt(arguments, options, long_options)
    except getopt.error, msg:
        print usage()
        print '-'*70
        print msg
        sys.exit (1)
450

451
452
    return opts, args

453

454
455
456
def processCmdLine():
    opts, args = parseCmdLine()

457
    global VTK_DATA_ROOT, VTK_BASELINE_ROOT, VTK_TEMP_DIR
458
459
460
461
462
463
464
465
466
467
468
469
470
    global _VERBOSE, _NO_IMAGE, _INTERACT

    # setup defaults
    try:
        VTK_DATA_ROOT = os.environ['VTK_DATA_ROOT']
    except KeyError:
        VTK_DATA_ROOT = os.path.normpath("../../../../VTKData")

    try:
        VTK_BASELINE_ROOT = os.environ['VTK_BASELINE_ROOT']
    except KeyError:
        pass

471
472
473
474
475
    try:
        VTK_TEMP_DIR = os.environ['VTK_TEMP_DIR']
    except KeyError:
        VTK_TEMP_DIR = os.path.normpath("../../../Testing/Temporary")

476
477
478
479
480
    for o, a in opts:
        if o in ('-D', '--data-dir'):
            VTK_DATA_ROOT = os.path.abspath(a)
        if o in ('-B', '--baseline-root'):
            VTK_BASELINE_ROOT = os.path.abspath(a)
481
482
        if o in ('-T', '--temp-dir'):
            VTK_TEMP_DIR = os.path.abspath(a)
483
484
485
486
487
488
489
490
491
492
493
        if o in ('-n', '--no-image'):
            _NO_IMAGE = 1
        if o in ('-I', '--interact'):
            _INTERACT = 1
        if o in ('-v', '--verbose'):
            try:
                _VERBOSE = int(a)
            except:
                msg="Verbosity should be an integer.  0, 1, 2 are valid."
                print msg
                sys.exit(1)
494
        if o in ('-h', '--help'):
495
496
497
498
499
500
501
502
503
            print usage()
            sys.exit()

    if not VTK_BASELINE_ROOT: # default value.
        VTK_BASELINE_ROOT = VTK_DATA_ROOT



if __name__ == "__main__":
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
    ######################################################################
    # A Trivial test case to illustrate how this module works.
    class SampleTest(vtkTest):
        obj = vtk.vtkActor()
        def testParse(self):
            "Test if class is parseable"
            self._testParse(self.obj)

        def testGetSet(self):
            "Testing Get/Set methods"
            self._testGetSet(self.obj)

        def testBoolean(self):
            "Testing Boolean methods"
            self._testBoolean(self.obj)

520
521
522
    # Test with the above trivial sample test.
    main( [ (SampleTest, 'test') ] )