benchCompare.py 4.33 KB
Newer Older
1 2 3 4 5 6 7 8 9
#!/usr/bin/env python3
#
# Compares the output from BenchmarkDeviceAdapter from the serial
# device to a parallel device and prints a table containing the results.
#
# Example usage:
#
# $ BenchmarkDeviceAdapter_SERIAL > serial.out
# $ BenchmarkDeviceAdapter_TBB > tbb.out
10
# $ benchCompare.py serial.out tbb.out
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
#
#
# The number of threads (optional -- only used to generate the "Warn" column)
maxThreads = 4
#
# Print debugging output:
doDebug = False
#
# End config options.

import re
import sys

assert(len(sys.argv) == 3)

def debug(str):
  if (doDebug): print(str)

# Parses "*** vtkm::Float64 ***************" --> vtkm::Float64
30
typeParser = re.compile("\\*{3} ([^*]+) on device ([^*]+) \\*{15}")
31 32 33 34 35 36 37 38

# Parses "Benchmark 'Benchmark name' results:" --> Benchmark name
nameParser = re.compile("Benchmark '([^-]+)' results:")

# Parses "mean = 0.0125s" --> 0.0125
meanParser = re.compile("\\s+mean = ([0-9.Ee+-]+)s")

# Parses "std dev = 0.0125s" --> 0.0125
39
stdDevParser = re.compile("\\s+std dev = ([naN0-9.Ee+-]+)s")
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157

serialFilename = sys.argv[1]
parallelFilename = sys.argv[2]

serialFile = open(serialFilename, 'r')
parallelFile = open(parallelFilename, 'r')

class BenchKey:
  def __init__(self, name_, type_):
    self.name = name_
    self.type = type_

  def __eq__(self, other):
    return self.name == other.name and self.type == other.type

  def __lt__(self, other):
    if self.name < other.name: return True
    elif self.name > other.name: return False
    else: return self.type < other.type

  def __hash__(self):
    return (self.name + self.type).__hash__()

class BenchData:
  def __init__(self, mean_, stdDev_):
    self.mean = mean_
    self.stdDev = stdDev_

def parseFile(f, benchmarks):
  type = ""
  bench = ""
  mean = -1.
  stdDev = -1.
  for line in f:
    debug("Line: {}".format(line))

    typeRes = typeParser.match(line)
    if typeRes:
      type = typeRes.group(1)
      debug("Found type: {}".format(type))
      continue

    nameRes = nameParser.match(line)
    if nameRes:
      name = nameRes.group(1)
      debug("Found name: {}".format(name))
      continue

    meanRes = meanParser.match(line)
    if meanRes:
      mean = float(meanRes.group(1))
      debug("Found mean: {}".format(mean))
      continue

    stdDevRes = stdDevParser.match(line)
    if stdDevRes:
      stdDev = float(stdDevRes.group(1))
      debug("Found stddev: {}".format(stdDev))

      # stdDev is always the last parse for a given benchmark, add entry now
      benchmarks[BenchKey(name, type)] = BenchData(mean, stdDev)
      debug("{} records found.".format(len(benchmarks)))

      mean = -1.
      stdDev = -1.

      continue

serialBenchmarks = {}
parallelBenchmarks = {}

parseFile(serialFile, serialBenchmarks)
parseFile(parallelFile, parallelBenchmarks)

serialKeys = set(serialBenchmarks.keys())
parallelKeys = set(parallelBenchmarks.keys())

commonKeys = sorted(list(serialKeys.intersection(parallelKeys)))

serialOnlyKeys = sorted(list(serialKeys.difference(parallelKeys)))
parallelOnlyKeys = sorted(list(parallelKeys.difference(serialKeys)))

debug("{} serial keys\n{} parallel keys\n{} common keys\n{} serialOnly keys\n{} parallelOnly keys.".format(
        len(serialKeys), len(parallelKeys), len(commonKeys), len(serialOnlyKeys), len(parallelOnlyKeys)))

if len(serialOnlyKeys) > 0:
  print("Keys found only in serial:")
  for k in serialOnlyKeys:
    print("%s (%s)"%(k.name, k.type))
  print("")

if len(parallelOnlyKeys) > 0:
  print("Keys found only in parallel:")
  for k in parallelOnlyKeys:
    print("%s (%s)"%(k.name, k.type))
  print("")

print("Comparison:")
print("| %7s | %4s | %8s    %8s | %8s    %8s | %s (%s) |"%(
        "Speedup", "Warn", "serial", "", "parallel", "", "Benchmark", "Type"))
print("|-%7s-|-%4s-|-%8s----%8s-|-%8s----%8s-|-%s--%s--|"%(
        "-"*7, "-"*4, "-"*8, "-"*8, "-"*8, "-"*8, "-"*9, "-"*4))
for key in commonKeys:
  sData = serialBenchmarks[key]
  pData = parallelBenchmarks[key]
  speedup = sData.mean / pData.mean if pData.mean != 0. else 0.
  if speedup > maxThreads * .9:
    flag = "    "
  elif speedup > maxThreads * .75:
    flag = "!   "
  elif speedup > maxThreads * .5:
    flag = "!!  "
  elif speedup > maxThreads * .25:
    flag = "!!! "
  else:
    flag = "!!!!"
  print("| %7.3f | %4s | %08.6f +- %08.6f | %08.6f +- %08.6f | %s (%s) |"%(
          speedup, flag, sData.mean, sData.stdDev, pData.mean, pData.stdDev, key.name, key.type))