#!/bin/env python
#*****************************************************************************
#
# Copyright (c) 2000 - 2018, Lawrence Livermore National Security, LLC
# Produced at the Lawrence Livermore National Laboratory
# LLNL-CODE-442911
# All rights reserved.
#
# This file is  part of VisIt. For  details, see https://visit.llnl.gov/.  The
# full copyright notice is contained in the file COPYRIGHT located at the root
# of the VisIt distribution or at http://www.llnl.gov/visit/copyright.html.
#
# Redistribution  and  use  in  source  and  binary  forms,  with  or  without
# modification, are permitted provided that the following conditions are met:
#
#  - Redistributions of  source code must  retain the above  copyright notice,
#    this list of conditions and the disclaimer below.
#  - Redistributions in binary form must reproduce the above copyright notice,
#    this  list of  conditions  and  the  disclaimer (as noted below)  in  the
#    documentation and/or other materials provided with the distribution.
#  - Neither the name of  the LLNS/LLNL nor the names of  its contributors may
#    be used to endorse or promote products derived from this software without
#    specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT  HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR  IMPLIED WARRANTIES, INCLUDING,  BUT NOT  LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND  FITNESS FOR A PARTICULAR  PURPOSE
# ARE  DISCLAIMED. IN  NO EVENT  SHALL LAWRENCE  LIVERMORE NATIONAL  SECURITY,
# LLC, THE  U.S.  DEPARTMENT OF  ENERGY  OR  CONTRIBUTORS BE  LIABLE  FOR  ANY
# DIRECT,  INDIRECT,   INCIDENTAL,   SPECIAL,   EXEMPLARY,  OR   CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT  LIMITED TO, PROCUREMENT OF  SUBSTITUTE GOODS OR
# SERVICES; LOSS OF  USE, DATA, OR PROFITS; OR  BUSINESS INTERRUPTION) HOWEVER
# CAUSED  AND  ON  ANY  THEORY  OF  LIABILITY,  WHETHER  IN  CONTRACT,  STRICT
# LIABILITY, OR TORT  (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY  WAY
# OUT OF THE  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
# DAMAGE.
#*****************************************************************************
"""
VisIt's test suite.
"""

import sys
import os
import math
import glob
import re
import shutil
import subprocess
import multiprocessing
import time
import math
import datetime
import pickle
import json
import traceback



from os.path import join as pjoin
from optparse import OptionParser

from visit_test_common import *
from visit_test_reports import *
from visit_test_ctest import *

# ----------------------------------------------------------------------------
#  Method: visit_root
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def visit_root():
    return abs_path(test_path(),"../..")


# ----------------------------------------------------------------------------
#  Method: test_path
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def test_path():
    return os.path.abspath(os.path.split(__file__)[0])


# ----------------------------------------------------------------------------
#  Method: check_skip
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
#
# Modifications:
#  Cyrus Harrison, Tue May 28 12:05:45 PDT 2013
#  Add support for platform based skips.
#
#  Kathleen Biagas, Thu Nov  8 10:30:36 PST 2018
#  Assume entire category is skipped it 'file' key is missing.
#
# ----------------------------------------------------------------------------
def check_skip(skip_list,test_modes,test_cat,test_file):
    if skip_list is None:
        return False
    # look for modes that match
    for v in skip_list['skip_list']:
        if v['mode'] == test_modes:
            for test in v['tests']:
                # check for platform restrictions
                if test.has_key("platform"):
                    tplat = test["platform"].lower()
                    splat = sys.platform.lower()
                    # win,linux,osx
                    # ignore this entry if we are on the wrong platform
                    # else, use std logic
                    if not splat.startswith(tplat):
                        continue
                if test['category'] == test_cat:
                    if test.has_key("file"):
                        if test['file'] == test_file:
                            if not test.has_key("cases"):
                            # skip the entire file if
                            # there are no specific cases
                                return True
                    else:
                        # skip the entire category
                        return True
    return False
# ----------------------------------------------------------------------------
#  Method: parse_test_specific_vargs
#
#  Programmer: Cyrus Harrison
#  Date:       Tues April 29, 2014
# ----------------------------------------------------------------------------
def parse_test_specific_vargs(test_file):
    lines = [l.strip() for l in open(test_file).readlines()]
    # check for pattern "# ... VARGS: {visit command line args}
    vargs = [l[1:] for l in lines if len(l) > 0 and l[0] == "#" and l.count("VARGS:")  == 1]
    vargs = " ".join(vargs)
    vargs = vargs.replace("VARGS:","")
    return vargs

# ----------------------------------------------------------------------------
#  Method: parse_test_specific_limit
#
#  Programmer: Kathleen Biagas 
#  Date:       Thu Nov 8, 2018
# ----------------------------------------------------------------------------
def parse_test_specific_limit(test_file):
    lines = [l.strip() for l in open(test_file).readlines()]
    # check for pattern "# ... LIMIT: {limit}
    limline = [l[1:] for l in lines if len(l) > 0 and l[0] == "#" and l.count("LIMIT:")  == 1]
    if len(limline) > 0:
        limline = " ".join(limline)
        limline = limline.replace("LIMIT:","")
        return int(limline)
    return -1

# ----------------------------------------------------------------------------
#  Method: launch_visit_test
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
#
#  Modifications:
#    Brad Whitlock, Thu Nov  7 14:00:28 PST 2013
#    Pass width and height to test.
#
#    Kathleen Biagas, Thu Feb  6 14:08:00 PST 2014
#    Pass 'ctest' to test.
#
#    Kathleen Biagas, Fri May 16 15:23:13 PDT 2014
#    Set up sim_dir based off executable location.
#
#    Burlen Loring, Mon May 26 15:36:26 PDT 2014
#    Addedd command line option to use threshold based image diff
#
#    Eric Brugger, Fri Aug 15 10:04:27 PDT 2014
#    I added the ability to specify the parallel launch method.
#
#    Kathleen Biagas, Thu Sep 4 16:45:39 MST 2014
#    Use exe path only for sim_dir on Windows.
#
#    Burlen Loring, Wed Oct 21 15:44:24 PDT 2015
#    Added an option (--display-failed) to display current,
#    baseline, and diff in a popup window as the test runs.
#
#    Kathleen Biagas, Thu Nov  8 10:31:51 PST 2018
#    Added src_dir and cmake_cmd, for plugin-vs-install tests.
#    If test has specified its own 'LIMIT' use it for the kill limit.
#
# ----------------------------------------------------------------------------
def launch_visit_test(args):
    """
    Runs a single VisIt test.
    """
    idx  =  args[0]
    test =  args[1]
    opts =  args[2]
    top_dir     = visit_root()
    test_script = abs_path(test_path(),"visit_test_main.py")
    test_dir, test_file = os.path.split(test)
    test_cat  = os.path.split(test_dir)[1]
    test_base = os.path.splitext(test_file)[0]
    rcmd  =  opts["executable"] + " "
    rcmd +=  opts["vargs"] + " "
    # check for vargs embedded in the test file header
    rcmd +=  parse_test_specific_vargs(test) + " "
    rcmd +=  "-exec-timeout %d -idle-timeout %d " % (opts["limit"],opts["limit"])
    rcmd +=  "-numrestarts 0 "
    if not opts["interactive"]:
        rcmd +=  "-nowin "
    if not opts["use_pil"]:
        rcmd +=  "-noPIL "
    if not opts["no_timings"]:
        rcmd +=  "-timing"
    rcmd += " -cli "
    cfile = pjoin(test_dir,test_base + ".config")
    if os.path.isfile(cfile):
        rcmd += "-config " + cfile + " "
    else:
        rcmd += "-noconfig "
    rcmd += "-geometry %dx%d+32+32 " % (opts["width"],opts["height"])
    rcmd += " -s %s " % os.path.abspath(test_script)
    modes_list = opts["modes"].split(",")
    if "dlb" in modes_list:
        rcmd += " -allowdynamic "
    if "icet" in modes_list:
        rcmd += " -icet "
    modes = ""
    if opts["verbose"]:
        rcmd += " -verbose "
    fuzzy = opts["fuzzy"]
    if "serial" in modes_list:
        modes = "serial"
    else:
        if "scalable" in modes_list:
            modes = "scalable"
            if not opts["no_fuzzy"]:
                fuzzy = True
        if "parallel" in modes_list:
            if len(modes) > 0:
                modes +=","
            modes +="parallel"
        if "icet" in modes_list:
            if len(modes) > 0:
                modes +=","
            modes +="icet"
    run_dir = pjoin(opts["result_dir"],"_run","_%s_%s" % (test_cat, test_base))
    # set opts vars
    tparams = {}
    tparams["script"]         = test
    tparams["category"]       = test_cat
    tparams["name"]           = test_base
    tparams["file"]           = test_file
    tparams["modes"]          = modes
    tparams["run_dir"]        = run_dir
    tparams["result_dir"]     = opts["result_dir"]
    tparams["fuzzy_match"]    = fuzzy
    tparams["skip_file"]      = None
    tparams["interactive"]    = opts["interactive"]
    tparams["use_pil"]        = opts["use_pil"]
    tparams["threshold_diff"] = opts["threshold_diff"]
    tparams["threshold_error"]= opts["threshold_error"]
    tparams["pixdiff"]        = opts["pixdiff"]
    tparams["avgdiff"]        = opts["avgdiff"]
    tparams["numdiff"]        = opts["numdiff"]
    tparams["top_dir"]        = top_dir
    tparams["data_dir"]       = opts["data_dir"]
    tparams["src_dir"]        = opts["src_dir"]
    tparams["data_host"]      = opts["data_host"]
    tparams["baseline_dir"]   = opts["baseline_dir"]
    tparams["tests_dir"]      = opts["tests_dir"]
    tparams["visit_bin"]      = opts["executable"]
    tparams["width"]          = opts["width"]
    tparams["height"]         = opts["height"]
    tparams["ctest"]          = opts["ctest"]
    tparams["display_failed"] = opts["display_failed"]
    tparams["parallel_launch"]= opts["parallel_launch"]
    tparams["host_profile_dir"]   = opts["host_profile_dir"]
    tparams["sessionfiles"]   = opts["sessionfiles"]
    tparams["cmake_cmd"]      = opts["cmake_cmd"]

    exe_dir, exe_file = os.path.split(tparams["visit_bin"])
    if sys.platform.startswith("win"):
        tparams["sim_dir"] = os.path.abspath(exe_dir)
    else:
        tparams["sim_dir"] = os.path.abspath(os.path.join(exe_dir, ".."))

    if not opts["no_skip"]:
        tparams["skip_file"]  = opts["skip_file"]
    skip  =  check_skip(opts["skip_list"],modes,test_cat,test_file)
    if skip:
        Log("[Skipping: %s/%s (found in skip list)]" % (test_cat,test_file))
        result = TestScriptResult(idx,
                                  test_cat,
                                  test_base,
                                  test_file,
                                  116, # full skip return code
                                  0,
                                  0)
    else:
        Log("[Launching: %s/%s]" % (test_cat,test_file))
        # run the test in a unique sub dir
        if os.path.isdir(run_dir):
            try:
                shutil.rmtree(run_dir)
            except OSError as e:
                Log("<Error Cleaning up Script Run Directory before launch> %s" % run_dir)
        os.mkdir(run_dir)
        pfile = open(pjoin(run_dir,"params.json"),"w")
        pfile.write("%s" % json_dumps(tparams))
        pfile.close()
        rcmd += " --params=%s" % os.path.abspath(pjoin(run_dir,"params.json"))
        # get start timestamp
        stime = time.time()
        # change to working dir to our run dir
        curr_dir = os.getcwd()
        os.chdir(run_dir)
        rcode = 0
        test_specific_limit = parse_test_specific_limit(test)
        if test_specific_limit != -1:
            use_limit = test_specific_limit
        else:
            use_limit = opts["limit"]
        sexe_res = sexe(rcmd,
                        suppress_output=(not (opts["verbose"] or opts["less_verbose"])),
                        echo=opts["verbose"],
                        timeout=use_limit * 1.1) # proc kill switch at 110% of the selected timeout
        json_res_file = pjoin(opts["result_dir"],"json","%s_%s.json" %(test_cat,test_base))
        if os.path.isfile(json_res_file):
            results = json_load(json_res_file)
            if results.has_key("result_code"):
                rcode = results["result_code"]
            # os.mkdir(run_dir)
        if sexe_res["killed"]:
            Log("<Limit killed> %s" % os.path.abspath(test))
            # wait for process chain to die
            time.sleep(1)
        # get end timestamp
        etime = time.time()
        dtime = math.ceil(etime - stime)
        # wait for any logs to flush
        time.sleep(1)
        # change back to prev working dir
        os.chdir(curr_dir)
        # check for core files
        ncores_files = process_cores(run_dir,opts["result_dir"],test_cat,test_base)
        # move logs and timings to the html output dir
        process_runtime_logs(run_dir,opts["result_dir"],test_cat,test_base)
        # parse any output files
        result = TestScriptResult(idx,
                                  test_cat,
                                  test_base,
                                  test_file,
                                  rcode,
                                  ncores_files,
                                  dtime)
        if opts["cleanup"]:
            try:
                shutil.rmtree(run_dir)
            except OSError as e:
                Log("<Error Removing Script Run Directory> %s" % run_dir)
    log_test_result(opts["result_dir"],result)
    return result


# ----------------------------------------------------------------------------
#  Method: process_cores
#
#  Programmer: Cyrus Harrison
#  Date:       Thu Aug  9 2012
# ----------------------------------------------------------------------------
def process_cores(run_dir,res_dir,test_cat,test_name):
    MAX_CORE_SIZE = 262144000 # 250 mbytes
    core_files    = glob.glob(pjoin(run_dir,"*core*"))
    num_cores     = len(core_files)
    if num_cores > 0:
        cores_base_dir = pjoin(res_dir,"cores")
        cores_cat_dir  = pjoin(cores_base_dir,test_cat)
        cores_test_dir = pjoin(cores_cat_dir,test_name)
        # make sure the log dirs exists
        for dname in [cores_base_dir, cores_cat_dir, cores_test_dir]:
            if not os.path.isdir(dname):
                os.mkdir(dname)
        # move any cores into this dir
        for cfile in core_files:
            # make sure we don't copy huge core files.
            csize = os.path.getsize(cfile)
            if csize < MAX_CORE_SIZE:
                shutil.copy2(cfile, cores_test_dir)
    return num_cores


# ----------------------------------------------------------------------------
#  Method: process_runtime_logs
#
#  Programmer: Cyrus Harrison
#  Date:       Thu Aug  9 2012
# ----------------------------------------------------------------------------
def process_runtime_logs(run_dir,res_dir,test_cat,test_name):
    rlog_files = glob.glob(pjoin(run_dir,"*vlog"))
    rlog_files.extend(glob.glob(pjoin(run_dir,"*.timings")))
    if len(rlog_files) > 0:
        logs_base_dir = pjoin(res_dir,"logs")
        logs_cat_dir  = pjoin(logs_base_dir,test_cat)
        logs_test_dir = pjoin(logs_cat_dir ,test_name)
        # make sure the log dirs exists
        for dname in [logs_base_dir, logs_cat_dir, logs_test_dir]:
            if not os.path.isdir(dname):
                os.mkdir(dname)
        for rlog in rlog_files:
            shutil.copy2(rlog, logs_test_dir)


# ----------------------------------------------------------------------------
#  Method: log_test_result
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def log_test_result(result_dir,result):
    Log(result.message())
    HTMLIndex(result_dir).add_result(result)
    JSONIndex(pjoin(result_dir,"results.json")).add_result(result.to_dict())


# ----------------------------------------------------------------------------
#  Method: default_suite_options
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 8 2013
#
#  Modifications:
#    Eric Brugger, Fri Aug 15 10:04:27 PDT 2014
#    I added the ability to specify the parallel launch method.
#
#    Brad Whitlock, Mon Dec 15 15:42:32 PST 2014
#    Added --data-host, --host-profile-dir.
#
#    Burlen Loring, Wed Oct 21 15:44:24 PDT 2015
#    Added an option (--display-failed) to display current,
#    baseline, and diff in a popup window as the test runs.
#
#    Mark C. Miller, Tue Sep  6 18:54:31 PDT 2016
#    Added sessionfiles option to rigoursly test session files
#
#    Kathleen Biagas, Thu Nov  8 10:33:45 PST 2018
#    Added src_dir and cmake_cmd.  
#
#    Eric Brugger, Wed Dec  5 13:05:18 PST 2018
#    Changed the definition of tests_dir_def to the new location of the
#    test directory.
#
#    Kathleen Biagas, Thu Dec 13 10:51:54 PST 2018
#    Changed definition of data_dir to new location.  Assumes in-src build,
#    which is used for nightly regression tests.
#
# ----------------------------------------------------------------------------
def default_suite_options():
    data_dir_def    = abs_path(visit_root(),"src","testdata")
    base_dir_def    = abs_path(visit_root(),"test","baseline")
    tests_dir_def   = abs_path(visit_root(),"src","test","tests")
    visit_exe_def   = abs_path(visit_root(),"src","bin","visit")
    src_dir_def     = abs_path(visit_root(),"src")
    skip_def        = pjoin(test_path(),"skip.json")
    nprocs_def      = multiprocessing.cpu_count()
    opts_full_defs = {
                      "use_pil":True,
                      "threshold_diff":False,
                      "threshold_error":{},
                      "src_dir":      src_dir_def,
                      "data_dir":     data_dir_def,
                      "baseline_dir": base_dir_def,
                      "tests_dir":    tests_dir_def,
                      "result_dir":   test_path(),
                      "post":False,
                      "verbose":False,
                      "less_verbose":False,
                      "quiet":False,
                      "width": 300,
                      "height":300,
                      "modes":"serial",
                      "classes":"nightly",
                      "limit":600,
                      "skip_file":skip_def,
                      "no_skip":False,
                      "check_data":True,
                      "fuzzy":False,
                      "no_fuzzy":False,
                      "cleanup":True,
                      "cleanup_delay":10,
                      "executable":visit_exe_def,
                      "data_host":"localhost",
                      "interactive":False,
                      "pixdiff":0.0,
                      "avgdiff":0,
                      "numdiff":0.0,
                      "vargs": "",
                      "host_profile_dir": "",
                      "retry":False,
                      "index":None,
                      "timeout":3600,
                      "nprocs":nprocs_def,
                      "ctest":False,
                      "display_failed":False,
                      "parallel_launch":"mpirun",
                      "sessionfiles":False,
                      "cmake_cmd":"cmake",
                      "no_timings":False,
                      "rsync_post":None}
    return opts_full_defs

def finalize_options(opts):
    opts["executable"]   = abs_path(opts["executable"])
    opts["result_dir"]   = abs_path(opts["result_dir"])
    opts["src_dir"]      = abs_path(opts["src_dir"])
    opts["data_dir"]     = abs_path(opts["data_dir"])
    opts["tests_dir"]    = abs_path(opts["tests_dir"])
    opts["baseline_dir"] = abs_path(opts["baseline_dir"])
    if isinstance(opts["classes"],basestring):
        opts["classes"]  = opts["classes"].split(",")
    opts["skip_list"]    = None
    if not opts["skip_file"] is None and os.path.isfile(opts["skip_file"]):
        try:
            if opts["no_skip"] == False:
                opts["skip_list"] = json_load(opts["skip_file"])
        except:
            opts["skip_list"] = None

# ----------------------------------------------------------------------------
#  Method: parse_args
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
#
#  Modifications:
#    Eric Brugger, Fri Aug 15 10:04:27 PDT 2014
#    I added the ability to specify the parallel launch method.
#
#    Kathleen Biagas, Wed Nov 5 14:32:21 PST 2014
#    On windows, glob any '*.py' tests names.
#
#    Matthew Wheeler, Mon Dec 15 12:56:00 GMT 2014
#    Changed pixdiff % to be a float rather than an int
#
#    Burlen Loring, Wed Oct 21 15:44:24 PDT 2015
#    Added an option (--display-failed) to display current,
#    baseline, and diff in a popup window as the test runs.
#
#    Mark C. Miller, Tue Sep  6 18:54:31 PDT 2016
#    Added sessionfiles option to rigoursly test session files
#
#    Kathleen Biagas, Thu Nov  8 10:34:27 PST 2018
#    Added '--src' for specifying src_dir, and --cmake for specifying
#    cmake_cmd, used for plugin-vs-install tests.
#
# ----------------------------------------------------------------------------
def parse_args():
    """
    Parses arguments to runtest.
    """
    parser = OptionParser()
    defs = default_suite_options()
    parser.add_option("-r",
                      "--run-only",
                      dest="use_pil",
                      default=True,
                      action="store_false",
                      help="no image differencing (no PIL)")
    parser.add_option( "--src",
                      dest="src_dir",
                      default=defs["src_dir"],
                      help="path to src directory [default=%s]" % defs["src_dir"])
    parser.add_option("-d",
                      "--data-dir",
                      dest="data_dir",
                      default=defs["data_dir"],
                      help="path to data directory [default=%s]" % defs["data_dir"])
    parser.add_option("-b",
                      "--baseline-dir",
                      dest="baseline_dir",
                      default=defs["baseline_dir"],
                      help="path to baseline directory [default=%s]" % defs["baseline_dir"])
    parser.add_option("-t",
                      "--tests-dir",
                      dest="tests_dir",
                      default=defs["tests_dir"],
                      help="path to tests directory [default=%s]" % defs["tests_dir"])
    parser.add_option("-o",
                      "--output-dir",
                      dest="result_dir",
                      default=defs["result_dir"],
                      help="path to output directory [default=%s]" % defs["result_dir"])
    parser.add_option("-p",
                      "--post",
                      default=defs["post"],
                      action="store_true",
                      help="post results to web")
    parser.add_option("-q",
                      "--quiet",
                      default=defs["quiet"],
                      action="store_true",
                      help="suppress all text output")
    parser.add_option("-v",
                      "--verbose",
                      default=defs["verbose"],
                      action="store_true",
                      help="extra test output")
    parser.add_option("--lessverbose",
                      default=defs["less_verbose"],
                      dest="less_verbose",
                      action="store_true",
                      help="extra test output without progress messages")
    parser.add_option("--width",
                      type="int",
                      default=defs["width"],
                      help="set image width")
    parser.add_option("--height",
                      type="int",
                      default=defs["height"],
                      help="set image height")
    parser.add_option("-m",
                      "--modes",
                      default=defs["modes"],
                      help="specify mode in which to run tests"
                           " [choose from 'parallel','serial','scalable', "
                           " 'dlb','hdf5', 'icet', and combinations such as"
                           " 'scalable,parallel']")
    parser.add_option("-c",
                      "--classes",
                      default=defs["classes"],
                      help="list of classes [choose from 'nightly']")
    parser.add_option("-l",
                      "--limit",
                      type="int",
                      default=defs["limit"],
                      help="set maximum elapsed run-time for each test")
    parser.add_option("-s",
                      "--skiplist",
                      dest="skip_file",
                      default=defs["skip_file"],
                      help="specify a skip list file [default=%s]" % defs["skip_file"])
    parser.add_option("--no-skiplist",
                      dest="no_skip",
                      default=defs["no_skip"],
                      action = "store_true",
                      help="Do not use a skip list file")
    parser.add_option("--no-data-check",
                      dest="check_data",
                      default=defs["check_data"],
                      action = "store_false",
                      help="Skip build sanity check on input data files")
    parser.add_option("--fuzzy",
                      dest="fuzzy",
                      default=defs["fuzzy"],
                      action = "store_true",
                      help="Use fuzzy image matching [default=False]")
    parser.add_option("--no-fuzzy",
                      dest="no_fuzzy",
                      default=defs["no_fuzzy"],
                      action = "store_true",
                      help="Do not use fuzzy image matching for scalable mode"),
    parser.add_option("--no-cleanup",
                      dest="cleanup",
                      default=defs["cleanup"],
                      action = "store_false",
                      help="Do not remove the _run directory"),
    parser.add_option("--cleanup-delay",
                      dest="cleanup_delay",
                      default=defs["cleanup_delay"],
                      type="int",
                      help="# of seconds to wait for processes to finish "
                           "before cleaning up. [default=%d]" % defs["cleanup_delay"]),
    parser.add_option("-e",
                      "--executable",
                      default=defs["executable"],
                      help="specify executable version of visit to run. "
                           "For example, use \"-e '/usr/gapps/visit/bin/visit -v "
                           " [default = %s]" % defs["executable"])
    parser.add_option("--data-host",
                      dest="data_host",
                      default=defs["data_host"],
                      help="Specify remote host to use for data and compute engine. "
                           "For example, use \"--data-host vulcan.llnl.gov "
                           " [default = %s]" % defs["data_host"])
    parser.add_option("-i",
                      "--interactive",
                      action = "store_true",
                      default=defs["interactive"],
                      help="don't redirect test .py file into visit."
                           "Just bring up CLI and let user Source() "
                           "the .py file explicitly.")
    parser.add_option("--pixdiff",
                      type="float",
                      default=defs["pixdiff"],
                      help="allowed % of pixels different [default = 0.0%]")
    parser.add_option("--avgdiff",
                      type="int",
                      default=defs["avgdiff"],
                      help="if pixdiff exceeded, allowed mean grayscale diff "
                           "[default = 0]")
    parser.add_option("--numdiff",
                      type="float",
                      default=defs["numdiff"],
                      help="allowed relative numerical difference in text "
                           "results [default = 0.0]")
    parser.add_option("--vargs",
                      default=defs["vargs"],
                      help="arguments to pass directly to VisIt "
                           "(surround them \" or ')")
    parser.add_option("--host-profile-dir",
                      default=defs["host_profile_dir"],
                      help="Specify a directory from which host profiles will be read.")
    parser.add_option("--retry",
                      default=defs["retry"],
                      action="store_true",
                      help="retry failed test cases from last index file")
    parser.add_option("--index",
                      dest="index",
                      default=defs["index"],
                      help="load test cases from a specific index file")
    parser.add_option("--timeout",
                      type="int",
                      default=defs["timeout"], # total timeout of one hour
                      help="total test suite timeout in seconds [default = 3600]")
    parser.add_option("-n","--num-processes",
                      dest="nprocs",
                      type=int,
                      default=defs["nprocs"],
                      help="number of tests to launch simultaneously [default =%d]" % defs["nprocs"])
    parser.add_option("--ctest",
                      default=defs["ctest"],
                      action="store_true",
                      help="generate ctest compatible output")
    parser.add_option("--threshold-diff",
                      default=defs["threshold_diff"],
                      dest="threshold_diff",
                      action="store_true",
                      help="use threshold based image diff")
    parser.add_option("--threshold-error",
                      default=defs["threshold_error"],
                      dest="threshold_error",
                      type=str,
                      action='callback',
                      callback=ParseThresholdOverride,
                      help="Per case overide of the max allowable error for threshold based image diff")
    parser.add_option("--display-failed",
                      default=defs["display_failed"],
                      action="store_true",
                      help="use image magick to display the current baseline and diff for each failure")
    parser.add_option("--parallel-launch",
                      default=defs["parallel_launch"],
                      help="specify the parallel launch method. "
                           "Options are mpirun and srun, or a space-separated "
                           "list of parallel launch options.")
    parser.add_option("--sessionfiles",
                      dest="sessionfiles",
                      action = "store_true",
                      default=defs["sessionfiles"],
                      help="Rigoursly test session files by saving a session, "
                           "validating the XML and then perturbing and "
                           "restoring the session prior to each image save.")
    parser.add_option("--no-timings",
                      dest="no_timings",
                      default=defs["no_timings"],
                      action = "store_true",
                      help="Do not generate timing files.")
    parser.add_option("--rsync-post",
                      default=defs["rsync_post"],
                      help="Post results via rsync")
    parser.add_option( "--cmake",
                      dest="cmake_cmd",
                      default=defs["cmake_cmd"],
                      help="path to cmake executable [default=%s]" % defs["cmake_cmd"])

    # parse args
    opts, tests = parser.parse_args()
    # note: we want a dict b/c the values could be passed without using optparse
    opts = vars(opts)
    if sys.platform.startswith("win"):
        # use glob to match any *.py
        expandedtests = []
        for t in tests:
           if not '*' in t:
              expandedtests.append(t)
           else:
              for match in glob.iglob(t):
                 expandedtests.append(match)
        if len(expandedtests) > 0:
            tests = expandedtests
    return opts, tests

# ----------------------------------------------------------------------------
#  Method: ParseThresholdOverride
#
#  Programmer: Burlen Loring
#  Date:       Tue May 27 11:11:22 PDT 2014
# ----------------------------------------------------------------------------
def ParseThresholdOverride(option, opt, value, parser):
    """
    Convert threshold overrides encodeed in a string like
    --threshold-error=a:b,c:d to a dictionary. Parse
    errors are intentionally fatal.
    """
    d={}
    for pair in value.split(','):
        k,v = pair.split(':')
        d[k] = float(v)
    setattr(parser.values, option.dest, d)

# ----------------------------------------------------------------------------
#  Method: find_test_cases
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def find_test_cases(tests_dir,test_classes):
    """
    Finds test suite scripts.
    """
    Log("[Finding test scripts]")
    if len(test_classes)> 0:
        Log("[Test Classes: %s]" % str(test_classes))
    res    = []
    tfiles = glob.glob(pjoin(tests_dir,"*","*.py"))
    tfiles = [ os.path.abspath(tf) for tf in tfiles]
    if len(test_classes) == 0 or test_classes == "ANY":
        res = tfiles
    else:
        for tf in tfiles:
            test_lines = open(tf).readlines()
            match = False
            for l in test_lines:
                if l.count("#  CLASSES:"):
                    for c in test_classes:
                        if l.count(c):
                            match = True
            if match:
                res.append(tf)
            else:
                Log("Excluding: %s" %tf)
    return res

# ----------------------------------------------------------------------------
#  Method: load_test_cases_from_index
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def load_test_cases_from_index(tests_dir,result_idx,only_failures=False):
    """
    Finds test suite scripts.
    """
    Log("[Loading test cases from index file = %s)]" % result_idx)
    res = []
    tests = JSONIndex.load_results(result_idx)
    for t in tests:
        tsr = TestSuiteResult(**t)
        if only_failures and tsr.error():
            res.append(pjoin(tests_dir,tsr.category,tsr.file))
        else:
            res.append(pjoin(tests_dir,tsr.category,tsr.file))
    return res

# ----------------------------------------------------------------------------
#  Method: prepare_result_dirs
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def prepare_result_dirs(res_dir=None):
    """
    Creates output directories for test suite results
    """
    if res_dir is None:
        res_dir = test_path()
    else:
        if not os.path.isdir(res_dir):
            os.mkdir(res_dir)
    run_dir = pjoin(res_dir,"_run")
    if not os.path.isdir(run_dir):
        os.mkdir(run_dir)
    for d in ["html","json","current","diff"]:
        full_dir = pjoin(res_dir,d)
        if os.path.isdir(full_dir):
            shutil.rmtree(full_dir)
        os.mkdir(full_dir)

# ----------------------------------------------------------------------------
#  Method: prepare_data_dir
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
#
#  Modifications:
#    Kathleen Biagas, Mon Jan 27 13:02:45 MST 2014
#    Change 'make test' to 'make testdata'.
# ----------------------------------------------------------------------------
def prepare_data_dir(data_dir):
    """
    Simple check of test data.
    """
    dfile = pjoin(data_dir,"silo_hdf5_test_data","globe.silo")
    Log("[Checking test data]")
    if not os.path.isfile(dfile):
        Log("[Rebuilding test data based "
            "on the fact that '%s' doesn't exist]" % dfile)
        if sys.platform.startswith("win"):
            # TODO: What should we do on windows if the data doesn't exist?
            pass
        else:
            cwd = os.getcwd()
            os.chdir(data_dir)
            cmd = "make testdata"
            sexe(cmd)
            os.chdir(cwd)
    else:
        Log("[Test data file '%s' exists.]" % dfile)

# ----------------------------------------------------------------------------
#  Method: cleanup
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def cleanup(res_dir,delay):
    Log("[Cleanup: Waiting for delayed writes]")
    time.sleep(delay)
    Log("[Cleanup: Removing _run directory]")
    run_dir = pjoin(res_dir,"_run")
    try:
        shutil.rmtree(run_dir)
    except OSError as e:
        Log("<Error Removing _run Directory> %s" % run_dir)

# ----------------------------------------------------------------------------
#  Method: launch_tests
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
# ----------------------------------------------------------------------------
def launch_tests(opts,tests):
    error = False
    results = []
    ntests = len(tests)
    if ntests == 1:
        Log("[Running %d test case]" % ntests)
    else:
        Log("[Running %d test cases]" % ntests)
    if opts["no_skip"] == False and os.path.isfile(opts["skip_file"]):
        Log("[Using skip list file: '%s']" % opts["skip_file"])
    test_args = [(idx,tests[idx],opts) for idx in range(ntests)]
    # save the input list
    test_list = json_dumps([(idx,tests[idx]) for idx in range(ntests)])
    open(pjoin(opts["result_dir"],"tests.json"),"w").write(test_list)
    if len(test_args) < opts["nprocs"]:
        opts["nprocs"] = len(test_args)
    if opts["nprocs"] < 2:
        Log("[Using %s test process]" % opts["nprocs"])
        for args in test_args:
            results.append(launch_visit_test(args))
    else:
        Log("[Using %s test processes]" % opts["nprocs"])
        pool = multiprocessing.Pool(processes=opts["nprocs"])
        p = pool.map_async(launch_visit_test,test_args)
        try:
            #results = p.get(0xFFFF) # used to make sure ctl-c doesn't hang
            results = p.get(opts["timeout"]) # used to make sure ctl-c doesn't hang
        except KeyboardInterrupt:
            Log("<Test Suite Run aborted by user keyboard interrupt.>")
            pool.terminate()
            error = True
        except Exception as e:
            traceback.print_exc()
            Log("<Unknown Exception>:%s" % str(e))
            pool.terminate()
            error = True
    if len(results) != len(test_list):
        for t in test_args:
            if not t[0] in [ r.index for r in results]:
                test_dir, test_file = os.path.split(t[1])
                test_cat  = os.path.split(test_dir)[1]
                test_base = os.path.splitext(test_file)[0]
                Log("<Missing result from test %s/%s, assuming Killed>" % (test_cat,test_base))
                res  = TestScriptResult(t[0],
                                        test_cat,
                                        test_base,
                                        test_file,
                                        115, # killed return code
                                        0,
                                        0)
                results.append(res)
    error = error or True in [r.error() for r in results]
    return error, results

# ----------------------------------------------------------------------------
#  Method: rsync_post
#
#  Programmer: Cyrus Harrison
#  Date:       ue Nov 25 11:23:12 PST 2014
#
#  Modifications:
# ----------------------------------------------------------------------------
def rsync_post(src_dir,rsync_dest):
    rsync_dest = pjoin(rsync_dest,timestamp(sep="_"))
    rsync_cmd =  "rsync -vur %s/ %s" % (src_dir,rsync_dest)
    Log("[rsyncing results to %s]" % rsync_dest)
    exe_res = sexe(rsync_cmd,
                   suppress_output=(not (opts["verbose"] or opts["less_verbose"])),
                    echo=opts["verbose"])



# ----------------------------------------------------------------------------
#  Method: main
#
#  Programmer: Cyrus Harrison
#  Date:       Wed May 30 2012
#
#  Modifications:
#   Kathleen Biagas, Thu Feb  6 14:08:00 PST 2014
#   Only do ctest logging if ctest is enabled.
#
#   Burlen Loring, Fri Oct  2 09:56:26 PDT 2015
#   Make ctest output for each subtest. Moved the timer code
#   into the ctest module so that I could track time spent
#   in the sub test.
#
# ----------------------------------------------------------------------------
def main(opts,tests):
    """
    Main entry point for the test suite.
    """
    finalize_options(opts)
    Log("[[VisIt Test Suite]]")
    if opts["check_data"]:
        prepare_data_dir(opts["data_dir"])
    if opts["index"]:
        ridx  = opts["index"]
        tests = load_test_cases_from_index(opts["tests_dir"],ridx)
    elif opts["retry"]:
        Log("[Retrying failures from previous run]")
        ridx  = pjoin(opts["result_dir"],"results.json")
        tests = load_test_cases_from_index(opts["tests_dir"],ridx,True)
    elif len(tests) == 0:
        tests = find_test_cases(opts["tests_dir"],opts["classes"]) 
    tests = [ abs_path(pjoin(opts["tests_dir"], "..",t)) for t in tests]
    prepare_result_dirs(opts["result_dir"])
    ststamp = timestamp(sep=":")
    stime   = time.time()
    if opts["ctest"]:
        ctestInitTestTimer()
    Log("[Starting test suite run @ %s]" % ststamp)
    html_index = HTMLIndex(opts["result_dir"])
    html_index.write_header(opts["modes"],ststamp)
    json_index = JSONIndex(pjoin(opts["result_dir"],"results.json"))
    json_index.write_header(opts,tests,ststamp)
    error, results = launch_tests(opts,tests)
    etstamp = timestamp(sep=":")
    etime   = time.time()
    rtime   = str(math.ceil(etime - stime))
    html_index.write_footer(etstamp,rtime)
    json_index.finalize(etstamp,rtime)
    nskip   = len([ r.skip()  for r in results if r.skip() == True])
    Log("[Test suite run complete @ %s (wall time = %s)]" % (etstamp,rtime))
    if nskip > 0:
        Log("-- %d files due to skip list." % nskip)
    if not error:
        Log("++ Test suite run finished with NO errors.")
    else:
        nerrors = len([ r.error() for r in results if r.error() == True])
        if nerrors == 1:
            Log("!! Test suite run finished with %d error." % nerrors)
        else:
            Log("!! Test suite run finished with %d errors." % nerrors)
    if opts["cleanup"]:
        cleanup(opts["result_dir"],opts["cleanup_delay"])
    if opts["rsync_post"] is not None:
        rsync_post(opts["result_dir"],opts["rsync_post"])
    if opts['ctest']:
        if not error:
            sys.exit(0)
        else:
            sys.exit(1)
    return pjoin(opts["result_dir"],"results.json")


# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
#  Method: run_visit_test and run_visit_tests
#
#  Programmer: Cyrus Harrison
#  Date:       Tue May 28 13:47:08 PDT 2013
#
#  Note: These are the two functions that comprise the module interface
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------

def run_visit_test(script_file,
             data_dir=None,
             baseline_dir=None,
             output_dir=None,
             visit_bin=None,
             nprocs=None,
             verbose=False):
    tests = [script_file]
    return run_visit_tests(tests,
                           data_dir,
                           baseline_dir,
                           output_dir,
                           visit_bin,
                           nprocs,
                           verbose)

def run_visit_tests(tests,
                    data_dir=None,
                    baseline_dir=None,
                    output_dir=None,
                    visit_bin=None,
                    nprocs=None,
                    verbose=False):
    opts = default_suite_options()
    if not data_dir is None:
        opts["data_dir"]     = data_dir
    if not baseline_dir is None:
        opts["baseline_dir"] = baseline_dir
    if not output_dir is None:
        opts["result_dir"] = output_dir
    # Future: try to find "visit" in path?
    if not visit_bin is None:
        opts["executable"] = visit_bin
    # override other default options
    opts["check_data"]    = False
    opts["cleanup_delay"] = 1
    if verbose:
        opts["verbose"] = True
    if not nprocs is None:
        opts["nprocs"] = nprocs
    else:
        opts["nprocs"] = 1 # default to 1 for now
    opts["test_dir"] = os.path.split(os.path.abspath(__file__))[0]
    print opts["test_dir"]
    res_file  = main(opts,tests)
    return JSONIndex.load_results(res_file,True)


if __name__ == "__main__":
    opts, tests = parse_args()
    main(opts,tests)

