diff --git a/.gitignore b/.gitignore index d5140b7ce16f9e8bb59017cbfcb8dd59240d40a6..7b9923ae0ccc272b44b2199f0a5cc2a5a66b5928 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,11 @@ -# Basic gitignore config +## Basic gitignore config *.swp *.pyc *.egg-info +## CLI gitignore config # Commands history diva_evaluation_cli/bin/private_src/implementation/status/command_history.json + +# Virtual environments +python_env/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000000000000000000000000000000000..53bc7106eb55f76160f5f8e668abfede52c3cc3b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "diva_evaluation_cli/bin/private_src/implementation/validate_execution/ActEV_Scorer"] + path = diva_evaluation_cli/bin/private_src/implementation/validate_execution/ActEV_Scorer + url = https://github.com/usnistgov/ActEV_Scorer.git diff --git a/diva_evaluation_cli/bin/cli.py b/diva_evaluation_cli/bin/cli.py index 803e11af5b35cbda6bb500d56a67beb35334d316..d0e91e8fc968807d62dc47b098e2821b665f8d85 100644 --- a/diva_evaluation_cli/bin/cli.py +++ b/diva_evaluation_cli/bin/cli.py @@ -34,13 +34,14 @@ from diva_evaluation_cli.bin.commands.actev_experiment_cleanup import ActevExper from diva_evaluation_cli.bin.commands.actev_merge_chunks import ActevMergeChunks from diva_evaluation_cli.bin.commands.actev_exec import ActevExec from diva_evaluation_cli.bin.commands.actev_status import ActevStatus - +from diva_evaluation_cli.bin.commands.actev_validate_execution import ActevValidateExecution private_subcommands = [ ActevGetSystem(), ActevValidateSystem(), ActevExec(), -ActevStatus() +ActevStatus(), +ActevValidateExecution() ] public_subcommands = [ diff --git a/diva_evaluation_cli/bin/commands/actev_validate_execution.py b/diva_evaluation_cli/bin/commands/actev_validate_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..b5594375b15da857ceb04a8dbe281bfa82006fec --- /dev/null +++ b/diva_evaluation_cli/bin/commands/actev_validate_execution.py @@ -0,0 +1,45 @@ +""" +USAGE + +ActEV validation-execution +Description +----------- +Test the execution of the system on each validation data set provided in container_output directory. +Compare the newly generated to the expected output and the reference. + +Args +---- +output of o: path to experiment output json file +reference or r: path to reference json file +file-index or f: path to file index json file for test set +activity-index or a: path to activity index json file for test set +result or -R: path to result of the ActEV scorer + +Warning: this file should not be modified: see src/entry_points to add your source code. +""" +import logging + +from diva_evaluation_cli.bin.commands.actev_command import ActevCommand +from diva_evaluation_cli.bin.private_src.entry_points.actev_validate_execution import entry_point + + +class ActevValidateExecution(ActevCommand): + + def __init__(self): + super(ActevValidateExecution, self).__init__('validate-execution', entry_point) + + def cli_parser(self, arg_parser): + """ Configure the description and the arguments (positional and optional) to parse. + + @param arg_parser: python arg parser to describe how to parse the command + """ + arg_parser.description = "Test the execution of the system on each validation data set provided" + required_named = arg_parser.add_argument_group('required named arguments') + + required_named.add_argument("-o", "--output", help="path to experiment output json file", required=True) + required_named.add_argument("-r", "--reference", help="path to reference json file", required=True) + required_named.add_argument("-a", "--activity-index", help="path to activity index json file", required=True) + required_named.add_argument("-f", "--file-index", help="path to file index json file", required=True) + required_named.add_argument("-R", "--result", help="path to result of the ActEV scorer", required=True) + + arg_parser.set_defaults(func=ActevValidateExecution.command, object=self) diff --git a/diva_evaluation_cli/bin/private_src/entry_points/actev_validate_execution.py b/diva_evaluation_cli/bin/private_src/entry_points/actev_validate_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..f2f5ff25cec1c606975fb2b125d41d12d8257b61 --- /dev/null +++ b/diva_evaluation_cli/bin/private_src/entry_points/actev_validate_execution.py @@ -0,0 +1,30 @@ +""" +ENTRY POINT + +This file should not be modified. +""" +import os + +def entry_point(output, reference, activity_index, file_index, result): + """ Private entry points. + """ + # go into the right directory to execute the script + path = os.path.dirname(__file__) + execution_validation_dir = os.path.join(path, '../implementation/validate_execution') + + installation_script = os.path.join(execution_validation_dir, 'install.sh') + script = os.path.join(execution_validation_dir, 'score.sh') + script += " " + output + \ + " " + reference + \ + " " + activity_index + \ + " " + file_index + \ + " " + result + + # execute the script + # status is the exit status code returned by the program + status = os.system('cd ' + execution_validation_dir + \ + ';. ' + installation_script + \ + ';' + script) + if status != 0: + raise Exception("Error occured in install.sh or score.sh") + diff --git a/diva_evaluation_cli/bin/private_src/implementation/validate_execution/ActEV_Scorer b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/ActEV_Scorer new file mode 160000 index 0000000000000000000000000000000000000000..6fc44aa6ea9513edc5278fe27ee4bf9bbcbfa931 --- /dev/null +++ b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/ActEV_Scorer @@ -0,0 +1 @@ +Subproject commit 6fc44aa6ea9513edc5278fe27ee4bf9bbcbfa931 diff --git a/diva_evaluation_cli/bin/private_src/implementation/validate_execution/install.sh b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/install.sh new file mode 100755 index 0000000000000000000000000000000000000000..1ad58298e62e3c5f1f50992b9c40eed700bd0aee --- /dev/null +++ b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/install.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +env_dir="python_env" +path=`pwd` +path_to_env_dir="$path/$env_dir" + +if [ -d $path_to_env_dir ];then + . ./$env_dir/bin/activate +else + virtualenv -p /usr/bin/python2 $env_dir + . ./$env_dir/bin/activate + python -m pip --no-cache-dir install -r requirements.txt +fi diff --git a/diva_evaluation_cli/bin/private_src/implementation/validate_execution/requirements.txt b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b67fea25d957069b09e420399c1c80fac90199e2 --- /dev/null +++ b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/requirements.txt @@ -0,0 +1,4 @@ +munkres==1.0.12 +scipy==1.0.0 +matplotlib==2.0.2 +jsonschema==2.5.1 diff --git a/diva_evaluation_cli/bin/private_src/implementation/validate_execution/score.sh b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/score.sh new file mode 100755 index 0000000000000000000000000000000000000000..8cbe3d13d029a92a6aebd8dd185d19970094eb41 --- /dev/null +++ b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/score.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +cd "$(dirname "$0")" + +# Get the ActEV scorer submodule +git submodule update --init --recursive +if [ $? -ne 0 ];then + exit 1 +fi + +# Configure the scorer with the right arguments +output=$1 +reference=$2 +activity=$3 +file=$4 +result=$5 + +# Execute ActEV Scorer +cd ActEV_Scorer + +python2 ActEV_Scorer.py \ + ActEV18_AD \ + -s $output \ + -r $reference \ + -a $activity \ + -f $file \ + -o $result \ + -v diff --git a/diva_evaluation_cli/bin/private_src/implementation/validate_system/expected_container_output.txt b/diva_evaluation_cli/bin/private_src/implementation/validate_system/expected_container_output.txt index 219eec13d1eb1c857385e04bbaa0573feeeff915..d7ba3c5823ba536b2e1a432d4784aece237eb041 100644 --- a/diva_evaluation_cli/bin/private_src/implementation/validate_system/expected_container_output.txt +++ b/diva_evaluation_cli/bin/private_src/implementation/validate_system/expected_container_output.txt @@ -1,4 +1,4 @@ file.json activity.json output.json -chunks.json +chunk.json diff --git a/diva_evaluation_cli/container_output/cli_validation_mini/chunks.json b/diva_evaluation_cli/container_output/cli_validation_mini/chunk.json similarity index 100% rename from diva_evaluation_cli/container_output/cli_validation_mini/chunks.json rename to diva_evaluation_cli/container_output/cli_validation_mini/chunk.json