From 1ad164b911dcbaeea30675eb2ffe1443790cfbdb Mon Sep 17 00:00:00 2001 From: alexandreB Date: Fri, 9 Nov 2018 23:08:54 +0000 Subject: [PATCH 1/6] add ActEV Scorer as a submodule --- .gitmodules | 3 ++ diva_evaluation_cli/bin/cli.py | 5 +-- .../commands/actev_execution_validation.py | 34 +++++++++++++++++++ .../actev_execution_validation.py | 11 ++++++ .../execution_validation/ActEV_Scorer | 1 + 5 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 .gitmodules create mode 100644 diva_evaluation_cli/bin/commands/actev_execution_validation.py create mode 100644 diva_evaluation_cli/bin/private_src/entry_points/actev_execution_validation.py create mode 160000 diva_evaluation_cli/bin/private_src/implementation/execution_validation/ActEV_Scorer diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..2dcb90c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "diva_evaluation_cli/bin/private_src/implementation/execution_validation/ActEV_Scorer"] + path = diva_evaluation_cli/bin/private_src/implementation/execution_validation/ActEV_Scorer + url = https://github.com/usnistgov/ActEV_Scorer.git diff --git a/diva_evaluation_cli/bin/cli.py b/diva_evaluation_cli/bin/cli.py index 803e11a..f6752d7 100644 --- a/diva_evaluation_cli/bin/cli.py +++ b/diva_evaluation_cli/bin/cli.py @@ -34,13 +34,14 @@ from diva_evaluation_cli.bin.commands.actev_experiment_cleanup import ActevExper from diva_evaluation_cli.bin.commands.actev_merge_chunks import ActevMergeChunks from diva_evaluation_cli.bin.commands.actev_exec import ActevExec from diva_evaluation_cli.bin.commands.actev_status import ActevStatus - +from diva_evaluation_cli.bin.commands.actev_execution_validation import ActevExecutionValidation private_subcommands = [ ActevGetSystem(), ActevValidateSystem(), ActevExec(), -ActevStatus() +ActevStatus(), +ActevExecutionValidation() ] public_subcommands = [ diff --git a/diva_evaluation_cli/bin/commands/actev_execution_validation.py b/diva_evaluation_cli/bin/commands/actev_execution_validation.py new file mode 100644 index 0000000..ae1cb88 --- /dev/null +++ b/diva_evaluation_cli/bin/commands/actev_execution_validation.py @@ -0,0 +1,34 @@ +""" +USAGE + +ActEV execution-validation +Description +----------- +Test the execution of the system on each validation data set provided in container_output directory. +Compare the newly generated to the expected output and the reference. + + +Warning: this file should not be modified: see src/entry_points to add your source code. +""" +import logging + +from diva_evaluation_cli.bin.commands.actev_command import ActevCommand +from diva_evaluation_cli.bin.private_src.entry_points.actev_execution_validation import entry_point + + +class ActevExecutionValidation(ActevCommand): + + def __init__(self): + super(ActevExecutionValidation, self).__init__('execution-validation', entry_point) + + def cli_parser(self, arg_parser): + """ Configure the description and the arguments (positional and optional) to parse. + + @param arg_parser: python arg parser to describe how to parse the command + """ + arg_parser.description = "Test the execution of the system on each validation data set provided" + required_named = arg_parser.add_argument_group('required named arguments') + + required_named.add_argument("-o", "--output", help="path to experiment result json file", required=True) + + arg_parser.set_defaults(func=ActevExecutionValidation.command, object=self) diff --git a/diva_evaluation_cli/bin/private_src/entry_points/actev_execution_validation.py b/diva_evaluation_cli/bin/private_src/entry_points/actev_execution_validation.py new file mode 100644 index 0000000..5242502 --- /dev/null +++ b/diva_evaluation_cli/bin/private_src/entry_points/actev_execution_validation.py @@ -0,0 +1,11 @@ +""" +ENTRY POINT + +This file should not be modified. +""" +import os + +def before_entry_point(output): + """ Private entry points. + """ + pass diff --git a/diva_evaluation_cli/bin/private_src/implementation/execution_validation/ActEV_Scorer b/diva_evaluation_cli/bin/private_src/implementation/execution_validation/ActEV_Scorer new file mode 160000 index 0000000..6fc44aa --- /dev/null +++ b/diva_evaluation_cli/bin/private_src/implementation/execution_validation/ActEV_Scorer @@ -0,0 +1 @@ +Subproject commit 6fc44aa6ea9513edc5278fe27ee4bf9bbcbfa931 -- GitLab From c3759aac428b39c689f06c2439d731557349f900 Mon Sep 17 00:00:00 2001 From: alexandreB Date: Thu, 15 Nov 2018 21:56:38 +0000 Subject: [PATCH 2/6] add actev execution-validation --- .gitignore | 6 +++- .../commands/actev_execution_validation.py | 13 ++++++++- .../actev_execution_validation.py | 23 +++++++++++++-- .../execution_validation/install.sh | 13 +++++++++ .../execution_validation/requirements.txt | 4 +++ .../execution_validation/score.sh | 28 +++++++++++++++++++ 6 files changed, 83 insertions(+), 4 deletions(-) create mode 100755 diva_evaluation_cli/bin/private_src/implementation/execution_validation/install.sh create mode 100644 diva_evaluation_cli/bin/private_src/implementation/execution_validation/requirements.txt create mode 100755 diva_evaluation_cli/bin/private_src/implementation/execution_validation/score.sh diff --git a/.gitignore b/.gitignore index d5140b7..7b9923a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,11 @@ -# Basic gitignore config +## Basic gitignore config *.swp *.pyc *.egg-info +## CLI gitignore config # Commands history diva_evaluation_cli/bin/private_src/implementation/status/command_history.json + +# Virtual environments +python_env/ diff --git a/diva_evaluation_cli/bin/commands/actev_execution_validation.py b/diva_evaluation_cli/bin/commands/actev_execution_validation.py index ae1cb88..fdbfd6d 100644 --- a/diva_evaluation_cli/bin/commands/actev_execution_validation.py +++ b/diva_evaluation_cli/bin/commands/actev_execution_validation.py @@ -7,6 +7,13 @@ Description Test the execution of the system on each validation data set provided in container_output directory. Compare the newly generated to the expected output and the reference. +Args +---- +output of o: path to experiment output json file +reference or r: path to reference json file +file-index or f: path to file index json file for test set +activity-index or a: path to activity index json file for test set +result or -R: path to result of the ActEV scorer Warning: this file should not be modified: see src/entry_points to add your source code. """ @@ -29,6 +36,10 @@ class ActevExecutionValidation(ActevCommand): arg_parser.description = "Test the execution of the system on each validation data set provided" required_named = arg_parser.add_argument_group('required named arguments') - required_named.add_argument("-o", "--output", help="path to experiment result json file", required=True) + required_named.add_argument("-o", "--output", help="path to experiment output json file", required=True) + required_named.add_argument("-r", "--reference", help="path to reference json file", required=True) + required_named.add_argument("-a", "--activity-index", help="path to activity index json file", required=True) + required_named.add_argument("-f", "--file-index", help="path to file index json file", required=True) + required_named.add_argument("-R", "--result", help="path to result of the ActEV scorer", required=True) arg_parser.set_defaults(func=ActevExecutionValidation.command, object=self) diff --git a/diva_evaluation_cli/bin/private_src/entry_points/actev_execution_validation.py b/diva_evaluation_cli/bin/private_src/entry_points/actev_execution_validation.py index 5242502..b41c1dc 100644 --- a/diva_evaluation_cli/bin/private_src/entry_points/actev_execution_validation.py +++ b/diva_evaluation_cli/bin/private_src/entry_points/actev_execution_validation.py @@ -5,7 +5,26 @@ This file should not be modified. """ import os -def before_entry_point(output): +def entry_point(output, reference, activity_index, file_index, result): """ Private entry points. """ - pass + # go into the right directory to execute the script + path = os.path.dirname(__file__) + execution_validation_dir = os.path.join(path, '../implementation/execution_validation') + + installation_script = os.path.join(execution_validation_dir, 'install.sh') + script = os.path.join(execution_validation_dir, 'score.sh') + script += " " + output + \ + " " + reference + \ + " " + activity_index + \ + " " + file_index + \ + " " + result + + # execute the script + # status is the exit status code returned by the program + status = os.system('cd ' + execution_validation_dir + \ + ';. ' + installation_script + \ + ';' + script) + if status != 0: + raise Exception("Error occured in install_and_score.sh") + diff --git a/diva_evaluation_cli/bin/private_src/implementation/execution_validation/install.sh b/diva_evaluation_cli/bin/private_src/implementation/execution_validation/install.sh new file mode 100755 index 0000000..1ad5829 --- /dev/null +++ b/diva_evaluation_cli/bin/private_src/implementation/execution_validation/install.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +env_dir="python_env" +path=`pwd` +path_to_env_dir="$path/$env_dir" + +if [ -d $path_to_env_dir ];then + . ./$env_dir/bin/activate +else + virtualenv -p /usr/bin/python2 $env_dir + . ./$env_dir/bin/activate + python -m pip --no-cache-dir install -r requirements.txt +fi diff --git a/diva_evaluation_cli/bin/private_src/implementation/execution_validation/requirements.txt b/diva_evaluation_cli/bin/private_src/implementation/execution_validation/requirements.txt new file mode 100644 index 0000000..b67fea2 --- /dev/null +++ b/diva_evaluation_cli/bin/private_src/implementation/execution_validation/requirements.txt @@ -0,0 +1,4 @@ +munkres==1.0.12 +scipy==1.0.0 +matplotlib==2.0.2 +jsonschema==2.5.1 diff --git a/diva_evaluation_cli/bin/private_src/implementation/execution_validation/score.sh b/diva_evaluation_cli/bin/private_src/implementation/execution_validation/score.sh new file mode 100755 index 0000000..8cbe3d1 --- /dev/null +++ b/diva_evaluation_cli/bin/private_src/implementation/execution_validation/score.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +cd "$(dirname "$0")" + +# Get the ActEV scorer submodule +git submodule update --init --recursive +if [ $? -ne 0 ];then + exit 1 +fi + +# Configure the scorer with the right arguments +output=$1 +reference=$2 +activity=$3 +file=$4 +result=$5 + +# Execute ActEV Scorer +cd ActEV_Scorer + +python2 ActEV_Scorer.py \ + ActEV18_AD \ + -s $output \ + -r $reference \ + -a $activity \ + -f $file \ + -o $result \ + -v -- GitLab From 30b93916b85cb9f5b7e29af19ac1603b59a1e9f6 Mon Sep 17 00:00:00 2001 From: alexandreB Date: Fri, 16 Nov 2018 14:46:29 +0000 Subject: [PATCH 3/6] change name of the actev execution-validation command --- diva_evaluation_cli/bin/cli.py | 4 ++-- ...ution_validation.py => actev_validate_execution.py} | 10 +++++----- ...ution_validation.py => actev_validate_execution.py} | 4 ++-- .../implementation/validate_execution/ActEV_Scorer | 1 + .../install.sh | 0 .../requirements.txt | 0 .../score.sh | 0 7 files changed, 10 insertions(+), 9 deletions(-) rename diva_evaluation_cli/bin/commands/{actev_execution_validation.py => actev_validate_execution.py} (82%) rename diva_evaluation_cli/bin/private_src/entry_points/{actev_execution_validation.py => actev_validate_execution.py} (90%) create mode 160000 diva_evaluation_cli/bin/private_src/implementation/validate_execution/ActEV_Scorer rename diva_evaluation_cli/bin/private_src/implementation/{execution_validation => validate_execution}/install.sh (100%) rename diva_evaluation_cli/bin/private_src/implementation/{execution_validation => validate_execution}/requirements.txt (100%) rename diva_evaluation_cli/bin/private_src/implementation/{execution_validation => validate_execution}/score.sh (100%) diff --git a/diva_evaluation_cli/bin/cli.py b/diva_evaluation_cli/bin/cli.py index f6752d7..d0e91e8 100644 --- a/diva_evaluation_cli/bin/cli.py +++ b/diva_evaluation_cli/bin/cli.py @@ -34,14 +34,14 @@ from diva_evaluation_cli.bin.commands.actev_experiment_cleanup import ActevExper from diva_evaluation_cli.bin.commands.actev_merge_chunks import ActevMergeChunks from diva_evaluation_cli.bin.commands.actev_exec import ActevExec from diva_evaluation_cli.bin.commands.actev_status import ActevStatus -from diva_evaluation_cli.bin.commands.actev_execution_validation import ActevExecutionValidation +from diva_evaluation_cli.bin.commands.actev_validate_execution import ActevValidateExecution private_subcommands = [ ActevGetSystem(), ActevValidateSystem(), ActevExec(), ActevStatus(), -ActevExecutionValidation() +ActevValidateExecution() ] public_subcommands = [ diff --git a/diva_evaluation_cli/bin/commands/actev_execution_validation.py b/diva_evaluation_cli/bin/commands/actev_validate_execution.py similarity index 82% rename from diva_evaluation_cli/bin/commands/actev_execution_validation.py rename to diva_evaluation_cli/bin/commands/actev_validate_execution.py index fdbfd6d..b559437 100644 --- a/diva_evaluation_cli/bin/commands/actev_execution_validation.py +++ b/diva_evaluation_cli/bin/commands/actev_validate_execution.py @@ -1,7 +1,7 @@ """ USAGE -ActEV execution-validation +ActEV validation-execution Description ----------- Test the execution of the system on each validation data set provided in container_output directory. @@ -20,13 +20,13 @@ Warning: this file should not be modified: see src/entry_points to add your sour import logging from diva_evaluation_cli.bin.commands.actev_command import ActevCommand -from diva_evaluation_cli.bin.private_src.entry_points.actev_execution_validation import entry_point +from diva_evaluation_cli.bin.private_src.entry_points.actev_validate_execution import entry_point -class ActevExecutionValidation(ActevCommand): +class ActevValidateExecution(ActevCommand): def __init__(self): - super(ActevExecutionValidation, self).__init__('execution-validation', entry_point) + super(ActevValidateExecution, self).__init__('validate-execution', entry_point) def cli_parser(self, arg_parser): """ Configure the description and the arguments (positional and optional) to parse. @@ -42,4 +42,4 @@ class ActevExecutionValidation(ActevCommand): required_named.add_argument("-f", "--file-index", help="path to file index json file", required=True) required_named.add_argument("-R", "--result", help="path to result of the ActEV scorer", required=True) - arg_parser.set_defaults(func=ActevExecutionValidation.command, object=self) + arg_parser.set_defaults(func=ActevValidateExecution.command, object=self) diff --git a/diva_evaluation_cli/bin/private_src/entry_points/actev_execution_validation.py b/diva_evaluation_cli/bin/private_src/entry_points/actev_validate_execution.py similarity index 90% rename from diva_evaluation_cli/bin/private_src/entry_points/actev_execution_validation.py rename to diva_evaluation_cli/bin/private_src/entry_points/actev_validate_execution.py index b41c1dc..f2f5ff2 100644 --- a/diva_evaluation_cli/bin/private_src/entry_points/actev_execution_validation.py +++ b/diva_evaluation_cli/bin/private_src/entry_points/actev_validate_execution.py @@ -10,7 +10,7 @@ def entry_point(output, reference, activity_index, file_index, result): """ # go into the right directory to execute the script path = os.path.dirname(__file__) - execution_validation_dir = os.path.join(path, '../implementation/execution_validation') + execution_validation_dir = os.path.join(path, '../implementation/validate_execution') installation_script = os.path.join(execution_validation_dir, 'install.sh') script = os.path.join(execution_validation_dir, 'score.sh') @@ -26,5 +26,5 @@ def entry_point(output, reference, activity_index, file_index, result): ';. ' + installation_script + \ ';' + script) if status != 0: - raise Exception("Error occured in install_and_score.sh") + raise Exception("Error occured in install.sh or score.sh") diff --git a/diva_evaluation_cli/bin/private_src/implementation/validate_execution/ActEV_Scorer b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/ActEV_Scorer new file mode 160000 index 0000000..6fc44aa --- /dev/null +++ b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/ActEV_Scorer @@ -0,0 +1 @@ +Subproject commit 6fc44aa6ea9513edc5278fe27ee4bf9bbcbfa931 diff --git a/diva_evaluation_cli/bin/private_src/implementation/execution_validation/install.sh b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/install.sh similarity index 100% rename from diva_evaluation_cli/bin/private_src/implementation/execution_validation/install.sh rename to diva_evaluation_cli/bin/private_src/implementation/validate_execution/install.sh diff --git a/diva_evaluation_cli/bin/private_src/implementation/execution_validation/requirements.txt b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/requirements.txt similarity index 100% rename from diva_evaluation_cli/bin/private_src/implementation/execution_validation/requirements.txt rename to diva_evaluation_cli/bin/private_src/implementation/validate_execution/requirements.txt diff --git a/diva_evaluation_cli/bin/private_src/implementation/execution_validation/score.sh b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/score.sh similarity index 100% rename from diva_evaluation_cli/bin/private_src/implementation/execution_validation/score.sh rename to diva_evaluation_cli/bin/private_src/implementation/validate_execution/score.sh -- GitLab From 532378b0634016d63de8e67e39e8ede93d558021 Mon Sep 17 00:00:00 2001 From: alexandreB Date: Fri, 16 Nov 2018 14:58:13 +0000 Subject: [PATCH 4/6] rename chunks.json into chunk.json to be consistent with the plan --- .../validate_system/expected_container_output.txt | 2 +- .../cli_validation_mini/{chunks.json => chunk.json} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename diva_evaluation_cli/container_output/cli_validation_mini/{chunks.json => chunk.json} (100%) diff --git a/diva_evaluation_cli/bin/private_src/implementation/validate_system/expected_container_output.txt b/diva_evaluation_cli/bin/private_src/implementation/validate_system/expected_container_output.txt index 219eec1..d7ba3c5 100644 --- a/diva_evaluation_cli/bin/private_src/implementation/validate_system/expected_container_output.txt +++ b/diva_evaluation_cli/bin/private_src/implementation/validate_system/expected_container_output.txt @@ -1,4 +1,4 @@ file.json activity.json output.json -chunks.json +chunk.json diff --git a/diva_evaluation_cli/container_output/cli_validation_mini/chunks.json b/diva_evaluation_cli/container_output/cli_validation_mini/chunk.json similarity index 100% rename from diva_evaluation_cli/container_output/cli_validation_mini/chunks.json rename to diva_evaluation_cli/container_output/cli_validation_mini/chunk.json -- GitLab From ee8b0463809ead391c71ba3a58d6deb60c998b58 Mon Sep 17 00:00:00 2001 From: alexandreB Date: Fri, 16 Nov 2018 15:23:09 +0000 Subject: [PATCH 5/6] ActEV scorer submodule --- .gitmodules | 4 ++-- .../implementation/execution_validation/ActEV_Scorer | 1 - .../private_src/implementation/validate_execution/.gitmodules | 3 +++ 3 files changed, 5 insertions(+), 3 deletions(-) delete mode 160000 diva_evaluation_cli/bin/private_src/implementation/execution_validation/ActEV_Scorer create mode 100644 diva_evaluation_cli/bin/private_src/implementation/validate_execution/.gitmodules diff --git a/.gitmodules b/.gitmodules index 2dcb90c..53bc710 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "diva_evaluation_cli/bin/private_src/implementation/execution_validation/ActEV_Scorer"] - path = diva_evaluation_cli/bin/private_src/implementation/execution_validation/ActEV_Scorer +[submodule "diva_evaluation_cli/bin/private_src/implementation/validate_execution/ActEV_Scorer"] + path = diva_evaluation_cli/bin/private_src/implementation/validate_execution/ActEV_Scorer url = https://github.com/usnistgov/ActEV_Scorer.git diff --git a/diva_evaluation_cli/bin/private_src/implementation/execution_validation/ActEV_Scorer b/diva_evaluation_cli/bin/private_src/implementation/execution_validation/ActEV_Scorer deleted file mode 160000 index 6fc44aa..0000000 --- a/diva_evaluation_cli/bin/private_src/implementation/execution_validation/ActEV_Scorer +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6fc44aa6ea9513edc5278fe27ee4bf9bbcbfa931 diff --git a/diva_evaluation_cli/bin/private_src/implementation/validate_execution/.gitmodules b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/.gitmodules new file mode 100644 index 0000000..190ef66 --- /dev/null +++ b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ActEV_Scorer"] + path = ActEV_Scorer + url = https://github.com/usnistgov/ActEV_Scorer.git -- GitLab From a21136bcf744960bffd9d5d4d0c7c54ded8c0787 Mon Sep 17 00:00:00 2001 From: alexandreB Date: Fri, 16 Nov 2018 15:26:24 +0000 Subject: [PATCH 6/6] delete useless file --- .../private_src/implementation/validate_execution/.gitmodules | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 diva_evaluation_cli/bin/private_src/implementation/validate_execution/.gitmodules diff --git a/diva_evaluation_cli/bin/private_src/implementation/validate_execution/.gitmodules b/diva_evaluation_cli/bin/private_src/implementation/validate_execution/.gitmodules deleted file mode 100644 index 190ef66..0000000 --- a/diva_evaluation_cli/bin/private_src/implementation/validate_execution/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "ActEV_Scorer"] - path = ActEV_Scorer - url = https://github.com/usnistgov/ActEV_Scorer.git -- GitLab