Updated to use score flag

diva_evaluation_cli/bin/commands/actev_validate_execution.py

@@ -23,6 +23,7 @@ class ActevValidateExecution(ActevCommand):
         * file-index or f:     path to file index json file for test set
         * activity-index or a: path to activity index json file for test set
         * result or -R:        path to result of the ActEV scorer
+        * score or -s:         sets flag to score system
 
     """
     def __init__(self):
@@ -39,9 +40,10 @@ class ActevValidateExecution(ActevCommand):
         required_named = arg_parser.add_argument_group('required named arguments')
 
         required_named.add_argument("-o", "--output", help="path to experiment output json file", required=True)
-        required_named.add_argument("-r", "--reference", help="path to reference json file", required=True)
+        required_named.add_argument("-r", "--reference", help="path to reference json file", required=False)
         required_named.add_argument("-a", "--activity-index", help="path to activity index json file", required=True)
         required_named.add_argument("-f", "--file-index", help="path to file index json file", required=True)
-        required_named.add_argument("-R", "--result", help="path to result of the ActEV scorer", required=True)
+        required_named.add_argument("-R", "--result", help="path to result of the ActEV scorer", required=False)
+        required_named.add_argument("-s", "--score", help="sets flag to score system", required=False, action='store_true')
 
         arg_parser.set_defaults(func=ActevValidateExecution.command, object=self)

diva_evaluation_cli/bin/private_src/entry_points/actev_validate_execution.py

@@ -4,7 +4,7 @@ This file should not be modified.
 """
 import os
 
-def entry_point(output, reference, activity_index, file_index, result):
+def entry_point(output, reference, activity_index, file_index, result, score):
     """Private entry point.
 
     Test the execution of the system on each validation data set provided in container_output directory
@@ -20,14 +20,18 @@ def entry_point(output, reference, activity_index, file_index, result):
     # go into the right directory to execute the script
     path = os.path.dirname(__file__)
     execution_validation_dir = os.path.join(path, '../implementation/validate_execution')
-
+    if score:
+        s = "true"
+    else:
+        s = "false"
     installation_script = os.path.join(execution_validation_dir, 'install.sh')
     script = os.path.join(execution_validation_dir, 'score.sh')
     script += " " + output + \
               " " + reference + \
               " " + activity_index + \
               " " + file_index + \
-              " " + result
+              " " + result + \
+              " " + s
 
     # execute the script
     # status is the exit status code returned by the program

diva_evaluation_cli/bin/private_src/implementation/validate_execution/score.sh

@@ -14,15 +14,24 @@ reference=$2
 activity=$3
 file=$4
 result=$5
-
+score=$6
 # Execute ActEV Scorer
 cd ActEV_Scorer
-
-python2 ActEV_Scorer.py \
-	ActEV18_AD \
-	-s $output \
-	-r $reference \
-	-a $activity \
-	-f $file \
-	-o $result \
-	-v 
+if [ $score == "true" ] ; then
+    python2 ActEV_Scorer.py \
+	    ActEV18_AD \
+	    -s $output \
+	    -r $reference \
+	    -a $activity \
+	    -f $file \
+	    -o $result \
+	    -v
+else
+    python2 ActEV_Scorer.py \
+	    ActEV18_AD \
+	    -s $output \
+	    -a $activity \
+	    -f $file \
+	    -v \
+	    -V
+fi

diva_evaluation_cli/bin/private_src/implementation/validate_system/expected_validation_result.txt

 actev_system_setup - ()
+actev_validate_execution - (output, reference, activity_index, file_index, result, score)
 actev_design_chunks - (file_index, activity_index, output, nb_videos_per_chunk)
 actev_experiment_init - (file_index, activity_index, chunks, video_location, system_cache_dir, config_file=None)
 actev_pre_process_chunk - (chunk_id, system_cache_dir)

doc/cli_commands/actev_validate_execution.md

@@ -12,11 +12,11 @@ This command requires the following parameters:
 | Name      | Id         | Required | Definition                 |
 |-----------|------------|---------|----------------------------|
 | output         | o | True    | path to experiment output json file    |
-| reference      | r | True    | path to reference json file            |
+| reference      | r | False   | path to reference json file            |
 | file-index     | f | True    | path to file index json file           |
 | activity-index | a | True    | path to activity index json file       |
-| result         | R | True    | path to result of the ActEV scorer     |   
-
+| result         | R | False   | path to result of the ActEV scorer     |   
+| score          | s | False   | sets flag to score system              |
 
 ## Usage
 
@@ -29,7 +29,8 @@ actev validate-execution -o <path to output result> -r <path to reference file>
 Example:
 
 ```
-actev validate-execution ~/output.json -r diva_evaluation_cli/container_output/dataset/output.json -a ~/activity.json -f ~/file.json -R ~/result.json
+actev validate-execution ~/output.json -r diva_evaluation_cli/container_output/dataset/output.json -a ~/activity.json -f ~/file.json -R ~/result.json --score
+
 ```