Add a similarity output log to the validate-execution command using the --score mode

diva_evaluation_cli/bin/private_src/entry_points/actev_validate_execution.py

@@ -2,7 +2,11 @@
 
 This file should not be modified.
 """
-import os
+import csv, logging, os
+
+from collections import Counter
+
+logger = logging.getLogger(__name__)
 
 def entry_point(output, reference, activity_index, file_index, result, score):
     """Private entry point.
@@ -15,6 +19,7 @@ def entry_point(output, reference, activity_index, file_index, result, score):
         file_index (str): Path to file index json file for test set
         activity_index (str): Path to activity index json file for test set
         result (str): Path to result of the ActEV scorer
+        score (bool): Whether scoring the system output against a reference
 
     """
     # go into the right directory to execute the script
@@ -40,3 +45,22 @@ def entry_point(output, reference, activity_index, file_index, result, score):
     if status != 0:
         raise Exception("Error occured in install.sh or score.sh")
 
+    # If the system output is scored, make it also check the alignments
+    # Produced by the scorer. This is a reproducibility check made to compare
+    # A system output against another.
+    if score and result:
+        # Count the CD/FA/MD labels in the second column
+        # (CorrectDetection/FalseAlarm/MissedDetection)
+        # And calculate and display a similarity score
+        alignments_filepath = os.path.join(result, 'alignment.csv')
+        with open(alignments_filepath) as alignments_file:
+            alignments_cursor = csv.reader(alignments_file, delimiter='|')
+            classification_results = Counter([ row[1] for row in alignments_cursor ])
+            cd = classification_results.get('CD', 0)
+            fa = classification_results.get('FA', 0)
+            md = classification_results.get('MD', 0)
+            similarity_score = (fa + md) / (fa + md + cd)
+            logger.info('''Similarity score:
+            (FP + FN) / (FP + FN + TP): ({} + {}) / ({} + {} + {}) = {}
+            '''.format(fa, md, fa, md, cd, similarity_score)
+            )
\ No newline at end of file