Commit 6c4188ca authored by Maxime Hubert's avatar Maxime Hubert
Browse files

Add a similarity output log to the validate-execution command using the --score mode

parent b00bf90e
Pipeline #143845 passed with stage
in 1 minute and 11 seconds
...@@ -2,7 +2,11 @@ ...@@ -2,7 +2,11 @@
This file should not be modified. This file should not be modified.
""" """
import os import csv, logging, os
from collections import Counter
logger = logging.getLogger(__name__)
def entry_point(output, reference, activity_index, file_index, result, score): def entry_point(output, reference, activity_index, file_index, result, score):
"""Private entry point. """Private entry point.
...@@ -15,6 +19,7 @@ def entry_point(output, reference, activity_index, file_index, result, score): ...@@ -15,6 +19,7 @@ def entry_point(output, reference, activity_index, file_index, result, score):
file_index (str): Path to file index json file for test set file_index (str): Path to file index json file for test set
activity_index (str): Path to activity index json file for test set activity_index (str): Path to activity index json file for test set
result (str): Path to result of the ActEV scorer result (str): Path to result of the ActEV scorer
score (bool): Whether scoring the system output against a reference
""" """
# go into the right directory to execute the script # go into the right directory to execute the script
...@@ -40,3 +45,22 @@ def entry_point(output, reference, activity_index, file_index, result, score): ...@@ -40,3 +45,22 @@ def entry_point(output, reference, activity_index, file_index, result, score):
if status != 0: if status != 0:
raise Exception("Error occured in install.sh or score.sh") raise Exception("Error occured in install.sh or score.sh")
# If the system output is scored, make it also check the alignments
# Produced by the scorer. This is a reproducibility check made to compare
# A system output against another.
if score and result:
# Count the CD/FA/MD labels in the second column
# (CorrectDetection/FalseAlarm/MissedDetection)
# And calculate and display a similarity score
alignments_filepath = os.path.join(result, 'alignment.csv')
with open(alignments_filepath) as alignments_file:
alignments_cursor = csv.reader(alignments_file, delimiter='|')
classification_results = Counter([ row[1] for row in alignments_cursor ])
cd = classification_results.get('CD', 0)
fa = classification_results.get('FA', 0)
md = classification_results.get('MD', 0)
similarity_score = (fa + md) / (fa + md + cd)
logger.info('''Similarity score:
(FP + FN) / (FP + FN + TP): ({} + {}) / ({} + {} + {}) = {}
'''.format(fa, md, fa, md, cd, similarity_score)
)
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment