Commit 4867b83f authored by Maxime Hubert's avatar Maxime Hubert
Browse files

Enhance validation with two additional checks, move file index and activity...

Enhance validation with two additional checks, move file index and activity index in a partitions directory, move baseline sytem_output to another location
parent 223d4a0c
stages:
- build
- build-test
- test
- release
- documentation
build:
stage: build
build-test:
stage: build-test
image: python:3.5-stretch
script:
- python3 -m pip install -r ./requirements.txt
- python3 -m pip install -e .
- actev -h
- actev validate-system
- python3 -m unittest discover -s test/
......@@ -26,5 +26,8 @@ class ActevValidateSystem(ActevCommand):
arg_parser(:obj:`ArgParser`): Python arg parser to describe how to parse the command
"""
arg_parser.description = "Checks the structure of the directory after ActEV-system-setup is run"
arg_parser.description = """Checks the structure of the directory after ActEV-system-setup is run.
Also, checks for self-reported system outputs."""
arg_parser.add_argument("--strict", help="Exits with an error in case of an invalid system.", action="store_true", default=False)
arg_parser.set_defaults(func=ActevValidateSystem.command, object=self)
......@@ -5,11 +5,16 @@ This file should not be modified.
import os
from diva_evaluation_cli.bin.private_src.implementation.validate_system.validate_system import validate_system
def entry_point():
def entry_point(strict):
"""Private entry point.
Test the execution of the system on each validation data set provided in container_output directory
Args:
strict (bool): Whether to cause a failure in case of an error or not
Checks the structure of the directory after ActEV-system-setup is run. Checks for expected API contents, etc.
"""
validate_system()
validate_system(strict)
......@@ -8,21 +8,39 @@ import logging
import os
import sys
import glob
import filecmp
root_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../../')
def validate_system():
def handle_error(msg, strict):
"""Error handler
msg (str): An error message to display as a warning or as an Exception
strict (bool): Whether to throw an exception with the provided message
"""
if strict:
raise Exception('System validation failed: {}'.format(msg))
logging.warning('{}'.format(msg))
def validate_system(strict):
"""Main validation method
strict (bool): Whether to cause a failure in case of an error or not
"""
validate_structure()
validate_cli()
validate_container_output()
logging.info("System is valid")
validate_structure(strict)
validate_cli(strict)
validate_container_output(strict)
if strict:
logging.info("System is valid")
else:
logging.warning("System validation not performed yet. Fix any WARNING and validate again using the --strict flag")
############################### structure #####################################
def validate_structure():
def validate_structure(strict):
"""Validate structure of the CLI: should contain required directories.
strict (bool): Whether to cause a failure in case of an error or not
"""
directories = os.listdir(root_path)
content = import_expected_result('expected_structure.txt')
......@@ -30,15 +48,18 @@ def validate_structure():
for directory in content:
if not directory in directories:
raise Exception("System validation failed, missing {} directory".format(directory))
handle_error("missing {} directory".format(directory), strict)
if not os.path.isdir(root_path + '/' + directory):
raise Exception("System validation failed, {} not a directory".format(directory))
handle_error("{} not a directory".format(directory), strict)
logging.info(" .. {} is valid".format(directory))
################################## cli ########################################
def validate_cli():
def validate_cli(strict):
""" Import the entry points method and try to compare them with the expected result
strict (bool): Whether to cause a failure in case of an error or not
"""
from diva_evaluation_cli.bin.cli import public_subcommands
......@@ -54,17 +75,18 @@ def validate_cli():
try:
entry_point_module = importlib.import_module('diva_evaluation_cli.src.entry_points.{}'.format(actev_command))
except:
raise Exception("System validation failed, {} entry_point method removed".format(actev_command))
handle_error("{} entry_point method removed".format(actev_command), strict)
entry_point_function = getattr(entry_point_module, 'entry_point')
result = "{} - {}".format(actev_command,inspect.signature(entry_point_function))
# Is entry point contain correct arguments
if content[i] != result:
raise Exception("System validation failed, {} entry_point method changed".format(actev_command))
handle_error(("{} entry_point method changed".format(actev_command)))
i += 1
logging.info(" .. {} is valid".format(actev_command))
def import_expected_result(file_name):
""" Import expected validation result
......@@ -82,21 +104,68 @@ def import_expected_result(file_name):
############################# container output ################################
def validate_container_output():
def dataset_name_from_dirname(dirname, dataset_names):
"""Get the dataset name from the dirname
"""
for dataset_name in dataset_names:
if dirname.startswith(dataset_name):
return dataset_name
return None
def is_valid_container_output_dirname(dirname, dataset_names, strict):
"""Get the dataset name and check its format, and whether it exists
"""
if not dirname:
handle_error('system output format should be in the following format:<dataset-name>*', strict)
if not any([dirname.startswith(dataset_name) for dataset_name in dataset_names]):
return False
return True
def validate_container_output(strict):
""" Check that container output directory is present.
For each datasetID directory, chunks, activity, file and output should be present too.
For each subdirectory, an *output.json file should be present.
strict (bool): Whether to cause a failure in case of an error or not
"""
container_output_dir = os.path.join(root_path, 'container_output')
container_output_dir = os.path.normpath(os.path.join(root_path, 'container_output'))
logging.info("Container output validation")
for dataset_id in os.listdir(container_output_dir):
dataset_id_path = os.path.join(container_output_dir, dataset_id)
if os.path.isdir(dataset_id_path):
content = import_expected_result('expected_container_output.txt')
for file_name in content:
dataset_filename = '*' + file_name
files = os.path.join(dataset_id_path, dataset_filename)
if not glob.glob(files):
raise Exception("System validation failed, {} not present in {}".format(dataset_filename, dataset_id))
logging.info(" .. {} is valid".format(dataset_id))
# Gather all system outputs paths
system_outputs_paths = [ dirname for dirname in os.listdir(container_output_dir) if os.path.isdir(os.path.join(container_output_dir, dirname)) ]
# Check whether the container-outputs dir is empty
if not system_outputs_paths:
handle_error('no system output in {}'.format(container_output_dir), strict)
for system_output_dir_path in system_outputs_paths:
system_output_name = os.path.basename(system_output_dir_path)
logging.info('Checking system output {}'.format(system_output_dir_path))
# 1st check: check if an output.json is present (right now only one file is required)
expected_files = import_expected_result('expected_container_output.txt')
for expected_file in expected_files:
dataset_filename = '*' + expected_file
files = os.path.join(container_output_dir, system_output_name, dataset_filename)
if not glob.glob(files):
handle_error("{} not present in {}".format(files, system_output_name), strict)
# 2nd check: check that the system output name matches one of the partitions
baseline_results_path = os.path.join(os.path.dirname(__file__), 'baseline_outputs')
dataset_names = [ path for path in os.listdir(baseline_results_path) if os.path.isdir(os.path.join(baseline_results_path, path)) and path ]
if is_valid_container_output_dirname(system_output_name, dataset_names, strict):
# 3rd check: check whether the output is identical to the baseline output
dataset_name = dataset_name_from_dirname(system_output_name, dataset_names)
baseline_result_path = os.path.join(baseline_results_path, dataset_name, 'output.json')
system_output_path_glob = os.path.join(container_output_dir, system_output_name, '*output.json')
system_output_path = glob.glob(system_output_path_glob)
if system_output_path:
if filecmp.cmp(baseline_result_path, system_output_path[0]):
handle_error('container output file {} is identical to the baseline output on the same dataset.'.format(system_output_name, dataset_names), strict)
else:
handle_error('dataset name couldnt be extracted from "{}", it has to be one of {}'.format(system_output_name, dataset_names), strict)
{
"Chunk1": {
"activities": [
"Open_Trunk",
"Exiting",
"specialized_talking_phone",
"Entering",
"specialized_texting_phone",
"Closing_Trunk",
"Opening",
"Transport_HeavyCarry",
"vehicle_u_turn",
"Closing",
"Loading",
"vehicle_turning_right",
"activity_carrying",
"vehicle_turning_left",
"Pull",
"Unloading",
"Talking",
"Riding"
],
"files": [
"VIRAT_S_040003_02_000197_000552.mp4",
"VIRAT_S_000206_02_000294_000327.mp4",
"2018-03-14.07-30-04.07-35-04.G336.avi",
"VIRAT_S_040100_03_000496_000559.mp4"
]
}
}
\ No newline at end of file
{
"Chunk1": {
"files": [
"VIRAT_S_000206_02_000294_000327.mp4",
"VIRAT_S_040003_02_000197_000552.mp4",
"2018-03-14.07-30-04.07-35-04.school.G336.avi",
"VIRAT_S_040100_03_000496_000559.mp4"
],
"activities": [
"Pull",
"Exiting",
"Entering",
"Open_Trunk",
"specialized_texting_phone",
"activity_carrying",
"Closing",
"specialized_talking_phone",
"vehicle_u_turn",
"vehicle_turning_left",
"vehicle_turning_right",
"Transport_HeavyCarry",
"Riding",
"Talking",
"Loading",
"Closing_Trunk",
"Opening",
"Unloading"
]
},
"Chunk2": {
"files": [
"2018-03-15.15-55-01.16-00-01.admin.G326.avi"
],
"activities": [
"Pull",
"Exiting",
"Entering",
"Open_Trunk",
"specialized_texting_phone",
"activity_carrying",
"Closing",
"specialized_talking_phone",
"vehicle_u_turn",
"vehicle_turning_left",
"vehicle_turning_right",
"Transport_HeavyCarry",
"Riding",
"Talking",
"Loading",
"Closing_Trunk",
"Opening",
"Unloading"
]
}
}
\ No newline at end of file
## Self-reported system output
Please use this folder to report your personal system outputs.
Create a subdirectory using the following pattern for its name:
```
<dataset-id>*/
```
This subdirectory may then contain your system output using the following structure:
```
<dataset-id>*/
*output.json
```
......@@ -2,9 +2,22 @@
## Description
Checks the structure of the directory after ActEV-system-setup is run. Checks for expected API contents, etc.
Checks the structure of the directory after ActEV-system-setup is run. Checks for global directory structure, expected API contents, and system outputs directory and integrity.
The `--strict` flag causes the command to throw an exception on an error. Otherwise only WARNING logs are displayed.
The command contains the following subcommands:
In order to succed:
* The repository contains the required sub-directories,
* The signatures of the .src.** functions have to remain unchanged,
* The container_outputs directory contains at least one system output,
* Each reported system output is different from the baseline's,
* Names of the directories inside `container_outputs/` start with a dataset name.
## Parameters
| Name | Id | Required | Definition |
|-----------|------------|---------|----------------------------|
| strict | --strict | False | Whether to throw an exception on an error. |
## Usage
......@@ -12,3 +25,8 @@ The command contains the following subcommands:
actev validate-system
```
Example:
```
actev validate-system --strict
```
\ No newline at end of file
To trigger the execution of the unittests:
```
python3 -m unittest discover -s test/
```
# -*- coding: utf-8 -*-
'''
This module implements unit tests of the submission validation.
'''
import os, sys, glob
import unittest
from diva_evaluation_cli.bin.cli import cli_parser
class TestValidateSystem(unittest.TestCase):
"""
Test the actev validate-system CLI call, using valid and invalid directories
"""
def setUp(self):
current_path = os.path.abspath(__file__)
self.test_dir_path = os.path.dirname(current_path)
self.test_examples_path = os.path.join(self.test_dir_path, 'validate_system/validate_container_output')
self.system_outputs_dir = os.path.join(self.test_dir_path, '../diva_evaluation_cli/container_output')
valid_test_examples_dirs = os.listdir(os.path.join(self.test_examples_path, 'valid'))
self.valid_test_examples = [os.path.join(self.test_examples_path, 'valid', os.path.basename(subdir))
for subdir in valid_test_examples_dirs]
invalid_test_examples_dirs = os.listdir(os.path.join(self.test_examples_path, 'invalid'))
self.invalid_test_examples = [os.path.join(self.test_examples_path, 'invalid', os.path.basename(subdir))
for subdir in invalid_test_examples_dirs]
def tearDown(self):
# Remove all symbolic links
test_examples = self.valid_test_examples + self.invalid_test_examples
for test_example_dir in test_examples:
link_path = os.path.join(self.system_outputs_dir, os.path.basename(test_example_dir))
if os.path.islink(link_path):
os.unlink(link_path)
def test_valid_container_outputs(self):
assert len(self.valid_test_examples) > 0
for test_example_dir in self.valid_test_examples:
new_system_output = os.path.join(self.system_outputs_dir, os.path.basename(test_example_dir))
print('Linking {} to {}'.format(test_example_dir, new_system_output))
os.symlink(test_example_dir, new_system_output)
sys.argv[1:] = ['validate-system', '--strict']
try:
cli_parser()
except Exception:
self.fail("Strict validator failed on valid submission {}".format(test_example_dir))
def test_invalid_container_outputs(self):
assert len(self.invalid_test_examples) > 0
for test_example_dir in self.invalid_test_examples:
new_system_output = os.path.join(self.system_outputs_dir, os.path.basename(test_example_dir))
print('Linking {} to {}'.format(test_example_dir, new_system_output))
os.symlink(test_example_dir, new_system_output)
# Test the validator in a strict mode
sys.argv[1:] = ['validate-system', '--strict']
with self.assertRaises(SystemExit):
cli_parser()
self.fail("Validator succedded on invalid system output {}".format(test_example_dir))
# Test the validator in a non strict mode
sys.argv[1:] = ['validate-system']
try:
cli_parser()
except Exception:
self.fail("Non strict validator produced an error on invalid submission {}".format(test_example_dir))
if __name__ == '__main__':
unittest.main()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment