Commit 8ea8a532 authored by Baptiste CHOCOT's avatar Baptiste CHOCOT
Browse files

Update validation pipeline commands

parent c734c0d6
SDL-pipelinecommands-validation.txt
December 18, 2020
April 23, 2021
Overview:
......@@ -30,6 +30,8 @@ Variables:
system_git_checkout -if the system is submitted via a git repo, this is the branch or commit to checkout
system_tar -if the system is submitted via tar, this is the url to download from
evaluation_dataset -sequestered dataset used for evaluation
proposals_output -full path to write the proposals output
localization_output -full path to write the localization output
Steps and Commands:
......@@ -45,8 +47,15 @@ Steps and Commands:
None
STEP: actev nist install
purpose: retrieves the base CLI and installs the required packages
purpose: install conda, then retrieves the base CLI and installs the required packages
commands:
# CONDA-RELATED STUFF
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o conda.sh
bash conda.sh -b -p ~/miniconda && rm conda.sh
eval "$(~/miniconda/bin/conda shell.bash hook)" && conda init && conda update -y conda
echo "$(conda shell.bash hook)" >> ~/.bashrc
cat ~/.bashrc | sed s/return// > tmp && mv tmp ~/.bashrc
# THEN PULLING THE CLI
sed -ie "5 i PATH=$PATH:/home/ubuntu/.local/bin" /home/ubuntu/.bashrc
rm -rf ~/tmpnist;
mkdir -p ~/tmpnist
......@@ -55,6 +64,7 @@ Steps and Commands:
cd diva_evaluation_cli
sudo apt-get update
diva_evaluation_cli/bin/install.sh –all
echo "conda activate ~/.conda_envs/cli.env" >> ~/.bashrc
STEP: actev get system
......@@ -63,7 +73,7 @@ Steps and Commands:
# IF THE SYSTEM IS A TAR FILE
actev get-system archive -u ${system_tar} -l ~ -n diva_evaluation_cli
#IF THE SYSTEM IS A GIT REPO
# IF THE SYSTEM IS A GIT REPO
actev get-system git -u ${system_git_repo} -l ~ -n diva_evaluation_cli -s ${system_git_checkout}
local_diva_src=$(find ~ | grep -v ~/tmpnist | grep -v "__MACOSX" | grep diva_evaluation_cli/src | head -n 1)
......@@ -71,12 +81,17 @@ Steps and Commands:
ln -s $local_diva_src ~/tmpnist/diva_evaluation_cli/diva_evaluation_cli/
rm -rf ~/tmpnist/diva_evaluation_cli/diva_evaluation_cli/container_output
cp -R ${local_diva_src}/../*_output ~/tmpnist/diva_evaluation_cli/diva_evaluation_cli/container_output
# IF THE SYSTEM USES PIP
python3 -m pip install -r ${local_diva_src}/../../requirements.txt --upgrade
# IF THE SYSTEM USES CONDA
${local_diva_src}/../bin/install.sh
sudo apt remove python3-apt --yes;
chmod -R 777 ~/tmpnist $local_diva_src
cp ${local_diva_src}/../../LICENSE.txt ~/tmpnist/diva_evaluation_cli/diva_evaluation_cli/LICENSE.txt
STEP: actev test cmd
purpose: tests the 'actev' command to verify it's installed properly
......@@ -156,6 +171,10 @@ Steps and Commands:
commands:
actev experiment-init -f /datasets/indexes/file-index.json -a /datasets/indexes/activity-index.json -c /outputs/exec_results/${validation_dataset}/${validation_dataset}_designed_chunks.json -v /datasets/video -s /mnt
# IF PREPARE PROPOSALS OUTPUT
command = command + " --prepare-proposal-outputs"
# IF PREPARE LOCALIZATION OUTPUT
command = command + " --prepare-localization-outputs"
STEP: actev process chunks
purpose: Begins processing the videos of the validation dataset. The for loop is run on the main instance, while the "actev" commands are run on the node.
......@@ -172,12 +191,18 @@ Steps and Commands:
commands:
actev merge-chunks -c /outputs/exec_results/${validation_dataset}/${validation_dataset}_chunk_result_summary.json -r /mnt -o /outputs/exec_results/${validation_dataset}/${validation_dataset}_output.json
# IF PREPARE PROPOSALS OUTPUT
command = command + " -p ${proposals_output}"
# IF PREPARE LOCALIZATION OUTPUT
command = command + " -l ${localization_outputs}"
STEP: actev validate execution:
purpose: validates the merged output file
commands:
actev validate-execution -f /datasets/indexes/file-index.json -a /datasets/indexes/activity-index.json -o /outputs/exec_results/${validation_dataset}/${validation_dataset}_output.json
# IF LOCALIZATION OUTPUT
command = command + " -l ${localization_outputs}"
STEP: actev collect results
purpose: retrieves the system output and resource monitoring report
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment