Development

.gitignore

 *.swp
 *.pyc
+*.egg-info

README.md

@@ -9,8 +9,19 @@ Clone the repository:
 git clone https://gitlab.kitware.com/alexandreB/diva_evaluation_cli.git
 ```
 
-Create an editable package
---------------------------
+Automatic installation
+----------------------
+
+Go into the project directory and execute `install.sh`:
+
+```
+bin/install.sh
+```
+
+Manual installation
+-------------------
+
+### Create an editable package
 
 In order to test your code with the CLI, you need to create a python package of the project.
 An editable package is the best option to avoid to package the project at each modification of the code.
@@ -20,6 +31,8 @@ Go into the project directory and run the following command to install the CLI:
 python -m pip install -e . --user
 ```
 
+### Configure the PATH variable
+
 Check that the $PATH environment variable contains `~/.local/bin`:
 ```
 echo $PATH

diva_evaluation_cli/bin/install.sh

+#!/bin/bash
+
+#####################################################
+# Command Line Interface actev: installation script #
+#####################################################
+
+echo $PATH | grep /.local/bin
+if [ $? -ne 0 ];then
+  PATH="${PATH}:${HOME}/.local/bin"
+  export $PATH
+fi
+
+python3 -m pip install setuptools --user
+python3 -m pip install -e . --user -U
+

diva_evaluation_cli/bin/private_src/exec/exec.sh

@@ -13,19 +13,48 @@ output=$8
 chunks_result=$9
 
 tmp_file="chunk_ids.tmp"
+
+function should_exit {
+  exit_status=$1
+  if [ $exit_status -ne 0  ];then
+    exit 1
+  fi
+}
+
+function should_cleanup_exit {
+  exit_status=$1
+  if [ $exit_status -ne 0  ];then
+    actev experiment-cleanup
+    exit 1
+  fi
+}
+
+function should_continue {
+  exit_status=$1
+  chunk_id=$2
+  if [ $exit_status -ne 0  ];then
+    actev reset-chunk -i $chunk_id
+    continue
+  fi
+}
+
 actev design-chunks -f $file_index -a $activity_index -o $chunks -n $number_of_videos
+should_exit $?
 actev experiment-init -f $file_index -a $activity_index -c $chunks -v $video_location -s $system_cache_dir
+should_cleanup_exit $?
 
 # Get chunks
 python3 get_chunks_ids.py $chunks $tmp_file
+should_cleanup_exit $? "actev experiment-cleanup"
 
 for chunk_id in $(cat $tmp_file); do
   actev pre-process-chunk -i $chunk_id
+  should_continue $? $chunk_id
   actev process-chunk -i $chunk_id
+  should_continue $? $chunk_id
   actev post-process-chunk -i $chunk_id
-  actev reset-chunk -i $chunk_id
+  should_continue $? $chunk_id
 done
-
 rm $tmp_file
 
 actev merge-chunks -o $output -c $chunks_result -r $system_cache_dir

diva_evaluation_cli/bin/private_src/private_entry_points/actev_get_system.py

@@ -6,7 +6,7 @@ This file should not be modified.
 import os
 from diva_evaluation_cli.bin.private_src.system_types.system_types_definition import system_types
 
-def entry_point(url, system_type, location="None", user="None", password="None", token="None"):
+def entry_point(url, system_type, location="None", user="None", password="None", token="None", install_cli=False):
     """ Private entry points.
     """
     try:
@@ -23,6 +23,10 @@ def entry_point(url, system_type, location="None", user="None", password="None",
         password = "None"
     if not token:
         token = "None"
+    if install_cli:
+        install_cli = "True"
+    else:
+        install_cli = "False"
 
     # go into the right directory to execute the script
     path = os.path.dirname(__file__)
@@ -31,10 +35,11 @@ def entry_point(url, system_type, location="None", user="None", password="None",
               " " + location + \
               " " + user + \
               " " + password + \
-              " " + token
+              " " + token + \
+              " " + install_cli
 
     # execute the script
     # status is the exit status code returned by the program
     status = os.system(script)
     if status != 0:
-        raise Exception("Error occured in exec.sh")
+        raise Exception("Error occured in %s" % command.entry_point)

diva_evaluation_cli/bin/private_src/system_types/commands/docker_command.py

@@ -29,5 +29,6 @@ class ActevGetSystemDocker(ActevCommand):
         arg_parser.description= "Downloads a docker image"
         required_named = arg_parser.add_argument_group('required named arguments')
         arg_parser.add_argument("-U", "--user", help="username to access the url")
-        arg_parser.add_argument("-p", "--password", help="password to access the url")
+        arg_parser.add_argument("-p", "--password", help="password to access the url."
+                                "Warning: if password starts with \'-\', use this: --password=<your password>")
 

diva_evaluation_cli/bin/private_src/system_types/commands/git_command.py

@@ -8,10 +8,11 @@ Clones a git repository
 
 Args
 ----
-location or l: path to store the system
-user or U:     url to get the system
-password or p: password to access the url
-token or t:    token to access the url
+location or l:    path to store the system
+user or U:        url to get the system
+password or p:    password to access the url
+token or t:       token to access the url
+install-cli or i: install the cli to use it
 
 Warning: this file should not be modified.
 """
@@ -31,8 +32,12 @@ class ActevGetSystemGit(ActevCommand):
         arg_parser.description= "Downloads a git repository"
         required_named = arg_parser.add_argument_group('required named arguments')
         arg_parser.add_argument("-U", "--user", help="username to access the url")
-        arg_parser.add_argument("-p", "--password", help="password to access the url")
+        arg_parser.add_argument("-p", "--password", help="password to access the url"
+                                "Warning: if password starts with \'-\', use this: --password=<your password>")
         arg_parser.add_argument("-l", "--location", help="path to store the system")
-        arg_parser.add_argument("-t", "--token", help="token to access the url", type=str)
+        arg_parser.add_argument("-t", "--token", help="token to access the url"
+                                "Warning: if token starts with \'-\', use this: --token=<your token>",
+                                type=str)
+        arg_parser.add_argument("-i", "--install-cli", help="install the cli to use it", action='store_true')
 
 

diva_evaluation_cli/bin/private_src/system_types/commands/other_command.py

@@ -30,9 +30,11 @@ class ActevGetSystemOther(ActevCommand):
         """
         arg_parser.description= "Downloads a web resources as a tar file"
         required_named = arg_parser.add_argument_group('required named arguments')
-        arg_parser.add_argument("-U", "--user", help="username to access the url")
+        arg_parser.add_argument("-U", "--user", help="username to access the url"
+                                "Warning: if password starts with \'-\', use this: --password=<your password>")
         arg_parser.add_argument("-p", "--password", help="password to access the url")
         arg_parser.add_argument("-l", "--location", help="path to store the system")
-        arg_parser.add_argument("-t", "--token", help="token to access the url")
+        arg_parser.add_argument("-t", "--token", help="token to access the url"
+                                "Warning: if token starts with \'-\', use this: --token=<your token>")
 
 

diva_evaluation_cli/bin/private_src/system_types/get/get_git.sh

@@ -5,7 +5,9 @@ location=$2
 user=$3
 password=$4
 token=$5
+cli=$6
 
+# If there are credentials
 if [ $user != "None" ];then
   if [ $password != "None" ]; then
     credentials="${user}:${password}@"
@@ -23,8 +25,20 @@ if [ $user != "None" ];then
   url="${http}//${credentials}${git}/${end}"
 fi
 
+# If there is a location
 if [ $location != "None" ];then
   cd $location
 fi
 
 git clone $url
+
+if [ $? -eq 0 ];then
+  # If the system has to be installed
+  if [ $cli == "True" ];then
+    repo_name=`echo $url | rev | cut -d '.' -f2 | rev`
+    repo_name=`echo $repo_name | rev | cut -d '/' -f1 | rev`
+    cd $repo_name
+
+    diva_evaluation_cli/bin/install.sh
+  fi
+fi

diva_evaluation_cli/bin/private_src/validation/expected_container_output.txt

+file.json
+activity.json
+output.json
+chunks.json

diva_evaluation_cli/bin/private_src/validation/expected_structure.txt

+bin
+src
+container_output

diva_evaluation_cli/bin/private_src/validation/validate_system.py

@@ -7,25 +7,51 @@ import inspect
 import importlib
 import logging
 import os
+import sys
 
+root_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../')
 
 def validate_system():
+    validate_structure()
+    validate_cli()
+    validate_container_output()
+    logging.info("System is valid")
+
+############################### structure #####################################
+
+def validate_structure():
+    directories = os.listdir(root_path)
+    content = import_expected_result('expected_structure.txt')
+    logging.info("Structure validation")
+
+    for directory in content:
+        if not directory in directories:
+            raise Exception("System validation failed, missing {} directory".format(directory))
+        if not os.path.isdir(root_path + '/'  + directory):
+            raise Exception("System validation failed, {} not a directory".format(directory))
+        logging.info("    .. {} is valid".format(directory))
+
+################################## cli ########################################
+
+def validate_cli():
     """ Import the entry points method and try to compare them with the expected result
     """
     from diva_evaluation_cli.bin.cli import public_subcommands
 
     # Open expected result
-    content = import_expected_result('expected_validation_result.txt') 
+    content = import_expected_result('expected_validation_result.txt')
     i = 0
+    logging.info("CLI validation")
+
     for subcommand in public_subcommands:
         actev_command = "actev_" + subcommand.command.replace('-', '_')
-        
+
         # Is entry point exist
         try:
             entry_point_module = importlib.import_module('diva_evaluation_cli.src.entry_points.{}'.format(actev_command))
         except:
             raise Exception("System validation failed, {} entry_point method removed".format(actev_command))
-        
+
         entry_point_function = getattr(entry_point_module, 'entry_point')
         result = "{} - {}".format(actev_command,inspect.signature(entry_point_function))
 
@@ -33,16 +59,33 @@ def validate_system():
         if content[i] != result:
             raise Exception("System validation failed, {} entry_point method changed".format(actev_command))
         i += 1
-        logging.info("{} is valid".format(actev_command))
+        logging.info("    .. {} is valid".format(actev_command))
 
 def import_expected_result(file_name):
     """ Import expected validation result
     """
     content = []
     path = os.path.dirname(__file__)
-    expected_validation_result = os.path.join(path, file_name)
-    with open(expected_validation_result, 'r') as f:
+    expected_result = os.path.join(path, file_name)
+    with open(expected_result, 'r') as f:
         content = f.readlines()
     content = [line.strip() for line in content]
     return content
-    
+
+############################# container output ################################
+
+def validate_container_output():
+    """ Check that container output directory is present.
+    For each datasetID directory, chunks, activity, file and output should be present too.
+    """
+    container_output_dir = os.path.join(root_path, 'container_output')
+    logging.info("Container output validation")
+
+    for dataset_id in os.listdir(container_output_dir):
+        files = os.listdir(container_output_dir + '/'  + dataset_id)
+        content = import_expected_result('expected_container_output.txt')
+        for file_name in content:
+            if not file_name in files:
+                raise Exception("System validation failed, {} not present in {}".format(file_name, dataset_id))
+        logging.info("    .. {} is valid".format(dataset_id))
+

diva_evaluation_cli/container_output/cli_validation_mini/activity.json

+{
+  "Closing": {
+  "objectTypes": [
+  "Door",
+  "Person",
+  "Vehicle"
+  ]
+  },
+  "Closing_Trunk": {
+  "objectTypes": [
+  "Person",
+  "Vehicle"
+  ]
+  },
+  "Entering": {
+  "objectTypes": [
+  "Door",
+  "Person",
+  "Vehicle"
+  ]
+  },
+  "Exiting": {
+  "objectTypes": [
+  "Door",
+  "Person",
+  "Vehicle"
+  ]
+  },
+  "Loading": {
+  "objectTypes": [
+  "Person",
+  "Vehicle",
+  "Prop"
+  ]
+  }
+}

diva_evaluation_cli/container_output/cli_validation_mini/chunks.json

+{
+  "Chunk2": {
+    "activities": [
+      "Exiting",
+      "Loading",
+      "Closing",
+      "Closing_Trunk",
+      "Entering"
+    ],
+    "files": [
+      "VIRAT_S_000000.mp4"
+    ]
+  },
+  "Chunk1": {
+    "activities": [
+      "Exiting",
+      "Loading",
+      "Closing",
+      "Closing_Trunk",
+      "Entering"
+    ],
+    "files": [
+      "VIRAT_S_000001.mp4"
+    ]
+  }
+}
\ No newline at end of file

diva_evaluation_cli/container_output/cli_validation_mini/file.json

+{
+  "VIRAT_S_000000.mp4": {
+  "framerate": 30,
+  "selected": {
+    "1": 1,
+    "20941": 0
+  }
+},
+"VIRAT_S_000001.mp4": {
+  "framerate": 30,
+  "selected": {
+  "11": 1,
+  "201": 0,
+  "300": 1,
+  "20656": 0
+  }
+}
+}

diva_evaluation_cli/src/init_experiment.sh

 #!/bin/bash
 
+cd "$(dirname "$0")"
+
 FILE_INDEX=$1
 ACTIVITY_INDEX=$2
 CHUNK=$3
 VIDEO_LOCATION=$4
 SYSTEM_CACHE_DIRECTORY=$5
 
+docker ps -a | grep rc3d 1> /dev/null
+if [ $? -eq 0  ];then
+  ./clean_up.sh
+fi
+
 nvidia-docker run -itd --name rc3d \
   -v ${VIDEO_LOCATION}:/data/diva/v1-frames \
   -v ${SYSTEM_CACHE_DIRECTORY}:/data/diva/system-cache \
@@ -14,3 +21,4 @@ nvidia-docker run -itd --name rc3d \
 nvidia-docker cp ${FILE_INDEX} rc3d:/data/diva/nist-json/file-index.json 
 nvidia-docker cp ${ACTIVITY_INDEX} rc3d:/data/diva/nist-json/activity-index.json 
 nvidia-docker cp ${CHUNK} rc3d:/data/diva/nist-json/chunk.json
+

diva_evaluation_cli/src/setup.sh

 #!/bin/bash
 
+docker pull gitlab.kitware.com:4567/diva-baseline/diva-baseline:eval_cli