WIP add get-system, validate-system, exec

diva_evaluation_cli/bin/actev_exec.py

+"""
+USAGE
+
+ActEV exec
+Description
+-----------
+Calls a team-implemented API. Captures time stamps, resource usage, etc.
+
+Args
+----
+file-index or f:         path to file index json file for test set
+activity-index or a:     path to activity index json file for test set
+chunks or c:             path to chunks json file
+nb_video-per-chunk or n: number of videos in the chunk
+video-location or v:     path to videos content
+system-cache-dir or s:   path to system cache directory
+config-file or C:        path to config file
+output_file:                  path to merge chunks command result
+chunk_result:            path to chunks json file after merge chunks execution
+
+Warning: this file should not be modified: see src/entry_points to add your source code.
+"""
+import logging
+
+from diva_evaluation_cli.bin.actev_command import ActevCommand
+from diva_evaluation_cli.bin.private_src.private_entry_points.actev_exec import entry_point
+
+class ActevExec(ActevCommand):
+
+    def __init__(self):
+        super(ActevExec, self).__init__('exec', entry_point)
+
+    def cli_parser(self, arg_parser):
+        """ Configure the description and the arguments (positional and optional) to parse.
+
+        @param arg_parser: python arg parser to describe how to parse the command
+        """
+        arg_parser.description= "Calls a team-implemented API. Captures time stamps, resource usage, etc."
+        required_named = arg_parser.add_argument_group('required named arguments')
+
+        required_named.add_argument("-f", "--file_index", help="path to file index json file", required=True)
+        required_named.add_argument("-a", "--activity_index", help="path to activity index json file", required=True)
+        required_named.add_argument("-c", "--chunks", help="path to chunks json file", required=True)
+        arg_parser.add_argument("-n", "--nb_videos_per_chunk", help="number of videos in a chunk")
+        
+        required_named.add_argument("-v", "--video_location", help="path to videos content", required=True)
+        required_named.add_argument("-s", "--system_cache_dir", help="path to system cache directory", required=True)
+        arg_parser.add_argument("-C", "--config-file", help="path to config file")
+        
+        required_named.add_argument("-o", "--output_file", help="path to merge chunks command result", required=True)
+        required_named.add_argument("-r", "--chunks_result", help="path to chunks json file after merge chunks execution", required=True)
+        arg_parser.set_defaults(func=ActevExec.command, object=self)
+

diva_evaluation_cli/bin/actev_get_system.py

@@ -8,7 +8,7 @@ Downloads a credentialed, web-accessible content into src
 
 Args
 ----
-url or u:           url to get the system
+url or u:      url to get the system
 user or U:     username to access the url
 password or p: password to access the url
 token or t:    token to access the url
@@ -18,7 +18,7 @@ Warning: this file should not be modified: see src/entry_points to add your sour
 import logging
 
 from diva_evaluation_cli.bin.actev_command import ActevCommand
-from diva_evaluation_cli.src.entry_points.actev_experiment_init import entry_point
+from diva_evaluation_cli.bin.private_src.private_entry_points.actev_get_system import entry_point
 
 class ActevGetSystem(ActevCommand):
 

diva_evaluation_cli/bin/actev_validate_system.py

+"""
+USAGE
+
+ActEV validate-system
+Description
+-----------
+Checks the structure of the  directory after ActEV-system-setup is run. Checks for expected API contents, etc.
+
+Warning: this file should not be modified: see src/entry_points to add your source code.
+"""
+import logging
+
+from diva_evaluation_cli.bin.actev_command import ActevCommand
+from diva_evaluation_cli.bin.entry_points.actev_validate_system import entry_point
+
+
+class ActevValidateSystem(ActevCommand):
+
+    def __init__(self):
+        super(ActevValidateSystem, self).__init__('validate-system', entry_point)
+
+    def cli_parser(self, arg_parser):
+        """ Configure the description and the arguments (positional and optional) to parse.
+
+        @param arg_parser: python arg parser to describe how to parse the command
+        """
+        arg_parser.description = "Checks the structure of the  directory after ActEV-system-setup is run"
+        arg_parser.set_defaults(func=ActevValidateSystem.command, object=self)

diva_evaluation_cli/bin/cli.py

@@ -30,6 +30,7 @@ from diva_evaluation_cli.bin.actev_post_process_chunk import ActevPostProcessChu
 from diva_evaluation_cli.bin.actev_reset_chunk import ActevResetChunk
 from diva_evaluation_cli.bin.actev_experiment_cleanup import ActevExperimentCleanup
 from diva_evaluation_cli.bin.actev_merge_chunks import ActevMergeChunks
+from diva_evaluation_cli.bin.actev_exec import ActevExec
 
 subcommands = [
 ActevSystemSetup(),
@@ -40,7 +41,8 @@ ActevProcessChunk(),
 ActevPostProcessChunk(),
 ActevResetChunk(),
 ActevMergeChunks(),
-ActevExperimentCleanup()
+ActevExperimentCleanup(),
+ActevExec()
 ]
 
 def cli_parser():

diva_evaluation_cli/bin/private_src/exec.sh

+#!/bin/bash
+
+cd "$(dirname "$0")"
+
+file_index=$1
+activity_index=$2
+chunks=$3
+number_of_videos=$4
+video_location=$5
+system_cache_dir=$6
+config=$7
+output=$8
+chunks_result=$9
+
+tmp_file="chunk_ids.tmp"
+actev system-setup
+actev design-chunks -f $file_index -a $activity_index -o $chunks -n $number_of_videos
+actev experiment-init -f $file_index -a $activity_index -c $chunks -v $video_location -s $system_cache_dir
+
+# Get chunks
+python3 get_chunks_ids.py $chunks $tmp_file
+
+for chunk_id in $(cat $tmp_file); do
+  actev pre-process-chunk -i $chunk_id
+  actev process-chunk -i $chunk_id
+  actev post-process-chunk -i $chunk_id
+  actev reset-chunk -i $chunk_id
+done
+
+rm $tmp_file
+
+actev merge-chunks -o $output -c $chunks_result -r $system_cache_dir
+actev experiment-cleanup

diva_evaluation_cli/bin/private_src/get_chunks_ids.py

+"""
+SOURCE
+
+
+"""
+import json
+import sys
+
+def get_chunks_ids(chunk_file, output):
+    """ Get chunk ids from a chunk json file
+    """
+    chunk_ids = []
+    chunks = json.load(open(chunk_file, 'r'))
+    with open(output, "w") as f:
+        for chunk_id in chunks:
+            f.write(chunk_id + "\n")
+
+
+if __name__ == '__main__':
+    if len(sys.argv) == 3:
+        get_chunks_ids(sys.argv[1], sys.argv[2])
+
+
+  

diva_evaluation_cli/bin/private_src/private_entry_points/actev_exec.py

+"""
+ENTRY POINT
+
+This file should not be modified.
+"""
+import os
+
+def entry_point(file_index, activity_index, chunks, nb_videos_per_chunk, 
+    video_location, system_cache_dir, output_file, chunks_result, config_file=None):
+    """ Private entry points.
+    """
+    if not nb_videos_per_chunk:
+        nb_video_per_chunk = 96
+
+    if not config_file:
+        config_file = "None"
+
+    # go into the right directory to execute the script
+    path = os.path.dirname(__file__)
+    script = os.path.join(path, '../exec.sh')
+    script += " " + file_index + \
+              " " + activity_index + \
+              " " + chunks + \
+              " " + nb_videos_per_chunk + \
+              " " + video_location + \
+              " " + system_cache_dir + \
+              " " + config_file + \
+              " " + output_file + \
+              " " + chunks_result
+
+    # execute the script
+    # status is the exit status code returned by the program
+    status = os.system(script)
+    if status != 0:
+        raise Exception("Error occured in exec.sh")

diva_evaluation_cli/bin/private_src/private_entry_points/actev_get_system.py

+"""
+ENTRY POINT
+
+This file should not be modified.
+"""
+import os
+
+system_types = {
+    'docker': 'get_docker.sh',
+    'git': 'get_git.sh',
+    'other': 'get_other.sh'
+}
+
+def entry_point(url, system_type, location, user, password, token):
+    """ Private entry points.
+    """
+    try:
+        script = systems_types[system_type]
+    except:
+        raise Exception("Unknown system type")
+    
+    # go into the right directory to execute the script
+    path = os.path.dirname(__file__)
+    script = os.path.join(path, '../system_types/' + script)
+    script += " " + url + \
+              " " + location + \
+              " " + user + \
+              " " + password + \
+              " " + token + \
+    
+    # execute the script
+    # status is the exit status code returned by the program
+    status = os.system(script)
+    if status != 0:
+        raise Exception("Error occured in exec.sh")

diva_evaluation_cli/bin/private_src/private_entry_points/actev_validate_system.py

+"""
+ENTRY POINT
+
+This file should not be modified.
+"""
+import os
+from diva_evaluation_cli.bin.private_src.validate_system import validate_system
+
+def entry_point():
+    """ Private entry points.
+    """
+    validate_system()
+

diva_evaluation_cli/bin/private_src/system_types/get_docker.sh

+#!/bin/bash
+
+url=$1
+location=$2
+user=$3
+password=$4
+token=$5
+
+if [ $user != "None" ] && [ $password != "None"];then 
+  docker login --username $user --password $password
+fi
+docker pull $url

diva_evaluation_cli/bin/private_src/system_types/get_git.sh

+#!/bin/bash
+
+url=$1
+location=$2
+user=$3
+password=$4
+token=$5
+
+if [ $user != "None" ] && [ [ $password != "None"] || [ $token != "None"  ] ];then
+  if [ $password == "None" ]; then
+    credentials="${user}:${password}"
+  else
+    credentials="${user}:${token}"
+  fi
+
+  http=`echo $url | cut -d '/' -f1`
+  git=`echo $url | cut -d '/' -f3`
+  end=`echo $url | cut -d '/' -f4-`
+
+  url="${http}//${credentials}@${git}/${end}"
+fi
+git clone $url

diva_evaluation_cli/bin/private_src/system_types/get_other.sh

+#!/bin/bash
+
+url=$1
+location=$2
+user=$3
+password=$4
+token=$5
+
+if [ $user != "None" ] && [ $password != "None"];then 
+  curl -u $user:$password $url
+else
+  if [ $user != "None" ] && [ $password == "None"];then
+    curl -u $user $url
+  else
+    curl $url
+  fi
+fi
+

diva_evaluation_cli/bin/private_src/validate_system.py

+"""
+SOURCE
+
+
+"""
+
+def validate_system():
+    pass
+
+def is_cli_complete():
+    pass
+
+def is_cli_executable():
+    pass
+