Commit 49f3445d authored by Alexandre Boyer's avatar Alexandre Boyer
Browse files

WIP add get-system, validate-system, exec

parent cb9879cf
"""
USAGE
ActEV exec
Description
-----------
Calls a team-implemented API. Captures time stamps, resource usage, etc.
Args
----
file-index or f: path to file index json file for test set
activity-index or a: path to activity index json file for test set
chunks or c: path to chunks json file
nb_video-per-chunk or n: number of videos in the chunk
video-location or v: path to videos content
system-cache-dir or s: path to system cache directory
config-file or C: path to config file
output_file: path to merge chunks command result
chunk_result: path to chunks json file after merge chunks execution
Warning: this file should not be modified: see src/entry_points to add your source code.
"""
import logging
from diva_evaluation_cli.bin.actev_command import ActevCommand
from diva_evaluation_cli.bin.private_src.private_entry_points.actev_exec import entry_point
class ActevExec(ActevCommand):
def __init__(self):
super(ActevExec, self).__init__('exec', entry_point)
def cli_parser(self, arg_parser):
""" Configure the description and the arguments (positional and optional) to parse.
@param arg_parser: python arg parser to describe how to parse the command
"""
arg_parser.description= "Calls a team-implemented API. Captures time stamps, resource usage, etc."
required_named = arg_parser.add_argument_group('required named arguments')
required_named.add_argument("-f", "--file_index", help="path to file index json file", required=True)
required_named.add_argument("-a", "--activity_index", help="path to activity index json file", required=True)
required_named.add_argument("-c", "--chunks", help="path to chunks json file", required=True)
arg_parser.add_argument("-n", "--nb_videos_per_chunk", help="number of videos in a chunk")
required_named.add_argument("-v", "--video_location", help="path to videos content", required=True)
required_named.add_argument("-s", "--system_cache_dir", help="path to system cache directory", required=True)
arg_parser.add_argument("-C", "--config-file", help="path to config file")
required_named.add_argument("-o", "--output_file", help="path to merge chunks command result", required=True)
required_named.add_argument("-r", "--chunks_result", help="path to chunks json file after merge chunks execution", required=True)
arg_parser.set_defaults(func=ActevExec.command, object=self)
......@@ -8,7 +8,7 @@ Downloads a credentialed, web-accessible content into src
Args
----
url or u: url to get the system
url or u: url to get the system
user or U: username to access the url
password or p: password to access the url
token or t: token to access the url
......@@ -18,7 +18,7 @@ Warning: this file should not be modified: see src/entry_points to add your sour
import logging
from diva_evaluation_cli.bin.actev_command import ActevCommand
from diva_evaluation_cli.src.entry_points.actev_experiment_init import entry_point
from diva_evaluation_cli.bin.private_src.private_entry_points.actev_get_system import entry_point
class ActevGetSystem(ActevCommand):
......
"""
USAGE
ActEV validate-system
Description
-----------
Checks the structure of the directory after ActEV-system-setup is run. Checks for expected API contents, etc.
Warning: this file should not be modified: see src/entry_points to add your source code.
"""
import logging
from diva_evaluation_cli.bin.actev_command import ActevCommand
from diva_evaluation_cli.bin.entry_points.actev_validate_system import entry_point
class ActevValidateSystem(ActevCommand):
def __init__(self):
super(ActevValidateSystem, self).__init__('validate-system', entry_point)
def cli_parser(self, arg_parser):
""" Configure the description and the arguments (positional and optional) to parse.
@param arg_parser: python arg parser to describe how to parse the command
"""
arg_parser.description = "Checks the structure of the directory after ActEV-system-setup is run"
arg_parser.set_defaults(func=ActevValidateSystem.command, object=self)
......@@ -30,6 +30,7 @@ from diva_evaluation_cli.bin.actev_post_process_chunk import ActevPostProcessChu
from diva_evaluation_cli.bin.actev_reset_chunk import ActevResetChunk
from diva_evaluation_cli.bin.actev_experiment_cleanup import ActevExperimentCleanup
from diva_evaluation_cli.bin.actev_merge_chunks import ActevMergeChunks
from diva_evaluation_cli.bin.actev_exec import ActevExec
subcommands = [
ActevSystemSetup(),
......@@ -40,7 +41,8 @@ ActevProcessChunk(),
ActevPostProcessChunk(),
ActevResetChunk(),
ActevMergeChunks(),
ActevExperimentCleanup()
ActevExperimentCleanup(),
ActevExec()
]
def cli_parser():
......
#!/bin/bash
cd "$(dirname "$0")"
file_index=$1
activity_index=$2
chunks=$3
number_of_videos=$4
video_location=$5
system_cache_dir=$6
config=$7
output=$8
chunks_result=$9
tmp_file="chunk_ids.tmp"
actev system-setup
actev design-chunks -f $file_index -a $activity_index -o $chunks -n $number_of_videos
actev experiment-init -f $file_index -a $activity_index -c $chunks -v $video_location -s $system_cache_dir
# Get chunks
python3 get_chunks_ids.py $chunks $tmp_file
for chunk_id in $(cat $tmp_file); do
actev pre-process-chunk -i $chunk_id
actev process-chunk -i $chunk_id
actev post-process-chunk -i $chunk_id
actev reset-chunk -i $chunk_id
done
rm $tmp_file
actev merge-chunks -o $output -c $chunks_result -r $system_cache_dir
actev experiment-cleanup
"""
SOURCE
"""
import json
import sys
def get_chunks_ids(chunk_file, output):
""" Get chunk ids from a chunk json file
"""
chunk_ids = []
chunks = json.load(open(chunk_file, 'r'))
with open(output, "w") as f:
for chunk_id in chunks:
f.write(chunk_id + "\n")
if __name__ == '__main__':
if len(sys.argv) == 3:
get_chunks_ids(sys.argv[1], sys.argv[2])
"""
ENTRY POINT
This file should not be modified.
"""
import os
def entry_point(file_index, activity_index, chunks, nb_videos_per_chunk,
video_location, system_cache_dir, output_file, chunks_result, config_file=None):
""" Private entry points.
"""
if not nb_videos_per_chunk:
nb_video_per_chunk = 96
if not config_file:
config_file = "None"
# go into the right directory to execute the script
path = os.path.dirname(__file__)
script = os.path.join(path, '../exec.sh')
script += " " + file_index + \
" " + activity_index + \
" " + chunks + \
" " + nb_videos_per_chunk + \
" " + video_location + \
" " + system_cache_dir + \
" " + config_file + \
" " + output_file + \
" " + chunks_result
# execute the script
# status is the exit status code returned by the program
status = os.system(script)
if status != 0:
raise Exception("Error occured in exec.sh")
"""
ENTRY POINT
This file should not be modified.
"""
import os
system_types = {
'docker': 'get_docker.sh',
'git': 'get_git.sh',
'other': 'get_other.sh'
}
def entry_point(url, system_type, location, user, password, token):
""" Private entry points.
"""
try:
script = systems_types[system_type]
except:
raise Exception("Unknown system type")
# go into the right directory to execute the script
path = os.path.dirname(__file__)
script = os.path.join(path, '../system_types/' + script)
script += " " + url + \
" " + location + \
" " + user + \
" " + password + \
" " + token + \
# execute the script
# status is the exit status code returned by the program
status = os.system(script)
if status != 0:
raise Exception("Error occured in exec.sh")
"""
ENTRY POINT
This file should not be modified.
"""
import os
from diva_evaluation_cli.bin.private_src.validate_system import validate_system
def entry_point():
""" Private entry points.
"""
validate_system()
#!/bin/bash
url=$1
location=$2
user=$3
password=$4
token=$5
if [ $user != "None" ] && [ $password != "None"];then
docker login --username $user --password $password
fi
docker pull $url
#!/bin/bash
url=$1
location=$2
user=$3
password=$4
token=$5
if [ $user != "None" ] && [ [ $password != "None"] || [ $token != "None" ] ];then
if [ $password == "None" ]; then
credentials="${user}:${password}"
else
credentials="${user}:${token}"
fi
http=`echo $url | cut -d '/' -f1`
git=`echo $url | cut -d '/' -f3`
end=`echo $url | cut -d '/' -f4-`
url="${http}//${credentials}@${git}/${end}"
fi
git clone $url
#!/bin/bash
url=$1
location=$2
user=$3
password=$4
token=$5
if [ $user != "None" ] && [ $password != "None"];then
curl -u $user:$password $url
else
if [ $user != "None" ] && [ $password == "None"];then
curl -u $user $url
else
curl $url
fi
fi
"""
SOURCE
"""
def validate_system():
pass
def is_cli_complete():
pass
def is_cli_executable():
pass
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment