diff --git a/diva_evaluation_cli/src/entry_points/actev_design_chunks.py b/diva_evaluation_cli/src/entry_points/actev_design_chunks.py index f8e0620dfc11d0c03761f7fb826a35bf4d07e031..9dc34976b8c9eb8e4030c45ce11c6df2840d56bc 100644 --- a/diva_evaluation_cli/src/entry_points/actev_design_chunks.py +++ b/diva_evaluation_cli/src/entry_points/actev_design_chunks.py @@ -5,7 +5,7 @@ Implements the entry-point by using Python or any other languages. """ import argparse import json - +from diva_evaluation_cli.src.implementation.design_chunks import design_chunks def entry_point(file_index, activity_index, output, nb_videos_per_chunk): """Method to complete: you have to raise an exception if an error occured in the program. @@ -19,4 +19,7 @@ def entry_point(file_index, activity_index, output, nb_videos_per_chunk): nb_video_per_chunk (int): Number of videos in the chunk """ - raise NotImplementedError("You should implement the entry_point method.") + if not nb_videos_per_chunk: + nb_videos_per_chunk = 96 + design_chunks(file_index, activity_index, output, nb_videos_per_chunk) + diff --git a/diva_evaluation_cli/src/entry_points/actev_experiment_cleanup.py b/diva_evaluation_cli/src/entry_points/actev_experiment_cleanup.py index 7ad5a65280117962822235b799bef05d88f34265..c84e5a9820e6d95e5545fa196e051a28c915bfc6 100644 --- a/diva_evaluation_cli/src/entry_points/actev_experiment_cleanup.py +++ b/diva_evaluation_cli/src/entry_points/actev_experiment_cleanup.py @@ -14,4 +14,12 @@ def entry_point(system_cache_dir): system_cache_dir (str): Path to system cache directory """ - raise NotImplementedError("You should implement the entry_point method.") + # go into the right directory to execute the script + path = os.path.dirname(__file__) + script = os.path.join(path, '../implementation/reset_experiment.sh') + + # execute the script + # status is the exit status code returned by the program + status = os.system(script) + if status != 0: + raise Exception("Error occured in clean_up.sh") diff --git a/diva_evaluation_cli/src/entry_points/actev_experiment_init.py b/diva_evaluation_cli/src/entry_points/actev_experiment_init.py index 9b945562b30f02383f15740ef5341a878d61e0fa..f1712aa2c5cb888ccc8189a281bd61da48724dd9 100644 --- a/diva_evaluation_cli/src/entry_points/actev_experiment_init.py +++ b/diva_evaluation_cli/src/entry_points/actev_experiment_init.py @@ -20,4 +20,17 @@ def entry_point(file_index, activity_index, chunks, config_file (str, optional): Path to config file """ - raise NotImplementedError("You should implement the entry_point method.") + # go into the right directory to execute the script + path = os.path.dirname(__file__) + script = os.path.join(path, '../implementation/init_experiment.sh') + script += " " + file_index + \ + " " + activity_index + \ + " " + chunks + \ + " " + video_location + \ + " " + system_cache_dir + + # execute the script + # status is the exit status code returned by the program + status = os.system(script) + if status != 0: + raise Exception("Error occured in init_experiment.sh") diff --git a/diva_evaluation_cli/src/entry_points/actev_merge_chunks.py b/diva_evaluation_cli/src/entry_points/actev_merge_chunks.py index 1e36c1881a99b5e8b1f21872a6f21d34093b346d..1259f8cdcde0cf097eff0025d2f2338253a7d622 100644 --- a/diva_evaluation_cli/src/entry_points/actev_merge_chunks.py +++ b/diva_evaluation_cli/src/entry_points/actev_merge_chunks.py @@ -6,6 +6,7 @@ Implements the entry-point by using Python or any other languages. import argparse import json +from diva_evaluation_cli.src.implementation.merge_chunks import merge_chunks def entry_point(result_location, output_file, chunks_file, chunk_ids): """Method to complete: you have to raise an exception if an error occured in the program. @@ -19,4 +20,4 @@ def entry_point(result_location, output_file, chunks_file, chunk_ids): chunk_ids (:obj:`list`): List of chunk ids """ - raise NotImplementedError("You should implement the entry_point method.") + merge_chunks(result_location, output_file, chunks_file, chunk_ids) diff --git a/diva_evaluation_cli/src/entry_points/actev_post_process_chunk.py b/diva_evaluation_cli/src/entry_points/actev_post_process_chunk.py index 907807c1ac15dac7a005c34d6e170318f9c740cb..51ce9f759761b2f6d80f6721a583604d5a71abca 100644 --- a/diva_evaluation_cli/src/entry_points/actev_post_process_chunk.py +++ b/diva_evaluation_cli/src/entry_points/actev_post_process_chunk.py @@ -15,4 +15,12 @@ def entry_point(chunk_id, system_cache_dir): system_cache_dir (str): Path to system cache directory """ - raise NotImplementedError("You should implement the entry_point method.") + # go into the right directory to execute the script + path = os.path.dirname(__file__) + script = os.path.join(path, '../implementation/post_process_chunk.sh') + + # execute the script + # status is the exit status code returned by the program + status = os.system("%s %s" % (script, chunk_id)) + if status != 0: + raise Exception("Error occured in post_process_chunk.sh") diff --git a/diva_evaluation_cli/src/entry_points/actev_pre_process_chunk.py b/diva_evaluation_cli/src/entry_points/actev_pre_process_chunk.py index 79b78ff355ba3b264c1582ae67d5305286777f06..26a1c38bb9f6c7bdb2ffb2f79e3df689ee35c3fb 100644 --- a/diva_evaluation_cli/src/entry_points/actev_pre_process_chunk.py +++ b/diva_evaluation_cli/src/entry_points/actev_pre_process_chunk.py @@ -13,6 +13,10 @@ def entry_point(chunk_id, system_cache_dir): Args: chunk_id (str): Chunk id system_cache_dir (str): Path to system cache directory - """ - raise NotImplementedError("You should implement the entry_point method.") + script_path = os.path.join(os.path.dirname(__file__), '../implementation/pre_process_chunk.sh') + status = os.system("{0} {1}".format(script_path, chunk_id)) + if status != 0: + raise Exception("Preprocessing {0} failed".format(chunk_id)) + + diff --git a/diva_evaluation_cli/src/entry_points/actev_process_chunk.py b/diva_evaluation_cli/src/entry_points/actev_process_chunk.py index 8e701adfadc7d3f036b7a935d03bc9cff91339c4..8d59a9a891f1c6383b969b4c1324337c620222ef 100644 --- a/diva_evaluation_cli/src/entry_points/actev_process_chunk.py +++ b/diva_evaluation_cli/src/entry_points/actev_process_chunk.py @@ -4,7 +4,6 @@ Implements the entry-point by using Python or any other languages. """ import os -import re def entry_point(chunk_id, system_cache_dir): """Method to complete: you have to raise an exception if an error occured in the program. @@ -16,4 +15,7 @@ def entry_point(chunk_id, system_cache_dir): system_cache_dir (str): Path to system cache directory """ - raise NotImplementedError("You should implement the entry_point method.") + script_path = os.path.join(os.path.dirname(__file__), '../implementation/process_chunk.sh') + status = os.system("{0} {1}".format(script_path, chunk_id)) + if status != 0: + raise Exception("Processing {0} failed".format(chunk_id)) diff --git a/diva_evaluation_cli/src/entry_points/actev_reset_chunk.py b/diva_evaluation_cli/src/entry_points/actev_reset_chunk.py index 7f3708ad156d9c805dd3697eae66a31d80427569..3562d3ae4b477d688acd5851a50bc92205c1b93a 100644 --- a/diva_evaluation_cli/src/entry_points/actev_reset_chunk.py +++ b/diva_evaluation_cli/src/entry_points/actev_reset_chunk.py @@ -14,4 +14,15 @@ def entry_point(chunk_id, system_cache_dir): chunk_id (str): Chunk id system_cache_dir (str): Path to system cache directory """ - raise NotImplementedError("You should implement the entry_point method.") + if not system_cache_dir: + raise Exception("reset_chunk.sh needs system-cache-dir argument, please provide it") + + # go into the right directory to execute the script + path = os.path.dirname(__file__) + script = os.path.join(path, '../implementation/reset_chunk.sh') + + # execute the script + # status is the exit status code returned by the program + status = os.system("%s %s %s" % (script, chunk_id, system_cache_dir)) + if status != 0: + raise Exception("Error occured in reset_chunk.sh") diff --git a/diva_evaluation_cli/src/entry_points/actev_system_setup.py b/diva_evaluation_cli/src/entry_points/actev_system_setup.py index bc2eaab7a81b535229e75ee85fa2b746f3d174c0..4003ae4412e8bea34d8780ba5ed64066ffe69261 100644 --- a/diva_evaluation_cli/src/entry_points/actev_system_setup.py +++ b/diva_evaluation_cli/src/entry_points/actev_system_setup.py @@ -11,4 +11,9 @@ def entry_point(): Run any compilation/preparation steps for the system. """ - raise NotImplementedError("You should implement the entry_point method.") + path = os.path.dirname(__file__) + script = os.path.join(path, '../implementation/setup.sh') + + status = os.system(script) + if status != 0: + raise Exception("Error occured in setup.sh") diff --git a/diva_evaluation_cli/src/implementation/__init__.py b/diva_evaluation_cli/src/implementation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/diva_evaluation_cli/src/implementation/design_chunks.py b/diva_evaluation_cli/src/implementation/design_chunks.py new file mode 100644 index 0000000000000000000000000000000000000000..6ed60926f3d9fb259ab4bd531269209fc6f91036 --- /dev/null +++ b/diva_evaluation_cli/src/implementation/design_chunks.py @@ -0,0 +1,32 @@ +"""Design chunks source code + +""" +import json + +def design_chunks(file_index_file, activity_index_file, save_path, num_videos_per_chunk): + """ Merge file_index with activity_index content into different chunks written in a file + + Args: + file_index_file (str): Path to file index json file + activity_index_file (str): Path to activity index json file + save_path (str): Path to save chunks file + num_videos_per_chunks(int): number of videos in a chunk + """ + file_index = json.load(open(file_index_file, 'r')) + activity_index = json.load(open(activity_index_file, 'r')) + chunk_dict = {} + chunk_count = 0 + chunk_prefix = "Chunk" + all_activities = list(activity_index.keys()) + + for index, file_name in enumerate(file_index.keys()): + if index % num_videos_per_chunk == 0: + # start a new chunk + chunk_count += 1 + chunk_name = chunk_prefix + str(chunk_count) + chunk_dict[chunk_name] = {"activities": all_activities, + "files": []} + chunk_name = chunk_prefix + str(chunk_count) + chunk_dict[chunk_name]["files"].append(file_name) + with open(save_path, 'w') as f: + json.dump(chunk_dict, f, indent=2) diff --git a/diva_evaluation_cli/src/implementation/init_experiment.sh b/diva_evaluation_cli/src/implementation/init_experiment.sh new file mode 100755 index 0000000000000000000000000000000000000000..c44ad75877d96f7f8412db5d3ff47da1ea272cba --- /dev/null +++ b/diva_evaluation_cli/src/implementation/init_experiment.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +cd "$(dirname "$0")" + +FILE_INDEX=$1 +ACTIVITY_INDEX=$2 +CHUNK=$3 +VIDEO_LOCATION=$4 +SYSTEM_CACHE_DIRECTORY=$5 + +# Start the rc3d container +nvidia-docker run -itd --name rc3d \ + --entrypoint bash \ + -v ${VIDEO_LOCATION}:/data/diva/frames \ + -v ${SYSTEM_CACHE_DIRECTORY}:/data/diva/system-cache \ + kitware/rc3d-pipeline:latest + +nvidia-docker exec rc3d mkdir -p /diva/nist-json +nvidia-docker cp ${FILE_INDEX} rc3d:/diva/nist-json/file-index.json +nvidia-docker cp ${ACTIVITY_INDEX} rc3d:/diva/nist-json/activity-index.json +nvidia-docker cp ${CHUNK} rc3d:/diva/nist-json/chunk.json + diff --git a/diva_evaluation_cli/src/implementation/merge_chunks.py b/diva_evaluation_cli/src/implementation/merge_chunks.py new file mode 100755 index 0000000000000000000000000000000000000000..01fe6376ace53e43ce2bd94a2bd1d1b99f2b823a --- /dev/null +++ b/diva_evaluation_cli/src/implementation/merge_chunks.py @@ -0,0 +1,71 @@ +"""Merge chunk source code + +""" +import json +import os +import re + +def merge_chunks(result_location, output_file, chunks_file, chunk_ids=None): + """Merge a list of chunk results files inside an output file, and a chunks file. + + Args: + result_location (str): Path to chunk results files + output_file (str): Path to output file where chunks are merged + chunks_file (str): Path to a global chunk file created after merging + chunk_ids (:obj: `list`, optional): List of chunk ids to merge, if empty, merge every chunk results files + + """ + output_dict = {"filesProcessed": [], "activities": []} + chunks_dict = {} + + chunk_files = [] + if chunk_ids == None: + # Find all the chunks + chunk_files = get_chunk_files("Chunk", result_location) + else: + # Find the specified chunk file + for chunk_id in chunk_ids: + chunk_files.extend(get_chunk_files(chunk_id, result_location)) + + for chunk_file in chunk_files: + chunk = json.load(open(chunk_file, 'r')) + + # Remove out_of_bounds activities + activities_clean = [] + for activity in chunk["activities"]: + if activity["activity"] != "out_of_bounds": + activities_clean.append(activity) + + # Generate output_file + output_dict["activities"].extend(activities_clean) + for file_name in chunk["filesProcessed"]: + if file_name not in output_dict["filesProcessed"]: + output_dict["filesProcessed"].append(file_name) + + # Generate chunks_file + chunk_key = re.search('(Chunk[0-9]+)', chunk_file).group(1) + chunks_dict[chunk_key] = {} + activities = [] + for activity in activities_clean: + if activity["activity"] not in activities: + activities.append(activity["activity"]) + chunks_dict[chunk_key].update({"activities": activities, "files": chunk["filesProcessed"]}) + + with open(output_file, 'w') as f: + json.dump(output_dict, f, indent=2) + + with open(chunks_file, 'w') as f: + json.dump(chunks_dict, f, indent=2) + +def get_chunk_files(chunk_id, result_location): + """Get a chunk results file according to a given chunk id and a result location + + Args: + chunk_id (str): Chunk id + result_location (str): Path to chunk results files + """ + chunk_files = [] + for chunk_file in os.listdir(result_location): + if chunk_id in chunk_file: + chunk_files.append(os.path.join(result_location, chunk_file)) + return chunk_files diff --git a/diva_evaluation_cli/src/implementation/post_process_chunk.sh b/diva_evaluation_cli/src/implementation/post_process_chunk.sh new file mode 100755 index 0000000000000000000000000000000000000000..051b036acb66550558e1ce0173031153ae202e63 --- /dev/null +++ b/diva_evaluation_cli/src/implementation/post_process_chunk.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +CHUNK_ID=$1 + +nvidia-docker start rc3d + +nvidia-docker exec -it rc3d /bin/bash -c "python /diva_install/lib/python2.7/dist-packages/diva/scripts/cli_helpers/merge_videos.py --chunk-id=${CHUNK_ID} --chunk-json=/diva/nist-json/chunk.json --video-root /data/diva/system-cache --out-chunk-root /data/diva/system-cache" diff --git a/diva_evaluation_cli/src/implementation/pre_process_chunk.sh b/diva_evaluation_cli/src/implementation/pre_process_chunk.sh new file mode 100755 index 0000000000000000000000000000000000000000..f1503dfef67bc6f9082c6350abaca1e177440d8b --- /dev/null +++ b/diva_evaluation_cli/src/implementation/pre_process_chunk.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +CHUNK_ID=$1 + +nvidia-docker start rc3d + +nvidia-docker exec -it rc3d /bin/bash -c "python /diva_install/lib/python2.7/dist-packages/diva/scripts/cli_helpers/generate_experiments.py --chunk-id=${CHUNK_ID} --chunk-json=/diva/nist-json/chunk.json --data-root=/data/diva/frames --experiment-root /data/diva/system-cache --use-videos" diff --git a/diva_evaluation_cli/src/implementation/process_chunk.sh b/diva_evaluation_cli/src/implementation/process_chunk.sh new file mode 100755 index 0000000000000000000000000000000000000000..9bdf798b3a4fc8fe40400f0c9d659ffb7a7dbc5c --- /dev/null +++ b/diva_evaluation_cli/src/implementation/process_chunk.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +CHUNK_ID=$1 + +nvidia-docker start rc3d + +nvidia-docker exec -it rc3d /bin/bash -c "python /R-C3D/cli-helpers/chunk_runner.py --chunk-id=${CHUNK_ID} --chunk-json=/diva/nist-json/chunk.json --experiment-root=/data/diva/system-cache --out-video-root=/data/diva/system-cache --activity-index=/diva/nist-json/activity-index.json" diff --git a/diva_evaluation_cli/src/implementation/reset_chunk.sh b/diva_evaluation_cli/src/implementation/reset_chunk.sh new file mode 100755 index 0000000000000000000000000000000000000000..5cb5438441028d2824c17d19028ddd9f37560920 --- /dev/null +++ b/diva_evaluation_cli/src/implementation/reset_chunk.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +CHUNK_ID=$1 + +nvidia-docker start rc3d + +nvidia-docker exec -it rc3d /bin/bash -c "python /diva_install/lib/python2.7/dist-packages/diva/scripts/cli_helpers/cleanup_chunk.py --chunk-id=${CHUNK_ID} --chunk-json=/diva/nist-json/chunk.json --cache-dir /data/diva/system-cache" diff --git a/diva_evaluation_cli/src/implementation/reset_experiment.sh b/diva_evaluation_cli/src/implementation/reset_experiment.sh new file mode 100755 index 0000000000000000000000000000000000000000..ee89b01ddad4bd905dcc9fef7dc6360c13dc7554 --- /dev/null +++ b/diva_evaluation_cli/src/implementation/reset_experiment.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +nvidia-docker start rc3d + +nvidia-docker exec -it rc3d /bin/bash -c "python /diva_install/lib/python2.7/dist-packages/diva/scripts/cli_helpers/cleanup_experiment.py --chunk-json /data/diva/system-cache/chunk.json --cache-dir /data/diva/system-cache" +sleep 1 +nvidia-docker stop rc3d +sleep 1 +nvidia-docker rm rc3d + + diff --git a/diva_evaluation_cli/src/implementation/setup.sh b/diva_evaluation_cli/src/implementation/setup.sh new file mode 100755 index 0000000000000000000000000000000000000000..31763ac6f3503ed1e8f72a5985625a863df8ff2e --- /dev/null +++ b/diva_evaluation_cli/src/implementation/setup.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +sudo apt-get update +# Set up a crontab to execute system-setup after a reboot +crontab -l | grep system-setup +if [ $? -ne 0 ];then + (crontab -l 2>/dev/null; echo "@reboot . /home/ubuntu/.profile; actev system-setup >> /home/ubuntu/setup.log") | crontab +fi + +USER="ubuntu" + +# Install ffmpeg +ffmpeg -h +if [ $? -ne 0 ];then + sudo apt-get install ffmpeg -y +fi + +# Install docker-ce +docker -h +if [ $? -ne 0 ];then + sudo apt-get install \ + apt-transport-https \ + ca-certificates \ + curl \ + software-properties-common -y + + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo apt-key fingerprint 0EBFCD88 + + sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + + sudo apt-get install docker-ce -y + + sudo usermod -a -G docker $USER + sudo reboot +else + docker image ls + if [ $? -ne 0 ];then + exit 1 + fi +fi + +# Install nvidia-drivers +nvidia-smi -h +if [ $? -ne 0 ];then + sudo add-apt-repository ppa:graphics-drivers + sudo apt-get update + sudo apt-get install nvidia-390 -y + sudo reboot +else + nvidia-smi + if [ $? -ne 0 ];then + exit 1 + fi +fi + +# Install nvidia-docker +nvidia-docker -h +if [ $? -ne 0 ];then + curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | \ + + sudo apt-key add - + distribution=$(. /etc/os-release;echo $ID$VERSION_ID) + curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | \ + sudo tee /etc/apt/sources.list.d/nvidia-docker.list + sudo apt-get update + + sudo apt-get install -y nvidia-docker2 -y + sudo pkill -SIGHUP dockerd + sudo usermod -a -G docker $USER + sudo reboot +else + docker image ls + if [ $? -ne 0 ];then + exit 1 + fi +fi + +# Install Docker python sdk +pip install docker + +# Install RC3D framework image +docker pull gitlab.kitware.com:4567/kwiver/r-c3d/framework-cli:latest