Commit 5b7338ef authored by Alexandre Boyer's avatar Alexandre Boyer
Browse files

modify docstring entry points in bin/private_src/implementation

parent f0edb7da
Pipeline #124126 passed with stage
in 28 seconds
"""
SOURCE
"""Exec module: utils
"""
import json
import sys
def get_chunks_ids(chunk_file, output):
""" Get chunk ids from a chunk json file
""" Get chunk ids from a chunk json file.
Args:
chunk_file (str): Path to a json chunk file
output (str): Path to the output file containing chunk ids
"""
chunk_ids = []
chunks = json.load(open(chunk_file, 'r'))
......@@ -21,4 +24,4 @@ if __name__ == '__main__':
get_chunks_ids(sys.argv[1], sys.argv[2])
"""
SOURCE
"""Experiment init module
"""
import os
......@@ -8,6 +6,11 @@ import json
def experiment_validation(file_index, video_location):
""" Check that every video files enumerated in file_index exists, is readable and has content.
Args:
file_index (str): Path to file index in json
video_location (str): Path to directory containing videos
"""
with open(file_index) as f:
file_dict = json.load(f)
......@@ -17,7 +20,7 @@ def experiment_validation(file_index, video_location):
for video_filename in file_dict:
file_index_exists = False
for dirpath, dirnames, filenames in os.walk(video_location):
for filename in filenames:
if filename == video_filename:
......
"""
SOURCE
"""Get system module: system types definition
Dictionary of system types:
key: name of the system type
value: command to download the system
* key: name of the system type
* value: command to download the system
"""
from diva_evaluation_cli.bin.commands.actev_get_system_subcommands.docker_command import ActevGetSystemDocker
......
......@@ -10,9 +10,7 @@ Attributes:
"""
from pynvml import *
from . import utils
import logging
logger = logging.getLogger('MONITOR')
......
"""Utils module for resource monitoring
"""Monitor module: utils module for resource monitoring
"""
import psutil
......@@ -39,15 +39,15 @@ def psutil_snapshot_process(pid, snap_children=True):
snap_children (boolean): Whether to recurse the function on the process' children
Returns:
dict: resources usage for a given pid and its children
dict: resources usage for a given pid and its children::
if snap_children::
{'command':'some_command', ...}
else::
{'command':'some_command', ... {'children': [
if snap_children
{'command':'some_command', ...}
else:
{'command':'some_command', ... {'children': [
{'command':'some_command', ... {'children': {} }}},
... ]
}}
}}
"""
proc = psutil.Process(pid)
......
"""
SOURCE
"""Status module: check status
Contains methods to check status of each type of queries: system, experiment and chunks
"""
import json
......@@ -9,7 +9,22 @@ import os
###### Check chunk ############################################################
def check_status_chunk(query_type, command_history_dict, status_dict, **kwargs):
""" Check status of a specific chunk id during experiment processing
"""Check status of a specific chunk id during experiment processing.
Use the process-chunk state to get the status of a given chunk id.
Args:
query_type (str): Type of status query, chunk in this case
command_history_dict (:obj: `dict`): Dictionary of this form::
{'command name': [{'status': 'start', id: '98', args: {...}} ...], ...}
status_dict (:obj: `dict`): Dictionary of status according to a query type, a command name and a state
**kwargs: Arbitrary keyword arguments
Return:
str: status of the chunk id
"""
chunk_id = kwargs.get('chunk_id', None)
status = 'Not processed'
......@@ -21,7 +36,7 @@ def check_status_chunk(query_type, command_history_dict, status_dict, **kwargs):
while process_chunk['status'] != 'not defined':
if chunk_id == process_chunk['args']['chunk_id']:
if process_chunk['id'] > experiment_init_id:
status = get_status(query_type, process_chunk_command, process_chunk_status, status_dict)
status = get_status(query_type, process_chunk_command, process_chunk_status, status_dict)
break
process_chunk = command_history_dict['process-chunk'].pop()
......@@ -30,9 +45,25 @@ def check_status_chunk(query_type, command_history_dict, status_dict, **kwargs):
###### Check experiment #######################################################
def check_status_experiment(query_type, command_history_dict, status_dict, **kwargs):
""" Check status of the experiment during the process
"""Check status of the experiment during the process.
Use the experiment-init and cleanup and commands between (except pre/post/reset/process-chunk) to
get status of the experiment.
Args:
query_type (str): Type of status query, chunk in this case
command_history_dict (:obj: `dict`): Dictionary of this form::
{'command name': [{'status': 'start', id: '98', args: {...}} ...], ...}
status_dict (:obj: `dict`): Dictionary of status according to a query type, a command name and a state
**kwargs: Arbitrary keyword arguments
Return:
str: status of the experiment
"""
experiment_init_command = 'experiment-init'
experiment_init_command = 'experiment-init'
experiment_cleanup_command = 'experiment-cleanup'
# Determine status of experiment-init first
......@@ -45,7 +76,7 @@ def check_status_experiment(query_type, command_history_dict, status_dict, **kwa
experiment_cleanup_status = experiment_cleanup['status']
experiment_cleanup_id = experiment_cleanup['id']
status = get_status(query_type, experiment_init_command, experiment_init_status, status_dict)
status = get_status(query_type, experiment_init_command, experiment_init_status, status_dict)
max_id = experiment_init_id
# Determine status of all the commands between experiment-init and cleanup
......@@ -84,18 +115,34 @@ def check_status_experiment(query_type, command_history_dict, status_dict, **kwa
def check_status_system(query_type, command_history_dict, status_dict, **kwargs):
""" Check status of the system.
Use the system-setup state to get the status of the system.
Args:
query_type (str): Type of status query, chunk in this case
command_history_dict (:obj: `dict`): Dictionary of this form::
{'command name': [{'status': 'start', id: '98', args: {...}} ...], ...}
status_dict (:obj: `dict`): Dictionary of status according to a query type, a command name and a state
**kwargs: Arbitrary keyword arguments
Return:
str: status of the system
"""
command = 'system-setup'
system_status = command_history_dict[command].pop()['status']
system_status = command_history_dict[command].pop()['status']
status = get_status(query_type, command, system_status, status_dict)
return status
###### Main check function ####################################################
def check_status(query_type, **kwargs):
""" Check status of a component of the system to know wether it is running properly
""" Check status of a component of the system to know wether it is running properly.
@param query_type: string representing the type of the component to check_status
Args:
query_type (str): Type of the component whose status has to be checked
"""
status_methods = {'chunk-query': check_status_chunk,
'experiment-query': check_status_experiment,
......@@ -108,7 +155,7 @@ def check_status(query_type, **kwargs):
with open(command_history_json) as f:
command_history_dict = json.load(f)
with open(status_json) as f:
with open(status_json) as f:
status_dict = json.load(f)
status = status_methods[query_type](query_type, command_history_dict, status_dict, **kwargs)
......@@ -116,6 +163,15 @@ def check_status(query_type, **kwargs):
def get_status(query_type, command, execution_status, status_dict):
""" Get the status code of a command according to a query
Args:
query_type (str): Type of status query, chunk in this case
command (str): Command name
execution_status: State of the last given command execution
status_dict (:obj: `dict`): Dictionary of status according to a query type, a command name and a state
Return:
str: status according to query type, command and execution status
"""
return status_dict[query_type][command][execution_status]
"""
SOURCE
"""Status module: query types definition
Dictionary of system types:
key: name of the query type
value: command to get status
* key: name of the query type
* value: command to get status
"""
from diva_evaluation_cli.bin.commands.actev_status_subcommands.chunk_query_command import ActevStatusChunkQuery
......
"""
SOURCE
"""Status module
"""
import json
import os
class StatusFactory():
"""Generate state of a given command each time it is called.
"""
@staticmethod
def generateStatus(command, status, args):
""" Generate a dictionary inside a json file containing status of the commands
"""Generate a dictionary inside a json file containing status of the commands
Args:
command (str): command name
status (str): state of the command ('Starting', 'Done', 'Not defined', 'Issue')
"""
from diva_evaluation_cli.bin.cli import public_subcommands
status_id = 0
path = os.path.dirname(__file__)
status_log = os.path.join(path, './command_history.json')
......@@ -23,13 +29,13 @@ class StatusFactory():
json_status[subcommand.command] = [{'status': 'not defined', 'id': status_id, 'args': None}]
status_id += 1
json_status['id'] = status_id
else:
else:
with open(status_log, 'r') as f:
json_status = json.load(f)
if command in public_subcommands:
json_status['id'] += 1
json_status[command.command].append({'status': status, 'id': json_status['id'], 'args': args})
with open(status_log, 'w+') as f:
json.dump(json_status, f)
"""
UTILS
"""Utils module
Can be used by any other module of the CLI
Determine if python is running in a virtual environment
"""
import sys
......
"""
SOURCE
"""Validate system module
Used by the command validate-system
"""
......@@ -12,6 +11,8 @@ import sys
root_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../../')
def validate_system():
"""Main validation method
"""
validate_structure()
validate_cli()
validate_container_output()
......@@ -20,6 +21,8 @@ def validate_system():
############################### structure #####################################
def validate_structure():
"""Validate structure of the CLI: should contain required directories.
"""
directories = os.listdir(root_path)
content = import_expected_result('expected_structure.txt')
logging.info("Structure validation")
......@@ -63,6 +66,10 @@ def validate_cli():
def import_expected_result(file_name):
""" Import expected validation result
Args:
file_name (str): Path to file to open in order to extract lines inside a list
"""
content = []
path = os.path.dirname(__file__)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment