Commit 267b7f72 authored by Mayeul Chassagnard's avatar Mayeul Chassagnard

ENH: Add ExternalData Management System

Download Data from Girder according to data key files contained in iMSTK/Data

This uses ExternalData.cmake (Requires CMake >= 2.8.11 release).

Add TEST: Add ExternalDataTest to verify downloaded files from Girder
* Be sure the data requiered for the test is not there (delete it)
* Add the imstk_add_data(${target} ${data_list}) function in CMakeList.txt
which calls ExternalData_expand_arguments().
* It downloads the data provided by the data_list (relative path from
iMSTK/Data)

Add DOC: Add documentation on ExternalData Management
* How to add data to iMSTK from Grider
* Update data
* Use data
parent 021f9f03
This diff is collapsed.
set(ExternalData_OBJECT_STORES "@ExternalData_OBJECT_STORES@")
set(ExternalData_URL_TEMPLATES "@ExternalData_URL_TEMPLATES@")
set(ExternalData_TIMEOUT_INACTIVITY "@ExternalData_TIMEOUT_INACTIVITY@")
set(ExternalData_TIMEOUT_ABSOLUTE "@ExternalData_TIMEOUT_ABSOLUTE@")
set(ExternalData_NO_SYMLINKS "@ExternalData_NO_SYMLINKS@")
@_ExternalData_CONFIG_CODE@
if(EXISTS ${FileToCheck})
message(STATUS "${FileToCheck} exists.")
else()
message(SEND_ERROR "${FileToCheck} doesn't exist.")
endif()
get_filename_component(_imstkExternalData_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
include(${_imstkExternalData_DIR}/ExternalData.cmake)
if(NOT ExternalData_OBJECT_STORES)
# Use ExternalData_OBJECT_STORES from environment as default.
set(ExternalData_OBJECT_STORES_DEFAULT "")
if(DEFINED "ENV{ExternalData_OBJECT_STORES}")
file(TO_CMAKE_PATH "$ENV{ExternalData_OBJECT_STORES}" ExternalData_OBJECT_STORES_DEFAULT)
endif()
endif()
set(ExternalData_OBJECT_STORES "${ExternalData_OBJECT_STORES_DEFAULT}" CACHE STRING
"Semicolon-separated list of local directories holding data objects in the layout %(algo)/%(hash).")
mark_as_advanced(ExternalData_OBJECT_STORES)
if(NOT ExternalData_OBJECT_STORES)
set(ExternalData_OBJECT_STORES "${CMAKE_BINARY_DIR}/ExternalData/Objects")
file(MAKE_DIRECTORY "${ExternalData_OBJECT_STORES}")
endif()
list(APPEND ExternalData_OBJECT_STORES
# Local data store populated by the iMSTK pre-commit hook
"${CMAKE_SOURCE_DIR}/.ExternalData"
)
set(ExternalData_BINARY_ROOT ${CMAKE_BINARY_DIR}/ExternalData)
# Expands %(algo:lower)
set(ExternalData_URL_ALGO_MD5_lower md5)
set(ExternalData_URL_TEMPLATES "" CACHE STRING
"Additional URL templates for the ExternalData CMake script to look for testing data. E.g.
file:///var/bigharddrive/%(algo)/%(hash)")
mark_as_advanced(ExternalData_URL_TEMPLATES)
list(APPEND ExternalData_URL_TEMPLATES
# Data published on Girder
"https://data.kitware.com:443/api/v1/file/hashsum/%(algo)/%(hash)/download")
# Tell ExternalData commands to transform raw files to content links.
# TODO: Condition this feature on presence of our pre-commit hook.
set(ExternalData_LINK_CONTENT MD5)
# Emscripten currently has difficulty reading symlinks.
if(EMSCRIPTEN)
set(ExternalData_NO_SYMLINKS 1)
endif()
# Match series of the form <base>.<ext>, <base>.<n>.<ext> such that <base> may
# end in a (test) number that is not part of any series numbering.
set(ExternalData_SERIES_PARSE "()(\\.[^./]*)$")
set(ExternalData_SERIES_MATCH "(\\.[0-9]+)?")
# Sometimes we want to download very large files.
set(ExternalData_TIMEOUT_ABSOLUTE 900)
# Configure the default iMSTK_DATA_CONTENT_LINKS_PATH for the location of iMSTK Data content links.
set(iMSTK_DATA_CONTENT_LINKS_PATH "${iMSTK_SOURCE_DIR}/Data")
# Define the path to the data root directory
add_definitions( -DiMSTK_DATA_ROOT=\"${ExternalData_BINARY_ROOT}/Data\")
# Function to upload data from list of data file
function(imstk_add_data target)
# Download data
foreach(file IN LISTS ARGN)
set(datalist "${datalist} DATA{${iMSTK_DATA_CONTENT_LINKS_PATH}/${file}}")
endforeach()
ExternalData_expand_arguments(${target}ExternalData
COMMAND ${datalist}
)
# Add a build target to populate the real data.
ExternalData_Add_Target(${target}ExternalData)
endfunction()
......@@ -13,6 +13,11 @@
* [Removing previous commit changes](#removing-previous-commit-changes)
* [Editing older commits](#editing-older-commits)
* [Syncing repositories](#syncing-repositories)
* [Data Management](#data-management)
* [Add Data](#add-data)
* [Fetch Data for your examples](#fetch-data-for-your-examples)
* [Use external data in your examples](#use-external-data-in-your-examples)
* [Update your Data](#update-your-data)
* [Dashboards](#dashboards)
......@@ -272,6 +277,38 @@ This process consists of rewriting the branch history, therefore re-syncing it w
```
## Data Management
------
* ### Add Data
The proper way to add data to iMSTK, is to:
- Contact a project manager to upload your data in the [Girder iMSTK collection](https://data.kitware.com/#collection/58ab34918d777f073240dc02).
- Add the key file to `iMSTK-Source/Data` (Download the key file from girder as shown on the following picture).
![Download Key File](https://data.kitware.com/api/v1/item/58acc5588d777f0aef5cff96/download?contentDisposition=inline)
* ### Fetch Data for your examples
To fetch the external data before building your example, you will need to list it using `imstk_add_data` in your example CMakeLists.txt:
```cmake
imstk_add_data(exampleExecutableName dataList)
```
> * `exampleExecutableName` is usually `${PROJECT_NAME}` in an example CMakeLists.txt.
> * `dataList` need to contain the list of data names which are needed in the example. Those names are relative to `iMSTK-Source/Data`.
> * Regular expressions can be used to populate the file list using [REGEX](https://cmake.org/cmake/help/v3.8/module/ExternalData.html#referencing-associated-files).
The data will be downloaded from Girder to `iMSTK-build/Innerbuild/ExternalData/Data`: the `Objects` directory is the datastore, and the `Data` directory holds symbolic links to that data with expected file names.
* ### Use external data in your examples
The path to the data directory is defined by iMSTK_DATA_ROOT which can be used in your C++ code as shown below :
```c++
imstk::MeshIO::read(iMSTK_DATA_ROOT"/relative/path/to/mesh.vtk");
```
* ### Update your Data
- Contact a project manager to update your data in the [Girder iMSTK collection](https://data.kitware.com/#collection/58ab34918d777f073240dc02).
- Update the key file to `iMSTK-Source/Data` as described in the [Add Data section](#add-data).
## Dashboards
------
......
cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e
\ No newline at end of file
#-----------------------------------------------------------------------------
# ExternalData module
#-----------------------------------------------------------------------------
include(imstkExternalData)
#-----------------------------------------------------------------------------
# Add ExternalData
#-----------------------------------------------------------------------------
# Remove the data if already here
set(hashStampPath "${ExternalData_BINARY_ROOT}/Data/Testing/ExternalDataTest.txt-hash-stamp")
if(EXISTS hashStampPath)
file(READ hashStampPath hash)
string(STRIP "${hash}" hash)
endif()
execute_process(
COMMAND rm ${ExternalData_BINARY_ROOT}/Data/Testing/ExternalDataTest.txt
COMMAND rm ${ExternalData_BINARY_ROOT}/Data/Testing/ExternalDataTest.txt-hash-stamp
COMMAND rm ${ExternalData_BINARY_ROOT}/Objects/SHA512/${hash}
OUTPUT_QUIET
ERROR_QUIET)
# Download the data
imstk_add_data(ExternalDataTest "Testing/ExternalDataTest.txt")
#-----------------------------------------------------------------------------
# Add Test
#-----------------------------------------------------------------------------
# Check if the data has been correctly downloaded
add_test(
NAME imstkExternalDataTest
COMMAND ${CMAKE_COMMAND}
-DFileToCheck=${ExternalData_BINARY_ROOT}/Data/Testing/ExternalDataTest.txt
-P ${CMAKE_SOURCE_DIR}/CMake/Utilities/imstkCheckFileExists.cmake
)
#-----------------------------------------------------------------------------
# Add Sandbox subdirectory
#-----------------------------------------------------------------------------
add_subdirectory(Sandbox)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment