Merge branch 'feature/lfs-external' into 'master'

Feature/lfs external

See merge request !628

CMake/External/External_iMSTKData.cmake

+#-----------------------------------------------------------------------------
+# Add External Project
+#-----------------------------------------------------------------------------
+set(iMSTKData_PREFIX ${CMAKE_BINARY_DIR}/External/iMSTKData)
+
+find_package(GitLFS QUIET)
+if (NOT GitLFS_FOUND)
+  message(FATAL_ERROR " Building the tests or the examples requires the iMSTK data, which in turn\
+  requires git-lfs, make sure it is installed on your system, on linux please run `sudo apt-get install git-lfs`")
+endif()
+
+set(copy_data_command
+  ${CMAKE_COMMAND} -E copy_directory
+  ${iMSTKData_PREFIX}/src/Data
+  ${CMAKE_INSTALL_PREFIX}/data
+  )
+
+# HS - Due to an issue where it seems that the repository does not get updated
+# we need to call fetch here to refresh the index, this way the checkout can 
+# succeed for the add_external_project call
+execute_process(
+  COMMAND ${GIT_EXECUTABLE} fetch --all
+  WORKING_DIRECTORY ${iMSTKData_PREFIX}/src
+  RESULT_VARIABLE error_code
+  )
+
+include(imstkAddExternalProject)
+imstk_add_external_project( iMSTKData
+  GIT_REPOSITORY  "https://gitlab.kitware.com/iMSTK/imstk-data.git"
+  GIT_TAG  "14824e3d53328ed6be481981959780f32881030b"
+  CONFIGURE_COMMAND ${SKIP_STEP_COMMAND}
+  BUILD_COMMAND ${SKIP_STEP_COMMAND}
+  INSTALL_COMMAND COMMAND ${copy_data_command}
+  DEPENDENCIES ""
+  #VERBOSE
+)
+

CMake/Utilities/ExternalData_config.cmake.in

-set(ExternalData_OBJECT_STORES "@ExternalData_OBJECT_STORES@")
-set(ExternalData_URL_TEMPLATES "@ExternalData_URL_TEMPLATES@")
-set(ExternalData_TIMEOUT_INACTIVITY "@ExternalData_TIMEOUT_INACTIVITY@")
-set(ExternalData_TIMEOUT_ABSOLUTE "@ExternalData_TIMEOUT_ABSOLUTE@")
-set(ExternalData_NO_SYMLINKS "@ExternalData_NO_SYMLINKS@")
-@_ExternalData_CONFIG_CODE@

CMake/Utilities/FindGitLFS.cmake

+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+#[=======================================================================[.rst:
+FindGit
+-------
+
+The module defines the following variables:
+
+``GIT_LFS_EXECUTABLE``
+  Path to Git command-line client.
+``GitLFS_FOUND``, ``GITLFS_FOUND``
+  True if the Git command-line client was found.
+``GIT_LFS_VERSION_STRING``
+  The version of Git found.
+
+.. versionadded:: 3.14
+  The module defines the following ``IMPORTED`` targets (when
+  :prop_gbl:`CMAKE_ROLE` is ``PROJECT``):
+
+``Git::Git``
+  Executable of the Git command-line client.
+
+Example usage:
+
+.. code-block:: cmake
+
+   find_package(Git)
+   if(GitLFS_FOUND)
+     message("Git found: ${GIT_LFS_EXECUTABLE}")
+   endif()
+#]=======================================================================]
+
+# Look for 'git-lfs' or 'eg' (easy git)
+#
+set(git_names git-lfs)
+
+# Prefer .cmd variants on Windows unless running in a Makefile
+# in the MSYS shell.
+#
+if(CMAKE_HOST_WIN32)
+  if(NOT CMAKE_GENERATOR MATCHES "MSYS")
+    set(git_names git-lfs.cmd git-lfs)
+    # GitHub search path for Windows
+    file(GLOB github_path
+      "$ENV{LOCALAPPDATA}/Github/PortableGit*/cmd"
+      "$ENV{LOCALAPPDATA}/Github/PortableGit*/bin"
+      )
+    # SourceTree search path for Windows
+    set(_git_sourcetree_path "$ENV{LOCALAPPDATA}/Atlassian/SourceTree/git_local/bin")
+  endif()
+endif()
+
+# First search the PATH and specific locations.
+find_program(GIT_LFS_EXECUTABLE
+  NAMES ${git_names}
+  PATHS ${github_path} ${_git_sourcetree_path}
+  DOC "Git command line client"
+  )
+
+if(CMAKE_HOST_WIN32)
+  # Now look for installations in Git/ directories under typical installation
+  # prefixes on Windows.  Exclude PATH from this search because VS 2017's
+  # command prompt happens to have a PATH entry with a Git/ subdirectory
+  # containing a minimal git not meant for general use.
+  find_program(GIT_LFS_EXECUTABLE
+    NAMES ${git_names}
+    PATH_SUFFIXES Git/cmd Git/bin
+    NO_SYSTEM_ENVIRONMENT_PATH
+    DOC "Git command line client"
+    )
+endif()
+
+mark_as_advanced(GIT_LFS_EXECUTABLE)
+
+unset(git_names)
+unset(_git_sourcetree_path)
+
+if(GIT_LFS_EXECUTABLE)
+  # Avoid querying the version if we've already done that this run. For
+  # projects that use things like ExternalProject or FetchContent heavily,
+  # this saving can be measurable on some platforms.
+  #
+  # This is an internal property, projects must not try to use it.
+  # We don't want this stored in the cache because it might still change
+  # between CMake runs, but it shouldn't change during a run for a given
+  # git executable location.
+  set(__doGitVersionCheck TRUE)
+  get_property(__gitVersionProp GLOBAL
+    PROPERTY _CMAKE_FindGit_GIT_LFS_EXECUTABLE_VERSION
+  )
+  if(__gitVersionProp)
+    list(GET __gitVersionProp 0 __gitExe)
+    list(GET __gitVersionProp 1 __gitVersion)
+    if(__gitExe STREQUAL GIT_LFS_EXECUTABLE AND NOT __gitVersion STREQUAL "")
+      set(GIT_LFS_VERSION_STRING "${__gitVersion}")
+      set(__doGitVersionCheck FALSE)
+    endif()
+    unset(__gitExe)
+    unset(__gitVersion)
+  endif()
+  unset(__gitVersionProp)
+
+  if(__doGitVersionCheck)
+    execute_process(COMMAND ${GIT_LFS_EXECUTABLE} --version
+                    OUTPUT_VARIABLE git_version
+                    ERROR_QUIET
+                    OUTPUT_STRIP_TRAILING_WHITESPACE)
+    if (git_version MATCHES "^git/[0-9\.]?")
+      string(REPLACE "git version " "" GIT_LFS_VERSION_STRING "${git_version}")
+      set_property(GLOBAL PROPERTY _CMAKE_FindGit_GIT_LFS_EXECUTABLE_VERSION
+        "${GIT_LFS_EXECUTABLE};${GIT_LFS_VERSION_STRING}"
+      )
+    endif()
+    unset(git_version)
+  endif()
+  unset(__doGitVersionCheck)
+
+  get_property(_findgit_role GLOBAL PROPERTY CMAKE_ROLE)
+  if(_findgit_role STREQUAL "PROJECT" AND NOT TARGET Git::GitLFS)
+    add_executable(Git::GitLFS IMPORTED)
+    set_property(TARGET Git::GitLFS PROPERTY IMPORTED_LOCATION "${GIT_LFS_EXECUTABLE}")
+  endif()
+  unset(_findgit_role)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(GitLFS
+                                  REQUIRED_VARS GIT_LFS_EXECUTABLE
+                                  VERSION_VAR GIT_LFS_VERSION_STRING)

CMake/Utilities/imstkExternalData.cmake

-include(ExternalData)
-
-if(NOT ExternalData_OBJECT_STORES)
-  # Use ExternalData_OBJECT_STORES from environment as default.
-  set(ExternalData_OBJECT_STORES_DEFAULT "")
-  if(DEFINED "ENV{ExternalData_OBJECT_STORES}")
-    file(TO_CMAKE_PATH "$ENV{ExternalData_OBJECT_STORES}" ExternalData_OBJECT_STORES_DEFAULT)
-  endif()
-endif()
-
-set(ExternalData_OBJECT_STORES "${ExternalData_OBJECT_STORES_DEFAULT}" CACHE STRING
-  "Semicolon-separated list of local directories holding data objects in the layout %(algo)/%(hash).")
-mark_as_advanced(ExternalData_OBJECT_STORES)
-if(NOT ExternalData_OBJECT_STORES)
-  set(ExternalData_OBJECT_STORES "${CMAKE_BINARY_DIR}/ExternalData/Objects")
-  file(MAKE_DIRECTORY "${ExternalData_OBJECT_STORES}")
-endif()
-list(APPEND ExternalData_OBJECT_STORES
-  # Local data store populated by the iMSTK pre-commit hook
-  "${CMAKE_SOURCE_DIR}/.ExternalData"
-  )
-
-set(ExternalData_URL_TEMPLATES "" CACHE STRING
-  "Additional URL templates for the ExternalData CMake script to look for testing data. E.g.
-file:///var/bigharddrive/%(algo)/%(hash)")
-mark_as_advanced(ExternalData_URL_TEMPLATES)
-list(APPEND ExternalData_URL_TEMPLATES
-  # Data published on Girder
-  "https://data.kitware.com:443/api/v1/file/hashsum/%(algo)/%(hash)/download")
-
-# Tell ExternalData commands to transform raw files to content links.
-# TODO: Condition this feature on presence of our pre-commit hook.
-set(ExternalData_LINK_CONTENT SHA512)
-
-# Emscripten currently has difficulty reading symlinks.
-if(EMSCRIPTEN)
-  set(ExternalData_NO_SYMLINKS 1)
-endif()
-
-# Match series of the form <base>.<ext>, <base>.<n>.<ext> such that <base> may
-# end in a (test) number that is not part of any series numbering.
-set(ExternalData_SERIES_PARSE "()(\\.[^./]*)$")
-set(ExternalData_SERIES_MATCH "(\\.[0-9]+)?")
-
-# Sometimes we want to download very large files.
-set(ExternalData_TIMEOUT_INACTIVITY 60)
-set(ExternalData_SOURCE_ROOT "${iMSTK_SOURCE_DIR}/Data")
-set(ExternalData_BINARY_ROOT "${CMAKE_INSTALL_PREFIX}/data/")
-
-# Define the path to the data root directory
-add_definitions( -DiMSTK_DATA_ROOT=\"${ExternalData_BINARY_ROOT}\")
-
-# Function to upload data from list of data file
-function(imstk_add_data target)
-  # Download data
-  foreach(file IN LISTS ARGN)
-    set(datalist "${datalist} DATA{${ExternalData_SOURCE_ROOT}/${file}}")
-  endforeach()
-
-  ExternalData_expand_arguments(${target}ExternalData
-    COMMAND ${datalist}
-  )
-
-  # Add a build target to populate the real data.
-  ExternalData_Add_Target(${target}ExternalData)
-  
-  #-----------------------------------------------------------------------------
-  # Add the target to Examples/Data folder
-  #-----------------------------------------------------------------------------
-  SET_TARGET_PROPERTIES (${target}ExternalData PROPERTIES FOLDER Examples/Data)
-endfunction()

CMake/Utilities/imstkExternalDataDownloadTest.cmake

-#-----------------------------------------------------------------------------
-# Add ExternalData
-#-----------------------------------------------------------------------------
-# Remove the data if already here
-set(hashStampPath "${ExternalData_BINARY_ROOT}/Data/Testing/ExternalDataTest.txt-hash-stamp")
-if(EXISTS hashStampPath)
-  file(READ hashStampPath hash)
-  string(STRIP "${hash}" hash)
-endif()
-execute_process(
-    COMMAND rm ${ExternalData_BINARY_ROOT}/Data/Testing/ExternalDataTest.txt
-    COMMAND rm ${ExternalData_BINARY_ROOT}/Data/Testing/ExternalDataTest.txt-hash-stamp
-    COMMAND rm ${ExternalData_BINARY_ROOT}/Objects/SHA512/${hash}
-    OUTPUT_QUIET
-    ERROR_QUIET)
-
-# Download the data
-imstk_add_data(ExternalDataTest "Testing/ExternalDataTest.txt")
-
-#-----------------------------------------------------------------------------
-# Add Test
-#-----------------------------------------------------------------------------
-# Check if the data has been correctly downloaded
-add_test(
-  NAME imstkExternalDataTest
-  COMMAND ${CMAKE_COMMAND}
-  -DFileToCheck=${ExternalData_BINARY_ROOT}/Testing/ExternalDataTest.txt
-  -P ${CMAKE_SOURCE_DIR}/CMake/Utilities/imstkCheckFileExists.cmake
-)

CMakeLists.txt

@@ -134,7 +134,12 @@ if(${PROJECT_NAME}_SUPERBUILD)
 
   option(${PROJECT_NAME}_USE_Uncrustify "Use Uncrustify as a code style beautifier." ON)
   if(${PROJECT_NAME}_USE_Uncrustify)
-    imstk_define_dependency(Uncrustify)
+    find_package (Python COMPONENTS Interpreter)
+    if(Python_Interpreter_FOUND)
+      imstk_define_dependency(Uncrustify)
+    else()
+      message(WARN "Uncrustify requires a python interpreter, please install python and rerun configure")
+    endif()
   endif()
 
   option(${PROJECT_NAME}_USE_OpenHaptics "Use OpenHaptic Support." OFF)
@@ -184,6 +189,10 @@ if(${PROJECT_NAME}_SUPERBUILD)
     )
   endif()
 
+  if (${PROJECT_NAME}_BUILD_TESTING OR ${PROJECT_NAME}_BUILD_EXAMPLES)
+    imstk_define_dependency(iMSTKData)
+  endif()
+
   #-----------------------------------------------------------------------------
   # Solve project dependencies
   #-----------------------------------------------------------------------------
@@ -313,6 +322,21 @@ else(Uncrustify_EXECUTABLE)
   message(WARNING "uncrustify not found! Cannot run code-style test.")
 endif(Uncrustify_EXECUTABLE)
 
+
+#-----------------------------------------------------------------------------
+# Data
+#----------------------------------------------------------------------
+if (${PROJECT_NAME}_BUILD_TESTING OR ${PROJECT_NAME}_BUILD_EXAMPLES)
+  add_definitions( -DiMSTK_DATA_ROOT=\"${CMAKE_INSTALL_PREFIX}/data/\")
+endif()
+
+# Note if the target name or data name changes this has to be changed as well
+add_custom_target(CopyDataFiles ALL
+    COMMAND ${CMAKE_COMMAND} -E copy_directory
+    ${CMAKE_BINARY_DIR}/../External/iMSTKData/src/Data
+    ${CMAKE_INSTALL_PREFIX}/data
+    )
+
 #-----------------------------------------------------------------------------
 # Testing
 #----------------------------------------------------------------------
@@ -335,21 +359,6 @@ if(${PROJECT_NAME}_COVERAGE)
   )
 endif()
 
-#-----------------------------------------------------------------------------
-# Download the external data needed for both testing and examples
-#-----------------------------------------------------------------------------
-if(${PROJECT_NAME}_BUILD_TESTING OR ${PROJECT_NAME}_BUILD_EXAMPLES)
-  include(imstkExternalData)
-  include(imstkExternalDataDownloadTest)
-  # Create a target for all the data files
-  file(GLOB_RECURSE FILE_LIST
-    LIST_DIRECTORIES FALSE
-    RELATIVE "${CMAKE_SOURCE_DIR}/Data/"
-    "${CMAKE_SOURCE_DIR}/Data/*")
-  string(REGEX REPLACE "\.sha512" "" FILE_LIST "${FILE_LIST}")
-  imstk_add_data(${PROJECT_NAME} ${FILE_LIST})
-endif()
-
 # Folder name to put our headers/cmake config files under
 set(${PROJECT_NAME}_INSTALL_FOLDER ${PROJECT_NAME}-${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR})
 

Data/IBL/roomBRDF.png.sha512

-1bb1fc2e26993f1528b23b7aff5ca36b45ebfa9dbbaac906704fb446327f506eeb8208dbd165891c949e68fc2ee0519da2a20bd2f182413db7d5e3bee0c8c23f

Data/IBL/roomIrradiance.dds.sha512

-45b101fee95153ce5f74d019b45f13639c1d4983c7d9972e16e3cceda9e2cefa505cd0a7572642bcbe66533d7a3bd984b01bd0918b403770980fe98372c8b84e

Data/IBL/roomRadiance.dds.sha512

-f67966d950f179347629b215213f0301ae5ca7fec72ce295b92c89f24938e2668343a2fb1de2ca30522779cf12f4bc10ef8755bdd1424c78282d4698dd2bb530

Data/PhysiologyState/StandardMale.json.sha512

-c9b1becf3219bf869568993199be4267203518aef23f7e08520c1bfa5f783d8f25157a7d113d37373f52f107d344e23046eecc06e49b3fdd79d76630ab4cc56c

Data/Surgical Instruments/Instrument Table/Instrument_Table.dae.sha512

-722f997b73f0cce2e7120a45350779536131143256a69a29ef599694f2ff2480ec1a576aa0016200d7a060d0c2b538b709cc6acc8cb17ae4828dda3c81ac4c53
\ No newline at end of file

Data/Surgical Instruments/OR Table/OR_Table.dae.sha512

-3cfd2815477823753116abe1faf56dd3f0764dc2740a58d8bfd9d39039fa3dd52bf2f48f6f419fee5f4b54790f03f2ef9fef5d227c84815b1e2a0df3b40280d2
\ No newline at end of file

Data/Surgical Instruments/Scalpel/Scalpel_Albedo.png.sha512

-ad0495eb4b68162adeff1cd199ae87896e1cb9707d99109160fc339fc06b7410b629d4044d537e632d4b30c2c170def129c2f729e5fb3ef9441f7a6c32c35e4d
\ No newline at end of file

Data/Surgical Instruments/Scalpel/Scalpel_Blade10.dae.sha512

-de81db98099d447f73a068553b91cf13b86067126a0f356248325cd9bbcdae466405b6eca7dce5b874de7dfc11979270fe53622437a3ee65b8182bd86e96c5b7
\ No newline at end of file

Data/Surgical Instruments/Scalpel/Scalpel_Blade11.dae.sha512

-35de8f05bdb4e0ae0e93c4caea21d3644fdcaa59a607e2a1e6bbcda76e0dbeabf364ad39dbd747b3e2ee1776e9cc1a938903f0c216e64f6ceaec85e5a4c59e43
\ No newline at end of file

Data/Surgical Instruments/Scalpel/Scalpel_Blade15.dae.sha512

-632f9e33d3279c8116ead86c785fb5758229b477169233f2eb92c3c7ebd796c0f68abdb18a5e0b10b42d6e540931319cc0a5419d8a57f5fab96c2ece0976a48b
\ No newline at end of file

Data/Surgical Instruments/Scalpel/Scalpel_Handle.dae.sha512

-1dfb0e86575555b41abfabdf456fc614036f7acfc7034456e17fe91e7c91d04b888d2084c425127bde4adf9b2d9e292d360682468f9839a6f7be53b9b44f7171
\ No newline at end of file

Data/Surgical Instruments/Scalpel/Scalpel_Hull_Subdivided.stl.sha512

-43595d92c9f570a2a35f1bde65239335efc4ffb5625bd3416e0d5023b7e07b34f8e430e1d09c986a05cb8cdfc35e58d1813e969eed685b72f9865eaade52e1e0
\ No newline at end of file

Data/Surgical Instruments/Scalpel/Scalpel_Hull_Subdivided_Shifted.stl.sha512

-51c7a8bc4249b3c07567895b283f13ebdd2bb8a3e1cc6e0315f298958b16aae96c4c4fd2b2630b650441fb416018324d39bfc2206ebe066a0e4678ba0e20c5bf
\ No newline at end of file

Data/Testing/ExternalDataTest.txt.sha512

-cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e
\ No newline at end of file