diff --git a/.readthedocs.yml b/.readthedocs.yml index de7f9fc..2e5a04c 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,4 +1,4 @@ -# Read the Docs configuration file for IMAS-Matlab +# Read the Docs configuration file for IMAS-MATLAB # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details # Required @@ -10,21 +10,6 @@ build: tools: python: "3.11" - jobs: - post_checkout: - # Clone IMAS-Core (only if not already present) - - | - if [ ! -d "IMAS-Core" ]; then - git clone --depth 1 --branch develop \ - https://github.com/iterorganization/IMAS-Core.git \ - IMAS-Core - fi - - ls -la - - ln -sfn $PWD/IMAS-Core/common/doc_common $PWD/doc/doc_common - - ls -la doc/ - - ls -la doc/doc_common/ - - echo "Created symlink doc/doc_common -> IMAS-Core/common/doc_common" - # Build documentation in the "doc" directory with Sphinx sphinx: configuration: doc/conf.py @@ -35,11 +20,11 @@ formats: - pdf - epub -# Optional but recommended, declare the Python requirements required +# Optional but Python requirements required recommended, declare the # to build your documentation python: install: - - requirements: IMAS-Core/common/doc_common/requirements.txt + - requirements: doc/doc_common/requirements.txt # Submodules configuration (optional) submodules: diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d3a520..9bc9a89 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,76 +20,70 @@ if(WIN32) endif() endif() -# Configuration options for common assets +# Local paths for IMAS-MATLAB ################################################################################ +# Include local paths to eliminate AL_COMMON_PATH dependency +# Use full path since CMAKE_MODULE_PATH hasn't been configured yet +include(${CMAKE_CURRENT_SOURCE_DIR}/common/cmake/ALLocalPaths.cmake) option( AL_DOWNLOAD_DEPENDENCIES "Automatically download assets from the AL git repository" ON ) set( AL_CORE_GIT_REPOSITORY "https://github.com/iterorganization/IMAS-Core.git" CACHE STRING "Git repository of AL-core" ) set( AL_CORE_VERSION "main" CACHE STRING "Git commit/tag/branch of AL-core" ) + include(FetchContent) -# Load common assets +# Load AL-core library (used for the shared library) ################################################################################ -if( DEFINED ENV{AL_COMMON_PATH} ) - # Take common assets from the path in this environment variable instead of al-core - set( AL_COMMON_PATH $ENV{AL_COMMON_PATH} ) -else() - if(WIN32) - if( ${AL_DOWNLOAD_DEPENDENCIES} ) - # FetchContent_Declare causing reursive call wrapped by vcpkg to _find_package - # So manually clone al-core here. Need fix in later release - set( al-core_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/_deps/al-core-src" ) - if( NOT EXISTS "${al-core_SOURCE_DIR}/.git" ) - message( STATUS "Cloning al-core from ${AL_CORE_GIT_REPOSITORY}" ) - execute_process( - COMMAND git clone "${AL_CORE_GIT_REPOSITORY}" "${al-core_SOURCE_DIR}" - RESULT_VARIABLE _GIT_CLONE_RESULT - ERROR_VARIABLE _GIT_CLONE_ERROR - ) - if( _GIT_CLONE_RESULT ) - message( FATAL_ERROR "Failed to clone al-core: ${_GIT_CLONE_ERROR}" ) - endif() - endif() - # Checkout the specified version - execute_process( - COMMAND git fetch origin - WORKING_DIRECTORY "${al-core_SOURCE_DIR}" - RESULT_VARIABLE _GIT_FETCH_RESULT - ) +if(WIN32) + if( ${AL_DOWNLOAD_DEPENDENCIES} ) + # FetchContent_Declare causing reursive call wrapped by vcpkg to _find_package + # So manually clone al-core here. Need fix in later release + set( al-core_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/_deps/al-core-src" ) + if( NOT EXISTS "${al-core_SOURCE_DIR}/.git" ) + message( STATUS "Cloning al-core from ${AL_CORE_GIT_REPOSITORY}" ) execute_process( - COMMAND git checkout "${AL_CORE_VERSION}" - WORKING_DIRECTORY "${al-core_SOURCE_DIR}" - RESULT_VARIABLE _GIT_CHECKOUT_RESULT - ERROR_VARIABLE _GIT_CHECKOUT_ERROR + COMMAND git clone "${AL_CORE_GIT_REPOSITORY}" "${al-core_SOURCE_DIR}" + RESULT_VARIABLE _GIT_CLONE_RESULT + ERROR_VARIABLE _GIT_CLONE_ERROR ) - if( _GIT_CHECKOUT_RESULT ) - message( FATAL_ERROR "Failed to checkout ${AL_CORE_VERSION}: ${_GIT_CHECKOUT_ERROR}" ) - endif() - else() - set( al-core_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../al-core" ) - if( NOT IS_DIRECTORY "${al-core_SOURCE_DIR}" ) - message( FATAL_ERROR "${al-core_SOURCE_DIR} does not exist. Set AL_DOWNLOAD_DEPENDENCIES=ON to download." ) + if( _GIT_CLONE_RESULT ) + message( FATAL_ERROR "Failed to clone al-core: ${_GIT_CLONE_ERROR}" ) endif() endif() - set( AL_COMMON_PATH "${al-core_SOURCE_DIR}/common" ) - else() - if( ${AL_DOWNLOAD_DEPENDENCIES} ) - # Download common assets from the ITER git: - FetchContent_Declare( - al-core - GIT_REPOSITORY "${AL_CORE_GIT_REPOSITORY}" - GIT_TAG "${AL_CORE_VERSION}" - ) - else() - FetchContent_Declare( - al-core - SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../al-core" - ) + # Checkout the specified version + execute_process( + COMMAND git fetch origin + WORKING_DIRECTORY "${al-core_SOURCE_DIR}" + RESULT_VARIABLE _GIT_FETCH_RESULT + ) + execute_process( + COMMAND git checkout "${AL_CORE_VERSION}" + WORKING_DIRECTORY "${al-core_SOURCE_DIR}" + RESULT_VARIABLE _GIT_CHECKOUT_RESULT + ERROR_VARIABLE _GIT_CHECKOUT_ERROR + ) + if( _GIT_CHECKOUT_RESULT ) + message( FATAL_ERROR "Failed to checkout ${AL_CORE_VERSION}: ${_GIT_CHECKOUT_ERROR}" ) endif() + elseif ( ${AL_DEVELOPMENT_LAYOUT} ) + set( al-core_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../al-core" ) + endif() +else() + if( ${AL_DOWNLOAD_DEPENDENCIES} ) + # Download common assets from the ITER git: + FetchContent_Declare( + al-core + GIT_REPOSITORY "${AL_CORE_GIT_REPOSITORY}" + GIT_TAG "${AL_CORE_VERSION}" + ) + FetchContent_MakeAvailable( al-core ) + elseif ( ${AL_DEVELOPMENT_LAYOUT} ) + FetchContent_Declare( + al-core + SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../al-core" + ) FetchContent_MakeAvailable( al-core ) - set( AL_COMMON_PATH "${al-core_SOURCE_DIR}/common" ) - endif() + endif() + endif() -add_subdirectory( ${AL_COMMON_PATH} _common ) - # Define project ################################################################################ @@ -120,8 +114,6 @@ include( ALCore ) if( AL_DOCS_ONLY ) return() endif() - - # Dependencies ################################################################################ find_package( Python3 REQUIRED COMPONENTS Interpreter ) @@ -130,8 +122,6 @@ if(WIN32) else() find_package( Matlab REQUIRED ) endif() - - # Utility sources and target ################################################################################ @@ -149,7 +139,6 @@ if(WIN32) target_link_libraries( al-mex PUBLIC al Matlab::mex Matlab::mx ) target_include_directories( al-mex PRIVATE src ) target_include_directories( al-mex PUBLIC ${Matlab_INCLUDE_DIRS} ) - target_include_directories( al-mex PUBLIC ${AL_COMMON_PATH}/../include ) # Use separate (non-interleaved) complex API for compatibility target_compile_definitions( al-mex PUBLIC) @@ -238,7 +227,7 @@ macro( GENERATE GENERATOR IS_C_GENERATOR OUTPUT_FILES ) add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${GENERATOR}_dummy.txt COMMAND - ${_SAXONCHE_VENV_PYTHON} "${AL_COMMON_PATH}/xsltproc.py" + ${_SAXONCHE_VENV_PYTHON} "${AL_LOCAL_XSLTPROC_SCRIPT}" -xsl ${CMAKE_CURRENT_SOURCE_DIR}/${GENERATOR}.xsl -s ${IDSDEF} -o ${CMAKE_CURRENT_BINARY_DIR}/${GENERATOR}_output.xml @@ -273,6 +262,8 @@ if(WIN32) target_include_directories( "mex-${NAME}" PRIVATE src ) # Use separate (non-interleaved) complex API for compatibility target_compile_definitions( "mex-${NAME}" PUBLIC) + # Add /bigobj flag for large generated code files + target_compile_options( "mex-${NAME}" PRIVATE /bigobj ) list( APPEND MEX_TARGETS "mex-${NAME}" ) endmacro() else() @@ -283,8 +274,6 @@ else() list( APPEND MEX_TARGETS "mex-${NAME}" ) endmacro() endif() - - # These XSLs generate files in triplets, which are then compiled into one mex target foreach( METHOD_NAME allocate delete gen get get_sample get_slice init put put_slice rand validate ) FILES_FROM_METHOD_NAME( OUTPUT_FILES ${METHOD_NAME} ) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 054e0a7..7fdb256 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,23 +1,23 @@ # Contributing guidelines -We welcome any kind of contribution to `IMAS-Matlab`, +We welcome any kind of contribution to `IMAS-MATLAB`, from a simple comment, a question or even a full fledged pull request. Please first make sure you read and follow the [Code of Conduct](CODE_OF_CONDUCT.md). ## You think you found a bug in the code, or have a question in its use -1. use the [issue search](https://github.com/iterorganization/IMAS-Matlab/issues) +1. use the [issue search](https://github.com/iterorganization/IMAS-MATLAB/issues) to check if someone already created a similar issue; 2. if not, make a **new issue** to describe your problem or question. In the case of a bug suspiscion, please try to give all the relevant information to allow reproducing the error or identifying -its root cause (version of the IMAS-Matlab, OS and relevant +its root cause (version of the IMAS-MATLAB, OS and relevant dependencies, snippet of code); 3. apply relevant labels to the issue. ## You want to make or ask some change to the code -1. use the [issue search](https://github.com/iterorganization/IMAS-Matlab/issues) +1. use the [issue search](https://github.com/iterorganization/IMAS-MATLAB/issues) to check if someone already proposed a similar idea/change; 2. if not, create a **new issue** to describe what change you would like to see implemented and specify it if you intend to work on it yourself or if some help diff --git a/ci/build_docs.sh b/ci/build_docs.sh index a4483f2..355d44a 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -51,7 +51,7 @@ CMAKE_ARGS=( -D AL_PLUGINS_VERSION=${AL_PLUGINS_VERSION:-main} -D AL_CORE_VERSION=${AL_CORE_VERSION:-main} -D DD_VERSION=${DD_VERSION:-main} - # Build only documentation + # Build only documentationv -D AL_HLI_DOCS=${AL_HLI_DOCS:-ON} -D AL_DOCS_ONLY=${AL_DOCS_ONLY:-ON} ) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt new file mode 100644 index 0000000..129ba5e --- /dev/null +++ b/common/CMakeLists.txt @@ -0,0 +1,27 @@ +# CMake configuration for AL common assets +# Intended for inclusion (using FetchContent_MakeAvailable) in other AL components + +# Add ./cmake to the module path in the parent scope: +set( CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake PARENT_SCOPE ) + +if ( ${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR} ) + message( FATAL_ERROR "In-source builds not supported. Please use a separate build directory, for example `cmake -B build [...]`" ) +endif() + +if( AL_PYTHON_BINDINGS ) + find_package( Python3 ) + set( PYVER "${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}" ) +endif() +configure_file( al_env.sh.in ${CMAKE_CURRENT_BINARY_DIR}/al_env.sh ) + +# Don't generate the environment file when building modules: +if( AL_DOWNLOAD_DEPENDENCIES OR AL_DEVELOPMENT_LAYOUT ) + install( + FILES ${CMAKE_CURRENT_BINARY_DIR}/al_env.sh + TYPE BIN + ) +endif() + +# al-common_SOURCE_DIR is used in some scripts in the cmake/ folder to refer to the +# current source directory: +set( al-common_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE ) diff --git a/common/al_env.sh.in b/common/al_env.sh.in new file mode 100644 index 0000000..e34bbf0 --- /dev/null +++ b/common/al_env.sh.in @@ -0,0 +1,7 @@ +export PATH="${CMAKE_INSTALL_PREFIX}/bin:$PATH" +export LD_LIBRARY_PATH="${CMAKE_INSTALL_PREFIX}/lib:$LD_LIBRARY_PATH" +export PKG_CONFIG_PATH="${CMAKE_INSTALL_PREFIX}/lib/pkgconfig:$PKG_CONFIG_PATH" + +if [ -d "${CMAKE_INSTALL_PREFIX}/mex" ]; then + export MATLABPATH="${CMAKE_INSTALL_PREFIX}/mex:$MATLABPATH" +fi diff --git a/common/cmake/ALBuildDataDictionary.cmake b/common/cmake/ALBuildDataDictionary.cmake new file mode 100644 index 0000000..f85e8e8 --- /dev/null +++ b/common/cmake/ALBuildDataDictionary.cmake @@ -0,0 +1,396 @@ +# Everything needed for building the Data Dictionary +# +# This script sets the following variables: +# IDSDEF Path to the generated IDSDef.xml +# IDS_NAMES List of IDSs that are available in the data dictionary +# DD_VERSION Version of the data dictionary +# DD_SAFE_VERSION DD version, safe to use as linker symbol + +if( AL_DOCS_ONLY ) + return() +endif() + +# Find Python for the xsltproc.py program +if(WIN32) + if(NOT Python3_FOUND AND NOT PYTHON_EXECUTABLE) + # Check if Python is in PATH + find_program(PYTHON_EXECUTABLE NAMES python3.exe python.exe python3 python DOC "Python interpreter") + if(NOT PYTHON_EXECUTABLE) + message(FATAL_ERROR "Could not find Python. Please ensure Python is installed and in PATH.") + endif() + else() + set(PYTHON_EXECUTABLE ${Python3_EXECUTABLE}) + endif() +else() + find_package(Python REQUIRED COMPONENTS Interpreter Development.Module) + set(PYTHON_EXECUTABLE ${Python_EXECUTABLE}) +endif() + +message(STATUS "Found Python: ${PYTHON_EXECUTABLE}") + +# Set up Python venv paths for saxonche (used for all XSLT transformations) +if(WIN32) + set(_VENV_PYTHON "${CMAKE_CURRENT_BINARY_DIR}/dd_build_env/Scripts/python.exe") + set(_VENV_PIP "${CMAKE_CURRENT_BINARY_DIR}/dd_build_env/Scripts/pip.exe") +else() + set(_VENV_PYTHON "${CMAKE_CURRENT_BINARY_DIR}/dd_build_env/bin/python") + set(_VENV_PIP "${CMAKE_CURRENT_BINARY_DIR}/dd_build_env/bin/pip") +endif() + +if( NOT AL_DOWNLOAD_DEPENDENCIES AND NOT AL_DEVELOPMENT_LAYOUT ) + # The DD easybuild module should be loaded, use that module: + # Create Python venv first and install imas_data_dictionary + if(NOT EXISTS "${_VENV_PYTHON}") + execute_process( + COMMAND ${PYTHON_EXECUTABLE} -m venv dd_build_env + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + RESULT_VARIABLE _VENV_EXITCODE + OUTPUT_VARIABLE _VENV_OUTPUT + ERROR_VARIABLE _VENV_ERROR + ) + + if(_VENV_EXITCODE) + message(STATUS "venv stdout: ${_VENV_OUTPUT}") + message(STATUS "venv stderr: ${_VENV_ERROR}") + message(FATAL_ERROR "Failed to create venv (exit code: ${_VENV_EXITCODE}). Ensure Python has venv module installed: python -m venv --help") + endif() + + if(DEFINED DD_VERSION) + execute_process( + COMMAND ${_VENV_PIP} install imas_data_dictionary==${DD_VERSION} + RESULT_VARIABLE _PIP_EXITCODE + OUTPUT_VARIABLE _PIP_OUTPUT + ERROR_VARIABLE _PIP_ERROR + ) + else() + execute_process( + COMMAND ${_VENV_PIP} install imas_data_dictionary + RESULT_VARIABLE _PIP_EXITCODE + OUTPUT_VARIABLE _PIP_OUTPUT + ERROR_VARIABLE _PIP_ERROR + ) + endif() + + if(_PIP_EXITCODE) + message(STATUS "imas_data_dictionary pip output: ${_PIP_OUTPUT}") + message(STATUS "imas_data_dictionary pip error: ${_PIP_ERROR}") + message(FATAL_ERROR "Failed to install imas_data_dictionary dependency (exit code: ${_PIP_EXITCODE}). Check network connectivity and Python wheel compatibility.") + endif() + + execute_process( + COMMAND ${_VENV_PIP} install saxonche + RESULT_VARIABLE _PIP_EXITCODE + OUTPUT_VARIABLE _PIP_OUTPUT + ERROR_VARIABLE _PIP_ERROR + ) + + if(_PIP_EXITCODE) + message(STATUS "saxonche pip output: ${_PIP_OUTPUT}") + message(STATUS "saxonche pip error: ${_PIP_ERROR}") + message(FATAL_ERROR "Failed to install saxonche dependency (exit code: ${_PIP_EXITCODE}). Check network connectivity and Python wheel compatibility.") + endif() + endif() +# Set up idsinfo command path +if(WIN32) + set(_IDSINFO_COMMAND "${CMAKE_CURRENT_BINARY_DIR}/dd_build_env/Scripts/idsinfo.exe") +else() + set(_IDSINFO_COMMAND "${CMAKE_CURRENT_BINARY_DIR}/dd_build_env/bin/idsinfo") +endif() + + # Use idsinfo idspath command from venv to get the path to IDSDef.xml or data_dictionary.xml + execute_process( + COMMAND ${_IDSINFO_COMMAND} idspath + OUTPUT_VARIABLE IDSDEF + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE _IDSINFO_EXITCODE + ) + + if( _IDSINFO_EXITCODE ) + message( FATAL_ERROR + "Failed to run 'idsinfo idspath' command. " + "Please ensure IMAS-Data-Dictionary module is loaded." + ) + endif() + + if( NOT EXISTS "${IDSDEF}" ) + message( FATAL_ERROR + "idsinfo idspath returned '${IDSDEF}' but file does not exist. " + "Please ensure IMAS-Data-Dictionary module is properly loaded." + ) + endif() + + message( STATUS "Found Data Dictionary: ${IDSDEF}" ) + + # Populate identifier source xmls based on the IDSDEF location + get_filename_component( DD_BASE_DIR "${IDSDEF}" DIRECTORY ) + + if( DD_BASE_DIR MATCHES "schemas$" ) + # DD 4.1.0+ layout: resources/schemas//*_identifier.xml + file( GLOB DD_IDENTIFIER_FILES "${DD_BASE_DIR}/*/*_identifier.xml" ) + else() + # DD 3.x/4.0.0 layout: dd_x.y.z/include//*_identifier.xml + file( GLOB DD_IDENTIFIER_FILES "${DD_BASE_DIR}/*/*_identifier.xml" ) + endif() + + if( NOT DD_IDENTIFIER_FILES ) + message( WARNING "No identifier XML files found in Data Dictionary at: ${IDSDEF}" ) + endif() + + # When using pre-installed DD, we still need venv for extracting IDS names and version + # Create Python venv and install saxonche if not already done + if(NOT EXISTS "${_VENV_PYTHON}") + execute_process( + COMMAND ${PYTHON_EXECUTABLE} -m venv dd_build_env + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + RESULT_VARIABLE _VENV_EXITCODE + OUTPUT_VARIABLE _VENV_OUTPUT + ERROR_VARIABLE _VENV_ERROR + ) + + if(_VENV_EXITCODE) + message(STATUS "venv stdout: ${_VENV_OUTPUT}") + message(STATUS "venv stderr: ${_VENV_ERROR}") + message(FATAL_ERROR "Failed to create venv (exit code: ${_VENV_EXITCODE}). Ensure Python has venv module installed: python -m venv --help") + endif() + + execute_process( + COMMAND ${_VENV_PIP} install saxonche + RESULT_VARIABLE _PIP_EXITCODE + OUTPUT_VARIABLE _PIP_OUTPUT + ERROR_VARIABLE _PIP_ERROR + ) + + if(_PIP_EXITCODE) + message(STATUS "saxonche pip output: ${_PIP_OUTPUT}") + message(STATUS "saxonche pip error: ${_PIP_ERROR}") + message(FATAL_ERROR "Failed to install saxonche dependency (exit code: ${_PIP_EXITCODE}). Check network connectivity and Python wheel compatibility.") + endif() + endif() +else() + if(WIN32) + # Build the DD from source using direct git commands: + if( AL_DOWNLOAD_DEPENDENCIES ) + # Download the Data Dictionary from the ITER git: + set( data-dictionary_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/_deps/data-dictionary-src" ) + if( NOT EXISTS "${data-dictionary_SOURCE_DIR}/.git" ) + message( STATUS "Cloning data-dictionary from ${DD_GIT_REPOSITORY}" ) + execute_process( + COMMAND git clone "${DD_GIT_REPOSITORY}" "${data-dictionary_SOURCE_DIR}" + RESULT_VARIABLE _GIT_CLONE_RESULT + ERROR_VARIABLE _GIT_CLONE_ERROR + ) + if( _GIT_CLONE_RESULT ) + message( FATAL_ERROR "Failed to clone data-dictionary: ${_GIT_CLONE_ERROR}" ) + endif() + endif() + # Checkout the specified version + execute_process( + COMMAND git fetch origin + WORKING_DIRECTORY "${data-dictionary_SOURCE_DIR}" + RESULT_VARIABLE _GIT_FETCH_RESULT + ) + execute_process( + COMMAND git checkout "${DD_VERSION}" + WORKING_DIRECTORY "${data-dictionary_SOURCE_DIR}" + RESULT_VARIABLE _GIT_CHECKOUT_RESULT + ERROR_VARIABLE _GIT_CHECKOUT_ERROR + ) + if( _GIT_CHECKOUT_RESULT ) + message( FATAL_ERROR "Failed to checkout ${DD_VERSION}: ${_GIT_CHECKOUT_ERROR}" ) + endif() + else() + # Look in ../data-dictionary for the data dictionary + if( NOT( AL_PARENT_FOLDER ) ) + set( AL_PARENT_FOLDER ${CMAKE_CURRENT_SOURCE_DIR}/.. ) + endif() + set( data-dictionary_SOURCE_DIR ${AL_PARENT_FOLDER}/data-dictionary ) + if( NOT IS_DIRECTORY ${data-dictionary_SOURCE_DIR} ) + message( FATAL_ERROR + "${data-dictionary_SOURCE_DIR} does not exist. Please clone the " + "data-dictionary repository or set AL_DOWNLOAD_DEPENDENCIES=ON." + ) + endif() + endif() + else() + # Build the DD from source: + include(FetchContent) + + if( AL_DOWNLOAD_DEPENDENCIES ) + # Download the Data Dictionary from the ITER git: + FetchContent_Declare( + data-dictionary + GIT_REPOSITORY ${DD_GIT_REPOSITORY} + GIT_TAG ${DD_VERSION} + ) + else() + # Look in ../data-dictionary for the data dictionary + if( NOT( AL_PARENT_FOLDER ) ) + set( AL_PARENT_FOLDER ${CMAKE_CURRENT_SOURCE_DIR}/.. ) + endif() + set( DD_SOURCE_DIRECTORY ${AL_PARENT_FOLDER}/data-dictionary ) + if( NOT IS_DIRECTORY ${DD_SOURCE_DIRECTORY} ) + message( FATAL_ERROR + "${DD_SOURCE_DIRECTORY} does not exist. Please clone the " + "data-dictionary repository or set AL_DOWNLOAD_DEPENDENCIES=ON." + ) + endif() + + FetchContent_Declare( + data-dictionary + SOURCE_DIR ${DD_SOURCE_DIRECTORY} + ) + set( DD_SOURCE_DIRECTORY ) # unset temporary var + endif() + FetchContent_MakeAvailable( data-dictionary ) + endif() + + + # get version of the data dictionary + execute_process( + COMMAND git describe --tags --always --dirty + WORKING_DIRECTORY ${data-dictionary_SOURCE_DIR} + OUTPUT_VARIABLE DD_GIT_DESCRIBE + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE _GIT_RESULT + ) + if(_GIT_RESULT) + execute_process( + COMMAND git rev-parse --short HEAD + WORKING_DIRECTORY ${data-dictionary_SOURCE_DIR} + OUTPUT_VARIABLE DD_GIT_DESCRIBE + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + endif() + + # We need the IDSDef.xml at configure time, ensure it is built + # Create Python venv and install saxonche if not already done + if(NOT EXISTS "${_VENV_PYTHON}") + execute_process( + COMMAND ${PYTHON_EXECUTABLE} -m venv dd_build_env + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + RESULT_VARIABLE _VENV_EXITCODE + OUTPUT_VARIABLE _VENV_OUTPUT + ERROR_VARIABLE _VENV_ERROR + ) + + if(_VENV_EXITCODE) + message(STATUS "venv stdout: ${_VENV_OUTPUT}") + message(STATUS "venv stderr: ${_VENV_ERROR}") + message(FATAL_ERROR "Failed to create venv (exit code: ${_VENV_EXITCODE}). Ensure Python has venv module installed: python -m venv --help") + endif() + + execute_process( + COMMAND ${_VENV_PIP} install saxonche + RESULT_VARIABLE _PIP_EXITCODE + OUTPUT_VARIABLE _PIP_OUTPUT + ERROR_VARIABLE _PIP_ERROR + ) + + if(_PIP_EXITCODE) + message(STATUS "saxonche pip output: ${_PIP_OUTPUT}") + message(STATUS "saxonche pip error: ${_PIP_ERROR}") + message(FATAL_ERROR "Failed to install saxonche dependency (exit code: ${_PIP_EXITCODE}). Check network connectivity and Python wheel compatibility.") + endif() + endif() + + execute_process( + COMMAND ${_VENV_PYTHON} "${AL_LOCAL_XSLTPROC_SCRIPT}" + -xsl "dd_data_dictionary.xml.xsl" + -o "IDSDef.xml" + -s "dd_data_dictionary.xml.xsd" + DD_GIT_DESCRIBE=${DD_GIT_DESCRIBE} + WORKING_DIRECTORY ${data-dictionary_SOURCE_DIR} + RESULT_VARIABLE _MAKE_DD_EXITCODE + OUTPUT_VARIABLE _MAKE_DD_OUTPUT + ERROR_VARIABLE _MAKE_DD_ERROR + ) + + if( _MAKE_DD_EXITCODE ) + message(STATUS "xsltproc.py output: ${_MAKE_DD_OUTPUT}") + message(STATUS "xsltproc.py error: ${_MAKE_DD_ERROR}") + message(FATAL_ERROR "Error while building the Data Dictionary (exit code: ${_MAKE_DD_EXITCODE}). Check paths and Saxon-HE configuration.") + endif() + + # Populate IDSDEF filename + set( IDSDEF "${data-dictionary_SOURCE_DIR}/IDSDef.xml" ) + + # Install IDSDEF (needed for some applications and for UDA backend) + get_filename_component( REAL_IDSDEF ${IDSDEF} REALPATH ) + install( FILES ${REAL_IDSDEF} DESTINATION include RENAME IDSDef.xml ) + + # Populate identifier source xmls + file( GLOB DD_IDENTIFIER_FILES "${data-dictionary_SOURCE_DIR}/*/*_identifier.xml" "${data-dictionary_SOURCE_DIR}/schemas/*/*_identifier.xml" ) +endif() + +# Find out which IDSs exist and populate IDS_NAMES +# Ensure saxonche is installed before using xsltproc.py +# Check if saxonche is available in the venv +execute_process( + COMMAND ${_VENV_PYTHON} -c "import saxonche" + RESULT_VARIABLE _SAXONCHE_CHECK + OUTPUT_QUIET + ERROR_QUIET +) + +if(_SAXONCHE_CHECK) + message(STATUS "Installing saxonche in venv...") + execute_process( + COMMAND ${_VENV_PIP} install saxonche + RESULT_VARIABLE _PIP_EXITCODE + OUTPUT_VARIABLE _PIP_OUTPUT + ERROR_VARIABLE _PIP_ERROR + ) + + if(_PIP_EXITCODE) + message(STATUS "saxonche pip output: ${_PIP_OUTPUT}") + message(STATUS "saxonche pip error: ${_PIP_ERROR}") + message(FATAL_ERROR "Failed to install saxonche dependency (exit code: ${_PIP_EXITCODE}). Check network connectivity and Python wheel compatibility.") + endif() +endif() + +set( list_idss_file ${CMAKE_SOURCE_DIR}/common/list_idss.xsl ) +set( CMAKE_CONFIGURE_DEPENDS ${CMAKE_CONFIGURE_DEPENDS};${list_idss_file};${IDSDEF} ) +set( ids_names_tmpfile "${CMAKE_CURRENT_BINARY_DIR}/ids_names_tmp.txt" ) +execute_process( COMMAND + ${_VENV_PYTHON} "${AL_LOCAL_XSLTPROC_SCRIPT}" + -xsl ${list_idss_file} + -s ${IDSDEF} + -o ${ids_names_tmpfile} + RESULT_VARIABLE _XSLT_RESULT + ERROR_VARIABLE _XSLT_ERROR +) +if(_XSLT_RESULT) + message(FATAL_ERROR "Failed to extract IDS names: ${_XSLT_ERROR}") +endif() +if(EXISTS ${ids_names_tmpfile}) + file(READ ${ids_names_tmpfile} IDS_NAMES) + string(STRIP "${IDS_NAMES}" IDS_NAMES) + file(REMOVE ${ids_names_tmpfile}) +else() + message(FATAL_ERROR "IDS names output file not created") +endif() +set( list_idss_file ) # unset temporary var + +# DD version +set( dd_version_file ${CMAKE_SOURCE_DIR}/common/dd_version.xsl ) +set( dd_version_tmpfile "${CMAKE_CURRENT_BINARY_DIR}/dd_version_tmp.txt" ) +execute_process( COMMAND + ${_VENV_PYTHON} "${AL_LOCAL_XSLTPROC_SCRIPT}" + -xsl ${dd_version_file} + -s ${IDSDEF} + -o ${dd_version_tmpfile} + RESULT_VARIABLE _XSLT_RESULT + ERROR_VARIABLE _XSLT_ERROR +) +if(_XSLT_RESULT) + message(FATAL_ERROR "Failed to extract DD version: ${_XSLT_ERROR}") +endif() +if(EXISTS ${dd_version_tmpfile}) + file(READ ${dd_version_tmpfile} DD_VERSION) + string(STRIP "${DD_VERSION}" DD_VERSION) + file(REMOVE ${dd_version_tmpfile}) +else() + message(FATAL_ERROR "DD version output file not created") +endif() +string( REGEX REPLACE "[+-]" "_" DD_SAFE_VERSION "${DD_VERSION}" ) +set( dd_version_file ) # unset temporary var diff --git a/common/cmake/ALBuildDocumentation.cmake b/common/cmake/ALBuildDocumentation.cmake new file mode 100644 index 0000000..c8cbb0e --- /dev/null +++ b/common/cmake/ALBuildDocumentation.cmake @@ -0,0 +1,22 @@ +# CMake config for building Sphinx docs +find_package( Python3 REQUIRED ) + +file( GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/doc.sh" + CONTENT "#!/bin/bash +# Set up python venv +${Python3_EXECUTABLE} -m venv '${CMAKE_CURRENT_BINARY_DIR}/doc_venv' +source '${CMAKE_CURRENT_BINARY_DIR}/doc_venv/bin/activate' +pip install -r '${CMAKE_CURRENT_SOURCE_DIR}/doc/doc_common/requirements.txt' + +# Instruct sphinx to treat all warnings as errors +export SPHINXOPTS='-W --keep-going' +# Build HTML documentation +make -C '${CMAKE_CURRENT_SOURCE_DIR}/doc' clean html +# Add a version file: +echo ${FULL_VERSION} > ${CMAKE_CURRENT_SOURCE_DIR}/doc/_build/html/version.txt +" +) + +add_custom_target( "${PROJECT_NAME}-docs" ALL + COMMAND bash "${CMAKE_CURRENT_BINARY_DIR}/doc.sh" +) diff --git a/common/cmake/ALCommonConfig.cmake b/common/cmake/ALCommonConfig.cmake new file mode 100644 index 0000000..45f2b00 --- /dev/null +++ b/common/cmake/ALCommonConfig.cmake @@ -0,0 +1,59 @@ +# Shared options + +# Note: default options are also listed in the docs: doc_common/building_installing.rst +# When changing default values, the documentation should be updated to reflect that. + +# Note: AL_DOWNLOAD_DEPENDENCIES is also shared, but needs to be set before this +# repository is available, so not listing it here. + +option( BUILD_SHARED_LIBS "Build shared libraries" ON ) +option( AL_EXAMPLES "Build and test examples" ON ) +option( AL_TESTS "Build tests and enable test framework" ON ) +option( AL_PLUGINS "Enable plugin framework for tests and examples" OFF ) +option( AL_HLI_DOCS "Build the Sphinx-based High Level Interface documentation" OFF ) +option( AL_DOCS_ONLY "Don't build anything, except the Sphinx-based High Level Interface documentation" OFF ) + +# Saxon XSLT processor has been replaced with Python saxonche +# No longer need to find SaxonHE - saxonche is installed automatically via pip in virtual environments + +# if( NOT AL_DOWNLOAD_DEPENDENCIES ) +# if( AL_DEVELOPMENT_LAYOUT ) +# set( _DEV ON ) +# else() +# set( _DEV OFF ) +# endif() +# option( AL_DEVELOPMENT_LAYOUT "Look into parent directories for dependencies" ${_DEV} ) +# endif() + +# Enable CTest? +if( AL_EXAMPLES OR AL_TESTS ) + include( CTest ) +endif() + +# Allow configuration of repositories and branches for dependent packages +################################################################################ + +if( AL_DOWNLOAD_DEPENDENCIES ) + if( ${AL_PLUGINS} ) + # AL plugins + ############################################################################## + set( + AL_PLUGINS_GIT_REPOSITORY "ssh://git@git.iter.org/imas/al-plugins.git" + CACHE STRING "Git repository of al-plugins" + ) + set( + AL_PLUGINS_VERSION "main" + CACHE STRING "al-plugins version (tag or branch name) to use for this build" + ) + endif() + # Data dictionary + ############################################################################## + set( + DD_GIT_REPOSITORY "https://github.com/iterorganization/IMAS-data-Dictionary.git" + CACHE STRING "Git repository of the Data Dictionary" + ) + set( + DD_VERSION "main" + CACHE STRING "Data dictionary version (tag or branch name) to use for this build" + ) +endif() diff --git a/common/cmake/ALCore.cmake b/common/cmake/ALCore.cmake new file mode 100644 index 0000000..21c6bc7 --- /dev/null +++ b/common/cmake/ALCore.cmake @@ -0,0 +1,189 @@ +# AL core and plugins + +if( NOT AL_DOWNLOAD_DEPENDENCIES AND NOT AL_DEVELOPMENT_LAYOUT ) + # The Access Layer core should be available as a module, use PkgConfig to create a + # target: + find_package( PkgConfig ) + pkg_check_modules( al REQUIRED IMPORTED_TARGET al-core ) + add_library( al ALIAS PkgConfig::al ) + set( AL_CORE_VERSION ${al_VERSION} ) + + # Stop processing + return() +endif() +if(WIN32) + if( AL_DOWNLOAD_DEPENDENCIES ) + # Download the AL core from the ITER git using direct git commands: + set( al-core_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/_deps/al-core-src" ) + if( NOT EXISTS "${al-core_SOURCE_DIR}/.git" ) + message( STATUS "Cloning al-core from ${AL_CORE_GIT_REPOSITORY}" ) + execute_process( + COMMAND git clone "${AL_CORE_GIT_REPOSITORY}" "${al-core_SOURCE_DIR}" + RESULT_VARIABLE _GIT_CLONE_RESULT + ERROR_VARIABLE _GIT_CLONE_ERROR + ) + if( _GIT_CLONE_RESULT ) + message( FATAL_ERROR "Failed to clone al-core: ${_GIT_CLONE_ERROR}" ) + endif() + endif() + # Checkout the specified version + execute_process( + COMMAND git fetch origin + WORKING_DIRECTORY "${al-core_SOURCE_DIR}" + RESULT_VARIABLE _GIT_FETCH_RESULT + ) + execute_process( + COMMAND git checkout "${AL_CORE_VERSION}" + WORKING_DIRECTORY "${al-core_SOURCE_DIR}" + RESULT_VARIABLE _GIT_CHECKOUT_RESULT + ERROR_VARIABLE _GIT_CHECKOUT_ERROR + ) + if( _GIT_CHECKOUT_RESULT ) + message( FATAL_ERROR "Failed to checkout ${AL_CORE_VERSION}: ${_GIT_CHECKOUT_ERROR}" ) + endif() + else() + # Look in ../al-core + set( al-core_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../al-core ) + if( NOT IS_DIRECTORY ${al-core_SOURCE_DIR} ) + # Repository used to be called "al-lowlevel", check this directory as well for + # backwards compatibility: + set( al-core_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../al-lowlevel ) + if( NOT IS_DIRECTORY ${al-core_SOURCE_DIR} ) + message( FATAL_ERROR + "${al-core_SOURCE_DIR} does not exist. Please clone the " + "al-core repository or set AL_DOWNLOAD_DEPENDENCIES=ON." + ) + endif() + endif() + endif() +else() + include(FetchContent) + + if( AL_DOWNLOAD_DEPENDENCIES ) + # Download the AL core from the ITER git: + FetchContent_Declare( + al-core + GIT_REPOSITORY ${AL_CORE_GIT_REPOSITORY} + GIT_TAG ${AL_CORE_VERSION} + ) + else() + # Look in ../al-core + set( AL_SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../al-core ) + if( NOT IS_DIRECTORY ${AL_SOURCE_DIRECTORY} ) + # Repository used to be called "al-lowlevel", check this directory as well for + # backwards compatibility: + set( AL_SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../al-lowlevel ) + if( NOT IS_DIRECTORY ${AL_SOURCE_DIRECTORY} ) + message( FATAL_ERROR + "${AL_SOURCE_DIRECTORY} does not exist. Please clone the " + "al-core repository or set AL_DOWNLOAD_DEPENDENCIES=ON." + ) + endif() + endif() + + FetchContent_Declare( + al-core + SOURCE_DIR ${AL_SOURCE_DIRECTORY} + ) + set( AL_SOURCE_DIRECTORY ) # unset temporary var + endif() +endif() + +# Don't load the AL core when only building documentation +if( NOT AL_DOCS_ONLY ) + # Ensure vcpkg packages are found in the subdirectory + if(WIN32) + # On Windows, ensure vcpkg packages are available to the subdirectory + if(DEFINED VCPKG_INSTALLED_DIR AND DEFINED VCPKG_TARGET_TRIPLET) + # Add vcpkg installed directory to CMAKE_PREFIX_PATH for the subdirectory + set(CMAKE_PREFIX_PATH "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET};${CMAKE_PREFIX_PATH}") + # Pass vcpkg variables to subdirectory by setting them in parent scope + set(VCPKG_INSTALLED_DIR "${VCPKG_INSTALLED_DIR}" CACHE STRING "vcpkg installed dir" FORCE) + set(VCPKG_TARGET_TRIPLET "${VCPKG_TARGET_TRIPLET}" CACHE STRING "vcpkg triplet" FORCE) + message(STATUS "ALCore: Passing vcpkg paths to al-core subdirectory") + message(STATUS " VCPKG_INSTALLED_DIR: ${VCPKG_INSTALLED_DIR}") + message(STATUS " VCPKG_TARGET_TRIPLET: ${VCPKG_TARGET_TRIPLET}") + message(STATUS " CMAKE_PREFIX_PATH: ${CMAKE_PREFIX_PATH}") + endif() + add_subdirectory( ${al-core_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/_deps/al-core-build ) + else() + FetchContent_MakeAvailable( al-core ) + endif() + get_target_property( AL_CORE_VERSION al VERSION ) +endif() + + +if( ${AL_PLUGINS} ) + if(WIN32) + if( ${AL_DOWNLOAD_DEPENDENCIES} ) + # Download the AL plugins from the ITER git using direct git commands: + set( al-plugins_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/_deps/al-plugins-src" ) + if( NOT EXISTS "${al-plugins_SOURCE_DIR}/.git" ) + message( STATUS "Cloning al-plugins from ${AL_PLUGINS_GIT_REPOSITORY}" ) + execute_process( + COMMAND git clone "${AL_PLUGINS_GIT_REPOSITORY}" "${al-plugins_SOURCE_DIR}" + RESULT_VARIABLE _GIT_CLONE_RESULT + ERROR_VARIABLE _GIT_CLONE_ERROR + ) + if( _GIT_CLONE_RESULT ) + message( FATAL_ERROR "Failed to clone al-plugins: ${_GIT_CLONE_ERROR}" ) + endif() + endif() + # Checkout the specified version + execute_process( + COMMAND git fetch origin + WORKING_DIRECTORY "${al-plugins_SOURCE_DIR}" + RESULT_VARIABLE _GIT_FETCH_RESULT + ) + execute_process( + COMMAND git checkout "${AL_PLUGINS_VERSION}" + WORKING_DIRECTORY "${al-plugins_SOURCE_DIR}" + RESULT_VARIABLE _GIT_CHECKOUT_RESULT + ERROR_VARIABLE _GIT_CHECKOUT_ERROR + ) + if( _GIT_CHECKOUT_RESULT ) + message( FATAL_ERROR "Failed to checkout ${AL_PLUGINS_VERSION}: ${_GIT_CHECKOUT_ERROR}" ) + endif() + else() + # Look in ../plugins + set( al-plugins_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../al-plugins ) + if( NOT IS_DIRECTORY ${al-plugins_SOURCE_DIR} ) + message( FATAL_ERROR + "${al-plugins_SOURCE_DIR} does not exist. Please clone the " + "al-plugins repository or set AL_DOWNLOAD_DEPENDENCIES=ON." + ) + endif() + endif() + + else() + if( ${AL_DOWNLOAD_DEPENDENCIES} ) + # Download the AL plugins from the ITER git: + FetchContent_Declare( + al-plugins + GIT_REPOSITORY ${AL_PLUGINS_GIT_REPOSITORY} + GIT_TAG ${AL_PLUGINS_VERSION} + ) + else() + # Look in ../plugins + set( PLUGINS_SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../al-plugins ) + if( NOT IS_DIRECTORY ${PLUGINS_SOURCE_DIRECTORY} ) + message( FATAL_ERROR + "${PLUGINS_SOURCE_DIRECTORY} does not exist. Please clone the " + "al-plugins repository or set AL_DOWNLOAD_DEPENDENCIES=ON." + ) + endif() + + FetchContent_Declare( + al-plugins + SOURCE_DIR ${PLUGINS_SOURCE_DIRECTORY} + ) + set( PLUGINS_SOURCE_DIRECTORY ) # unset temporary var + endif() + FetchContent_MakeAvailable( al-plugins ) + endif() +endif() + +if( AL_HLI_DOCS ) + include( ALBuildDocumentation ) +endif() + diff --git a/common/cmake/ALDetermineVersion.cmake b/common/cmake/ALDetermineVersion.cmake new file mode 100644 index 0000000..5c4399e --- /dev/null +++ b/common/cmake/ALDetermineVersion.cmake @@ -0,0 +1,53 @@ +# Determine the version of the current AL component, based on git describe + +if( NOT GIT_ARCHIVE_DESCRIBE ) + message( FATAL_ERROR "GIT_ARCHIVE_DESCRIBE should be set before including ALDetermineVersion" ) +endif() + +if( NOT GIT_ARCHIVE_DESCRIBE MATCHES "^.Format:%.describe" ) + # We are part of an exported tarball and git-archive set the describe content: + set( _GIT_DESCRIBE_ERROR_CODE 0 ) + set( _GIT_DESCRIBE_OUTPUT "${GIT_ARCHIVE_DESCRIBE}" ) +else() + # Ask git for a describe: + find_package( Git ) + if( GIT_EXECUTABLE ) + # Generate a git-describe version string from Git repository tags + execute_process( + COMMAND ${GIT_EXECUTABLE} describe --tags --dirty + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE _GIT_DESCRIBE_OUTPUT + RESULT_VARIABLE _GIT_DESCRIBE_ERROR_CODE + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + endif() +endif() + +# Process git describe output: +if( _GIT_DESCRIBE_OUTPUT AND NOT _GIT_DESCRIBE_ERROR_CODE ) + # Git describe should return the version (MAJOR.MINOR.PATCH) and potentially + # a suffix "--g[-dirty]" + # Use a regex to extract all parts: + if( _GIT_DESCRIBE_OUTPUT MATCHES "([0-9]+)[.]([0-9]+)[.]*([0-9]+)(.*)" ) + set( VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" ) + if( CMAKE_MATCH_4 MATCHES "-([0-9]+)-(.*)" ) + # Use ncommits as fourth version component for the CMAKE project version + set( VERSION "${VERSION}.${CMAKE_MATCH_1}" ) + endif() + else() + message( FATAL_ERROR "Unexpected output of git describe: '${_GIT_DESCRIBE_OUTPUT}'") + endif() + + # Generate a version string that conforms to the Python standards + # e.g. 5.1.0-3-g7c620eb5-dirty becomes 5.1.0+3-g7c620eb5-dirty + string( REGEX REPLACE "-(.*)$" "+\\1" FULL_VERSION ${_GIT_DESCRIBE_OUTPUT} ) + message( VERBOSE "Determined project version: ${VERSION}" ) +endif() + +# Fallback: git not found, or git describe fails +# Set version to 0.0.0 and report a warning +if( NOT DEFINED VERSION ) + set( VERSION "0.0.0" ) + set( FULL_VERSION "0.0.0+unknown" ) + message( WARNING "Failed to determine VERSION from git tags. Falling back to default version '${VERSION}'" ) +endif() diff --git a/common/cmake/ALExampleUtilities.cmake b/common/cmake/ALExampleUtilities.cmake new file mode 100644 index 0000000..1631afd --- /dev/null +++ b/common/cmake/ALExampleUtilities.cmake @@ -0,0 +1,53 @@ +# CMake file providing common logic to test AL examples + +set( EXAMPLE_ENVIRONMENT_WITHOUT_PLUGINS + "IMAS_AL_ENABLE_PLUGINS=FALSE" +) + +if( AL_PLUGINS ) + get_target_property( PLUGIN_DIR al-plugins BINARY_DIR ) + set( EXAMPLE_ENVIRONMENT_WITH_PLUGINS + ${TEST_ENVIRONMENT} + "IMAS_AL_ENABLE_PLUGINS=TRUE" + "IMAS_AL_PLUGINS=${PLUGIN_DIR}" + ) +endif() + +function( set_al_example_properties TEST DISABLED USE_PLUGINS EXTRA_ENVIRONMENT ) + # Set common properties + set_tests_properties( ${TEST} PROPERTIES + # Case insensitive fault|error[^_]|exception|severe|abort|segmentation|fault|dump|logic_error|failed + FAIL_REGULAR_EXPRESSION "[Ff][Aa][Uu][Ll][Tt]|[Ee][Rr][Rr][Oo][Rr][^_]|[Ee][Xx][Cc][Ee][Pp][Tt][Ii][Oo][Nn]|[Ss][Ee][Vv][Ee][Rr][Ee]|[Aa][Bb][Oo][Rr][Tt]|[Ss][Ee][Gg][Mm][Ee][Nn][Tt][Aa][Tt][Ii][Oo][Nn]|[Ff][Aa][Uu][Ll][Tt]|[Dd][Uu][Mm][Pp]|[Ll][Oo][Gg][Ii][Cc]_[Ee][Rr][Rr][Oo][Rr]|[Ff][Aa][Ii][Ll][Ee][Dd]" + DISABLED ${DISABLED} + ) + + # Set environment variables + if( USE_PLUGINS ) + set( P_ENV ${EXAMPLE_ENVIRONMENT_WITH_PLUGINS} ) + else() + set( P_ENV ${EXAMPLE_ENVIRONMENT_WITHOUT_PLUGINS} ) + endif() + set_tests_properties( ${TEST} PROPERTIES ENVIRONMENT "${P_ENV};${EXTRA_ENVIRONMENT}" ) + + # Set fixtures: put/put_slice must run before get/get_slice + string( TOLOWER ${TEST} TEST_LOWER ) + string( REGEX REPLACE "put|get" "" FIXTURE_NAME ${TEST_LOWER} ) + if( TEST_LOWER MATCHES "put" ) + set_tests_properties( ${TEST} PROPERTIES FIXTURES_SETUP ${FIXTURE_NAME} ) + elseif( TEST_LOWER MATCHES "get" ) + set_tests_properties( ${TEST} PROPERTIES FIXTURES_REQUIRED ${FIXTURE_NAME} ) + endif() +endfunction() + + +function( error_on_missing_tests SOURCE_EXTENSION TESTS ) + file( GLOB _SRCS "*.${SOURCE_EXTENSION}" ) + foreach( _SRC IN LISTS _SRCS ) + string( REGEX REPLACE "^.*[/\\]|[.]${SOURCE_EXTENSION}$" "" _SRC_STEM ${_SRC} ) + list( FIND TESTS ${_SRC_STEM} FOUND ) + if( FOUND EQUAL -1 ) + message( SEND_ERROR "Source file ${_SRC} found, but no corresponding test exists" ) + endif() + endforeach() +endfunction() + diff --git a/common/cmake/ALLocalPaths.cmake b/common/cmake/ALLocalPaths.cmake new file mode 100644 index 0000000..f367d74 --- /dev/null +++ b/common/cmake/ALLocalPaths.cmake @@ -0,0 +1,14 @@ +# Local paths for IMAS-MATLAB +# This file defines local paths to eliminate AL_COMMON_PATH dependency + +# Directory containing CMake modules +set(AL_LOCAL_CMAKE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/common/cmake) + +# Directory containing documentation common files +set(AL_LOCAL_DOC_COMMON_DIR ${CMAKE_CURRENT_SOURCE_DIR}/doc/doc_common) + +# XSLT processor script +set(AL_LOCAL_XSLTPROC_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/common/xsltproc.py) + +# Add cmake modules to CMake module path +list(APPEND CMAKE_MODULE_PATH ${AL_LOCAL_CMAKE_DIR}) diff --git a/common/cmake/ALSetCompilerFlags.cmake b/common/cmake/ALSetCompilerFlags.cmake new file mode 100644 index 0000000..b821ffc --- /dev/null +++ b/common/cmake/ALSetCompilerFlags.cmake @@ -0,0 +1,116 @@ +# Compiler flags for building the Access Layer + +include_guard( DIRECTORY ) + +################################################################################ +# Compile definitions for Fortran interface + +if( CMAKE_SYSTEM_NAME STREQUAL Linux ) + add_compile_definitions( $<$:_Linux> ) +elseif( CMAKE_SYSTEM_NAME STREQUAL Darwin ) + add_compile_definitions( $<$:_MacOS> ) +elseif( CMAKE_SYSTEM_NAME STREQUAL Windows ) + add_compile_definitions( $<$:_Windows> ) +else() + message( WARNING "Unknown CMAKE_SYSTEM_NAME '${CMAKE_SYSTEM_NAME}', continuing..." ) +endif() + + +################################################################################ +# General options + +set( CMAKE_POSITION_INDEPENDENT_CODE ON ) + +# Set default build type to RelWithDebInfo (optimize build, keep debugging symbols) +if( NOT CMAKE_BUILD_TYPE ) + set( CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING + "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." + FORCE + ) +endif() + +################################################################################ +# C++ +if(NOT DEFINED CMAKE_CXX_STANDARD) + set( CMAKE_CXX_STANDARD 17 ) +endif() +if( CMAKE_CXX_COMPILER_ID STREQUAL "Intel" OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" ) + # icpc/icpx options + string( APPEND CMAKE_CXX_FLAGS + # " -Wall" + ) +elseif( CMAKE_CXX_COMPILER_ID STREQUAL "GNU" ) + # g++ options + string( APPEND CMAKE_CXX_FLAGS + # " -Wall -Wextra" + ) +elseif( CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" ) + # Visual Studio C++ options + string( APPEND CMAKE_CXX_FLAGS + # " -Wall" + ) +elseif( CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" ) + # Apple Clang C++ options + string( APPEND CMAKE_CXX_FLAGS + # " -Wall" + ) +else() + message( WARNING "Unsupported C++ compiler: ${CMAKE_CXX_COMPILER_ID}" ) +endif() + + +################################################################################ +# Fortran + +get_property(languages GLOBAL PROPERTY ENABLED_LANGUAGES) +if( "Fortran" IN_LIST languages ) + + set( CMAKE_Fortran_FORMAT FREE ) + set( CMAKE_Fortran_PREPROCESS ON ) + set( CMAKE_Fortran_MODULE_DIRECTORY include ) + + if( CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" ) + # ifort options + string( APPEND CMAKE_Fortran_FLAGS + " -r8 -assume no2underscore" + ) + # string( APPEND CMAKE_Fortran_FLAGS " -warn all" ) + elseif( CMAKE_Fortran_COMPILER_ID STREQUAL "GNU" ) + # gfortran options + string( APPEND CMAKE_Fortran_FLAGS + " -fdefault-real-8 -fdefault-double-8 -fno-second-underscore -ffree-line-length-none" + ) + # string( APPEND CMAKE_Fortran_FLAGS " -Wall -Wextra" ) + elseif( CMAKE_Fortran_COMPILER_ID STREQUAL "NAG" ) + # nagfort options + string( APPEND CMAKE_Fortran_FLAGS + " -maxcontin=4000 -w=unused -w=x95 -kind=byte -r8" + ) + # Set CONFIG options for NAG, CMake's definition keep them all empty + string( APPEND CMAKE_Fortran_FLAGS_DEBUG " -g" ) + string( APPEND CMAKE_Fortran_FLAGS_RELWITHDEBINFO " -O2 -g" ) + string( APPEND CMAKE_Fortran_FLAGS_RELEASE " -O3" ) + elseif( CMAKE_Fortran_COMPILER_ID STREQUAL "PGI" ) + # PGI / nvfortran options + string( APPEND CMAKE_Fortran_FLAGS + " -r8 -Mnosecond_underscore" + ) + else() + message( WARNING "Unsupported Fortran compiler: ${CMAKE_Fortran_COMPILER_ID}" ) + endif() + +endif() + +################################################################################ +# Windows support + +# TODO: salvaged from old CMakeLists +# Need to figure out if this is needed and functional before uncommenting + +# add_definitions( -D_CRT_SECURE_NO_WARNINGS ) +# add_compile_options( /bigobj ) + +# set( CMAKE_FIND_LIBRARY_SUFFIXES ".dll.lib" ".lib" ) +# set( CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /SAFESEH:NO" ) +# set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SAFESEH:NO /SUBSYSTEM:CONSOLE" ) +# set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++14 /Zc:__cplusplus" ) diff --git a/common/dd_version.xsl b/common/dd_version.xsl new file mode 100644 index 0000000..906f556 --- /dev/null +++ b/common/dd_version.xsl @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + diff --git a/common/identifiers.common.xsl b/common/identifiers.common.xsl new file mode 100644 index 0000000..af36725 --- /dev/null +++ b/common/identifiers.common.xsl @@ -0,0 +1,247 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #include + + + + use + + + + import + + + + import + + ; + + + + + + + + + + + + + + + + + + + + + + + + + + + + <?xml version="1.0" ?> + <section class="topic" id="imas_enum_types__ "> + <title> + + </title> + + + + <para> + <screen> + + + + + + </screen> + </para> + + + + + + + <para> This identifier is used in the following places in the ITER IDSs: + + <screen> + + </screen> + + </para> + + + + + <para> Fortran interface example: + <screen> use + + , only: get_type_index, get_type_name, get_type_description</screen> + </para> + + + + + + + + <table> + <tgroup cols="3"> + <colspec colwidth="45mm"/> + <colspec colwidth="60mm"/> + <colspec colwidth="68mm"/> + <thead> + <row> + <entry>Name</entry> + <entry>Value</entry> + <entry>Description</entry> + </row> + </thead> + <tbody> + + + + + <table> + <tgroup cols="3">. + + + <colspec colwidth="15mm"/> + <colspec colwidth="50mm"/> + <colspec colwidth="108mm"/> + <thead> + <row> + <entry>Flag</entry> + <entry>Id</entry> + <entry>Description</entry> + </row> + </thead> + <tbody> + + + + + + + + + + <row> + <entry> + + + </entry> + <entry> <mono> + + </mono> </entry> + + <entry> + + </entry> + </row> + + + + + </tbody> + </tgroup> + </table> + + + </section> + + + + diff --git a/common/list_idss.xsl b/common/list_idss.xsl new file mode 100644 index 0000000..d6d814a --- /dev/null +++ b/common/list_idss.xsl @@ -0,0 +1,18 @@ + + + + + + + + + + + + ; + + + + + diff --git a/common/xsltproc.py b/common/xsltproc.py new file mode 100644 index 0000000..009b7ad --- /dev/null +++ b/common/xsltproc.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# simple net.sf.saxon.Transform cli replacement via saxonche Python bindings +# example invokation: +# ./xsltproc.py -xsl IDSDef2MDSpreTree.xsl -s IDSDef.xml -o output.xml DD_GIT_DESCRIBE=1 AL_GIT_DESCRIBE=1 + +import argparse +import logging + +import saxonche + + +def parse_arguments() -> tuple: + """Parse arguments, similar to net.sf.saxon.Transform...""" + + parser = argparse.ArgumentParser( + prog="xsltproc.py", + description="Imitates Saxon-HE's net.sf.saxon.Transform.", + epilog="Additional arguments in format key=value will be set as xml parameters", + ) + parser.add_argument( + "-xsl", + "--stylesheet_file", + type=str, + required=True, + help="XSL style sheet file", + ) + parser.add_argument( + "-s", + "--source_file", + type=str, + required=True, + help="source XML document", + ) + parser.add_argument( + "-o", + "--output_file", + type=str, + required=True, + help="transformed output XML document", + ) + + args, other_args = parser.parse_known_args() + # Convert list of strings "key=value" into dict(key=value, ...) + other_kwargs = {k: v for k, v in map(lambda x: x.split("="), other_args)} + return (args, other_kwargs) + + +def saxon_xsltproc( + source_file: str, stylesheet_file: str, output_file: str, **kwargs +) -> None: + with saxonche.PySaxonProcessor(license=False) as proc: + xsltproc = proc.new_xslt30_processor() + for key, value in kwargs.items(): + string_value = proc.make_string_value(value) + xsltproc.set_parameter(key, string_value) + xsltproc.transform_to_file( + source_file=source_file, + stylesheet_file=stylesheet_file, + output_file=output_file, + ) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + args, other_kwargs = parse_arguments() + saxon_xsltproc( + source_file=args.source_file, + stylesheet_file=args.stylesheet_file, + output_file=args.output_file, + **other_kwargs, + ) diff --git a/create_matlab_toolbox.m b/create_matlab_toolbox.m index 209d33a..0243b59 100644 --- a/create_matlab_toolbox.m +++ b/create_matlab_toolbox.m @@ -1,8 +1,8 @@ function create_matlab_toolbox(installDir, outputDir, version, ddVersion) -% CREATE_MATLAB_TOOLBOX Package IMAS-Matlab as a MATLAB toolbox +% CREATE_MATLAB_TOOLBOX Package IMAS-MATLAB as a MATLAB toolbox % % Arguments: -% installDir - Directory containing installed IMAS-Matlab files +% installDir - Directory containing installed IMAS-MATLAB files % outputDir - Directory where .mltbx file will be created % version - Version string (e.g., '5.5.0') % ddVersion - Data Dictionary version (e.g., '4.1.0') @@ -56,10 +56,10 @@ function create_matlab_toolbox(installDir, outputDir, version, ddVersion) % Configure toolbox metadata fprintf(' Configuring toolbox metadata...\n'); % Create a valid identifier (alphanumeric and hyphens only) - toolboxId = sprintf('imas-matlab-%s', strrep(version, '.', '-')); + toolboxId = sprintf('IMAS-MATLAB-%s', strrep(version, '.', '-')); opts = matlab.addons.toolbox.ToolboxOptions(packDir, toolboxId); - opts.ToolboxName = 'IMAS-Matlab'; + opts.ToolboxName = 'IMAS-MATLAB'; opts.ToolboxVersion = version; opts.AuthorName = 'ITER Organization'; opts.Summary = sprintf('MATLAB interface for IMAS Access Layer - %s', fullVersionName); @@ -118,7 +118,7 @@ function create_startup_script(packDir, version, ddVersion, platform) fid = fopen(startupFile, 'w'); fprintf(fid, 'function imas_toolbox_startup()\n'); -fprintf(fid, '%% IMAS_TOOLBOX_STARTUP Initialize IMAS-Matlab toolbox\n'); +fprintf(fid, '%% IMAS_TOOLBOX_STARTUP Initialize IMAS-MATLAB toolbox\n'); fprintf(fid, '%% Auto-generated startup script for %s\n\n', fullVersionName); fprintf(fid, ' toolboxRoot = fileparts(mfilename(''fullpath''));\n\n'); fprintf(fid, ' %% Add to path\n'); diff --git a/doc/.gitignore b/doc/.gitignore index a56608d..1c84c65 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -1,4 +1,2 @@ _build -# Symlinks to other AL projects (created by CMake): -doc_common plugins diff --git a/doc/conf.py b/doc/conf.py index 80ceb6f..0bdc1de 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -51,8 +51,8 @@ # todo_include_todos = True -templates_path = ["./doc_common/templates"] # Note: exclude doc_common and plugins folders (which are symlinked by the CMake build) +# Also exclude myenv (Python virtual environment) exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "doc_common", "plugins"] # -- RST snippets to include in every page ----------------------------------- diff --git a/doc/conf.rst b/doc/conf.rst deleted file mode 100644 index 59227f5..0000000 --- a/doc/conf.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ./doc_common/conf.rst diff --git a/doc/doc_common/building_installing.rst b/doc/doc_common/building_installing.rst new file mode 100644 index 0000000..9166c47 --- /dev/null +++ b/doc/doc_common/building_installing.rst @@ -0,0 +1,299 @@ +Building and installing the IMAS-MATLAB +======================================== + +This page describes how to build and install the IMAS-MATLAB. + +Documentation for developers wishing to contribute to the IMAS-MATLAB can be found in +the :ref:`IMAS-MATLAB development guide`. Please refer to that guide if you wish to set +up a development environment. + +For more information about related components, see: + +- `IMAS Core Documentation `__ +- `IMAS Data Dictionary Documentation `__ + + +.. note:: + + For Windows-specific installation instructions, please refer to the + :doc:`MATLAB on Windows ` guide. + + +.. _`build prerequisites`: + +Prerequisites +------------- + +To build the IMAS-MATLAB you need: + +- Git +- A C++11 compiler (tested with GCC and Intel compilers) +- CMake (3.16 or newer) +- Boost C++ libraries (1.66 or newer) +- PkgConfig + +The following dependencies are only required for some of the components: + +- Backends + + - **HDF5 backend**: HDF5 C/C++ libraries (1.8.12 or newer) + - **MDSplus backend**: MDSplus libraries (7.84.8 or newer) + - **UDA backend**: `UDA `__ libraries + (2.7.5 or newer) [#uda_install]_ + +.. [#uda_install] When installing UDA, make sure you have + `Cap'n'Proto `__ installed in your system + and add its support by adding the CMake switch `-DENABLE_CAPNP=ON` when configuring UDA. + + +- MATLAB High Level Interface + + - **MATLAB High Level Interface**: A working MATLAB installation (tested with + version 2023b) + + + + +Standard environments: + +.. md-tab-set:: + + .. md-tab-item:: SDCC ``intel-2023b`` + + The following modules provide all the requirements when using the + ``intel-2023b`` toolchain: + + .. code-block:: bash + + module load intel-compilers/2023.2.1 CMake/3.27.6-GCCcore-13.2.0 Saxon-HE/12.4-Java-21 \ + Boost/1.83.0-iimpi-2023b HDF5/1.14.3-iimpi-2023b \ + MDSplus/7.132.0-GCCcore-13.2.0 \ + UDA/2.8.1-iimpi-2023b Blitz++/1.0.2-GCCcore-13.2.0 \ + MATLAB/2023b-r5-GCCcore-13.2.0 SciPy-bundle/2023.11-intel-2023b \ + scikit-build-core/0.9.3-GCCcore-13.2.0 + + .. md-tab-item:: SDCC ``foss-2023b`` + + The following modules provide all the requirements when using the + ``foss-2023b`` toolchain: + + .. code-block:: bash + + module load CMake/3.27.6-GCCcore-13.2.0 Saxon-HE/12.4-Java-21 \ + Boost/1.83.0-GCC-13.2.0 HDF5/1.14.3-gompi-2023b \ + MDSplus/7.132.0-GCCcore-13.2.0 \ + UDA/2.8.1-GCC-13.2.0 Blitz++/1.0.2-GCCcore-13.2.0 \ + MATLAB/2023b-r5-GCCcore-13.2.0 SciPy-bundle/2023.11-gfbf-2023b \ + build/1.0.3-foss-2023b scikit-build-core/0.9.3-GCCcore-13.2.0 + + .. admonition:: The MATLAB/2023b-r5 installation is lightly tweaked + + The installation at ITER uses `EB PR#20508 + `__ + and its tweak resolves `IMAS-5162 `__ + by removing ``libstdc++.so.6`` from the MATLAB installation. It also adds + ``extern/bin/glnxa64`` to ``LD_LIBRARY_PATH`` to address the + ``MatlabEngine not found`` issue. + + .. caution:: + + When using the HDF5 backend within MATLAB, depending on the HDF5 library being used + you may need to add ``LD_PRELOAD=/lib/libhdf5_hl.so`` when starting + MATLAB. + + .. md-tab-item:: Ubuntu 22.04 + + The following packages provide most requirements when using Ubuntu 22.04: + + .. code-block:: bash + + apt install git build-essential cmake libsaxonhe-java libboost-all-dev \ + pkg-config libhdf5-dev xsltproc libblitz0-dev gfortran \ + default-jdk-headless python3-dev python3-venv python3-pip + + The following dependencies are not available from the package repository, + you will need to install them yourself: + + - MDSplus: see their `GitHub repository + `__ or `home page + `__ for installation instructions. + - UDA: see their `GitHub repository `__ for more + details. + - MATLAB, which is not freely available. + + +Building and installing a single High Level Interface +----------------------------------------------------- + +This section explains how to install a Matlab High Level Interface. Please make sure you +have the :ref:`build prerequisites` installed. + + +Clone the repository +```````````````````` + +First you need to clone the repository of the High Level Interface you want to build: + +.. code-block:: bash + + # For the MATLAB HLI use: + git clone git@github.com:iterorganization/IMAS-MATLAB.git + + +Configuration +````````````` + +Once you have cloned the repository, navigate your shell to the folder and run cmake. +You can pass configuration options with ``-D OPTION=VALUE``. See below list for an +overview of configuration options. + +.. code-block:: bash + + cd al-matlab # al-fortran, al-java, al-cpp or al-python + cmake -B build -D CMAKE_INSTALL_PREFIX=$HOME/al-install -D OPTION1=VALUE1 -D OPTION2=VALUE2 [...] + +.. note:: + + CMake will automatically fetch dependencies from other IMAS-MATLAB GIT repositories + for you. You may need to provide credentials to clone the following repositories: + + - `imas-core (git@github.com:iterorganization/IMAS-Core.git) + `__ + - `al-plugins (https://github.com/iterorganization/al-plugins.git) + `__ + - `imas-data-dictionary (git@github.com:iterorganization/IMAS-Data-Dictionary.git) + `__ + + If you need to change the git repositories, for example to point to a mirror of the + repository or to use a HTTPS URL instead of the default SSH URLs, you can update the + :ref:`configuration options`. For example, add the following options to your + ``cmake`` command to download the repositories over HTTPS instead of SSH: + + .. code-block:: text + :caption: Use explicit options to download dependent repositories over HTTPS + + cmake -B build \ + -D AL_CORE_GIT_REPOSITORY=git@github.com:iterorganization/IMAS-Core.git \ + -D AL_PLUGINS_GIT_REPOSITORY=git@github.com:iterorganization/al-plugins.git \ + -D DD_GIT_REPOSITORY=git@github.com:iterorganization/IMAS-Data-Dictionary.git + + If you use CMake 3.21 or newer, you can also use the ``https`` preset: + + .. code-block:: text + :caption: Use CMake preset to set to download dependent repositories over HTTPS + + cmake -B build --preset=https + + +Choosing the compilers +'''''''''''''''''''''' + +You can instruct CMake to use compilers with the following environment variables: + +- ``CC``: C compiler, for example ``gcc`` or ``icc``. +- ``CXX``: C++ compiler, for example ``g++`` or ``icpc``. + +If you don't specify a compiler, CMake will take a default (usually from the Gnu +Compiler Collection). + +.. important:: + + These environment variables must be set before the first time you configure + ``cmake``! + + If you have an existing ``build`` folder and want to use a different compiler, you + should delete the ``build`` folder first, or use a differently named folder for the + build tree. + + +Configuration options +''''''''''''''''''''' + +For a complete list of available configuration options, please see the `IMAS Core Configuration Options `__. + + +Build the High Level Interface +`````````````````````````````` + +Use ``make`` to build everything. You can speed things up by using parallel compiling +as shown with the ``-j`` option. Be careful with the amount of parallel processes +though: it's easy to exhaust your machine's available hardware (CPU or memory) which may +cause the build to fail. This is especially the case with the C++ High Level Interface. + +.. code-block:: bash + + # Instruct make to build "all" in the "build" folder, using at most "8" parallel + # processes: + make -C build -j8 all + +.. note:: + + By default CMake on Linux will create ``Unix Makefiles`` for actually building + everything, as assumed in this section. + + You can select different generators (such as Ninja) if you prefer, but these are not + tested. See the `CMake documentation + `__ for more + details. + + +Optional: Test the High Level Interface +``````````````````````````````````````` + +If you set either of the options ``AL_EXAMPLES`` or ``AL_TESTS`` to ``ON``, you can run +the corresponding test programs as follows: + +.. code-block:: bash + + # Use make: + make -C build test + # Directly invoke ctest + ctest --test-dir build + +This executes ``ctest`` to run all test and example programs. Note that this may take a +long time to complete. + + +Install the High Level Interface +```````````````````````````````` + +Run ``make install`` to install the high level interface in the folder that you chose in +the configuration step above. + + +Use the High Level Interface +```````````````````````````` + +After installing the HLI, you need to ensure that your code can find the installed +IMAS-MATLAB. To help you with this, a file ``al_env.sh`` is installed. You can +``source`` this file to set all required environment variables: + +.. code-block:: bash + :caption: Set environment variables (replace ```` with your install folder) + + source /bin/al_env.sh + +You may want to add this to your ``$HOME/.bashrc`` file to automatically make the Access +Layer installation available for you. + +.. note:: + + To use a ``public`` dataset, you also need to set the ``IMAS_HOME`` environment + variable. For example, on SDCC, this would be ``export IMAS_HOME=/work/imas``. + + Some programs may rely on an environment variable ``IMAS_VERSION`` to detect which + version of the data dictionary is used in the current IMAS environment. You may set + it manually with the DD version you've build the HLI with, for example: ``export + IMAS_VERSION=3.41.0``. + +Once you have set the required environment variables, you may continue :ref:`Using the +IMAS-MATLAB`. + + +Troubleshooting +``````````````` + +**Problem:** ``Target Boost::log already has an imported location`` + This problem is known to occur with the ``2020b`` toolchain on SDCC. Add the CMake + configuration option ``-D Boost_NO_BOOST_CMAKE=ON`` to work around the problem. + diff --git a/doc/doc_common/dev_guide.rst b/doc/doc_common/dev_guide.rst new file mode 100644 index 0000000..2897f98 --- /dev/null +++ b/doc/doc_common/dev_guide.rst @@ -0,0 +1,178 @@ +IMAS-MATLAB development guide +============================= + + +Repositories +------------ + +The IMAS-MATLAB consists of a number of dependencies which are developed in separate +repositories: + +- `imas-core `__: the + IMAS core repository, MDSplus model generator and Python lowlevel + bindings. +- `data-dictionary + `__: the IMAS Data + Dictionary definitions, used for generating MDSplus models and the traditional High + Level Interfaces. +- `IMAS-Core-Plugins `__: Access + Layer plugins. +- Traditional (code-generated) High Level Interfaces + + - `IMAS-MATLAB `__: + MATLAB HLI + + +The documentation on this page covers everything except the Non-generated HLIs, those +are documented in their own projects. + + +Development environment +----------------------- + +See the :ref:`build prerequisites` section for an overview of modules you need to load +when on SDCC or packages to install when using Ubuntu 22.04. + +The recommended development folder layout is to clone all :ref:`Repositories` in a single root folder (``al-dev`` in below example, but the name of that +folder is not important). + +.. code-block:: text + + al-dev/ # Feel free to name this folder however you want + ├── al-core/ + ├── al-plugins/ # Optional + ├── al-matlab/ + └── data-dictionary/ + +Then, when you configure a project for building (see :ref:`Configuration`), set the +option ``-D AL_DOWNLOAD_DEPENDENCIES=OFF``. Instead of fetching requirements from the +ITER git, CMake will now use the repositories as they are checked out in your +development folders. + + With this setup, it is your responsibility to update the repositories to their + latest versions (if needed). The ``_VERSION`` configuration options are + ignored when ``AL_DOWNLOAD_DEPENDENCIES=OFF``. + +This setup allows you to develop in multiple repositories in parallel. + + +Dependency management +--------------------- + +With all IMAS-MATLAB dependencies spread over different repositories, managing +dependencies is more complex than before. Below diagram expresses the dependencies +between the different repositories: + +.. md-mermaid:: + :name: repository-dependencies + + flowchart + core[al-core] -->|"MDSplus
models"| dd[data-dictionary] + plugins[al-plugins] --> core + hli["al-{hli}"] --> core + hli --> dd + hli --> plugins + +To manage the "correct" version of each of the dependencies, the CMake configuration +specifies which branch to use from each repository: + +- Each HLI indicates which commit to use from the ``al-core`` repository. This is + defined by the ``AL_CORE_VERSION`` cache string in the main ``CMakeLists.txt`` of + the repository. + + The default version used is ``main``, which is the last stable release of + ``al-core``. +- Inside the ``al-core`` repository, the commits to use for the + ``al-plugins`` and ``data-dictionary`` are set in `ALCommonConfig.cmake + `__. + + The default versions used are ``main`` for ``al-plugins``, and ``main`` for + ``data-dictionary``. + + +.. info:: + + CMake supports setting branch names, tags and commit hashes for the dependencies. + + +CMake +----- + +We're using CMake for the build configuration. See `the CMake documentation +`__ for more details about CMake. + +The ``FetchContent`` CMake module for making :ref:`dependencies from other repositories +` available. For more information on this module we refer to the +`FetchContent CMake documentation +`__ + + +Documentation overview +---------------------- + +The documentation is generated with Sphinx. For more information on Sphinx, see the `Sphinx docs +`__ and the `documentation of the theme +(sphinx-immaterial) that we're using +`__. + +Documentation of the HLI is inside the ``doc`` folder of the repository. This folder +contains the configuration (``conf.py``), and documentation pages (``*.rst``). +Documentation that is common to all High Level Interfaces (such as this developer guide) +is in the `common/doc_common folder in the al-core repository +`__. + + +Building the documentation +'''''''''''''''''''''''''' + +Use the option ``-D AL_HLI_DOCS`` to enable building documentation. This will create a +target ``al--docs``, e.g. ``al-matlab-docs`` that will only build the +documentation. You could also use ``-D AL_DOCS_ONLY`` to only build the documentation, +and nothing else. + +.. code-block:: console + :caption: Example: building the documentation for the Python HLI + + al-dev$ cd al-matlab + al-matlab$ # Configure cmake to only create the documentation: + al-matlab$ cmake -B build -D AL_HLI_DOCS -D AL_DOCS_ONLY + [...] + al-matlab$ make -C build al-matlab-docs + [...] + + +GitHub Actions CI/CD pipeline +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In addition to the ITER CI systems, the IMAS-MATLAB repository uses `GitHub Actions +`__ for automated building and testing. The +workflow is defined in `.github/workflows/build-and-test.yml +`__. + +This workflow: + +- **Triggers**: Automatically runs on pushes to ``main``, ``develop``, and ``feature/**`` branches, + on all pull requests to ``main`` and ``develop``, on release tags (``v*``), and can be triggered + manually via ``workflow_dispatch``. + +- **Platforms**: Currently tests on Ubuntu 24.04 with GCC 14 compiler and MATLAB R2023b. + +- **Build steps**: + + 1. Sets up Python 3.11 environment + 2. Installs MATLAB using GitHub's official MATLAB action + 3. Installs system dependencies (build-essential, cmake, pkg-config, etc.) + 4. Caches Boost and pip packages for faster builds + 5. Builds and optionally installs external dependencies (UDA, HDF5, etc.) + 6. Configures the project with CMake + 7. Compiles the code + 8. Runs tests if enabled + +- **Backends tested**: Currently enables the HDF5 backend while MDSplus and UDA + backends are disabled to simplify testing. + +- **Build artifacts**: The workflow checks that the code compiles successfully and + that all tests pass. Build logs are available in the GitHub Actions tab of the repository. + +You can monitor the status of builds and tests in the +`Actions `__ tab of the GitHub repository. diff --git a/doc/doc_common/identifiers.rst b/doc/doc_common/identifiers.rst new file mode 100644 index 0000000..8dfcf8e --- /dev/null +++ b/doc/doc_common/identifiers.rst @@ -0,0 +1,34 @@ +Identifiers +=========== + +The "identifier" structure is used to provide an enumerated list of options. + +For a complete reference of all available identifiers, see the +`IMAS Data Dictionary Identifiers `__ documentation. + +.. csv-table:: Identifier examples (from part of the ``core_sources/source`` identifier) + :header-rows: 1 + + Index, Name, Description + 2, NBI, Source from Neutral Beam Injection + 3, EC, Sources from heating at the electron cyclotron heating and current drive + 4, LH, Sources from lower hybrid heating and current drive + 5, IC, Sources from heating at the ion cyclotron range of frequencies + 6, fusion, "Sources from fusion reactions, e.g. alpha particle heating" + +Using the identifiers library +----------------------------- + +|identifiers_link_instructions| + +Below examples illustrates how to use the identifiers in your Matlab programs. + +.. literalinclude:: code_samples/identifier_example1 + :caption: Matlab example 1: obtain identifier information of coordinate identifier ``phi`` + +.. literalinclude:: code_samples/identifier_example2 + :caption: Matlab example 2: Use the identifier library to fill the ``NBI`` label in the ``core_sources`` IDS + +.. literalinclude:: code_samples/identifier_example3 + :caption: Matlab example 3: Use the identifier library to fill the type of coordinate system used in the ``equilibrium`` IDS + diff --git a/doc/doc_common/imas.rst b/doc/doc_common/imas.rst new file mode 100644 index 0000000..dc10c94 --- /dev/null +++ b/doc/doc_common/imas.rst @@ -0,0 +1,4 @@ +IMAS overview +============= + +For more information about IMAS, see the `IMAS Data Dictionary Introduction `__. diff --git a/doc/doc_common/imas_uri.rst b/doc/doc_common/imas_uri.rst new file mode 100644 index 0000000..1c7bc02 --- /dev/null +++ b/doc/doc_common/imas_uri.rst @@ -0,0 +1,43 @@ +.. _data entry uris: + +IMAS Data Entry URIs +==================== + +This documentation covers how to specify where and how IMAS data is stored using URIs. + +For comprehensive details about IMAS Data Entry URIs and the URI scheme, please refer to the +official IMAS Data Dictionary documentation: + +.. seealso:: + :title: IMAS URI and Data Entry Documentation + + - `Data-Entry and Occurrence `__ + - Explains what a data-entry is and the concept of occurrences in IMAS + + - `IMAS Access-Layer URI Scheme `__ + - Complete reference for the IMAS URI structure including: + + - Scheme, host, backend, query, and fragment components + - Backend types and their options + - Query parameters and keys + - URI examples and legacy identifiers + +When you :ref:`load or store IMAS data `, you need to provide +a data entry URI according to these specifications. + +.. note:: + + For detailed information about all backends, query keys, and backend-specific options, + please refer to the `IMAS URI Scheme documentation + `__ + in the IMAS Data Dictionary. + + + + +.. note:: + + For detailed information about all backends, query keys, and backend-specific options, + please refer to the `IMAS URI Scheme documentation + `__ + in the IMAS Data Dictionary. diff --git a/doc/doc_common/load_store_ids.rst b/doc/doc_common/load_store_ids.rst new file mode 100644 index 0000000..7f4f446 --- /dev/null +++ b/doc/doc_common/load_store_ids.rst @@ -0,0 +1,261 @@ +Loading and storing IMAS data +============================= + +IMAS data is grouped together in Data Entries. A Data Entry is a collection of +:ref:`IDSs ` and their (potentially) multiple +occurrences, which groups and stores data over multiple IDSs as a single +dataset. The Data Entry concept is used whether the collection of IDSs is stored +in a database or only exists temporarily (for example for communication in an +integrated workflow). + +Loading and storing IMAS data happens through an IMAS Database Entry. A Database +Entry tracks the information required for locating where the Data Entry is (or +will be) stored on disk. In |lang| this object is modeled as |dbentry|. + +You may :ref:`open an existing IMAS Database Entry`, which you can use for +loading data that was stored previously. Alternatively you can :ref:`create a +new IMAS Database Entry` to store IDS data. + + +Open an existing IMAS Database Entry +------------------------------------ + +To open an IMAS Database Entry, you need to know the URI indicating where the +Access Layer can find the data. IMAS URIs start with ``imas:`` and indicate +the format and the location of the stored data. You can find a detailed +description of the IMAS URI syntax on the :ref:`Data entry URIs` page. + +.. literalinclude:: code_samples/dbentry_open + :caption: |lang| example: open an existing IMAS Database Entry + +.. seealso:: + + API documentation for |dbentry_open|. + + +Loading IMAS data +----------------- + +After you open a database entry, you can request to load data from disk. + +.. contents:: + :local: + + +Load an entire IDS +'''''''''''''''''' + +With |dbentry_get| you can load ("get") an entire IDS from the database entry. + +Multiple `occurrences` of an IDS may be stored in a data entry. By default, if +you don't specify an occurrence number, occurrence 0 is loaded. By providing an +occurrence number you can load a specific occurrence. How different occurrences +are used depends on the experiment. They could, for example, correspond to: + +- different methods for computing the physical quantities of the IDS, or +- different functionalities in a workflow (e.g. initial values, prescribed + values, values at next time step, …), or +- multiple subsystems (e.g. diagnostics) of the same type in an experiment, etc. + +.. todo:: extend docs after Task 2c. is implemented (get multiple occurrences) + +.. literalinclude:: code_samples/dbentry_get + :caption: |lang| example: get an IDS from an IMAS Database Entry + +.. seealso:: + + - API documentation for |dbentry_get|. + + +Load a single `time slice` of an IDS +'''''''''''''''''''''''''''''''''''' + +Instead of loading a full IDS from disk, the Access Layer allows you to load a +specific `time slice`. This is often useful when you're not interested in the +full time evolution, but instead want data of a specific time. You can use +|dbentry_getslice| for this. + +Most of the time there are no entries at that specific time, so you also need to +indicate an `interpolation method`. This determines what values the access layer +returns when your requested time is in between available time points in the +data. Three interpolation methods currently exist: + +|PREVIOUS_INTERP| + Returns the `previous` time slice if the requested time does not exactly + exist in the original IDS. + + For example, when data exists at :math:`t=\{1, 3, 4\}`, requesting + :math:`t_r=2.1` will give you the data at :math:`t=1`. + + .. csv-table:: Edge case behaviour. :math:`\{t_i\}, i=1..N` represents the time series stored in the IDS. + :header-rows: 1 + + Case, Behaviour + :math:`t_r \lt t_1`, Return data at :math:`t_1`. + :math:`t_r = t_i` [#equal_note]_, Return data at :math:`t_i`. + + +|CLOSEST_INTERP| + Returns the `closest` time slice in the original IDS. This can also be + `after` the requested time. + + For example, when data exists at :math:`t=\{1, 3, 4\}`, requesting + :math:`t=2.1` will give you the data at :math:`t=3`. + + .. csv-table:: Edge case behaviour. :math:`\{t_i\}, i=1..N` represents the time series stored in the IDS. + :header-rows: 1 + + Case, Behaviour + :math:`t_r \lt t_1`, Return data at :math:`t_1`. + :math:`t_r = t_i` [#equal_note]_, Return data at :math:`t_i`. + :math:`t_r - t_i = t_{i+1} - t_r` [#equal_note]_, Return data at :math:`t_{i+1}`. + +.. [#equal_note] Equality for floating point numbers is tricky. For example, + :code:`3.0/7.0 + 2.0/7.0 + 2.0/7.0` is not exactly equal to :code:`1.0`. It + is therefore advised not to depend on this behaviour. + +|LINEAR_INTERP| + Returns a linear interpolation between the existing slices before and after + the requested time. + + For example, when data exists at :math:`t=\{1, 3, 4\}`, requesting + :math:`t=2.1` will give you a linear interpolation of the data at + :math:`t=1` and the data at :math:`t=3`. + + Note that the linear interpolation will be successful only if, between the + two time slices of an interpolated dynamic array of structure, the same + leaves are populated and they have the same size. Otherwise + |dbentry_getslice| will interpolate all fields with a compatible size and + leave others empty. + + .. csv-table:: Edge case behaviour. :math:`\{t_i\}, i=1..N` represents the time series stored in the IDS. + :header-rows: 1 + + Case, Behaviour + :math:`t_r \lt t_1`, Return data at :math:`t_1`. + :math:`t_r \gt t_N`, Return data at :math:`t_N`. + +.. literalinclude:: code_samples/dbentry_getslice + :caption: |lang| example: get a time slice from an IMAS Database Entry + +.. note:: + + The access layer assumes that all time arrays are stored in increasing + order. |dbentry_getslice| may return unexpected results if your data does + not adhere to this assumption. + +.. seealso:: + + API documentation for |dbentry_getslice|. + + +.. include:: partial_get + + +Create a new IMAS Database Entry +-------------------------------- + +To create a new IMAS Database Entry, you need to provide the URI to indicate the +format and the location where you want to store the data. You can find a +detailed description of the IMAS URI syntax and the options available on the +:ref:`Data entry URIs` page. + +.. caution:: + + This function erases any existing database entry on the specified URI! + + +.. literalinclude:: code_samples/dbentry_create + :caption: |lang| example: create a new IMAS Database Entry + +.. seealso:: + + API documentation for |dbentry_create|. + + +Store IMAS data +--------------- + +After you have created an IMAS Database Entry, you can use it for storing IDS +data. There are two ways to do this: + +.. contents:: + :local: + + +Store an entire IDS +''''''''''''''''''' + +With |dbentry_put| you can store ("put") an entire IDS in a database entry. +First you need to have an IDS with data: you can create a new one or :ref:`load +an IDS ` which you modify. See :ref:`Use Interface Data +Structures` for more information on using and manipulating IDSs. + +.. caution:: + + This function erases the existing IDS in the data entry if any was already + stored previously. + +Multiple `occurrences` of an IDS may be stored in a data entry. By default, if +you don't specify an occurrence number, the IDS is stored as occurrence 0. By +providing an occurrence number you can store the IDS as a specific occurrence. + +.. note:: + + The MDS+ backend has a limitation on the number of occurrences of a given + IDS. This number is indicated in the |DD| documentation in the "Max. + occurrence number" column of the list of IDSs. This limitation doesn't apply + to other backends. + +.. literalinclude:: code_samples/dbentry_put + :caption: |lang| example: put an IDS to an IMAS Database Entry + +.. seealso:: + + API documentation for |dbentry_put|. + + +Append a time slice to an already-stored IDS +'''''''''''''''''''''''''''''''''''''''''''' + +With |dbentry_put_slice| you can append a time slice to an existing database +entry. This is useful when you generate data inside a time loop (for example in +simulations, or when taking measurements of an experiment). + +It means you can put a time slice with every iteration of your loop such that +you don't have to keep track of the complete time evolution in memory. Instead, +the Access Layer will keep appending the data to the Database Entry in the +storage backend. + +.. note:: + + Although being put progressively time slice by time slice, the final IDS + must be compliant with the data dictionary. A typical error when + constructing IDS variables time slice by time slice is to change the size of + the IDS fields during the time loop, which is not allowed but for the + children of an array of structure which has time as its coordinate. + +.. literalinclude:: code_samples/dbentry_put_slice + :caption: |lang| example: iteratively put time slices to an IMAS Database Entry + +.. seealso:: + + API documentation for |dbentry_put_slice|. + + +Listing all occurrences of an IDS from a backend +'''''''''''''''''''''''''''''''''''''''''''''''' + +With |list_all_occurrences| you can List all non-empty occurrences of an IDS +using its name in the dataset, and optionnally return the content of a +descriptive node path. + +.. note:: + + The MDS+ backend is storing IDS occurrences infos (pulse file metadata) + for AL version > 5.0.0. Pulse files created with AL version <= 5.0.0. + do not provide these informations (an exception will occur for such + pulse files when calling |list_all_occurrences|). + +.. literalinclude:: code_samples/dbentry_list_all_occurrences + :caption: |lang| example: listing all occurrences of a magnetics IDS from an IMAS Database Entry diff --git a/doc/doc_common/requirements.txt b/doc/doc_common/requirements.txt new file mode 100644 index 0000000..c738935 --- /dev/null +++ b/doc/doc_common/requirements.txt @@ -0,0 +1,11 @@ +# Sphinx and theme +sphinx >= 6.0, < 7.0 +sphinx_immaterial >= 0.11.4, < 0.12 + +six # un-listed dependency of sphinx-fortran + +# Matlab domain +sphinxcontrib-matlabdomain + +# Requirements for the HLI files we import for the Python docs +numpy diff --git a/doc/doc_common/static/.keep b/doc/doc_common/static/.keep new file mode 100644 index 0000000..e69de29 diff --git a/doc/doc_common/use_ids.rst b/doc/doc_common/use_ids.rst new file mode 100644 index 0000000..6f10b4b --- /dev/null +++ b/doc/doc_common/use_ids.rst @@ -0,0 +1,339 @@ +Use Interface Data Structures +============================= + +The Interface Data Structures (IDSs) are the main way to interact with IMAS +data. An IDS is a tree-like structure with one root element (the IDS) and +several branches with data at the leave nodes. + +Many types of IDSs exist: check out the documentation of the |DD| for a complete +overview. + + +Creating IDSs +------------- + +IDSs can be created in multiple ways: + +1. :ref:`Load an IDS from disk ` +2. :ref:`Create an empty IDS` +3. :ref:`Create a copy of an IDS` + + +Create an empty IDS +''''''''''''''''''' + +You can create an empty instance of an IDS by |create_ids_text| creates an empty +``core_profiles`` IDS. This initializes all items in the IDS to their +:ref:`default values`. + +.. literalinclude:: code_samples/ids_create + :caption: |lang| example: create an empty IDS + + +Create a copy of an IDS +''''''''''''''''''''''' + +You can create a copy of another IDS |copy_ids|. + +.. literalinclude:: code_samples/ids_copy + :caption: |lang| example: create a copy of an IDS + + +Deallocate an IDS +''''''''''''''''' + +If you no longer need an IDS, you can deallocate it so it releases the (memory) +resources in use by the data. |deallocate_ids_text| + +.. literalinclude:: code_samples/ids_deallocate + :caption: |lang| example: deallocate an IDS + + +Mandatory and recommended IDS attributes +---------------------------------------- + +Some attributes in an IDS are mandatory or recommended to always fill. Below +list provides a short overview: + +.. todo:: + + Link to DD documentation + +1. ``ids_properties/homogeneous_time`` `[mandatory]`: see :ref:`Time coordinates + and time handling`. +2. ``ids_properties/comment`` `[recommended]`: a comment describing the content + of this IDS. +3. ``ids_properties/provider`` `[recommended]`: name of the person in charge of + producing this data. +4. ``ids_properties/creation_date`` `[recommended]`: date at which this data has + been produced, recommended to use the `ISO 8601 + `_ ``YYYY-MM-DD`` format. + +.. note:: + + ``ids_properties/version_put`` is filled by the access layer when you + :ref:`put an IDS `. + + +Understanding the IDS structure +------------------------------- + +An IDS is a `tree structure +`_. You can think of it +similar as a directory structure with files: the IDS is the root "directory", +and inside it you can find "subdirectories" and "files" with data. + +We will use the general Computer Science terminology for tree structures and +call these "files" and "directories" of our IDSs `nodes`. IDSs can have a +limited number of different types of nodes: + +1. :ref:`Structure`: think of these as the directories of your file + system. Structures contain one or more child nodes (files and + subdirectories). Child nodes can be of any node type again. + +2. :ref:`Array of structures`: this is an array of structures (see + previous point). + +3. :ref:`Data`: this is a data element. Like files on your file system + these nodes contain the actual data stored in the IDS. + + +Structure +''''''''' + +Structure nodes in an IDS are a container for other nodes. In |lang| they are +implemented as a |structures_type|. You can address child nodes as +|structures_child_attribute|, see the code sample below. + +.. literalinclude:: code_samples/ids_structures_node + :caption: |lang| example: address the child node of an IDS structure node + + +Array of structures +''''''''''''''''''' + +Array of structure nodes in an IDS are one-dimensional arrays, containing structure +nodes. In |lang| they are implemented as a |aos_type|. The default value (for +example, when creating a new IDS) for these nodes is |aos_default|. + +.. literalinclude:: code_samples/ids_array_of_structures_node + :caption: |lang| example: address the child node of an IDS arrays of structure node + + +Resizing an array of structures +``````````````````````````````` + +You can resize an array of structures with |aos_resize_meth|. After calling +this, the array of structures will have ``n`` elements. + +.. caution:: + + Resizing an array of structures with |aos_resize_meth| will clear all data + inside the array of structure! Use |aos_resize_keep_meth| to keep existing + data. + +.. literalinclude:: code_samples/ids_array_of_structures_resize + :caption: |lang| example: resizing an array of structures + + +Data +'''' + +Data nodes in an IDS contain numerical or textual data. The data type and +dimensions of a node are defined in the |DD|. + +.. literalinclude:: code_samples/ids_data_node + :caption: |lang| example: get the data contained in a data node of an IDS + + +Data types +`````````` + +The following data types exist: + +- Textual data (|str_type|) +- Whole numbers (|int_type|) +- Floating point numbers (|double_type|) +- Complex floating point numbers (|complex_type|) + +Data nodes can be 0-dimensional, which means that the node accepts a single +value of the specified type. Multi-dimensional data nodes also exist: + +- Textual data: at most 1 dimension (|str_1d_type|) +- Whole numbers: 1-3 dimensions (|int_nd_type|) +- Floating point numbers: 1-6 dimensions (|double_nd_type|) +- Complex floating point numbers: 1-6 dimensions (|complex_nd_type|) + + +.. _Empty fields: + +Default values +`````````````` + +The default values for data fields (for example when creating an empty IDS) are +indicated in the following table. |isFieldValid| + +.. csv-table:: + :header-rows: 1 + :stub-columns: 1 + + , 0D, 1+ dimension + "Textual + + data", |str_default|, |str_1D_default| + "Whole + + numbers", |int_default|, |ND_default| + "Floating + + point + + numbers", |double_default|, |ND_default| + "Complex + + numbers", |complex_default|, |ND_default| + + +Time coordinates and time handling +'''''''''''''''''''''''''''''''''' + +Some quantities (and array of structures) are time dependent. In the |DD| +documentation this is indicated by a coordinate that refers to a time quantity. + +This time-dependent coordinate is treated specially in the access layer, and it +depends on the value of ``ids_properties/homogeneous_time``. There are three +valid values for this property: + +.. todo:: + + Add links to DD. + +1. |tm_heterogeneous| (=0): time-dependent quantities in the IDS may have + different time coordinates. The time coordinates are stored as indicated by + the path in the documentation. This is known as `heterogeneous time`. +2. |tm_homogeneous| (=1): All time-dependent quantities in this IDS use the same + time coordinate. This is known as `homogeneous time`. This time coordinate is + located in the root of the IDS, for example ``core_profiles/time``. The paths + time paths indicated in the documentation are unused in this case. +3. |tm_independent| (=2): The IDS stores no time-dependent data. + + + +IDS validation +-------------- + +The IDSs you fill should be consistent. To help you in validating that, the +Access Layer provides a validation method (|ids_validate|) that executes the +following checks. + +.. contents:: Validation checks + :local: + :depth: 1 + +If you call this method and your IDS fails validation, the Access Layer +|validate_error| explaining the problem. See the following example: + +.. literalinclude:: code_samples/ids_validate + :caption: |lang| example: call IDS validation + +The Access Layer automatically validates an IDS every time you do a +`put` or `put_slice`. To disable this feature, you must set the environment +variable ``IMAS_AL_DISABLE_VALIDATE`` to ``1``. + +.. seealso:: + + API documentation: |ids_validate| + + +Validate the time mode +'''''''''''''''''''''' + +The time mode of an IDS is stored in ``ids_properties.homogeneous_time``. This +property must be filled with a valid time mode (|tm_homogeneous|, +|tm_heterogeneous| or |tm_independent|). When the time +mode is |tm_independent|, all time-dependent quantities must be empty. + + +Validate coordinates +'''''''''''''''''''' + +If a quantity in your IDS has coordinates, then these coordinates must be filled. The +size of your data must match the size of the coordinates: + +.. todo:: link to DD docs + +1. Some dimensions must have a fixed size. This is indicated by the Data Dictionary + as, for example, ``1...3``. + + For example, in the ``magnetics`` IDS, ``b_field_pol_probe(i1)/bandwidth_3db`` has + ``1...2`` as coordinate 1. This means that, if you fill this data field, the first + (and only) dimension of this field must be of size 2. + +2. If the coordinate is another quantity in the IDS, then that coordinate must be + filled and have the same size as your data. + + For example, in the ``pf_active`` IDS, ``coil(i1)/current_limit_max`` is a + two-dimensional quantity with coordinates ``coil(i1)/b_field_max`` and + ``coil(i1)/temperature``. This means that, if you fill this data field, their + coordinate fields must be filled as well. The first dimension of + ``current_limit_max`` must have the same size as ``b_field_max`` and the second + dimension the same size as ``temperature``. + + Time coordinates are handled depending on the value of + ``ids_properties/homogeneous_time``: + + - When using |tm_homogeneous|, all time coordinates look at the root + ``time`` node of the IDS. + - When using |tm_heterogeneous|, all time coordinates look at the time + path specified as coordinate by the Data Dictionary. + + For dynamic array of structures, the time coordinates is a ``FLT_0D`` inside the + AoS (see, for example, ``profiles_1d`` in the ``core_profiles`` IDS). In such + cases the time node must be set to something different than ``EMPTY_FLOAT``. + This is the only case in which values of the coordinates are verified, in all + other cases only the sizes of coordinates are validated. + + .. rubric:: Alternative coordinates + + Version 4 of the Data Dictionary introduces alternative coordinates. An + example of this can be found in the ``core_profiles`` IDS in + ``profiles_1d(itime)/grid/rho_tor_norm``. Alternatives for this coordinate + are: + + - ``profiles_1d(itime)/grid/rho_tor`` + - ``profiles_1d(itime)/grid/psi`` + - ``profiles_1d(itime)/grid/volume`` + - ``profiles_1d(itime)/grid/area`` + - ``profiles_1d(itime)/grid/surface`` + - ``profiles_1d(itime)/grid/rho_pol_norm`` + + Multiple alternative coordinates may be filled (for example, an IDS might + fill both the normalized and non-normalized toroidal flux coordinate). In + that case, the size must be the same. + + When a quantity refers to this set of alternatives (for example + ``profiles_1d(itime)/electrons/temperature``), at least one of the + alternative coordinates must be set and its size match the size of the + quantity. + +3. The Data Dictionary can indicate exclusive alternative coordinates. See for + example the ``distribution(i1)/profiles_2d(itime)/density(:,:)`` quantity in the + ``distributions`` IDS, which has as first coordinate + ``distribution(i1)/profiles_2d(itime)/grid/r OR + distribution(i1)/profiles_2d(itime)/grid/rho_tor_norm``. This means that + either ``r`` or ``rho_tor_norm`` can be used as coordinate. + + Validation works the same as explained in the previous point, except that + exactly one of the alternative coordinate must be filled. Its size must, of + course, still match the size of the data in the specified dimension. + +4. Some quantites indicate a coordinate must be the same size as another quantity + through the property ``coordinateX_same_as``. In this case, the other quantity is + not a coordinate, but their data is related and must be of the same size. + + An example can be found in the ``edge_profiles`` IDS, quantity + ``ggd(itime)/neutral(i1)/velocity(i2)/diamagnetic``. This is a two-dimensional field + for which the first coordinate must be the same as + ``ggd(itime)/neutral(i1)/velocity(i2)/radial``. When the diamagnetic velocity + component is filled, the radial component must be filled as well, and have a + matching size. diff --git a/doc/doc_common/using_al.rst b/doc/doc_common/using_al.rst new file mode 100644 index 0000000..143f841 --- /dev/null +++ b/doc/doc_common/using_al.rst @@ -0,0 +1,21 @@ +Using the IMAS-MATLAB +====================== + +Making the IMAS-MATLAB available for use +----------------------------------------- + +When you're working with a local installation (see :ref:`Building and +installing the IMAS-MATLAB`), you can source the installed environment file: + +.. code-block:: bash + :caption: Set environment variables (replace ```` with the folder of your local install) + + source /bin/al_env.sh + + +.. + HLI-specific documentation should include the following items: + + - Code samples for loading the library (import, include, use, etc.) + - Example program printing the IMAS-MATLAB version used + - Instructions for (compiling if relevant) and running the example program diff --git a/doc/getting_started.rst b/doc/getting_started.rst new file mode 100644 index 0000000..924501a --- /dev/null +++ b/doc/getting_started.rst @@ -0,0 +1,173 @@ +Getting Started with IMAS-MATLAB +================================= + +Welcome! This 5-minute guide will get you up and running with the IMAS-MATLAB. + +**What is IMAS-MATLAB?** + +IMAS-MATLAB is the IMAS data access library (formerly known as the Access Layer) for Matlab users/developers. + + +Load the IMAS-MATLAB Module +------------------------------- + +On the ITER SDCC (supercomputing cluster), make the Access Layer available: + +.. code-block:: bash + + module load IMAS-MATLAB + +To see available versions: + +.. code-block:: bash + + module avail IMAS-MATLAB + +If you have a local installation, source the environment file instead: + +.. code-block:: bash + + source /bin/al_env.sh + + +Open MATLAB and Connect to Data +------------------------------------------- + +Start MATLAB and open a database entry using an IMAS URI. A URI tells the Access Layer +where your data is stored and in what format. + +.. code-block:: matlab + + % Open a database entry + uri = 'imas:hdf5?path=/path/to/data'; + ctx = imas_open(uri, 40); + + if ctx < 0 + error('Unable to open database'); + end + +**What's an IMAS URI?** + +URIs follow the format: ``imas:backend?query_options`` + +For example: +- ``imas:hdf5?path=/path/to/data`` – Read from HDF5 files +- ``imas:mdsplus?path=./test_db`` – Read from MDSplus +- ``imas:uda?backend=...`` – Read from UDA backend + +Learn more: :ref:`Data entry URIs` + + +Load and Display Dat +------------------------ + +Fetch an IDS from your database entry: + +.. code-block:: matlab + + % Load the magnetics IDS (occurrence 0) + magnetics = ids_get(ctx, 'magnetics'); + + % Explore the data + disp(magnetics.ids_properties); % Metadata + disp(magnetics.time); % Time points + disp(magnetics.flux_loop{1}.flux.data); % Access nested data + + +Modify and Store Data +------------------------ + +You can create new data, modify existing data, and store it back: + +.. code-block:: matlab + + % Create a new IDS or modify existing one + equilibrium = ids_init('equilibrium'); + equilibrium.time = [0, 1, 2, 3]; + equilibrium.q_profile.value.data = [1, 2, 3, 4]; + + % Store it to the database + ids_put(ctx, 'equilibrium', equilibrium); + + +Clean Up +----------- + +Always close the database entry when you're done: + +.. code-block:: matlab + + imas_close(ctx); + + +Key Functions Reference +----------------------- + ++====================================+=============================================+ +| Function | Purpose | ++====================================+=============================================+ +| ``imas_open(uri, version)`` | Open a database entry at the given URI | ++------------------------------------+---------------------------------------------+ +| ``imas_close(ctx)`` | Close the database entry | ++------------------------------------+---------------------------------------------+ +| ``ids_get(ctx, ids_name)`` | Load an entire IDS | ++------------------------------------+---------------------------------------------+ +| ``ids_put(ctx, ids_name, ids_obj)``| Store an IDS to disk | ++------------------------------------+---------------------------------------------+ +| ``ids_init(ids_name)`` | Create and initialize a new IDS | ++------------------------------------+---------------------------------------------+ +| ``ids_get_slice(ctx, ids_name, time)`` | Load a specific time slice | ++====================================+=============================================+ + + +Common Use Cases +---------------- + +**Load data and extract a single time slice:** + +.. code-block:: matlab + + % Use CLOSEST interpolation + data = ids_get_slice(ctx, 'equilibrium', 2.5, 'CLOSEST'); + + +**Check if data exists:** + +.. code-block:: matlab + + if ids_isdefined(magnetics.flux_loop{1}.flux) + disp('Flux data is defined'); + end + + +**Access MATLAB examples:** + +The repository contains several example scripts in the ``examples/`` directory: +- ``test_get.m`` Load and display data +- ``test_put.m`` Store new data +- ``test_get_sample_magnetics.m`` Practical magnetics data example + + +Next Steps +---------- + +- **Read more about IDSs**: :doc:`Use Interface Data Structures ` +- **Learn advanced loading/storing**: :doc:`Loading and storing IMAS data ` +- **Understand data storage**: :ref:`Data entry URIs` +- **Check the full API documentation**: See your installed IMATLAB help or visit the + `IMAS-MATLAB `__ + + +Common Issues +------------- + +**"Unable to open pulse" error:** +- Check that your URI is correct and the data path exists + +**IDS not found:** +- Verify the data entry contains this IDS +- Use ``ids_isdefined()`` to check existence first + +**Need help?** +- Check the :doc:`Using the Access Layer ` guide +- Consult the `IMAS-MATLAB `__ \ No newline at end of file diff --git a/doc/imas_uri.rst b/doc/imas_uri.rst index 4c48dce..d5a21ee 100644 --- a/doc/imas_uri.rst +++ b/doc/imas_uri.rst @@ -1 +1,7 @@ +.. _ascii backend: +.. _hdf5 backend: +.. _mdsplus backend: +.. _memory backend: +.. _uda backend: + .. include:: ./doc_common/imas_uri.rst diff --git a/doc/index.rst b/doc/index.rst index 123e87b..bf55cd1 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,46 +1,23 @@ Access Layer - MATLAB High Level Interface ========================================== -These pages document the MATLAB High Level Interface to the :ref:`IMAS ` Access Layer. All code samples and API documentation will show how to use the access layer in the `MATLAB `_ programming language. -.. seealso:: - :title: Access Layer documentation for different programming languages - :collapsible: - - Check out the following pages if you want to use the Access Layer with a - different programming language: - - - `Python High Level Interface - `_ - - `Fortran High Level Interface - `_ - - `C++ High Level Interface - `_ - - `Java High Level Interface - `_ - -.. todo:: - - Replace IDM links with sharepoint links to the respective HLIs - -.. todolist:: - .. toctree:: :maxdepth: 2 :caption: Contents: - imas + getting_started + imas using_al load_store_ids use_ids imas_uri identifiers - conf - plugins .. toctree:: :caption: Examples diff --git a/doc/plugins.rst b/doc/plugins.rst deleted file mode 100644 index e199c5a..0000000 --- a/doc/plugins.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. include:: ./doc_common/plugins.rst - - -.. toctree:: - :caption: Contents - :maxdepth: 2 - - plugins_architecture - plugins_examples diff --git a/doc/plugins_architecture.rst b/doc/plugins_architecture.rst deleted file mode 100644 index a0b402a..0000000 --- a/doc/plugins_architecture.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ./doc_common/plugins_architecture.rst diff --git a/doc/plugins_examples.rst b/doc/plugins_examples.rst deleted file mode 100644 index 22e6081..0000000 --- a/doc/plugins_examples.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ./doc_common/plugins_examples.rst