Fix config access due to race condition

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 100

.github/workflows/apply-formatting.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     if: startsWith(github.event.comment.body, '@par-hermes format')
     steps:
-    - uses: JoshuaSBrown/cpp-py-formatter/command@v0.2.3
+    - uses: JoshuaSBrown/cpp-py-formatter/command@v0.3.8
       with:
         botName: par-hermes
         clangFormatVersion: 10

.github/workflows/format-check.yml

@@ -1,37 +1 @@
-name: Format Check
-on: push
-jobs:
-  check-formatting:
-    name: Code Formatting
-    runs-on: ubuntu-latest
-    steps:
-    - uses: JoshuaSBrown/cpp-py-formatter/check@v0.2.3
-      with:
-        clangFormatVersion: 10
-        token: ${{ secrets.GITHUB_TOKEN }}
 
-  format-check:
-    runs-on: ubuntu-latest
-    if: ${{ always() }}
-    # needs: check-formatting
-    strategy:
-      matrix:
-        python-version: [ "3.9" ]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Update ubuntu
-        run: sudo apt-get update
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          sudo apt-get install -y python3-sphinx sphinx-rtd-theme-common
-          pip install sphinx_rtd_theme recommonmark flake8 black pytest pytest-cov black[jupyter]
-      - name: Black formatting checks
-        run: black  --exclude '/jupyter_notebooks/' --check .
-      - name: Lint with flake8
-        run: |
-          flake8 --count --select=E9,F63,F7,F82 --show-source --statistics --max-line-length 100 .
-          flake8 --count --statistics --max-line-length 100 .

.github/workflows/javascript-format.yml

@@ -0,0 +1,34 @@
+name: JavaScript Formatter
+on: push
+jobs:
+  format-javascript:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Step 1: Checkout the repository
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      # Step 2: Set up Node.js environment
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18' # Specify your Node.js version
+
+      # Step 3: Install Prettier and ESLint globally
+      - name: Install Prettier
+        run: |
+          npm install -g prettier
+
+      # Step 4: Run Prettier to format code
+      - name: Run prettier
+        run: |
+          prettier "**/*.js" --write
+          git diff
+          git reset --hard
+          prettier --check "**/*.js" 
+
+      # Step 5: Report status
+      - name: Complete
+        run: echo "Formatting completed successfully!"
+

.github/workflows/javascript-lint.yml

@@ -0,0 +1,28 @@
+name: JavaScript Linter
+on: push
+jobs:
+  lint-javascript:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Step 1: Checkout the repository
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      # Step 2: Set up Node.js environment
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18' # MOre up to date version needed for linter
+
+      # Step 3: Instal ESLint and run
+      - name: Install ESLint
+        run: |
+          npm init -y
+          npm install eslint@latest @babel/eslint-parser@latest eslint-define-config globals eslint-plugin-jsdoc --save
+          npx eslint "**/*.js"
+
+      # Step 4: Report status
+      - name: Complete
+        run: echo "Linting completed successfully!"
+

.github/workflows/jupyternotebook-lint.yml

@@ -0,0 +1,40 @@
+name: Jupyter Notebook Linter
+on:
+  push:
+    paths:
+      - '**/*.ipynb'
+      - '.github/workflows/jupyternotebook-lint.yml'
+
+jobs:
+  lint-jupyter:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Step 1: Checkout the repository
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      # Step 2: Set up Python environment
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'  # Specify the Python version
+
+      # Step 3: Install dependencies
+      - name: Install nbQA, flake8, and black
+        run: |
+          pip install nbqa flake8 black
+
+      # Step 4: Run flake8 and black on notebooks in ./jupyter_notebooks folder
+      - name: Lint Jupyter Notebooks with flake8
+        run: nbqa flake8 .
+        working-directory: jupyter_notebooks
+
+      - name: Format Jupyter Notebooks with black
+        run: nbqa black . --check
+        working-directory: jupyter_notebooks
+
+      # Step 5: Report status
+      - name: Complete
+        run: echo "Jupyter Notebook linting completed!"
+

.github/workflows/unit-tests.yml

@@ -2,20 +2,30 @@ name: Unit-Testing
 on: push
 jobs:
   unit-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     if: ${{ always() }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Update ubuntu
         run: sudo apt-get update
+      - name: Install software-properties-common
+        run: sudo apt install software-properties-common
+      - name: Add deadsnakes apt repository
+        run: sudo add-apt-repository ppa:deadsnakes/ppa
+      - name: Update to bring in deadsnakes packages
+        run: sudo apt update
+      - name: Install Python 3.9
+        run: sudo apt install python3.9 python3.9-dev python3.9-venv python3.9-distutils
+      - name: Ensure pip is installed
+        run: python3.9 -m ensurepip
       - name: Install dependencies
         run: |
           ./scripts/generate_datafed.sh
           sudo ./scripts/install_core_dependencies.sh
           ./scripts/generate_datafed.sh
       - name: Build
         run: |
-          cmake -S. -B build -DCMAKE_BUILD_TYPE=Debug
+          cmake -S. -B build -DCMAKE_BUILD_TYPE=Debug -DBUILD_WEB_SERVER=OFF
           cmake --build build -j4
       - name: Run tests
         run: |

.gitignore

@@ -1,4 +1,5 @@
 /.idea/
+.venv/
 build
 cmake/sodium_version
 cmake/curl_version
@@ -42,6 +43,7 @@ python/datafed_pkg/datafed/Version_pb2.py
 python/datafed_pkg/datafed/__pycache__/
 python/datafed_pkg/dist/
 web/package.json
+repository/gridftp/globus5/authz/mock_core/Version.hpp
 repository/gridftp/globus5/authz/source/Version.hpp
 repository/server/Version.hpp
 scripts/globus/datafed-home-repo.sh

.gitlab-ci.yml

@@ -22,6 +22,8 @@ include:
   - local: .gitlab/stage_provision_client.yml
   - local: .gitlab/stage_image_check.yml
   - local: .gitlab/stage_build.yml
+  - local: .gitlab/stage_test.yml
+  - local: .gitlab/stage_unit.yml
   - local: .gitlab/end_to_end.yml
 
 stages:
@@ -34,9 +36,12 @@ stages:
   - provision-client
   - image-check
   - build
+  - test # Required for gitlab sast templates
+  - unit
   - end-to-end-setup-arango
   - end-to-end-setup
   - end-to-end-test
+  - end-to-end-test-finish-signal
   - log
   - deploy-pypi-package
 
@@ -50,3 +55,4 @@ variables:
   HARBOR_USER: 'robot$$datafed+harbor_datafed_gitlab_ci_registry_2'
   REGISTRY: 'camden.ornl.gov'
   DATAFED_DEPENDENCIES_INSTALL_PATH: "/shared/install"
+  DOCKER_TLS_CERTDIR: "" # Required for running docker in docker

.gitlab/build/build_core_image.yml

@@ -9,10 +9,12 @@ build-core:
   extends: .docker_build_script
   stage: build
   variables:
-    IMAGE_TAG: "datafed/core"
+    PROJECT: "datafed"
+    COMPONENT: "core"
     GIT_STRATEGY: clone
     DOCKER_FILE_PATH: "core/docker/Dockerfile"
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - ci-datafed-core
     - docker
@@ -31,8 +33,21 @@ retag-image:
   extends: .docker_retag_image
   stage: build
   variables:
-    IMAGE_TAG: "datafed/core"
+    PROJECT: "datafed"
+    COMPONENT: "core"
     GIT_STRATEGY: clone
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - docker
+  rules:
+    - changes:
+        - docker/**/*
+        - scripts/**/*
+        - core/**/*
+        - common/**/*
+        - CMakeLists.txt
+        - cmake/**/*
+        - .gitlab-ci.yml
+      when: never
+    - when: on_success

.gitlab/build/build_foxx_image.yml

@@ -9,27 +9,46 @@ build-foxx:
   extends: .docker_build_script
   stage: build
   variables:
-    IMAGE_TAG: "datafed/foxx"
+    PROJECT: "datafed"
+    COMPONENT: "foxx"
     GIT_STRATEGY: clone
     DOCKER_FILE_PATH: "docker/Dockerfile.foxx"
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - docker
   rules:
     - changes:
         - docker/**/*
         - scripts/**/*
-        - web/**/*
+        - cmake/**/*
+        - core/database/**/*
+        - core/CMakeLists.txt
         - common/proto/**/*
         - .gitlab-ci.yml
+        - CMakeLists.txt
       when: on_success
 
 retag-image:
   extends: .docker_retag_image
   stage: build
   variables:
-    IMAGE_TAG: "datafed/foxx"
+    PROJECT: "datafed"
+    COMPONENT: "foxx"
     GIT_STRATEGY: clone
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - docker
+  rules:
+    - changes:
+        - docker/**/*
+        - scripts/**/*
+        - cmake/**/*
+        - core/database/**/*
+        - core/CMakeLists.txt
+        - common/proto/**/*
+        - .gitlab-ci.yml
+        - CMakeLists.txt
+      when: never
+    - when: on_success

.gitlab/build/build_gcs_base_image.yml

@@ -8,10 +8,12 @@ include:
 build-gcs-base:
   stage: build
   variables:
-    IMAGE_TAG: "datafed/gcs-base"
+    PROJECT: "datafed"
+    COMPONENT: "gcs-base"
     GIT_SUBMODULE_STRATEGY: recursive
     GIT_STRATEGY: clone
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - ci-datafed-globus
     - docker
@@ -31,28 +33,41 @@ build-gcs-base:
     - cd "external/globus-connect-server-deploy/docker"
     - git checkout "$DATAFED_GCS_SUBMODULE_VERSION"
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}"
-    - docker build --progress plain -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" - < "./docker-files/Dockerfile.ubuntu-20.04"
-    - docker tag "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:$CI_COMMIT_SHA"
-    - export DATAFED_HARBOR_REPOSITORY="${IMAGE_TAG}-${BRANCH_LOWER}"
+    - docker build --no-cache --progress plain -t "${REGISTRY}/${PROJECT}/${COMPONENT}-${BRANCH_LOWER}:latest" - < "./docker-files/Dockerfile.ubuntu-20.04"
+    - docker tag "${REGISTRY}/${PROJECT}/${COMPONENT}-${BRANCH_LOWER}:latest" "${REGISTRY}/${PROJECT}/${COMPONENT}-${BRANCH_LOWER}:$CI_COMMIT_SHA"
+    - export DATAFED_HARBOR_REPOSITORY="${COMPONENT}-${BRANCH_LOWER}"
     - export DATAFED_HARBOR_USERNAME="${HARBOR_USER}"
     - export DATAFED_HARBOR_PASSWORD="${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}"
-    - docker push "${REGISTRY}/${DATAFED_HARBOR_REPOSITORY}:latest"
-    - docker push "${REGISTRY}/${DATAFED_HARBOR_REPOSITORY}:$CI_COMMIT_SHA"
+    - docker push "${REGISTRY}/${PROJECT}/${DATAFED_HARBOR_REPOSITORY}:latest"
+    - docker push "${REGISTRY}/${PROJECT}/${DATAFED_HARBOR_REPOSITORY}:$CI_COMMIT_SHA"
+    - cd "${CI_PROJECT_DIR}"
     - |
-      while [ "$(${CI_PROJECT_DIR}/scripts/ci_harbor_artifact_count.sh)" == "0" ]; do
+      while [ "$(${CI_PROJECT_DIR}/scripts/ci_harbor_artifact_count.sh -r ${DATAFED_HARBOR_REPOSITORY})" == "0" ]; do
         echo "Artifact missing from harbor..."
-        docker push "${REGISTRY}/${DATAFED_HARBOR_REPOSITORY}:latest"
-        docker push "${REGISTRY}/${DATAFED_HARBOR_REPOSITORY}:$CI_COMMIT_SHA"
+        docker push "${REGISTRY}/${PROJECT}/${DATAFED_HARBOR_REPOSITORY}:latest"
+        docker push "${REGISTRY}/${PROJECT}/${DATAFED_HARBOR_REPOSITORY}:$CI_COMMIT_SHA"
         sleep 5  # Optional: Add a sleep to avoid busy waiting 
       done
-    - cat "./harbor_check.log"
+    - cat "${CI_PROJECT_DIR}/harbor_check.log"
 
 retag-image:
   extends: .docker_retag_image
   stage: build
   variables:
-    IMAGE_TAG: "datafed/gcs-base"
+    PROJECT: "datafed"
+    COMPONENT: "gcs-base"
     GIT_STRATEGY: clone
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - docker
+  rules:
+    - changes:
+        - docker/**/*
+        - scripts/**/*
+        - common/**/*
+        - .gitlab-ci.yml
+        - CMakeLists.txt
+        - cmake/**/*
+      when: never
+    - when: on_success