From 5da176f4faca56feb0ab254024b0c1036e119f41 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Jan 2026 15:14:34 -0700 Subject: [PATCH 1/8] Add external bash for all DOD HPCMP systems and @climbfuji's dev machines --- configs/sites/tier1/atlantis/packages.yaml | 4 + configs/sites/tier1/blueback/packages.yaml | 4 + configs/sites/tier1/narwhal/packages.yaml | 4 + configs/sites/tier1/nautilus/packages.yaml | 4 + configs/sites/tier2/blackpearl/packages.yaml | 4 + configs/sites/tier2/bounty/packages.yaml | 4 + configs/sites/tier2/jean/packages.yaml | 218 ++++++++++--------- configs/sites/tier2/wheat/packages.yaml | 4 + 8 files changed, 139 insertions(+), 107 deletions(-) diff --git a/configs/sites/tier1/atlantis/packages.yaml b/configs/sites/tier1/atlantis/packages.yaml index 4aff25223..ee26e29b6 100644 --- a/configs/sites/tier1/atlantis/packages.yaml +++ b/configs/sites/tier1/atlantis/packages.yaml @@ -15,6 +15,10 @@ packages: externals: - spec: automake@1.16.1 prefix: /usr + bash: + externals: + - spec: bash@4.4.20 + prefix: /usr binutils: externals: - spec: binutils@2.30.117 diff --git a/configs/sites/tier1/blueback/packages.yaml b/configs/sites/tier1/blueback/packages.yaml index f2ca2d561..571cccf0b 100644 --- a/configs/sites/tier1/blueback/packages.yaml +++ b/configs/sites/tier1/blueback/packages.yaml @@ -12,6 +12,10 @@ packages: externals: - spec: automake@1.15.1 prefix: /usr + bash: + externals: + - spec: bash@4.4.23 + prefix: /usr binutils: externals: - spec: binutils@2.43.1~gold+headers diff --git a/configs/sites/tier1/narwhal/packages.yaml b/configs/sites/tier1/narwhal/packages.yaml index 3918947d7..35a21ea36 100644 --- a/configs/sites/tier1/narwhal/packages.yaml +++ b/configs/sites/tier1/narwhal/packages.yaml @@ -12,6 +12,10 @@ packages: externals: - spec: automake@1.15.1 prefix: /usr + bash: + externals: + - spec: bash@4.4.23 + prefix: /usr binutils: externals: - spec: binutils@2.37.20211103 diff --git a/configs/sites/tier1/nautilus/packages.yaml b/configs/sites/tier1/nautilus/packages.yaml index 1822fd72c..9af694482 100644 --- a/configs/sites/tier1/nautilus/packages.yaml +++ b/configs/sites/tier1/nautilus/packages.yaml @@ -12,6 +12,10 @@ packages: externals: - spec: automake@1.16.1 prefix: /usr + bash: + externals: + - spec: bash@4.4.20 + prefix: /usr binutils: externals: - spec: binutils@2.30.117 diff --git a/configs/sites/tier2/blackpearl/packages.yaml b/configs/sites/tier2/blackpearl/packages.yaml index 992254c14..3c97c75f9 100644 --- a/configs/sites/tier2/blackpearl/packages.yaml +++ b/configs/sites/tier2/blackpearl/packages.yaml @@ -10,6 +10,10 @@ packages: externals: - spec: automake@1.16.2 prefix: /usr + bash: + externals: + - spec: bash@5.1.8 + prefix: /usr binutils: externals: - spec: binutils@2.35.2 diff --git a/configs/sites/tier2/bounty/packages.yaml b/configs/sites/tier2/bounty/packages.yaml index 8a6f8d470..e9cb5e735 100644 --- a/configs/sites/tier2/bounty/packages.yaml +++ b/configs/sites/tier2/bounty/packages.yaml @@ -10,6 +10,10 @@ packages: externals: - spec: automake@1.16.2 prefix: /usr + bash: + externals: + - spec: bash@5.1.8 + prefix: /usr binutils: externals: - spec: binutils@2.35.2~gold+headers diff --git a/configs/sites/tier2/jean/packages.yaml b/configs/sites/tier2/jean/packages.yaml index 655316440..82d0fd511 100644 --- a/configs/sites/tier2/jean/packages.yaml +++ b/configs/sites/tier2/jean/packages.yaml @@ -1,107 +1,111 @@ -packages: - # Modification of common packages - # Use system zlib instead of spack-built zlib-ng - all: - providers: - zlib-api:: [zlib] - zlib-api: - buildable: False - zlib: - buildable: False - externals: - - spec: zlib@1.2.11 - prefix: /usr - -# All other packages listed alphabetically - autoconf: - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - externals: - - spec: automake@1.16.1 - prefix: /usr - binutils: - externals: - - spec: binutils@2.30.125~gold~headers - prefix: /usr - #bison: - # externals: - # - spec: bison@3.0.4 - # prefix: /usr - coreutils: - externals: - - spec: coreutils@8.30 - prefix: /usr - #curl: - # externals: - # - spec: curl@7.61.1+gssapi+ldap+nghttp2 - # prefix: /usr - diffutils: - externals: - - spec: diffutils@3.6 - prefix: /usr - findutils: - externals: - - spec: findutils@4.6.0 - prefix: /usr - flex: - externals: - - spec: flex@2.6.1+lex - prefix: /usr - gawk: - externals: - - spec: gawk@4.2.1 - prefix: /usr - gettext: - externals: - - spec: gettext@0.19.8.1 - prefix: /usr - git: - externals: - - spec: git@2.43.5~tcltk - prefix: /usr - # DH* external git-lfs missing - groff: - externals: - - spec: groff@1.22.3 - prefix: /usr - gmake: - externals: - - spec: gmake@4.2.1 - prefix: /usr - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr - m4: - externals: - - spec: m4@1.4.18 - prefix: /usr - openssh: - externals: - - spec: openssh@8.0p1 - prefix: /usr - #- spec: openssh@10.0p2a - # prefix: /usr/local/krb5 - #openssl: - # externals: - # - spec: openssl@1.1.1k - # prefix: /usr - #perl: - # externals: - # - spec: perl@5.26.3~cpanm+opcode+open+shared+threads - # prefix: /usr - pkgconf: - externals: - - spec: pkgconf@1.4.2 - prefix: /usr - # DH* external qt5 missing - sed: - externals: - - spec: sed@4.5 - prefix: /usr - tar: - externals: - - spec: tar@1.30 - prefix: /usr +packages: + # Modification of common packages + # Use system zlib instead of spack-built zlib-ng + all: + providers: + zlib-api:: [zlib] + zlib-api: + buildable: False + zlib: + buildable: False + externals: + - spec: zlib@1.2.11 + prefix: /usr + +# All other packages listed alphabetically + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + automake: + externals: + - spec: automake@1.16.1 + prefix: /usr + bash: + externals: + - spec: bash@4.4.20 + prefix: /usr + binutils: + externals: + - spec: binutils@2.30.125~gold~headers + prefix: /usr + #bison: + # externals: + # - spec: bison@3.0.4 + # prefix: /usr + coreutils: + externals: + - spec: coreutils@8.30 + prefix: /usr + #curl: + # externals: + # - spec: curl@7.61.1+gssapi+ldap+nghttp2 + # prefix: /usr + diffutils: + externals: + - spec: diffutils@3.6 + prefix: /usr + findutils: + externals: + - spec: findutils@4.6.0 + prefix: /usr + flex: + externals: + - spec: flex@2.6.1+lex + prefix: /usr + gawk: + externals: + - spec: gawk@4.2.1 + prefix: /usr + gettext: + externals: + - spec: gettext@0.19.8.1 + prefix: /usr + git: + externals: + - spec: git@2.43.5~tcltk + prefix: /usr + # DH* external git-lfs missing + groff: + externals: + - spec: groff@1.22.3 + prefix: /usr + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + openssh: + externals: + - spec: openssh@8.0p1 + prefix: /usr + #- spec: openssh@10.0p2a + # prefix: /usr/local/krb5 + #openssl: + # externals: + # - spec: openssl@1.1.1k + # prefix: /usr + #perl: + # externals: + # - spec: perl@5.26.3~cpanm+opcode+open+shared+threads + # prefix: /usr + pkgconf: + externals: + - spec: pkgconf@1.4.2 + prefix: /usr + # DH* external qt5 missing + sed: + externals: + - spec: sed@4.5 + prefix: /usr + tar: + externals: + - spec: tar@1.30 + prefix: /usr diff --git a/configs/sites/tier2/wheat/packages.yaml b/configs/sites/tier2/wheat/packages.yaml index 7e6586230..e87fdd16e 100644 --- a/configs/sites/tier2/wheat/packages.yaml +++ b/configs/sites/tier2/wheat/packages.yaml @@ -21,6 +21,10 @@ packages: externals: - spec: automake@1.16.1 prefix: /usr + bash: + externals: + - spec: bash@4.4.20 + prefix: /usr binutils: externals: - spec: binutils@2.30.117 From 8e0c784629be7d063641f69cf82a226dc9fee54d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Jan 2026 15:15:56 -0700 Subject: [PATCH 2/8] Add variant gittools for neptune-python-env (packages py-pygithub, gh), enable in neptune-dev template --- configs/common/packages.yaml | 5 ++++- configs/common/packages_oneapi.yaml | 3 +++ configs/templates/neptune-dev/spack.yaml | 4 ++-- .../spack_stack/packages/neptune_python_env/package.py | 6 ++++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 6e9b15e4c..85ed7e4cb 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -374,6 +374,10 @@ packages: py-poetry-core: require: - '@1.7' + # Need at least py-pygithub@2.7 for self-hosted runner management + py-pygithub: + require: + - '@2.7:' # This version builds with Intel oneAPI compilers, newer versions don't py-pyogrio: require: @@ -394,7 +398,6 @@ packages: py-setuptools: require: - '@69' - ## When making changes here, also check the packages.yaml for atlantis, nautilus, blackpearl py-torch: require: - +custom-protobuf ~mkldnn diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index 30db8f8c7..93a7efbee 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -32,6 +32,9 @@ packages: gmake: require: - '%gcc' + go: + require: + - '%c,cxx=gcc' libbsd: require: - '%gcc' diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 96e1dfc9e..fb7d251eb 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -12,8 +12,8 @@ spack: - neptune-env +debug +openmp +espc +ncview ^esmf@=8.9.1 - neptune-env ~debug ~openmp +espc +ncview ^esmf@=8.9.1 - neptune-env +debug ~openmp +espc +ncview ^esmf@=8.9.1 - - neptune-python-env ^neptune-env ~debug +openmp +espc +ncview ^esmf@=8.9.1 - - jedi-neptune-env +adp ^neptune-env ~debug +openmp +espc +ncview ^esmf@=8.9.1 + - neptune-python-env +gittools ^neptune-env ~debug +openmp +espc +ncview ^esmf@=8.9.1 + - jedi-neptune-env +adp ^neptune-env ~debug +openmp +espc +ncview ^esmf@=8.9.1 - crtm@3.1.2 - crtm@v2.4.1-jedi.2 diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py index cc82c0bc6..bc9557ee4 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/neptune_python_env/package.py @@ -20,6 +20,8 @@ class NeptunePythonEnv(BundlePackage): version("1.5.0") + variant("gittools", default=False, description="Build additional tools for Git/GitHub") + depends_on("neptune-env", type="run") # Enable the Python variant for ESMF depends_on("esmf +python", type="run") @@ -44,4 +46,8 @@ class NeptunePythonEnv(BundlePackage): depends_on("met", type="run") depends_on("metplus", type="run") + with when("+gittools"): + depends_on("gh", type="run") + depends_on("py-pygithub", type="run") + # There is no need for install() since there is no code. From 66bd1caa28ab72f4fcec9912c42ae614c13e686e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Jan 2026 15:16:09 -0700 Subject: [PATCH 3/8] Add rank-run in repos/spack_stack/spack_repo/spack_stack/packages/base_env/package.py --- .../spack_repo/spack_stack/packages/base_env/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/base_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/base_env/package.py index 44726f643..2c2ceba57 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/base_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/base_env/package.py @@ -44,4 +44,7 @@ class BaseEnv(BundlePackage): depends_on("py-setuptools-scm", type="run") depends_on("py-pytest", type="run") + # Miscellaneous + depends_on("rank-run", type="run") + # There is no need for install() since there is no code. From 0005f5a0a13771f4c57e3dcd07f5ac1005082823 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Jan 2026 15:16:37 -0700 Subject: [PATCH 4/8] Try to find external bash in .github/workflows/ubuntu-ci-x86_64-*.yaml --- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 1 + .github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml | 1 + .github/workflows/ubuntu-ci-x86_64-oneapi.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 32bbb9e39..68f863aeb 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -101,6 +101,7 @@ jobs: spack external find --scope system wget spack external find --scope system texlive spack external find --scope system mysql + spack external find --scope system bash # Find compilers spack compiler find --scope system diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml index cf27ce5f8..fa149b001 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml @@ -101,6 +101,7 @@ jobs: spack external find --scope system wget spack external find --scope system texlive spack external find --scope system mysql + spack external find --scope system bash # Find compilers spack compiler find --scope system diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index 4dded738e..e5f4576f7 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -101,6 +101,7 @@ jobs: spack external find --scope system wget spack external find --scope system texlive spack external find --scope system mysql + spack external find --scope system bash # Find compilers spack compiler find --scope system From 179aae78c781e8a4373caa7cc53ab8c8785e9cf3 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Jan 2026 15:19:49 -0700 Subject: [PATCH 5/8] Update .gitmodules and submodule pointer for repos/builtin --- .gitmodules | 6 ++++-- repos/builtin | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9be4ae217..e97bc70cf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,5 +6,7 @@ branch = feature/update_spack-stack-dev_from_develop [submodule "repos/builtin"] path = repos/builtin - url = https://github.com/jcsda/spack-packages - branch = spack-stack-dev + #url = https://github.com/jcsda/spack-packages + #branch = spack-stack-dev + url = https://github.com/climbfuji/spack-packages + branch = feature/pygithub_rank_run \ No newline at end of file diff --git a/repos/builtin b/repos/builtin index adeba0b78..28fb7e7c6 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit adeba0b78cd5ddebabe6aa5a1fe87f196bc9e795 +Subproject commit 28fb7e7c67f2db8746f6bab620f3fc176495cdfb From 2930f1cc7d96077443decef2a8948f920eef4c90 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Jan 2026 15:21:00 -0700 Subject: [PATCH 6/8] Bug fix in .gitmodules: revert spack submodule pointer information --- .gitmodules | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index e97bc70cf..bc79ec45c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,7 @@ [submodule "spack"] path = spack - #url = https://github.com/jcsda/spack - #branch = spack-stack-dev - url = https://github.com/climbfuji/spack - branch = feature/update_spack-stack-dev_from_develop + url = https://github.com/jcsda/spack + branch = spack-stack-dev [submodule "repos/builtin"] path = repos/builtin #url = https://github.com/jcsda/spack-packages From be353869c168fa670afb9405b7d2718bdf640d3a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Jan 2026 19:38:35 -0700 Subject: [PATCH 7/8] Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index 28fb7e7c6..7b3789ce1 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 28fb7e7c67f2db8746f6bab620f3fc176495cdfb +Subproject commit 7b3789ce1ac59a6c7c73879e2d3dc93336857928 From 65b0830bac405763572dc29e589333173f6910d0 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 28 Jan 2026 11:28:17 -0700 Subject: [PATCH 8/8] Revert .gitmodules and update submodule pointer for repos/builtin --- .gitmodules | 6 ++---- repos/builtin | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index bc79ec45c..2f6cb0566 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,7 +4,5 @@ branch = spack-stack-dev [submodule "repos/builtin"] path = repos/builtin - #url = https://github.com/jcsda/spack-packages - #branch = spack-stack-dev - url = https://github.com/climbfuji/spack-packages - branch = feature/pygithub_rank_run \ No newline at end of file + url = https://github.com/jcsda/spack-packages + branch = spack-stack-dev diff --git a/repos/builtin b/repos/builtin index 7b3789ce1..70baff9fe 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 7b3789ce1ac59a6c7c73879e2d3dc93336857928 +Subproject commit 70baff9fea93d96f35d86b476978d22945ac1f57