From 549b89a883f332bb3fb2f1aede0e5b2367e10f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 12 Jan 2026 14:12:51 +0100 Subject: [PATCH 1/8] feat: enable sccache for cargo-pgrx builds in GitHub Actions Speed up rust builds by enabling sccache caching for cargo-pgrx builds in GitHub Actions. We mount a persistent disk at /var/cache/sccache and configure sccache to use it. --- .../actions/nix-install-ephemeral/action.yml | 5 ++ .github/workflows/nix-build.yml | 56 +++++++++++++++++++ nix/cargo-pgrx/buildPgrxExtension.nix | 18 +++++- 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index b30a31db0..a21f561e6 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -5,6 +5,10 @@ inputs: description: 'Whether to push build outputs to the Nix binary cache' required: false default: 'false' + enable-sccache-sandbox-path: + description: 'Whether to expose /nix/var/cache/sccache in the Nix sandbox' + required: false + default: 'false' aws-region: description: 'AWS region for the Nix binary cache S3 bucket' required: false @@ -48,4 +52,5 @@ runs: substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= ${{ inputs.push-to-cache == 'true' && 'post-build-hook = /etc/nix/upload-to-cache.sh' || '' }} + ${{ inputs.enable-sccache-sandbox-path == 'true' && 'extra-sandbox-paths = /nix/var/cache/sccache?' || '' }} max-jobs = 4 diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 2b8945d63..717c829f4 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -36,17 +36,31 @@ jobs: - name: Checkout Repo if: ${{ matrix.attr != '' }} uses: actions/checkout@v4 + - name: Mount sccache disk + if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }} + path: /nix/var/cache/sccache - name: Install nix (ephemeral) if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} uses: ./.github/actions/nix-install-ephemeral with: push-to-cache: 'true' + enable-sccache-sandbox-path: 'true' env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - name: Install nix (self-hosted) if: ${{ matrix.attr != '' && matrix.runs_on.group == 'self-hosted-runners-nix' }} uses: ./.github/actions/nix-install-self-hosted + - name: Allow sccache cache write access + if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + run: | + sudo chgrp nixbld /nix/var/cache/sccache + sudo chmod 777 /nix/var/cache/sccache + sudo chmod g+s /nix/var/cache/sccache + sudo setfacl -d -m u::rwX,g::rwX,o::rwX /nix/var/cache/sccache - name: nix build if: ${{ matrix.attr != '' }} shell: bash @@ -67,17 +81,31 @@ jobs: - name: Checkout Repo if: ${{ matrix.attr != '' }} uses: actions/checkout@v4 + - name: Mount sccache disk + if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }} + path: /nix/var/cache/sccache - name: Install nix (ephemeral) if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} uses: ./.github/actions/nix-install-ephemeral with: push-to-cache: 'true' + enable-sccache-sandbox-path: 'true' env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - name: Install nix (self-hosted) if: ${{ matrix.attr != '' && matrix.runs_on.group == 'self-hosted-runners-nix' }} uses: ./.github/actions/nix-install-self-hosted + - name: Allow sccache cache write access + if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + run: | + sudo chgrp nixbld /nix/var/cache/sccache + sudo chmod 777 /nix/var/cache/sccache + sudo chmod g+s /nix/var/cache/sccache + sudo setfacl -d -m u::rwX,g::rwX,o::rwX /nix/var/cache/sccache - name: nix build if: ${{ matrix.attr != '' }} shell: bash @@ -144,14 +172,28 @@ jobs: - name: Checkout Repo if: ${{ matrix.attr != '' }} uses: actions/checkout@v4 + - name: Mount sccache disk + if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }} + path: /nix/var/cache/sccache - name: Install nix if: ${{ matrix.attr != '' }} uses: ./.github/actions/nix-install-ephemeral with: + enable-sccache-sandbox-path: 'true' push-to-cache: 'true' env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + - name: Allow sccache cache write access + if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + run: | + sudo chgrp nixbld /nix/var/cache/sccache + sudo chmod 777 /nix/var/cache/sccache + sudo chmod g+s /nix/var/cache/sccache + sudo setfacl -d -m u::rwX,g::rwX,o::rwX /nix/var/cache/sccache - name: nix build if: ${{ matrix.attr != '' }} shell: bash @@ -172,14 +214,28 @@ jobs: - name: Checkout Repo if: ${{ matrix.attr != '' }} uses: actions/checkout@v4 + - name: Mount sccache disk + if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + uses: useblacksmith/stickydisk@v1 + with: + key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }} + path: /nix/var/cache/sccache - name: Install nix if: ${{ matrix.attr != '' }} uses: ./.github/actions/nix-install-ephemeral with: + enable-sccache-sandbox-path: 'true' push-to-cache: 'true' env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + - name: Allow sccache cache write access + if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + run: | + sudo chgrp nixbld /nix/var/cache/sccache + sudo chmod 777 /nix/var/cache/sccache + sudo chmod g+s /nix/var/cache/sccache + sudo setfacl -d -m u::rwX,g::rwX,o::rwX /nix/var/cache/sccache - name: nix build if: ${{ matrix.attr != '' }} shell: bash diff --git a/nix/cargo-pgrx/buildPgrxExtension.nix b/nix/cargo-pgrx/buildPgrxExtension.nix index 828dd7f02..f6d3139cb 100644 --- a/nix/cargo-pgrx/buildPgrxExtension.nix +++ b/nix/cargo-pgrx/buildPgrxExtension.nix @@ -35,6 +35,7 @@ darwin, writeShellScriptBin, defaultBindgenHook, + sccache, }: # The idea behind: Use it mostly like rustPlatform.buildRustPackage and so @@ -56,7 +57,7 @@ postgresql, # enable override to generate bindings using bindgenHook. # Some older versions of cargo-pgrx use a bindgenHook that is not compatible with the - # current clang version present in stdenv + # current clang version present in stdenv bindgenHook ? defaultBindgenHook, # cargo-pgrx calls rustfmt on generated bindings, this is not strictly necessary, so we avoid the # dependency here. Set to false and provide rustfmt in nativeBuildInputs, if you need it, e.g. @@ -135,12 +136,22 @@ let postgresql pkg-config bindgenHook + sccache ] ++ lib.optionals useFakeRustfmt [ fakeRustfmt ]; buildPhase = '' runHook preBuild + if [[ -d "/nix/var/cache/sccache" && -w "/nix/var/cache/sccache" ]]; then + echo "sccache: cache directory available, enabling" + export RUSTC_WRAPPER="${sccache}/bin/sccache" + export SCCACHE_DIR="/nix/var/cache/sccache" + export SCCACHE_CACHE_SIZE="500G" + else + echo "sccache: cache directory not available, skipping" + fi + echo "Executing cargo-pgrx buildPhase" ${preBuildAndTest} ${maybeEnterBuildAndTestSubdir} @@ -154,6 +165,11 @@ let --features "${builtins.concatStringsSep " " buildFeatures}" \ --out-dir "$out" + if [[ -n "''${RUSTC_WRAPPER:-}" ]]; then + echo "sccache stats:" + ${sccache}/bin/sccache --show-stats + fi + ${maybeLeaveBuildAndTestSubdir} runHook postBuild From ea771c88e4f2765c209761389972c1fd06bc1639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 13 Jan 2026 10:34:54 +0100 Subject: [PATCH 2/8] feat(sccache): enable sccache for pgrx builds with per-PG-version cache keys - Add sccache to buildPgrxExtension.nix nativeBuildInputs - Enable sccache conditionally when /nix/var/cache/sccache is available - Configure Nix with auto-allocate-uids and max-jobs=1 for consistent UID mapping - Add cache_key field to github-matrix output for stickydisk grouping - Fix postgresql_version extraction for packages with extra path levels - Use per-postgres-version cache keys to reduce stickydisk conflicts - Chown cache to UID 872415232 (auto-allocate-uids base) before builds --- .../actions/nix-install-ephemeral/action.yml | 5 +- .github/workflows/nix-build.yml | 36 ++++----- nix/cargo-pgrx/buildPgrxExtension.nix | 2 +- nix/packages/github-matrix/github_matrix.py | 46 ++++++----- .../github-matrix/tests/test_github_matrix.py | 78 +++++++++++++++++++ 5 files changed, 126 insertions(+), 41 deletions(-) diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index a21f561e6..29604b4d6 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -53,4 +53,7 @@ runs: trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= ${{ inputs.push-to-cache == 'true' && 'post-build-hook = /etc/nix/upload-to-cache.sh' || '' }} ${{ inputs.enable-sccache-sandbox-path == 'true' && 'extra-sandbox-paths = /nix/var/cache/sccache?' || '' }} - max-jobs = 4 + max-jobs = 1 + auto-allocate-uids = true + use-cgroups = true + experimental-features = nix-command flakes cgroups auto-allocate-uids diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 717c829f4..2028e2a6a 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -40,7 +40,7 @@ jobs: if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} uses: useblacksmith/stickydisk@v1 with: - key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }} + key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cache_key }} path: /nix/var/cache/sccache - name: Install nix (ephemeral) if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} @@ -57,10 +57,9 @@ jobs: - name: Allow sccache cache write access if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} run: | - sudo chgrp nixbld /nix/var/cache/sccache - sudo chmod 777 /nix/var/cache/sccache - sudo chmod g+s /nix/var/cache/sccache - sudo setfacl -d -m u::rwX,g::rwX,o::rwX /nix/var/cache/sccache + # With auto-allocate-uids, UID 872415232 (0x34000000) maps to nixbld inside sandbox + sudo chown -R 872415232 /nix/var/cache/sccache + sudo chmod -R 2777 /nix/var/cache/sccache - name: nix build if: ${{ matrix.attr != '' }} shell: bash @@ -85,7 +84,7 @@ jobs: if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} uses: useblacksmith/stickydisk@v1 with: - key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }} + key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cache_key }} path: /nix/var/cache/sccache - name: Install nix (ephemeral) if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} @@ -102,10 +101,9 @@ jobs: - name: Allow sccache cache write access if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} run: | - sudo chgrp nixbld /nix/var/cache/sccache - sudo chmod 777 /nix/var/cache/sccache - sudo chmod g+s /nix/var/cache/sccache - sudo setfacl -d -m u::rwX,g::rwX,o::rwX /nix/var/cache/sccache + # With auto-allocate-uids, UID 872415232 (0x34000000) maps to nixbld inside sandbox + sudo chown -R 872415232 /nix/var/cache/sccache + sudo chmod -R 2777 /nix/var/cache/sccache - name: nix build if: ${{ matrix.attr != '' }} shell: bash @@ -176,7 +174,7 @@ jobs: if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} uses: useblacksmith/stickydisk@v1 with: - key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }} + key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cache_key }} path: /nix/var/cache/sccache - name: Install nix if: ${{ matrix.attr != '' }} @@ -190,10 +188,9 @@ jobs: - name: Allow sccache cache write access if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} run: | - sudo chgrp nixbld /nix/var/cache/sccache - sudo chmod 777 /nix/var/cache/sccache - sudo chmod g+s /nix/var/cache/sccache - sudo setfacl -d -m u::rwX,g::rwX,o::rwX /nix/var/cache/sccache + # With auto-allocate-uids, UID 872415232 (0x34000000) maps to nixbld inside sandbox + sudo chown -R 872415232 /nix/var/cache/sccache + sudo chmod -R 2777 /nix/var/cache/sccache - name: nix build if: ${{ matrix.attr != '' }} shell: bash @@ -218,7 +215,7 @@ jobs: if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} uses: useblacksmith/stickydisk@v1 with: - key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }} + key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cache_key }} path: /nix/var/cache/sccache - name: Install nix if: ${{ matrix.attr != '' }} @@ -232,10 +229,9 @@ jobs: - name: Allow sccache cache write access if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} run: | - sudo chgrp nixbld /nix/var/cache/sccache - sudo chmod 777 /nix/var/cache/sccache - sudo chmod g+s /nix/var/cache/sccache - sudo setfacl -d -m u::rwX,g::rwX,o::rwX /nix/var/cache/sccache + # With auto-allocate-uids, UID 872415232 (0x34000000) maps to nixbld inside sandbox + sudo chown -R 872415232 /nix/var/cache/sccache + sudo chmod -R 2777 /nix/var/cache/sccache - name: nix build if: ${{ matrix.attr != '' }} shell: bash diff --git a/nix/cargo-pgrx/buildPgrxExtension.nix b/nix/cargo-pgrx/buildPgrxExtension.nix index f6d3139cb..4517bd76d 100644 --- a/nix/cargo-pgrx/buildPgrxExtension.nix +++ b/nix/cargo-pgrx/buildPgrxExtension.nix @@ -147,7 +147,7 @@ let echo "sccache: cache directory available, enabling" export RUSTC_WRAPPER="${sccache}/bin/sccache" export SCCACHE_DIR="/nix/var/cache/sccache" - export SCCACHE_CACHE_SIZE="500G" + export SCCACHE_CACHE_SIZE="50G" else echo "sccache: cache directory not available, skipping" fi diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 81385c215..348bc4e45 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -58,6 +58,7 @@ class GitHubActionPackage(TypedDict): system: System runs_on: RunsOnConfig postgresql_version: NotRequired[str] + cache_key: NotRequired[str] class NixEvalError(TypedDict): @@ -209,8 +210,7 @@ def run_nix_eval_jobs( def is_extension_pkg(pkg: NixEvalJobsOutput) -> bool: """Check if the package is a postgresql extension package.""" - attrs = pkg["attr"].split(".") - return attrs[-2] == "exts" + return ".exts." in pkg["attr"] # thank you buildbot-nix https://github.com/nix-community/buildbot-nix/blob/985d069a2a45cf4a571a4346107671adc2bd2a16/buildbot_nix/buildbot_nix/build_trigger.py#L297 @@ -247,6 +247,31 @@ def is_kvm_pkg(pkg: NixEvalJobsOutput) -> bool: return "kvm" in pkg.get("requiredSystemFeatures", []) +def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: + """Convert nix-eval-jobs output to GitHub Actions matrix package""" + runner = get_runner_for_package(pkg) + if runner is None: + raise ValueError(f"No runner configuration for system: {pkg['system']}") + returned_pkg: GitHubActionPackage = { + "attr": pkg["attr"], + "name": pkg["name"], + "system": pkg["system"], + "runs_on": runner, + } + if is_extension_pkg(pkg): + # Extract PostgreSQL version from attribute path + # e.g., legacyPackages.aarch64-linux.psql_17.exts.wrappers-pkgs.0_5_6 + # or legacyPackages.aarch64-linux.psql_17.exts.pg_graphql + attrs = pkg["attr"].split(".") + exts_idx = attrs.index("exts") + pg_version = attrs[exts_idx - 1].split("_")[-1] + returned_pkg["postgresql_version"] = pg_version + returned_pkg["cache_key"] = f"pg{pg_version}" + else: + returned_pkg["cache_key"] = "shared" + return returned_pkg + + def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig | None: """Determine the appropriate GitHub Actions runner for a package. @@ -294,23 +319,6 @@ def main() -> None: packages, warnings_list, errors_list = run_nix_eval_jobs(cmd) gh_action_packages = sort_pkgs_by_closures(packages) - def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: - """Convert nix-eval-jobs output to GitHub Actions matrix package""" - runner = get_runner_for_package(pkg) - if runner is None: - raise ValueError(f"No runner configuration for system: {pkg['system']}") - returned_pkg: GitHubActionPackage = { - "attr": pkg["attr"], - "name": pkg["name"], - "system": pkg["system"], - "runs_on": runner, - } - if is_extension_pkg(pkg): - # Extract PostgreSQL version from attribute path - attrs = pkg["attr"].split(".") - returned_pkg["postgresql_version"] = attrs[-3].split("_")[-1] - return returned_pkg - # Group packages by system and type (checks vs packages) packages_by_system: Dict[System, List[GitHubActionPackage]] = defaultdict(list) checks_by_system: Dict[System, List[GitHubActionPackage]] = defaultdict(list) diff --git a/nix/packages/github-matrix/tests/test_github_matrix.py b/nix/packages/github-matrix/tests/test_github_matrix.py index 17ce8132d..2546eb6ee 100644 --- a/nix/packages/github-matrix/tests/test_github_matrix.py +++ b/nix/packages/github-matrix/tests/test_github_matrix.py @@ -4,6 +4,7 @@ from github_matrix import ( NixEvalJobsOutput, + clean_package_for_output, get_runner_for_package, is_extension_pkg, is_kvm_pkg, @@ -260,3 +261,80 @@ def test_dependency_order(self): result = sort_pkgs_by_closures([pkg1, pkg2]) assert result == [pkg1, pkg2] + + +class TestCleanPackageForOutput: + def test_extension_package_simple_path(self): + """Test extension package like pg_graphql with simple attr path""" + pkg: NixEvalJobsOutput = { + "attr": "legacyPackages.aarch64-linux.psql_17.exts.pg_graphql", + "attrPath": [ + "legacyPackages", + "aarch64-linux", + "psql_17", + "exts", + "pg_graphql", + ], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "pg_graphql", + "system": "aarch64-linux", + } + result = clean_package_for_output(pkg) + assert result["postgresql_version"] == "17" + assert result["cache_key"] == "pg17" + + def test_extension_package_versioned_path(self): + """Test extension package like wrappers with version suffix in attr path""" + pkg: NixEvalJobsOutput = { + "attr": "legacyPackages.aarch64-linux.psql_17.exts.wrappers-pkgs.0_5_6", + "attrPath": [ + "legacyPackages", + "aarch64-linux", + "psql_17", + "exts", + "wrappers-pkgs", + "0_5_6", + ], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "wrappers-0.5.6", + "system": "aarch64-linux", + } + result = clean_package_for_output(pkg) + assert result["postgresql_version"] == "17" + assert result["cache_key"] == "pg17" + + def test_extension_package_pg15(self): + """Test extension package with PostgreSQL 15""" + pkg: NixEvalJobsOutput = { + "attr": "legacyPackages.x86_64-linux.psql_15.exts.pg_cron", + "attrPath": [ + "legacyPackages", + "x86_64-linux", + "psql_15", + "exts", + "pg_cron", + ], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "pg_cron", + "system": "x86_64-linux", + } + result = clean_package_for_output(pkg) + assert result["postgresql_version"] == "15" + assert result["cache_key"] == "pg15" + + def test_non_extension_package(self): + """Test non-extension package gets shared cache key""" + pkg: NixEvalJobsOutput = { + "attr": "legacyPackages.x86_64-linux.psql_15", + "attrPath": ["legacyPackages", "x86_64-linux", "psql_15"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgresql-15.0", + "system": "x86_64-linux", + } + result = clean_package_for_output(pkg) + assert "postgresql_version" not in result + assert result["cache_key"] == "shared" From 93716d2d8d29bd9a1e611790fce08113e2264578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 13 Jan 2026 18:20:13 +0100 Subject: [PATCH 3/8] feat(nix-install-ephemeral): add configurable max-jobs input Add max-jobs input parameter with empty default. When set, includes max-jobs in nix config. auto-allocate-uids and use-cgroups are now only enabled when enable-sccache-sandbox-path is true. --- .github/actions/nix-install-ephemeral/action.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index 29604b4d6..423fad271 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -9,6 +9,10 @@ inputs: description: 'Whether to expose /nix/var/cache/sccache in the Nix sandbox' required: false default: 'false' + max-jobs: + description: 'Maximum number of parallel Nix builds' + required: false + default: '' aws-region: description: 'AWS region for the Nix binary cache S3 bucket' required: false @@ -53,7 +57,7 @@ runs: trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= ${{ inputs.push-to-cache == 'true' && 'post-build-hook = /etc/nix/upload-to-cache.sh' || '' }} ${{ inputs.enable-sccache-sandbox-path == 'true' && 'extra-sandbox-paths = /nix/var/cache/sccache?' || '' }} - max-jobs = 1 - auto-allocate-uids = true - use-cgroups = true - experimental-features = nix-command flakes cgroups auto-allocate-uids + ${{ inputs.max-jobs != '' && format('max-jobs = {0}', inputs.max-jobs) || '' }} + ${{ inputs.enable-sccache-sandbox-path == 'true' && 'auto-allocate-uids = true' || '' }} + ${{ inputs.enable-sccache-sandbox-path == 'true' && 'use-cgroups = true' || '' }} + experimental-features = nix-command flakes ${{ inputs.enable-sccache-sandbox-path == 'true' && 'cgroups auto-allocate-uids' || '' }} From 7018ec980d5e565d009d82faa62fe92a0d8ecb24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 13 Jan 2026 18:20:19 +0100 Subject: [PATCH 4/8] feat(nix-build): enable sccache only for extension packages Only mount stickydisk, enable sccache sandbox path, and set max-jobs=1 for packages with postgresql_version (extension packages). Non-extension packages build without sccache and with default max-jobs. --- .github/workflows/nix-build.yml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 2028e2a6a..cda87a019 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -37,7 +37,7 @@ jobs: if: ${{ matrix.attr != '' }} uses: actions/checkout@v4 - name: Mount sccache disk - if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }} uses: useblacksmith/stickydisk@v1 with: key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cache_key }} @@ -47,7 +47,8 @@ jobs: uses: ./.github/actions/nix-install-ephemeral with: push-to-cache: 'true' - enable-sccache-sandbox-path: 'true' + enable-sccache-sandbox-path: ${{ matrix.postgresql_version && 'true' || 'false' }} + max-jobs: ${{ matrix.postgresql_version && '1' || '' }} env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} @@ -55,7 +56,7 @@ jobs: if: ${{ matrix.attr != '' && matrix.runs_on.group == 'self-hosted-runners-nix' }} uses: ./.github/actions/nix-install-self-hosted - name: Allow sccache cache write access - if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }} run: | # With auto-allocate-uids, UID 872415232 (0x34000000) maps to nixbld inside sandbox sudo chown -R 872415232 /nix/var/cache/sccache @@ -81,7 +82,7 @@ jobs: if: ${{ matrix.attr != '' }} uses: actions/checkout@v4 - name: Mount sccache disk - if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }} uses: useblacksmith/stickydisk@v1 with: key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cache_key }} @@ -91,7 +92,8 @@ jobs: uses: ./.github/actions/nix-install-ephemeral with: push-to-cache: 'true' - enable-sccache-sandbox-path: 'true' + enable-sccache-sandbox-path: ${{ matrix.postgresql_version && 'true' || 'false' }} + max-jobs: ${{ matrix.postgresql_version && '1' || '' }} env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} @@ -99,7 +101,7 @@ jobs: if: ${{ matrix.attr != '' && matrix.runs_on.group == 'self-hosted-runners-nix' }} uses: ./.github/actions/nix-install-self-hosted - name: Allow sccache cache write access - if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }} run: | # With auto-allocate-uids, UID 872415232 (0x34000000) maps to nixbld inside sandbox sudo chown -R 872415232 /nix/var/cache/sccache @@ -171,7 +173,7 @@ jobs: if: ${{ matrix.attr != '' }} uses: actions/checkout@v4 - name: Mount sccache disk - if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }} uses: useblacksmith/stickydisk@v1 with: key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cache_key }} @@ -180,13 +182,14 @@ jobs: if: ${{ matrix.attr != '' }} uses: ./.github/actions/nix-install-ephemeral with: - enable-sccache-sandbox-path: 'true' + enable-sccache-sandbox-path: ${{ matrix.postgresql_version && 'true' || 'false' }} + max-jobs: ${{ matrix.postgresql_version && '1' || '' }} push-to-cache: 'true' env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - name: Allow sccache cache write access - if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }} run: | # With auto-allocate-uids, UID 872415232 (0x34000000) maps to nixbld inside sandbox sudo chown -R 872415232 /nix/var/cache/sccache @@ -212,7 +215,7 @@ jobs: if: ${{ matrix.attr != '' }} uses: actions/checkout@v4 - name: Mount sccache disk - if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }} uses: useblacksmith/stickydisk@v1 with: key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cache_key }} @@ -221,13 +224,14 @@ jobs: if: ${{ matrix.attr != '' }} uses: ./.github/actions/nix-install-ephemeral with: - enable-sccache-sandbox-path: 'true' + enable-sccache-sandbox-path: ${{ matrix.postgresql_version && 'true' || 'false' }} + max-jobs: ${{ matrix.postgresql_version && '1' || '' }} push-to-cache: 'true' env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - name: Allow sccache cache write access - if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }} run: | # With auto-allocate-uids, UID 872415232 (0x34000000) maps to nixbld inside sandbox sudo chown -R 872415232 /nix/var/cache/sccache From cdb8dad6add2e3a17365a8066661f2d5f11812db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 13 Jan 2026 18:52:29 +0100 Subject: [PATCH 5/8] docs: document sccache CI integration --- nix/docs/README.md | 4 ++ nix/docs/sccache-ci.md | 88 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 nix/docs/sccache-ci.md diff --git a/nix/docs/README.md b/nix/docs/README.md index 0d7b080ee..11e9e3e18 100644 --- a/nix/docs/README.md +++ b/nix/docs/README.md @@ -35,6 +35,10 @@ learn how to play with `postgres` in the [build guide](./build-postgres.md). - **[Migration Tests](./migration-tests.md)** - Testing database migrations - **[Testing PG Upgrade Scripts](./testing-pg-upgrade-scripts.md)** - Testing PostgreSQL upgrades +## CI/CD + +- **[sccache in CI](./sccache-ci.md)** - Rust compilation caching for pgrx extensions + ## Reference - **[References](./references.md)** - Useful links and resources diff --git a/nix/docs/sccache-ci.md b/nix/docs/sccache-ci.md new file mode 100644 index 000000000..4e244027a --- /dev/null +++ b/nix/docs/sccache-ci.md @@ -0,0 +1,88 @@ +# sccache in CI + +This document explains how sccache is integrated into the Nix CI pipeline to accelerate Rust compilation for pgrx-based PostgreSQL extensions. + +## Overview + +Building pgrx extensions (pg_graphql, pg_jsonschema, wrappers, etc.) involves significant Rust compilation. +sccache caches compiled artifacts to speed up subsequent builds when source code hasn't changed. + +The integration required solving several challenges around Nix's sandboxed builds and CI runner persistence. + +## Architecture + +Ephemeral CI runners (Blacksmith) use stickydisk for persistent storage at `/nix/var/cache/sccache`. +Self-hosted Darwin runners use a local directory at the same path. + +The cache is keyed by PostgreSQL version to avoid cross-version cache pollution: +- Extension packages: `sccache-Linux-ARM64-pg17`, `sccache-Linux-ARM64-pg15`, etc. +- Non-extension packages: `sccache-Linux-ARM64-shared` + +| Platform | Runner type | sccache enabled | +|----------|-------------|-----------------| +| x86_64-linux | Blacksmith ephemeral | Yes (extensions only) | +| aarch64-linux | Blacksmith ephemeral | Yes (extensions only) | +| aarch64-linux | Self-hosted (KVM packages) | No | +| aarch64-darwin | Self-hosted | Yes (extensions only) | + +## Nix sandbox integration + +Nix builds run in a sandbox that isolates the build environment. +To allow sccache to persist across builds, the cache directory must be mounted into the sandbox via `extra-sandbox-paths`. + +### Linux with auto-allocate-uids + +On Linux, we use Nix's `auto-allocate-uids` feature which creates user namespaces for builds. +This introduces a UID mapping challenge. + +The base UID is 872415232 (0x34000000). +Inside the sandbox, processes run as root (UID 0), but outside the sandbox this maps to UID 872415232. +Files created inside the sandbox are owned by this mapped UID. + +The workflow changes ownership to UID 872415232 before builds to ensure cache access works correctly. + +### Darwin + +Darwin doesn't have user namespaces. +Builds run as nixbld users (members of the `nixbld` group). +The cache directory is owned by the nixbld group with setgid permissions. + +## Compromises and limitations + +### max-jobs = 1 for extensions + +With `auto-allocate-uids`, each parallel Nix build gets a different UID slot (872415232, 872480768, etc.). +When multiple builds run in parallel, they create files with different ownership, causing EPERM errors when one build tries to update another's cache files. + +To avoid this, extension builds use `max-jobs = 1`, serializing Nix builds so all use the same UID slot. +Non-extension packages don't use sccache and can build in parallel. + +### Blacksmith stickydisk last-writer-wins + +Blacksmith's stickydisk has "last writer wins" semantics for concurrent writes. +If multiple jobs write to the same cache key simultaneously, only the last job's writes persist. + +We mitigate this by using per-PostgreSQL-version cache keys, reducing concurrent writes to the same cache. + +### Extensions only + +sccache is only enabled for extension packages (those with `postgresql_version` in the matrix). +Non-extension packages (PostgreSQL itself, tooling) build without sccache to allow parallel builds. + +## Debugging + +To check sccache statistics, look for output at the end of extension builds: + +``` +sccache stats: +Compile requests 150 +Compile requests executed 148 +Cache hits 120 +Cache misses 28 +``` + +## References + +- [Nix auto-allocate-uids](https://nixos.org/manual/nix/stable/command-ref/conf-file.html#conf-auto-allocate-uids) +- [sccache](https://github.com/mozilla/sccache) +- [Blacksmith stickydisk](https://blacksmith.sh/docs/stickydisk) From 7f319cd643b63eaffac3af13a5babb0e47da193c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 13 Jan 2026 18:58:40 +0100 Subject: [PATCH 6/8] feat: build pg_graphql and pg_jsonschema extension packages separately in CI To be able to build extensions versions packages separately in CI, we expose them in a nested structure. We did this change in the past for wrappers packages. Now we do the same for pg_graphql and pg_jsonschema extension packages. --- nix/ext/pg_graphql/default.nix | 12 +++++++++--- nix/ext/pg_jsonschema/default.nix | 12 +++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/nix/ext/pg_graphql/default.nix b/nix/ext/pg_graphql/default.nix index a7f6d1065..bc91f7e14 100644 --- a/nix/ext/pg_graphql/default.nix +++ b/nix/ext/pg_graphql/default.nix @@ -125,9 +125,11 @@ let versions = lib.naturalSort (lib.attrNames supportedVersions); latestVersion = lib.last versions; numberOfVersions = builtins.length versions; - packages = builtins.attrValues ( - lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions - ); + packagesAttrSet = lib.mapAttrs' (name: value: { + name = lib.replaceStrings [ "." ] [ "_" ] name; + value = build name value.hash value.rust value.pgrx; + }) supportedVersions; + packages = builtins.attrValues packagesAttrSet; in (buildEnv { name = pname; @@ -175,6 +177,10 @@ in inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); + # Expose individual packages for CI to build separately + packages = packagesAttrSet // { + recurseForDerivations = true; + }; }; }).overrideAttrs (_: { diff --git a/nix/ext/pg_jsonschema/default.nix b/nix/ext/pg_jsonschema/default.nix index 1a2e8ee58..9dfda0952 100644 --- a/nix/ext/pg_jsonschema/default.nix +++ b/nix/ext/pg_jsonschema/default.nix @@ -126,9 +126,11 @@ let versions = lib.naturalSort (lib.attrNames supportedVersions); latestVersion = lib.last versions; numberOfVersions = builtins.length versions; - packages = builtins.attrValues ( - lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions - ); + packagesAttrSet = lib.mapAttrs' (name: value: { + name = lib.replaceStrings [ "." ] [ "_" ] name; + value = build name value.hash value.rust value.pgrx; + }) supportedVersions; + packages = builtins.attrValues packagesAttrSet; in (pkgs.buildEnv { name = pname; @@ -168,6 +170,10 @@ in inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); + # Expose individual packages for CI to build separately + packages = packagesAttrSet // { + recurseForDerivations = true; + }; }; }).overrideAttrs (_: { From 021a8aae3acb1d3e13eb622f85e3b64ba9870250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 13 Jan 2026 19:16:20 +0100 Subject: [PATCH 7/8] chore: trigger rebuild to check caching behavior --- nix/cargo-pgrx/buildPgrxExtension.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/cargo-pgrx/buildPgrxExtension.nix b/nix/cargo-pgrx/buildPgrxExtension.nix index 4517bd76d..2289d56b5 100644 --- a/nix/cargo-pgrx/buildPgrxExtension.nix +++ b/nix/cargo-pgrx/buildPgrxExtension.nix @@ -143,6 +143,7 @@ let buildPhase = '' runHook preBuild + if [[ -d "/nix/var/cache/sccache" && -w "/nix/var/cache/sccache" ]]; then echo "sccache: cache directory available, enabling" export RUSTC_WRAPPER="${sccache}/bin/sccache" From a32079200e561fb2f23eacaa5ed8d6c20e57ad33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 13 Jan 2026 20:02:19 +0100 Subject: [PATCH 8/8] fix: unwanted warnings in nix-eval-jobs --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index efd0b7e9e..70340fa76 100644 --- a/flake.nix +++ b/flake.nix @@ -28,7 +28,7 @@ git-hooks.inputs.nixpkgs.follows = "nixpkgs"; nixpkgs-go124.url = "github:Nixos/nixpkgs/d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0"; nixpkgs-pgbackrest.url = "github:nixos/nixpkgs/nixos-unstable-small"; - nix-eval-jobs.url = "github:nix-community/nix-eval-jobs"; + nix-eval-jobs.url = "github:jfroche/nix-eval-jobs/fix-warnings"; }; outputs =