feat: enable sccache for cargo-pgrx builds in GitHub Actions

.github/actions/nix-install-ephemeral/action.yml

@@ -5,6 +5,14 @@ inputs:
     description: 'Whether to push build outputs to the Nix binary cache'
     required: false
     default: 'false'
+  enable-sccache-sandbox-path:
+    description: 'Whether to expose /nix/var/cache/sccache in the Nix sandbox'
+    required: false
+    default: 'false'
+  max-jobs:
+    description: 'Maximum number of parallel Nix builds'
+    required: false
+    default: ''
   aws-region:
     description: 'AWS region for the Nix binary cache S3 bucket'
     required: false
@@ -48,4 +56,8 @@ runs:
           substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
           trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
           ${{ inputs.push-to-cache == 'true' && 'post-build-hook = /etc/nix/upload-to-cache.sh' || '' }}
-          max-jobs = 4
+          ${{ inputs.enable-sccache-sandbox-path == 'true' && 'extra-sandbox-paths = /nix/var/cache/sccache?' || '' }}
+          ${{ inputs.max-jobs != '' && format('max-jobs = {0}', inputs.max-jobs) || '' }}
+          ${{ inputs.enable-sccache-sandbox-path == 'true' && 'auto-allocate-uids = true' || '' }}
+          ${{ inputs.enable-sccache-sandbox-path == 'true' && 'use-cgroups = true' || '' }}
+          experimental-features = nix-command flakes ${{ inputs.enable-sccache-sandbox-path == 'true' && 'cgroups auto-allocate-uids' || '' }}

.github/workflows/nix-build.yml

@@ -36,17 +36,31 @@ jobs:
       - name: Checkout Repo
         if: ${{ matrix.attr != '' }}
         uses: actions/checkout@v4
+      - name: Mount sccache disk
+        if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }}
+        uses: useblacksmith/stickydisk@v1
+        with:
+          key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cache_key }}
+          path: /nix/var/cache/sccache
       - name: Install nix (ephemeral)
         if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }}
         uses: ./.github/actions/nix-install-ephemeral
         with:
           push-to-cache: 'true'
+          enable-sccache-sandbox-path: ${{ matrix.postgresql_version && 'true' || 'false' }}
+          max-jobs: ${{ matrix.postgresql_version && '1' || '' }}
         env:
           DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
           NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
       - name: Install nix (self-hosted)
         if: ${{ matrix.attr != '' && matrix.runs_on.group == 'self-hosted-runners-nix' }}
         uses: ./.github/actions/nix-install-self-hosted
+      - name: Allow sccache cache write access
+        if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }}
+        run: |
+          # With auto-allocate-uids, UID 872415232 (0x34000000) maps to nixbld inside sandbox
+          sudo chown -R 872415232 /nix/var/cache/sccache
+          sudo chmod -R 2777 /nix/var/cache/sccache
       - name: nix build
         if: ${{ matrix.attr != '' }}
         shell: bash
@@ -67,17 +81,31 @@ jobs:
       - name: Checkout Repo
         if: ${{ matrix.attr != '' }}
         uses: actions/checkout@v4
+      - name: Mount sccache disk
+        if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }}
+        uses: useblacksmith/stickydisk@v1
+        with:
+          key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cache_key }}
+          path: /nix/var/cache/sccache
       - name: Install nix (ephemeral)
         if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }}
         uses: ./.github/actions/nix-install-ephemeral
         with:
           push-to-cache: 'true'
+          enable-sccache-sandbox-path: ${{ matrix.postgresql_version && 'true' || 'false' }}
+          max-jobs: ${{ matrix.postgresql_version && '1' || '' }}
         env:
           DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
           NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
       - name: Install nix (self-hosted)
         if: ${{ matrix.attr != '' && matrix.runs_on.group == 'self-hosted-runners-nix' }}
         uses: ./.github/actions/nix-install-self-hosted
+      - name: Allow sccache cache write access
+        if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }}
+        run: |
+          # With auto-allocate-uids, UID 872415232 (0x34000000) maps to nixbld inside sandbox
+          sudo chown -R 872415232 /nix/var/cache/sccache
+          sudo chmod -R 2777 /nix/var/cache/sccache
       - name: nix build
         if: ${{ matrix.attr != '' }}
         shell: bash
@@ -144,14 +172,28 @@ jobs:
       - name: Checkout Repo
         if: ${{ matrix.attr != '' }}
         uses: actions/checkout@v4
+      - name: Mount sccache disk
+        if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }}
+        uses: useblacksmith/stickydisk@v1
+        with:
+          key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cache_key }}
+          path: /nix/var/cache/sccache
       - name: Install nix
         if: ${{ matrix.attr != '' }}
         uses: ./.github/actions/nix-install-ephemeral
         with:
+          enable-sccache-sandbox-path: ${{ matrix.postgresql_version && 'true' || 'false' }}
+          max-jobs: ${{ matrix.postgresql_version && '1' || '' }}
           push-to-cache: 'true'
         env:
           DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
           NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
+      - name: Allow sccache cache write access
+        if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }}
+        run: |
+          # With auto-allocate-uids, UID 872415232 (0x34000000) maps to nixbld inside sandbox
+          sudo chown -R 872415232 /nix/var/cache/sccache
+          sudo chmod -R 2777 /nix/var/cache/sccache
       - name: nix build
         if: ${{ matrix.attr != '' }}
         shell: bash
@@ -172,14 +214,28 @@ jobs:
       - name: Checkout Repo
         if: ${{ matrix.attr != '' }}
         uses: actions/checkout@v4
+      - name: Mount sccache disk
+        if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }}
+        uses: useblacksmith/stickydisk@v1
+        with:
+          key: ${{ github.repository }}-sccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cache_key }}
+          path: /nix/var/cache/sccache
       - name: Install nix
         if: ${{ matrix.attr != '' }}
         uses: ./.github/actions/nix-install-ephemeral
         with:
+          enable-sccache-sandbox-path: ${{ matrix.postgresql_version && 'true' || 'false' }}
+          max-jobs: ${{ matrix.postgresql_version && '1' || '' }}
           push-to-cache: 'true'
         env:
           DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
           NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
+      - name: Allow sccache cache write access
+        if: ${{ matrix.attr != '' && matrix.postgresql_version && matrix.runs_on.group != 'self-hosted-runners-nix' }}
+        run: |
+          # With auto-allocate-uids, UID 872415232 (0x34000000) maps to nixbld inside sandbox
+          sudo chown -R 872415232 /nix/var/cache/sccache
+          sudo chmod -R 2777 /nix/var/cache/sccache
       - name: nix build
         if: ${{ matrix.attr != '' }}
         shell: bash

flake.nix

@@ -28,7 +28,7 @@
     git-hooks.inputs.nixpkgs.follows = "nixpkgs";
     nixpkgs-go124.url = "github:Nixos/nixpkgs/d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0";
     nixpkgs-pgbackrest.url = "github:nixos/nixpkgs/nixos-unstable-small";
-    nix-eval-jobs.url = "github:nix-community/nix-eval-jobs";
+    nix-eval-jobs.url = "github:jfroche/nix-eval-jobs/fix-warnings";
   };
 
   outputs =

nix/cargo-pgrx/buildPgrxExtension.nix

@@ -35,6 +35,7 @@
   darwin,
   writeShellScriptBin,
   defaultBindgenHook,
+  sccache,
 }:
 
 # The idea behind: Use it mostly like rustPlatform.buildRustPackage and so
@@ -56,7 +57,7 @@
   postgresql,
   # enable override to generate bindings using bindgenHook.
   # Some older versions of cargo-pgrx use a bindgenHook that is not compatible with the
-  # current clang version present in stdenv
+  # current clang version present in stdenv
   bindgenHook ? defaultBindgenHook,
   # cargo-pgrx calls rustfmt on generated bindings, this is not strictly necessary, so we avoid the
   # dependency here. Set to false and provide rustfmt in nativeBuildInputs, if you need it, e.g.
@@ -135,12 +136,23 @@ let
         postgresql
         pkg-config
         bindgenHook
+        sccache
       ]
       ++ lib.optionals useFakeRustfmt [ fakeRustfmt ];
 
     buildPhase = ''
       runHook preBuild
 
+
+      if [[ -d "/nix/var/cache/sccache" && -w "/nix/var/cache/sccache" ]]; then
+        echo "sccache: cache directory available, enabling"
+        export RUSTC_WRAPPER="${sccache}/bin/sccache"
+        export SCCACHE_DIR="/nix/var/cache/sccache"
+        export SCCACHE_CACHE_SIZE="50G"
+      else
+        echo "sccache: cache directory not available, skipping"
+      fi
+
       echo "Executing cargo-pgrx buildPhase"
       ${preBuildAndTest}
       ${maybeEnterBuildAndTestSubdir}
@@ -154,6 +166,11 @@ let
         --features "${builtins.concatStringsSep " " buildFeatures}" \
         --out-dir "$out"
 
+      if [[ -n "''${RUSTC_WRAPPER:-}" ]]; then
+        echo "sccache stats:"
+        ${sccache}/bin/sccache --show-stats
+      fi
+
       ${maybeLeaveBuildAndTestSubdir}
 
       runHook postBuild

nix/docs/README.md

@@ -35,6 +35,10 @@ learn how to play with `postgres` in the [build guide](./build-postgres.md).
 - **[Migration Tests](./migration-tests.md)** - Testing database migrations
 - **[Testing PG Upgrade Scripts](./testing-pg-upgrade-scripts.md)** - Testing PostgreSQL upgrades
 
+## CI/CD
+
+- **[sccache in CI](./sccache-ci.md)** - Rust compilation caching for pgrx extensions
+
 ## Reference
 
 - **[References](./references.md)** - Useful links and resources

nix/docs/sccache-ci.md

@@ -0,0 +1,88 @@
+# sccache in CI
+
+This document explains how sccache is integrated into the Nix CI pipeline to accelerate Rust compilation for pgrx-based PostgreSQL extensions.
+
+## Overview
+
+Building pgrx extensions (pg_graphql, pg_jsonschema, wrappers, etc.) involves significant Rust compilation.
+sccache caches compiled artifacts to speed up subsequent builds when source code hasn't changed.
+
+The integration required solving several challenges around Nix's sandboxed builds and CI runner persistence.
+
+## Architecture
+
+Ephemeral CI runners (Blacksmith) use stickydisk for persistent storage at `/nix/var/cache/sccache`.
+Self-hosted Darwin runners use a local directory at the same path.
+
+The cache is keyed by PostgreSQL version to avoid cross-version cache pollution:
+- Extension packages: `sccache-Linux-ARM64-pg17`, `sccache-Linux-ARM64-pg15`, etc.
+- Non-extension packages: `sccache-Linux-ARM64-shared`
+
+| Platform | Runner type | sccache enabled |
+|----------|-------------|-----------------|
+| x86_64-linux | Blacksmith ephemeral | Yes (extensions only) |
+| aarch64-linux | Blacksmith ephemeral | Yes (extensions only) |
+| aarch64-linux | Self-hosted (KVM packages) | No |
+| aarch64-darwin | Self-hosted | Yes (extensions only) |
+
+## Nix sandbox integration
+
+Nix builds run in a sandbox that isolates the build environment.
+To allow sccache to persist across builds, the cache directory must be mounted into the sandbox via `extra-sandbox-paths`.
+
+### Linux with auto-allocate-uids
+
+On Linux, we use Nix's `auto-allocate-uids` feature which creates user namespaces for builds.
+This introduces a UID mapping challenge.
+
+The base UID is 872415232 (0x34000000).
+Inside the sandbox, processes run as root (UID 0), but outside the sandbox this maps to UID 872415232.
+Files created inside the sandbox are owned by this mapped UID.
+
+The workflow changes ownership to UID 872415232 before builds to ensure cache access works correctly.
+
+### Darwin
+
+Darwin doesn't have user namespaces.
+Builds run as nixbld users (members of the `nixbld` group).
+The cache directory is owned by the nixbld group with setgid permissions.
+
+## Compromises and limitations
+
+### max-jobs = 1 for extensions
+
+With `auto-allocate-uids`, each parallel Nix build gets a different UID slot (872415232, 872480768, etc.).
+When multiple builds run in parallel, they create files with different ownership, causing EPERM errors when one build tries to update another's cache files.
+
+To avoid this, extension builds use `max-jobs = 1`, serializing Nix builds so all use the same UID slot.
+Non-extension packages don't use sccache and can build in parallel.
+
+### Blacksmith stickydisk last-writer-wins
+
+Blacksmith's stickydisk has "last writer wins" semantics for concurrent writes.
+If multiple jobs write to the same cache key simultaneously, only the last job's writes persist.
+
+We mitigate this by using per-PostgreSQL-version cache keys, reducing concurrent writes to the same cache.
+
+### Extensions only
+
+sccache is only enabled for extension packages (those with `postgresql_version` in the matrix).
+Non-extension packages (PostgreSQL itself, tooling) build without sccache to allow parallel builds.
+
+## Debugging
+
+To check sccache statistics, look for output at the end of extension builds:
+
+```
+sccache stats:
+Compile requests                    150
+Compile requests executed           148
+Cache hits                          120
+Cache misses                         28
+```
+
+## References
+
+- [Nix auto-allocate-uids](https://nixos.org/manual/nix/stable/command-ref/conf-file.html#conf-auto-allocate-uids)
+- [sccache](https://github.com/mozilla/sccache)
+- [Blacksmith stickydisk](https://blacksmith.sh/docs/stickydisk)

nix/ext/pg_graphql/default.nix

@@ -125,9 +125,11 @@ let
   versions = lib.naturalSort (lib.attrNames supportedVersions);
   latestVersion = lib.last versions;
   numberOfVersions = builtins.length versions;
-  packages = builtins.attrValues (
-    lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions
-  );
+  packagesAttrSet = lib.mapAttrs' (name: value: {
+    name = lib.replaceStrings [ "." ] [ "_" ] name;
+    value = build name value.hash value.rust value.pgrx;
+  }) supportedVersions;
+  packages = builtins.attrValues packagesAttrSet;
 in
 (buildEnv {
   name = pname;
@@ -175,6 +177,10 @@ in
     inherit versions numberOfVersions pname;
     version =
       "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
+    # Expose individual packages for CI to build separately
+    packages = packagesAttrSet // {
+      recurseForDerivations = true;
+    };
   };
 }).overrideAttrs
   (_: {

nix/ext/pg_jsonschema/default.nix

@@ -126,9 +126,11 @@ let
   versions = lib.naturalSort (lib.attrNames supportedVersions);
   latestVersion = lib.last versions;
   numberOfVersions = builtins.length versions;
-  packages = builtins.attrValues (
-    lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions
-  );
+  packagesAttrSet = lib.mapAttrs' (name: value: {
+    name = lib.replaceStrings [ "." ] [ "_" ] name;
+    value = build name value.hash value.rust value.pgrx;
+  }) supportedVersions;
+  packages = builtins.attrValues packagesAttrSet;
 in
 (pkgs.buildEnv {
   name = pname;
@@ -168,6 +170,10 @@ in
     inherit versions numberOfVersions pname;
     version =
       "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
+    # Expose individual packages for CI to build separately
+    packages = packagesAttrSet // {
+      recurseForDerivations = true;
+    };
   };
 }).overrideAttrs
   (_: {