feat(benchmark): add k6 benchmarks for JSONB/encrypted query performance

.github/workflows/benchmark.yml

@@ -16,6 +16,8 @@ permissions:
   deployments: write
   # contents permission to update benchmark contents in gh-pages branch
   contents: write
+  # pull-requests permission to comment on PRs with benchmark alerts
+  pull-requests: write
 
 jobs:
   benchmark:
@@ -42,11 +44,45 @@ jobs:
       - run: |
           mise run postgres:up --extra-args "--detach --wait"
 
-      - name: Run benchmark
+      - name: Run k6 benchmark
         working-directory: tests/benchmark
         env:
           RUST_BACKTRACE: "1"
-        run: mise run benchmark:continuous
+        run: mise run k6:benchmark:continuous
+
+      # Store k6 throughput benchmark (higher is better)
+      - name: Store k6 throughput benchmark
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: 'k6 Throughput'
+          tool: 'customBiggerIsBetter'
+          output-file-path: tests/benchmark/results/k6-throughput.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          fail-on-alert: true
+          comment-on-alert: true
+          summary-always: true
+          auto-push: true
+          benchmark-data-dir-path: docs/k6/throughput
+
+      # Store k6 latency benchmark (lower is better)
+      - name: Store k6 latency benchmark
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: 'k6 Latency'
+          tool: 'customSmallerIsBetter'
+          output-file-path: tests/benchmark/results/k6-latency.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          fail-on-alert: true
+          comment-on-alert: true
+          summary-always: true
+          auto-push: true
+          benchmark-data-dir-path: docs/k6/latency
+
+      - name: Run pgbench benchmark
+        working-directory: tests/benchmark
+        env:
+          RUST_BACKTRACE: "1"
+        run: mise run benchmark_service --target=proxy --transaction=encrypted --protocol=extended --port=6432 --time=30 --clients=10
 
       # Download previous benchmark result from cache (if exists)
       - name: Download previous benchmark data
@@ -56,20 +92,18 @@ jobs:
           key: ${{ runner.os }}-benchmark
 
       # Run `github-action-benchmark` action
-      - name: Store benchmark result
+      - name: Store pgbench benchmark result
         uses: benchmark-action/github-action-benchmark@v1
         with:
-          # What benchmark tool the output.txt came from
+          name: 'pgbench'
           tool: 'customBiggerIsBetter'
-          # Where the output from the benchmark tool is stored
           output-file-path: tests/benchmark/results/output.json
-
           github-token: ${{ secrets.GITHUB_TOKEN }}
           fail-on-alert: true
           comment-on-alert: true
           summary-always: true
           auto-push: true
-          benchmark-data-dir-path: docs
+          benchmark-data-dir-path: docs/pgbench
 
       - uses: ./.github/actions/send-slack-notification
         with:

tests/benchmark/k6/Dockerfile

@@ -0,0 +1,12 @@
+FROM golang:1.23-alpine AS builder
+RUN apk add --no-cache git
+ENV GOTOOLCHAIN=auto
+RUN go install go.k6.io/xk6/cmd/xk6@latest
+WORKDIR /build
+RUN xk6 build --with github.com/grafana/xk6-sql@latest --with github.com/grafana/xk6-sql-driver-postgres@latest --output /build/k6
+
+FROM alpine:3.19
+RUN apk add --no-cache ca-certificates
+COPY --from=builder /build/k6 /usr/local/bin/k6
+WORKDIR /scripts
+ENTRYPOINT ["k6"]

tests/benchmark/k6/scripts/jsonb-large-payload.js

@@ -0,0 +1,64 @@
+// Large payload INSERT benchmark for memory/performance investigation
+// Uses encrypted_jsonb_extract (~250KB) and encrypted_jsonb_full (~500KB) columns
+// Replicates customer scenario with realistic credit report structures
+//
+// Uses xk6-sql API:
+//   sql.open(driver, connString)
+//   db.exec(sql, ...args)
+
+import sql from 'k6/x/sql';
+import driver from 'k6/x/sql/driver/postgres';
+import { getConnectionString, getDefaultOptions } from './lib/config.js';
+import { randomId, generateExtractPayload, generateFullPayload } from './lib/data.js';
+import { createSummaryHandler } from './lib/summary.js';
+
+const target = __ENV.K6_TARGET || 'proxy';
+const connectionString = getConnectionString(target);
+
+// Payload mode: 'extract' (~250KB), 'full' (~500KB), or 'dual' (both columns)
+const payloadMode = __ENV.K6_PAYLOAD_MODE || 'dual';
+
+export const options = getDefaultOptions({
+  'iteration_duration': ['p(95)<30000'],  // 30s for large payloads
+});
+
+const db = sql.open(driver, connectionString);
+
+export default function() {
+  const id = randomId();
+
+  if (payloadMode === 'extract') {
+    // ~250KB payload only
+    const extractPayload = generateExtractPayload(id);
+    db.exec(
+      `INSERT INTO benchmark_encrypted (id, encrypted_jsonb_extract) VALUES ($1, $2)`,
+      id,
+      JSON.stringify(extractPayload)
+    );
+  } else if (payloadMode === 'full') {
+    // ~500KB payload only
+    const fullPayload = generateFullPayload(id);
+    db.exec(
+      `INSERT INTO benchmark_encrypted (id, encrypted_jsonb_full) VALUES ($1, $2)`,
+      id,
+      JSON.stringify(fullPayload)
+    );
+  } else {
+    // Dual mode: insert both columns simultaneously (default)
+    // This replicates the customer scenario that caused 25s+ timeouts
+    const extractPayload = generateExtractPayload(id);
+    const fullPayload = generateFullPayload(id);
+    db.exec(
+      `INSERT INTO benchmark_encrypted (id, encrypted_jsonb_extract, encrypted_jsonb_full) VALUES ($1, $2, $3)`,
+      id,
+      JSON.stringify(extractPayload),
+      JSON.stringify(fullPayload)
+    );
+  }
+}
+
+export function teardown() {
+  db.close();
+}
+
+export const handleSummary = createSummaryHandler('jsonb-large-payload');

tests/benchmark/k6/scripts/jsonb-ste-vec-containment.js

@@ -0,0 +1,69 @@
+// JSONB containment (@>) benchmark
+//
+// The @> operator on eql_v2_encrypted is confirmed working via integration tests:
+//   packages/cipherstash-proxy-integration/src/select/jsonb_contains.rs
+//
+// Uses xk6-sql API:
+//   sql.open(driver, connString)
+//   db.exec(sql, ...args)
+
+import sql from 'k6/x/sql';
+import driver from 'k6/x/sql/driver/postgres';
+import { getConnectionString, getDefaultOptions } from './lib/config.js';
+import { createSummaryHandler } from './lib/summary.js';
+
+const target = __ENV.K6_TARGET || 'proxy';
+const connectionString = getConnectionString(target);
+
+// ID range for containment benchmark data (isolated from other tests)
+const ID_START = 2000000;
+const ID_COUNT = 100;
+
+export const options = getDefaultOptions({
+  'iteration_duration': ['p(95)<200'],
+});
+
+const db = sql.open(driver, connectionString);
+
+export function setup() {
+  // Clean up any leftover data from crashed runs before inserting
+  db.exec(`DELETE FROM benchmark_encrypted WHERE id BETWEEN $1 AND $2`, ID_START, ID_START + ID_COUNT - 1);
+
+  // Insert seed data with known values for containment queries
+  for (let i = 0; i < ID_COUNT; i++) {
+    const id = ID_START + i;
+    const jsonb = {
+      id: id,
+      string: `value${i % 10}`,
+      number: i % 10,
+      nested: { string: 'world', number: i },
+    };
+    db.exec(
+      `INSERT INTO benchmark_encrypted (id, encrypted_jsonb_with_ste_vec) VALUES ($1, $2)`,
+      id,
+      JSON.stringify(jsonb)
+    );
+  }
+}
+
+export default function() {
+  const i = Math.floor(Math.random() * 10);
+  const pattern = JSON.stringify({ string: `value${i}` });
+
+  // Use exec instead of query - we only need to verify the query runs, not inspect results
+  // Query uses @> containment operator on encrypted JSONB
+  db.exec(
+    `SELECT id FROM benchmark_encrypted WHERE encrypted_jsonb_with_ste_vec @> $1 AND id BETWEEN $2 AND $3`,
+    pattern,
+    ID_START,
+    ID_START + ID_COUNT - 1
+  );
+}
+
+export function teardown() {
+  // Clean up seed data
+  db.exec(`DELETE FROM benchmark_encrypted WHERE id BETWEEN $1 AND $2`, ID_START, ID_START + ID_COUNT - 1);
+  db.close();
+}
+
+export const handleSummary = createSummaryHandler('jsonb-ste-vec-containment');

tests/benchmark/k6/scripts/jsonb-ste-vec-insert.js

@@ -0,0 +1,37 @@
+// JSONB INSERT benchmark - primary CI benchmark for encrypted JSONB performance
+//
+// Uses xk6-sql API:
+//   sql.open(driver, connString)
+//   db.exec(sql, ...args)
+
+import sql from 'k6/x/sql';
+import driver from 'k6/x/sql/driver/postgres';
+import { getConnectionString, getDefaultOptions } from './lib/config.js';
+import { randomId, generateStandardJsonb } from './lib/data.js';
+import { createSummaryHandler } from './lib/summary.js';
+
+const target = __ENV.K6_TARGET || 'proxy';
+const connectionString = getConnectionString(target);
+
+export const options = getDefaultOptions({
+  'iteration_duration': ['p(95)<500'],
+});
+
+const db = sql.open(driver, connectionString);
+
+export default function() {
+  const id = randomId();
+  const jsonb = generateStandardJsonb(id);
+
+  db.exec(
+    `INSERT INTO benchmark_encrypted (id, encrypted_jsonb_with_ste_vec) VALUES ($1, $2)`,
+    id,
+    JSON.stringify(jsonb)
+  );
+}
+
+export function teardown() {
+  db.close();
+}
+
+export const handleSummary = createSummaryHandler('jsonb-ste-vec-insert');

tests/benchmark/k6/scripts/jsonb-ste-vec-large-payload.js

@@ -0,0 +1,37 @@
+// Large payload (500KB+) INSERT benchmark for memory/performance investigation
+//
+// Uses xk6-sql API:
+//   sql.open(driver, connString)
+//   db.exec(sql, ...args)
+
+import sql from 'k6/x/sql';
+import driver from 'k6/x/sql/driver/postgres';
+import { getConnectionString, getDefaultOptions } from './lib/config.js';
+import { randomId, generateLargeJsonb } from './lib/data.js';
+import { createSummaryHandler } from './lib/summary.js';
+
+const target = __ENV.K6_TARGET || 'proxy';
+const connectionString = getConnectionString(target);
+
+export const options = getDefaultOptions({
+  'iteration_duration': ['p(95)<30000'],  // 30s for large payloads
+});
+
+const db = sql.open(driver, connectionString);
+
+export default function() {
+  const id = randomId();
+  const jsonb = generateLargeJsonb(id);
+
+  db.exec(
+    `INSERT INTO benchmark_encrypted (id, encrypted_jsonb_with_ste_vec) VALUES ($1, $2)`,
+    id,
+    JSON.stringify(jsonb)
+  );
+}
+
+export function teardown() {
+  db.close();
+}
+
+export const handleSummary = createSummaryHandler('jsonb-ste-vec-large-payload');

tests/benchmark/k6/scripts/lib/config.js

@@ -0,0 +1,44 @@
+// Connection configuration for k6 benchmarks
+// Ports: postgres=5532, proxy=6432
+//
+// xk6-sql API:
+//   import sql from 'k6/x/sql';
+//   import driver from 'k6/x/sql/driver/postgres';
+//   const db = sql.open(driver, connString);
+//   db.exec(sql, ...args);
+//   const rows = db.query(sql, ...args);
+//   db.close();
+
+export const POSTGRES_PORT = 5532;
+export const PROXY_PORT = 6432;
+
+export function getConnectionString(target) {
+  // Default to host.docker.internal (works on macOS and Windows Docker)
+  // For Linux CI with --network=host, set K6_DB_HOST=127.0.0.1
+  const host = __ENV.K6_DB_HOST || 'host.docker.internal';
+  const port = target === 'proxy' ? PROXY_PORT : POSTGRES_PORT;
+  const user = __ENV.K6_DB_USER || 'cipherstash';
+  const password = __ENV.K6_DB_PASSWORD || 'p@ssword';
+  const database = __ENV.K6_DB_NAME || 'cipherstash';
+  // Default sslmode=disable for local/CI; override via K6_DB_SSLMODE if needed
+  const sslmode = __ENV.K6_DB_SSLMODE || 'disable';
+
+  return `postgres://${user}:${password}@${host}:${port}/${database}?sslmode=${sslmode}`;
+}
+
+export function getDefaultOptions(thresholds = {}) {
+  return {
+    scenarios: {
+      default: {
+        executor: 'constant-vus',
+        vus: parseInt(__ENV.K6_VUS || '10'),
+        duration: __ENV.K6_DURATION || '30s',
+      },
+    },
+    summaryTrendStats: ['min', 'avg', 'med', 'p(90)', 'p(95)', 'p(99)', 'max'],
+    thresholds: {
+      'iteration_duration': ['p(95)<500'],
+      ...thresholds,
+    },
+  };
+}