diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 128b5abd..6c08a161 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -16,6 +16,8 @@ permissions: deployments: write # contents permission to update benchmark contents in gh-pages branch contents: write + # pull-requests permission to comment on PRs with benchmark alerts + pull-requests: write jobs: benchmark: @@ -42,11 +44,45 @@ jobs: - run: | mise run postgres:up --extra-args "--detach --wait" - - name: Run benchmark + - name: Run k6 benchmark working-directory: tests/benchmark env: RUST_BACKTRACE: "1" - run: mise run benchmark:continuous + run: mise run k6:benchmark:continuous + + # Store k6 throughput benchmark (higher is better) + - name: Store k6 throughput benchmark + uses: benchmark-action/github-action-benchmark@v1 + with: + name: 'k6 Throughput' + tool: 'customBiggerIsBetter' + output-file-path: tests/benchmark/results/k6-throughput.json + github-token: ${{ secrets.GITHUB_TOKEN }} + fail-on-alert: true + comment-on-alert: true + summary-always: true + auto-push: true + benchmark-data-dir-path: docs/k6/throughput + + # Store k6 latency benchmark (lower is better) + - name: Store k6 latency benchmark + uses: benchmark-action/github-action-benchmark@v1 + with: + name: 'k6 Latency' + tool: 'customSmallerIsBetter' + output-file-path: tests/benchmark/results/k6-latency.json + github-token: ${{ secrets.GITHUB_TOKEN }} + fail-on-alert: true + comment-on-alert: true + summary-always: true + auto-push: true + benchmark-data-dir-path: docs/k6/latency + + - name: Run pgbench benchmark + working-directory: tests/benchmark + env: + RUST_BACKTRACE: "1" + run: mise run benchmark_service --target=proxy --transaction=encrypted --protocol=extended --port=6432 --time=30 --clients=10 # Download previous benchmark result from cache (if exists) - name: Download previous benchmark data @@ -56,20 +92,18 @@ jobs: key: ${{ runner.os }}-benchmark # Run `github-action-benchmark` action - - name: Store benchmark result + - name: Store pgbench benchmark result uses: benchmark-action/github-action-benchmark@v1 with: - # What benchmark tool the output.txt came from + name: 'pgbench' tool: 'customBiggerIsBetter' - # Where the output from the benchmark tool is stored output-file-path: tests/benchmark/results/output.json - github-token: ${{ secrets.GITHUB_TOKEN }} fail-on-alert: true comment-on-alert: true summary-always: true auto-push: true - benchmark-data-dir-path: docs + benchmark-data-dir-path: docs/pgbench - uses: ./.github/actions/send-slack-notification with: diff --git a/tests/benchmark/k6/Dockerfile b/tests/benchmark/k6/Dockerfile new file mode 100644 index 00000000..097d1e04 --- /dev/null +++ b/tests/benchmark/k6/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.23-alpine AS builder +RUN apk add --no-cache git +ENV GOTOOLCHAIN=auto +RUN go install go.k6.io/xk6/cmd/xk6@latest +WORKDIR /build +RUN xk6 build --with github.com/grafana/xk6-sql@latest --with github.com/grafana/xk6-sql-driver-postgres@latest --output /build/k6 + +FROM alpine:3.19 +RUN apk add --no-cache ca-certificates +COPY --from=builder /build/k6 /usr/local/bin/k6 +WORKDIR /scripts +ENTRYPOINT ["k6"] diff --git a/tests/benchmark/k6/scripts/jsonb-large-payload.js b/tests/benchmark/k6/scripts/jsonb-large-payload.js new file mode 100644 index 00000000..05aa28af --- /dev/null +++ b/tests/benchmark/k6/scripts/jsonb-large-payload.js @@ -0,0 +1,64 @@ +// Large payload INSERT benchmark for memory/performance investigation +// Uses encrypted_jsonb_extract (~250KB) and encrypted_jsonb_full (~500KB) columns +// Replicates customer scenario with realistic credit report structures +// +// Uses xk6-sql API: +// sql.open(driver, connString) +// db.exec(sql, ...args) + +import sql from 'k6/x/sql'; +import driver from 'k6/x/sql/driver/postgres'; +import { getConnectionString, getDefaultOptions } from './lib/config.js'; +import { randomId, generateExtractPayload, generateFullPayload } from './lib/data.js'; +import { createSummaryHandler } from './lib/summary.js'; + +const target = __ENV.K6_TARGET || 'proxy'; +const connectionString = getConnectionString(target); + +// Payload mode: 'extract' (~250KB), 'full' (~500KB), or 'dual' (both columns) +const payloadMode = __ENV.K6_PAYLOAD_MODE || 'dual'; + +export const options = getDefaultOptions({ + 'iteration_duration': ['p(95)<30000'], // 30s for large payloads +}); + +const db = sql.open(driver, connectionString); + +export default function() { + const id = randomId(); + + if (payloadMode === 'extract') { + // ~250KB payload only + const extractPayload = generateExtractPayload(id); + db.exec( + `INSERT INTO benchmark_encrypted (id, encrypted_jsonb_extract) VALUES ($1, $2)`, + id, + JSON.stringify(extractPayload) + ); + } else if (payloadMode === 'full') { + // ~500KB payload only + const fullPayload = generateFullPayload(id); + db.exec( + `INSERT INTO benchmark_encrypted (id, encrypted_jsonb_full) VALUES ($1, $2)`, + id, + JSON.stringify(fullPayload) + ); + } else { + // Dual mode: insert both columns simultaneously (default) + // This replicates the customer scenario that caused 25s+ timeouts + const extractPayload = generateExtractPayload(id); + const fullPayload = generateFullPayload(id); + db.exec( + `INSERT INTO benchmark_encrypted (id, encrypted_jsonb_extract, encrypted_jsonb_full) VALUES ($1, $2, $3)`, + id, + JSON.stringify(extractPayload), + JSON.stringify(fullPayload) + ); + } +} + +export function teardown() { + db.close(); +} + +export const handleSummary = createSummaryHandler('jsonb-large-payload'); diff --git a/tests/benchmark/k6/scripts/jsonb-ste-vec-containment.js b/tests/benchmark/k6/scripts/jsonb-ste-vec-containment.js new file mode 100644 index 00000000..65a3d167 --- /dev/null +++ b/tests/benchmark/k6/scripts/jsonb-ste-vec-containment.js @@ -0,0 +1,69 @@ +// JSONB containment (@>) benchmark +// +// The @> operator on eql_v2_encrypted is confirmed working via integration tests: +// packages/cipherstash-proxy-integration/src/select/jsonb_contains.rs +// +// Uses xk6-sql API: +// sql.open(driver, connString) +// db.exec(sql, ...args) + +import sql from 'k6/x/sql'; +import driver from 'k6/x/sql/driver/postgres'; +import { getConnectionString, getDefaultOptions } from './lib/config.js'; +import { createSummaryHandler } from './lib/summary.js'; + +const target = __ENV.K6_TARGET || 'proxy'; +const connectionString = getConnectionString(target); + +// ID range for containment benchmark data (isolated from other tests) +const ID_START = 2000000; +const ID_COUNT = 100; + +export const options = getDefaultOptions({ + 'iteration_duration': ['p(95)<200'], +}); + +const db = sql.open(driver, connectionString); + +export function setup() { + // Clean up any leftover data from crashed runs before inserting + db.exec(`DELETE FROM benchmark_encrypted WHERE id BETWEEN $1 AND $2`, ID_START, ID_START + ID_COUNT - 1); + + // Insert seed data with known values for containment queries + for (let i = 0; i < ID_COUNT; i++) { + const id = ID_START + i; + const jsonb = { + id: id, + string: `value${i % 10}`, + number: i % 10, + nested: { string: 'world', number: i }, + }; + db.exec( + `INSERT INTO benchmark_encrypted (id, encrypted_jsonb_with_ste_vec) VALUES ($1, $2)`, + id, + JSON.stringify(jsonb) + ); + } +} + +export default function() { + const i = Math.floor(Math.random() * 10); + const pattern = JSON.stringify({ string: `value${i}` }); + + // Use exec instead of query - we only need to verify the query runs, not inspect results + // Query uses @> containment operator on encrypted JSONB + db.exec( + `SELECT id FROM benchmark_encrypted WHERE encrypted_jsonb_with_ste_vec @> $1 AND id BETWEEN $2 AND $3`, + pattern, + ID_START, + ID_START + ID_COUNT - 1 + ); +} + +export function teardown() { + // Clean up seed data + db.exec(`DELETE FROM benchmark_encrypted WHERE id BETWEEN $1 AND $2`, ID_START, ID_START + ID_COUNT - 1); + db.close(); +} + +export const handleSummary = createSummaryHandler('jsonb-ste-vec-containment'); diff --git a/tests/benchmark/k6/scripts/jsonb-ste-vec-insert.js b/tests/benchmark/k6/scripts/jsonb-ste-vec-insert.js new file mode 100644 index 00000000..f1bfe23a --- /dev/null +++ b/tests/benchmark/k6/scripts/jsonb-ste-vec-insert.js @@ -0,0 +1,37 @@ +// JSONB INSERT benchmark - primary CI benchmark for encrypted JSONB performance +// +// Uses xk6-sql API: +// sql.open(driver, connString) +// db.exec(sql, ...args) + +import sql from 'k6/x/sql'; +import driver from 'k6/x/sql/driver/postgres'; +import { getConnectionString, getDefaultOptions } from './lib/config.js'; +import { randomId, generateStandardJsonb } from './lib/data.js'; +import { createSummaryHandler } from './lib/summary.js'; + +const target = __ENV.K6_TARGET || 'proxy'; +const connectionString = getConnectionString(target); + +export const options = getDefaultOptions({ + 'iteration_duration': ['p(95)<500'], +}); + +const db = sql.open(driver, connectionString); + +export default function() { + const id = randomId(); + const jsonb = generateStandardJsonb(id); + + db.exec( + `INSERT INTO benchmark_encrypted (id, encrypted_jsonb_with_ste_vec) VALUES ($1, $2)`, + id, + JSON.stringify(jsonb) + ); +} + +export function teardown() { + db.close(); +} + +export const handleSummary = createSummaryHandler('jsonb-ste-vec-insert'); diff --git a/tests/benchmark/k6/scripts/jsonb-ste-vec-large-payload.js b/tests/benchmark/k6/scripts/jsonb-ste-vec-large-payload.js new file mode 100644 index 00000000..45412904 --- /dev/null +++ b/tests/benchmark/k6/scripts/jsonb-ste-vec-large-payload.js @@ -0,0 +1,37 @@ +// Large payload (500KB+) INSERT benchmark for memory/performance investigation +// +// Uses xk6-sql API: +// sql.open(driver, connString) +// db.exec(sql, ...args) + +import sql from 'k6/x/sql'; +import driver from 'k6/x/sql/driver/postgres'; +import { getConnectionString, getDefaultOptions } from './lib/config.js'; +import { randomId, generateLargeJsonb } from './lib/data.js'; +import { createSummaryHandler } from './lib/summary.js'; + +const target = __ENV.K6_TARGET || 'proxy'; +const connectionString = getConnectionString(target); + +export const options = getDefaultOptions({ + 'iteration_duration': ['p(95)<30000'], // 30s for large payloads +}); + +const db = sql.open(driver, connectionString); + +export default function() { + const id = randomId(); + const jsonb = generateLargeJsonb(id); + + db.exec( + `INSERT INTO benchmark_encrypted (id, encrypted_jsonb_with_ste_vec) VALUES ($1, $2)`, + id, + JSON.stringify(jsonb) + ); +} + +export function teardown() { + db.close(); +} + +export const handleSummary = createSummaryHandler('jsonb-ste-vec-large-payload'); diff --git a/tests/benchmark/k6/scripts/lib/config.js b/tests/benchmark/k6/scripts/lib/config.js new file mode 100644 index 00000000..dd72fdf0 --- /dev/null +++ b/tests/benchmark/k6/scripts/lib/config.js @@ -0,0 +1,44 @@ +// Connection configuration for k6 benchmarks +// Ports: postgres=5532, proxy=6432 +// +// xk6-sql API: +// import sql from 'k6/x/sql'; +// import driver from 'k6/x/sql/driver/postgres'; +// const db = sql.open(driver, connString); +// db.exec(sql, ...args); +// const rows = db.query(sql, ...args); +// db.close(); + +export const POSTGRES_PORT = 5532; +export const PROXY_PORT = 6432; + +export function getConnectionString(target) { + // Default to host.docker.internal (works on macOS and Windows Docker) + // For Linux CI with --network=host, set K6_DB_HOST=127.0.0.1 + const host = __ENV.K6_DB_HOST || 'host.docker.internal'; + const port = target === 'proxy' ? PROXY_PORT : POSTGRES_PORT; + const user = __ENV.K6_DB_USER || 'cipherstash'; + const password = __ENV.K6_DB_PASSWORD || 'p@ssword'; + const database = __ENV.K6_DB_NAME || 'cipherstash'; + // Default sslmode=disable for local/CI; override via K6_DB_SSLMODE if needed + const sslmode = __ENV.K6_DB_SSLMODE || 'disable'; + + return `postgres://${user}:${password}@${host}:${port}/${database}?sslmode=${sslmode}`; +} + +export function getDefaultOptions(thresholds = {}) { + return { + scenarios: { + default: { + executor: 'constant-vus', + vus: parseInt(__ENV.K6_VUS || '10'), + duration: __ENV.K6_DURATION || '30s', + }, + }, + summaryTrendStats: ['min', 'avg', 'med', 'p(90)', 'p(95)', 'p(99)', 'max'], + thresholds: { + 'iteration_duration': ['p(95)<500'], + ...thresholds, + }, + }; +} diff --git a/tests/benchmark/k6/scripts/lib/data.js b/tests/benchmark/k6/scripts/lib/data.js new file mode 100644 index 00000000..edfbed55 --- /dev/null +++ b/tests/benchmark/k6/scripts/lib/data.js @@ -0,0 +1,338 @@ +// JSONB payload generators for k6 benchmarks +// Matches integration test fixtures in common.rs + +// PostgreSQL int4 (serial) max value - prevents "out of range for type integer" errors +const MAX_INT4 = 2147483647; + +export function randomId() { + return Math.floor(Math.random() * MAX_INT4); +} + +export function generateStandardJsonb(id) { + return { + id: id, + string: 'hello', + number: 42, + nested: { + number: 1815, + string: 'world', + }, + array_string: ['hello', 'world'], + array_number: [42, 84], + }; +} + +// Helper: generate random date string within range +function randomDate(startYear, endYear) { + const year = startYear + Math.floor(Math.random() * (endYear - startYear + 1)); + const month = String(Math.floor(Math.random() * 12) + 1).padStart(2, '0'); + const day = String(Math.floor(Math.random() * 28) + 1).padStart(2, '0'); + return `${year}-${month}-${day}`; +} + +// Helper: generate random alphanumeric string +function randomAlphanumeric(length) { + const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; + let result = ''; + for (let i = 0; i < length; i++) { + result += chars.charAt(Math.floor(Math.random() * chars.length)); + } + return result; +} + +// Account type options for tradelines +const ACCOUNT_TYPES = ['revolving', 'installment', 'mortgage', 'open', 'collection']; +const PAYMENT_STATUSES = ['current', 'late_30', 'late_60', 'late_90', 'late_120', 'charged_off']; +const INDUSTRY_CODES = ['bank', 'credit_union', 'finance_company', 'retail', 'utility', 'medical']; + +/** + * Generate extract payload (~250KB) + * Structure based on processed credit report data with: + * - 130 tradelines with 15 payment snapshots each + * - 45 inquiries with industry data + * - Credit scores with score reasons + */ +export function generateExtractPayload(id) { + // Generate 130 tradelines (tuned for ~250KB) + const tradelines = []; + for (let i = 0; i < 130; i++) { + // 15 payment snapshots per tradeline + const paymentHistory = []; + for (let m = 0; m < 15; m++) { + paymentHistory.push({ + period: `2023-${String(m + 1).padStart(2, '0')}`, + balance: Math.floor(Math.random() * 50000), + status: PAYMENT_STATUSES[Math.floor(Math.random() * PAYMENT_STATUSES.length)], + scheduledPayment: Math.floor(Math.random() * 2000), + actualPayment: Math.floor(Math.random() * 2000), + pastDueAmount: Math.floor(Math.random() * 1000), + }); + } + + tradelines.push({ + tradelineId: `TL-${id}-${i}`, + creditorName: `Creditor ${String.fromCharCode(65 + (i % 26))}${Math.floor(i / 26)}`, + accountNumber: randomAlphanumeric(16), + accountType: ACCOUNT_TYPES[i % ACCOUNT_TYPES.length], + accountStatus: i % 10 === 0 ? 'closed' : 'open', + openedDate: randomDate(2010, 2022), + closedDate: i % 10 === 0 ? randomDate(2022, 2024) : null, + creditLimit: 1000 + (i * 500), + highestBalance: 500 + Math.floor(Math.random() * 10000), + currentBalance: Math.floor(Math.random() * 8000), + monthlyPayment: 50 + Math.floor(Math.random() * 500), + lastActivityDate: randomDate(2023, 2024), + paymentHistory: paymentHistory, + termsMonths: [12, 24, 36, 48, 60][i % 5], + originalAmount: 1000 + (i * 1000), + responsibilityCode: ['individual', 'joint', 'authorized'][i % 3], + }); + } + + // Generate 45 inquiries (tuned for ~250KB) + const inquiries = []; + for (let i = 0; i < 45; i++) { + inquiries.push({ + inquiryId: `INQ-${id}-${i}`, + inquiryDate: randomDate(2022, 2024), + creditorName: `Bank ${String.fromCharCode(65 + (i % 26))}`, + industryCode: INDUSTRY_CODES[i % INDUSTRY_CODES.length], + inquiryType: i % 3 === 0 ? 'hard' : 'soft', + purposeCode: ['credit_card', 'auto_loan', 'mortgage', 'personal_loan'][i % 4], + }); + } + + // Generate score reasons + const scoreReasons = [ + { code: 'R01', description: 'Length of credit history' }, + { code: 'R02', description: 'Number of accounts with balances' }, + { code: 'R03', description: 'Proportion of balances to credit limits' }, + { code: 'R04', description: 'Recent account activity' }, + { code: 'R05', description: 'Number of recent inquiries' }, + ]; + + return { + reportId: `RPT-${id}`, + generatedAt: new Date().toISOString(), + consumer: { + consumerId: `CON-${id}`, + firstName: `FirstName${id % 1000}`, + lastName: `LastName${id % 1000}`, + dateOfBirth: randomDate(1950, 2000), + ssnMasked: `XXX-XX-${String(id % 10000).padStart(4, '0')}`, + }, + scores: [ + { + scoreType: 'primary', + scoreValue: 300 + Math.floor(Math.random() * 550), + scoreDate: randomDate(2024, 2024), + scoreReasons: scoreReasons.slice(0, 4), + }, + { + scoreType: 'industry', + scoreValue: 300 + Math.floor(Math.random() * 550), + scoreDate: randomDate(2024, 2024), + scoreReasons: scoreReasons.slice(1, 5), + }, + ], + tradelines: tradelines, + inquiries: inquiries, + publicRecords: [], + collections: [], + summary: { + totalAccounts: tradelines.length, + openAccounts: tradelines.filter(t => t.accountStatus === 'open').length, + closedAccounts: tradelines.filter(t => t.accountStatus === 'closed').length, + totalBalance: tradelines.reduce((sum, t) => sum + t.currentBalance, 0), + totalCreditLimit: tradelines.reduce((sum, t) => sum + t.creditLimit, 0), + hardInquiries: inquiries.filter(i => i.inquiryType === 'hard').length, + softInquiries: inquiries.filter(i => i.inquiryType === 'soft').length, + }, + }; +} + +/** + * Generate full payload (~500KB) + * Structure based on raw credit bureau data with: + * - 330 consumer records (addresses, employers, phone numbers) + * - 750 processing records with metadata + */ +export function generateFullPayload(id) { + // Generate 110 address records (tuned for ~500KB) + const addresses = []; + for (let i = 0; i < 110; i++) { + addresses.push({ + addressId: `ADDR-${id}-${i}`, + addressLine1: `${100 + i} Street ${String.fromCharCode(65 + (i % 26))}`, + addressLine2: i % 5 === 0 ? `Apt ${i}` : null, + city: `City${i % 50}`, + state: ['CA', 'TX', 'NY', 'FL', 'IL', 'PA', 'OH', 'GA', 'NC', 'MI'][i % 10], + zipCode: `${10000 + (i * 100)}`, + addressType: ['current', 'previous', 'mailing'][i % 3], + reportedDate: randomDate(2015, 2024), + verifiedDate: i % 2 === 0 ? randomDate(2023, 2024) : null, + sourceCode: `SRC${i % 10}`, + residencyMonths: Math.floor(Math.random() * 120), + }); + } + + // Generate 110 employer records (tuned for ~500KB) + const employers = []; + for (let i = 0; i < 110; i++) { + employers.push({ + employerId: `EMP-${id}-${i}`, + employerName: `Company ${String.fromCharCode(65 + (i % 26))}${Math.floor(i / 26)} Inc`, + occupation: ['engineer', 'manager', 'analyst', 'director', 'specialist'][i % 5], + industry: ['technology', 'finance', 'healthcare', 'retail', 'manufacturing'][i % 5], + employmentStatus: ['employed', 'self_employed', 'unemployed', 'retired'][i % 4], + startDate: randomDate(2010, 2022), + endDate: i % 4 === 2 ? randomDate(2022, 2024) : null, + income: 30000 + Math.floor(Math.random() * 170000), + incomeFrequency: ['annual', 'monthly', 'weekly'][i % 3], + verifiedDate: i % 3 === 0 ? randomDate(2023, 2024) : null, + sourceCode: `SRC${i % 10}`, + }); + } + + // Generate 110 phone number records (tuned for ~500KB) + const phoneNumbers = []; + for (let i = 0; i < 110; i++) { + phoneNumbers.push({ + phoneId: `PHN-${id}-${i}`, + phoneNumber: `${200 + (i % 800)}-${100 + (i % 900)}-${1000 + (i % 9000)}`, + phoneType: ['mobile', 'home', 'work'][i % 3], + isPrimary: i === 0, + reportedDate: randomDate(2018, 2024), + verifiedDate: i % 2 === 0 ? randomDate(2023, 2024) : null, + sourceCode: `SRC${i % 10}`, + }); + } + + // Generate 750 processing records (tuned for ~500KB) + const processingRecords = []; + for (let i = 0; i < 750; i++) { + processingRecords.push({ + recordId: `PROC-${id}-${i}`, + recordType: ['tradeline', 'inquiry', 'public_record', 'collection', 'consumer_statement'][i % 5], + sourceId: `SOURCE-${i % 20}`, + sourceType: ['bureau', 'creditor', 'public', 'consumer'][i % 4], + receivedAt: randomDate(2020, 2024) + 'T' + String(Math.floor(Math.random() * 24)).padStart(2, '0') + ':' + + String(Math.floor(Math.random() * 60)).padStart(2, '0') + ':' + + String(Math.floor(Math.random() * 60)).padStart(2, '0') + 'Z', + processedAt: randomDate(2020, 2024) + 'T' + String(Math.floor(Math.random() * 24)).padStart(2, '0') + ':' + + String(Math.floor(Math.random() * 60)).padStart(2, '0') + ':' + + String(Math.floor(Math.random() * 60)).padStart(2, '0') + 'Z', + status: ['processed', 'pending', 'error', 'archived'][i % 4], + validationScore: Math.floor(Math.random() * 100), + matchConfidence: Math.random(), + metadata: { + version: `v${1 + (i % 5)}.${i % 10}.0`, + checksum: randomAlphanumeric(32), + encoding: 'UTF-8', + compressionType: i % 3 === 0 ? 'gzip' : 'none', + sizeBytes: 100 + Math.floor(Math.random() * 10000), + processingTimeMs: Math.floor(Math.random() * 1000), + retryCount: i % 10 === 0 ? Math.floor(Math.random() * 3) : 0, + priority: ['low', 'medium', 'high', 'critical'][i % 4], + tags: [`tag${i % 10}`, `category${i % 5}`], + }, + rawData: { + originalFormat: ['xml', 'json', 'csv', 'fixed_width'][i % 4], + fieldCount: 10 + (i % 50), + nullFields: i % 10, + warningCount: i % 5, + transformations: [`transform_${i % 8}`], + }, + }); + } + + // Generate dispute records + const disputes = []; + for (let i = 0; i < 20; i++) { + disputes.push({ + disputeId: `DISP-${id}-${i}`, + disputeType: ['accuracy', 'identity', 'fraud', 'duplicate'][i % 4], + disputeStatus: ['open', 'investigating', 'resolved', 'rejected'][i % 4], + filedDate: randomDate(2022, 2024), + resolvedDate: i % 4 === 2 || i % 4 === 3 ? randomDate(2023, 2024) : null, + relatedRecordId: `PROC-${id}-${i * 10}`, + description: `Dispute regarding record ${i}`, + resolution: i % 4 === 2 ? 'corrected' : i % 4 === 3 ? 'verified_accurate' : null, + }); + } + + return { + rawReportId: `RAW-${id}`, + bureauCode: ['experian', 'equifax', 'transunion'][id % 3], + pullDate: new Date().toISOString(), + consumer: { + consumerId: `CON-${id}`, + firstName: `FirstName${id % 1000}`, + middleName: id % 3 === 0 ? `MiddleName${id % 100}` : null, + lastName: `LastName${id % 1000}`, + suffix: id % 20 === 0 ? ['Jr', 'Sr', 'III'][id % 3] : null, + dateOfBirth: randomDate(1950, 2000), + ssnFull: `${String(100 + (id % 900)).padStart(3, '0')}-${String(10 + (id % 90)).padStart(2, '0')}-${String(id % 10000).padStart(4, '0')}`, + addresses: addresses, + employers: employers, + phoneNumbers: phoneNumbers, + }, + processingRecords: processingRecords, + disputes: disputes, + metadata: { + version: '2.0.0', + schemaVersion: '1.5.0', + generatedBy: 'benchmark-generator', + processingNode: `node-${id % 10}`, + totalRecords: processingRecords.length, + totalConsumerRecords: addresses.length + employers.length + phoneNumbers.length, + checksums: { + consumer: randomAlphanumeric(64), + processing: randomAlphanumeric(64), + disputes: randomAlphanumeric(64), + }, + }, + }; +} + +export function generateLargeJsonb(id) { + // Credit report structure: 50 tradelines x 24 months = ~500KB + const tradelines = []; + for (let i = 0; i < 50; i++) { + const history = []; + for (let m = 0; m < 24; m++) { + history.push({ + month: m + 1, + balance: Math.floor(Math.random() * 10000), + status: 'current', + payment: Math.floor(Math.random() * 500), + }); + } + tradelines.push({ + creditor: `Creditor ${i}`, + account_number: `ACCT${id}${i}`.padStart(16, '0'), + account_type: 'revolving', + opened_date: '2020-01-15', + credit_limit: 5000 + (i * 100), + current_balance: Math.floor(Math.random() * 5000), + payment_history: history, + }); + } + + return { + id: id, + report_id: `RPT-${id}`, + subject: { + name: 'Test Subject', + ssn_last4: '1234', + dob: '1990-01-01', + }, + tradelines: tradelines, + inquiries: [ + { date: '2024-01-01', creditor: 'Bank A' }, + { date: '2024-02-15', creditor: 'Bank B' }, + ], + public_records: [], + score: 750, + }; +} diff --git a/tests/benchmark/k6/scripts/lib/summary.js b/tests/benchmark/k6/scripts/lib/summary.js new file mode 100644 index 00000000..e1313832 --- /dev/null +++ b/tests/benchmark/k6/scripts/lib/summary.js @@ -0,0 +1,75 @@ +// benchmark-action compatible output formatter +// Outputs separate JSON files for throughput (bigger is better) and latency (smaller is better) +// +// Usage in scripts: +// import { createSummaryHandler } from './lib/summary.js'; +// export const handleSummary = createSummaryHandler('script-name'); + +export function createSummaryHandler(scriptName) { + return function(data) { + const iterationsPerSecond = data.metrics.iterations + ? data.metrics.iterations.values.rate + : 0; + + const p95Duration = data.metrics.iteration_duration + ? data.metrics.iteration_duration.values['p(95)'] + : 0; + + const p99Duration = data.metrics.iteration_duration + ? data.metrics.iteration_duration.values['p(99)'] + : 0; + + // Throughput metrics (customBiggerIsBetter) + const throughputOutput = [ + { + name: `${scriptName}_rate`, + unit: 'iter/s', + value: Math.round(iterationsPerSecond * 100) / 100, + }, + ]; + + // Latency metrics (customSmallerIsBetter) + const latencyOutput = [ + { + name: `${scriptName}_p95`, + unit: 'ms', + value: Math.round(p95Duration * 100) / 100, + }, + { + name: `${scriptName}_p99`, + unit: 'ms', + value: Math.round(p99Duration * 100) / 100, + }, + ]; + + return { + 'stdout': textSummary(data), + [`results/k6/${scriptName}-throughput.json`]: JSON.stringify(throughputOutput, null, 2), + [`results/k6/${scriptName}-latency.json`]: JSON.stringify(latencyOutput, null, 2), + }; + }; +} + +// Minimal text summary (k6 doesn't export textSummary by default in extensions) +function textSummary(data) { + const lines = []; + lines.push(''); + lines.push('=== Summary ==='); + + if (data.metrics.iterations) { + lines.push(`iterations: ${data.metrics.iterations.values.count}`); + lines.push(`rate: ${data.metrics.iterations.values.rate.toFixed(2)}/s`); + } + + if (data.metrics.iteration_duration) { + const dur = data.metrics.iteration_duration.values; + lines.push(`duration min: ${dur.min.toFixed(2)}ms`); + lines.push(`duration avg: ${dur.avg.toFixed(2)}ms`); + lines.push(`duration p95: ${dur['p(95)'].toFixed(2)}ms`); + lines.push(`duration p99: ${dur['p(99)'].toFixed(2)}ms`); + lines.push(`duration max: ${dur.max.toFixed(2)}ms`); + } + + lines.push(''); + return lines.join('\n'); +} diff --git a/tests/benchmark/k6/scripts/text-equality.js b/tests/benchmark/k6/scripts/text-equality.js new file mode 100644 index 00000000..afb766b6 --- /dev/null +++ b/tests/benchmark/k6/scripts/text-equality.js @@ -0,0 +1,56 @@ +// Text equality benchmark - baseline matching pgbench encrypted transaction +// +// Uses xk6-sql API: +// sql.open(driver, connString) +// db.exec(sql, ...args) + +import sql from 'k6/x/sql'; +import driver from 'k6/x/sql/driver/postgres'; +import { getConnectionString, getDefaultOptions } from './lib/config.js'; +import { createSummaryHandler } from './lib/summary.js'; + +const target = __ENV.K6_TARGET || 'proxy'; +const connectionString = getConnectionString(target); + +// ID range for text-equality benchmark data (isolated from other tests) +const ID_START = 1000000; +const ID_COUNT = 100; + +export const options = getDefaultOptions({ + 'iteration_duration': ['p(95)<100'], +}); + +const db = sql.open(driver, connectionString); + +export function setup() { + // Clean up any leftover data from crashed runs before inserting + db.exec(`DELETE FROM benchmark_encrypted WHERE id BETWEEN $1 AND $2`, ID_START, ID_START + ID_COUNT - 1); + + // Insert seed data for queries + for (let i = 0; i < ID_COUNT; i++) { + const id = ID_START + i; + const email = `user${i}@example.com`; + db.exec( + `INSERT INTO benchmark_encrypted (id, username, email) VALUES ($1, $2, $3)`, + id, + `user${i}`, + email + ); + } +} + +export default function() { + const i = Math.floor(Math.random() * ID_COUNT); + const email = `user${i}@example.com`; + + // Use exec instead of query - we only need to verify the query runs, not inspect results + db.exec(`SELECT username FROM benchmark_encrypted WHERE email = $1`, email); +} + +export function teardown() { + // Clean up seed data + db.exec(`DELETE FROM benchmark_encrypted WHERE id BETWEEN $1 AND $2`, ID_START, ID_START + ID_COUNT - 1); + db.close(); +} + +export const handleSummary = createSummaryHandler('text-equality'); diff --git a/tests/benchmark/mise.toml b/tests/benchmark/mise.toml index c3d8b073..c0bcc66e 100644 --- a/tests/benchmark/mise.toml +++ b/tests/benchmark/mise.toml @@ -172,3 +172,97 @@ run = """ set -e echo docker compose up --build {{arg(name="service",default="pgcat")}} {{option(name="extra-args",default="")}} | bash """ + +# ==================================================================================================== +# k6 Benchmarks +# ==================================================================================================== + +[tasks."k6:build"] +description = "Build k6 Docker image with xk6-pgxpool" +run = """ +docker build -t k6-pgxpool {{config_root}}/k6/ +""" + +# k6:run task is defined in tasks/k6_run.sh + +[tasks."k6:benchmark"] +description = "Run full k6 benchmark suite" +run = """ +set -e + +echo +echo '###############################################' +echo '# k6 Benchmark Suite' +echo '###############################################' +echo + +mise run k6:build + +# Ensure benchmark schema exists (creates benchmark_encrypted table for text-equality) +mise run benchmark:setup + +# JSONB STE-VEC INSERT (primary) +echo "Running jsonb-ste-vec-insert..." +mise run k6_run --script=jsonb-ste-vec-insert --target=proxy --duration=30s + +# Text equality (baseline) +echo "Running text-equality..." +mise run k6_run --script=text-equality --target=proxy --duration=30s + +# JSONB large payload +echo "Running jsonb-large-payload..." +mise run k6_run --script=jsonb-large-payload --target=proxy --duration=30s + +echo +echo "Results written to results/k6/" +""" + +[tasks."k6:benchmark:continuous"] +description = "Run k6 benchmarks for CI (jsonb-large-payload)" +run = """ +set -e + +echo +echo '###############################################' +echo '# Preflight' +echo '###############################################' +echo + +mise run benchmark:clean + +# Ensure Postgres instances are running +mise run test:integration:preflight + +echo +echo '###############################################' +echo '# Setup' +echo '###############################################' +echo + +# Ensure EQL is set up before we try and start Proxy +mise --env tcp run postgres:setup + +mise run benchmark:setup + +mise --env tcp run proxy:up proxy --extra-args "--detach --wait" +mise --env tcp run test:wait_for_postgres_to_quack --port 6432 --max-retries 20 + +echo +echo '###############################################' +echo '# k6 Benchmarks' +echo '###############################################' +echo + +mise run k6:build + +echo +echo '# jsonb-large-payload via proxy' +echo + +mise run k6_run --script=jsonb-large-payload --target=proxy --vus=10 --duration=60s + +# Copy outputs for benchmark-action (separate files for throughput vs latency) +mkdir -p results +cp results/k6/jsonb-large-payload-throughput.json results/k6-throughput.json +cp results/k6/jsonb-large-payload-latency.json results/k6-latency.json +""" diff --git a/tests/benchmark/sql/benchmark-schema.sql b/tests/benchmark/sql/benchmark-schema.sql index c2024fe0..88a9972c 100644 --- a/tests/benchmark/sql/benchmark-schema.sql +++ b/tests/benchmark/sql/benchmark-schema.sql @@ -11,7 +11,10 @@ DROP TABLE IF EXISTS benchmark_encrypted; CREATE TABLE benchmark_encrypted ( id serial primary key, username text, - email eql_v2_encrypted + email eql_v2_encrypted, + encrypted_jsonb_extract eql_v2_encrypted, -- ~250KB: structured report data + encrypted_jsonb_full eql_v2_encrypted, -- ~500KB: raw report data + encrypted_jsonb_with_ste_vec eql_v2_encrypted -- STE-VEC benchmarks ); SELECT eql_v2.add_column( @@ -19,3 +22,23 @@ SELECT eql_v2.add_column( 'email' ); +SELECT eql_v2.add_column( + 'benchmark_encrypted', + 'encrypted_jsonb_extract', + 'jsonb' +); + +SELECT eql_v2.add_column( + 'benchmark_encrypted', + 'encrypted_jsonb_full', + 'jsonb' +); + +SELECT eql_v2.add_search_config( + 'benchmark_encrypted', + 'encrypted_jsonb_with_ste_vec', + 'ste_vec', + 'jsonb', + '{"prefix": "benchmark_encrypted/encrypted_jsonb_with_ste_vec"}' +); + diff --git a/tests/benchmark/tasks/k6_run.sh b/tests/benchmark/tasks/k6_run.sh new file mode 100755 index 00000000..4ae88854 --- /dev/null +++ b/tests/benchmark/tasks/k6_run.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +#MISE description="Run a single k6 benchmark script" +#USAGE flag "--script