From 5020bb8e459c9643e00b9325c5ef55d646919576 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 19 Jan 2026 17:27:28 +0100 Subject: [PATCH 1/9] feat: jsonata query filter --- core/lib/server/jsonata-query/compiler.ts | 20 + core/lib/server/jsonata-query/errors.ts | 31 + core/lib/server/jsonata-query/index.ts | 12 + .../jsonata-query/jsonata-query.db.test.ts | 655 ++++++++++++++++++ .../lib/server/jsonata-query/memory-filter.ts | 218 ++++++ core/lib/server/jsonata-query/parser.ts | 384 ++++++++++ core/lib/server/jsonata-query/sql-builder.ts | 304 ++++++++ core/lib/server/jsonata-query/types.ts | 131 ++++ core/package.json | 6 +- pnpm-lock.yaml | 100 +-- 10 files changed, 1812 insertions(+), 49 deletions(-) create mode 100644 core/lib/server/jsonata-query/compiler.ts create mode 100644 core/lib/server/jsonata-query/errors.ts create mode 100644 core/lib/server/jsonata-query/index.ts create mode 100644 core/lib/server/jsonata-query/jsonata-query.db.test.ts create mode 100644 core/lib/server/jsonata-query/memory-filter.ts create mode 100644 core/lib/server/jsonata-query/parser.ts create mode 100644 core/lib/server/jsonata-query/sql-builder.ts create mode 100644 core/lib/server/jsonata-query/types.ts diff --git a/core/lib/server/jsonata-query/compiler.ts b/core/lib/server/jsonata-query/compiler.ts new file mode 100644 index 000000000..e3848c705 --- /dev/null +++ b/core/lib/server/jsonata-query/compiler.ts @@ -0,0 +1,20 @@ +import type { ParsedCondition, ParsedQuery } from "./parser" + +import { parseJsonataQuery } from "./parser" + +export interface CompiledQuery { + condition: ParsedCondition + originalExpression: string +} + +/** + * compiles a jsonata expression into a query that can be used for + * both sql generation and in-memory filtering + */ +export function compileJsonataQuery(expression: string): CompiledQuery { + const parsed = parseJsonataQuery(expression) + return { + condition: parsed.condition, + originalExpression: expression, + } +} diff --git a/core/lib/server/jsonata-query/errors.ts b/core/lib/server/jsonata-query/errors.ts new file mode 100644 index 000000000..40dbe5764 --- /dev/null +++ b/core/lib/server/jsonata-query/errors.ts @@ -0,0 +1,31 @@ +export class JsonataQueryError extends Error { + constructor( + message: string, + public readonly expression?: string + ) { + super(message) + this.name = "JsonataQueryError" + } +} + +export class UnsupportedExpressionError extends JsonataQueryError { + constructor( + message: string, + public readonly nodeType?: string, + expression?: string + ) { + super(message, expression) + this.name = "UnsupportedExpressionError" + } +} + +export class InvalidPathError extends JsonataQueryError { + constructor( + message: string, + public readonly path?: string, + expression?: string + ) { + super(message, expression) + this.name = "InvalidPathError" + } +} diff --git a/core/lib/server/jsonata-query/index.ts b/core/lib/server/jsonata-query/index.ts new file mode 100644 index 000000000..9974e627d --- /dev/null +++ b/core/lib/server/jsonata-query/index.ts @@ -0,0 +1,12 @@ +export { compileJsonataQuery, type CompiledQuery } from "./compiler" +export { applyJsonataFilter, type SqlBuilderOptions } from "./sql-builder" +export { filterPubsWithJsonata, pubMatchesJsonataQuery } from "./memory-filter" +export { parseJsonataQuery, type ParsedQuery, type ParsedCondition } from "./parser" +export { JsonataQueryError, UnsupportedExpressionError, InvalidPathError } from "./errors" +export type { + PubFieldPath, + ComparisonCondition, + FunctionCondition, + LogicalCondition, + NotCondition, +} from "./types" diff --git a/core/lib/server/jsonata-query/jsonata-query.db.test.ts b/core/lib/server/jsonata-query/jsonata-query.db.test.ts new file mode 100644 index 000000000..944051256 --- /dev/null +++ b/core/lib/server/jsonata-query/jsonata-query.db.test.ts @@ -0,0 +1,655 @@ +import type { ProcessedPub } from "contracts" +import type { CommunitiesId, PubsId } from "db/public" + +import { describe, expect, it } from "vitest" + +import { CoreSchemaType } from "db/public" + +import { createSeed } from "~/prisma/seed/createSeed" +import { mockServerCode } from "../../__tests__/utils" +import { + compileJsonataQuery, + filterPubsWithJsonata, + parseJsonataQuery, + pubMatchesJsonataQuery, +} from "./index" +import { applyJsonataFilter } from "./sql-builder" + +const { createForEachMockedTransaction, testDb } = await mockServerCode() +const { getTrx } = createForEachMockedTransaction() + +const communitySlug = `${new Date().toISOString()}:jsonata-query-test` + +// test pub ids +const titlePubId = crypto.randomUUID() as PubsId +const title2PubId = crypto.randomUUID() as PubsId +const numberPubId = crypto.randomUUID() as PubsId +const number42PubId = crypto.randomUUID() as PubsId +const booleanPubId = crypto.randomUUID() as PubsId +const arrayPubId = crypto.randomUUID() as PubsId +const complexPubId = crypto.randomUUID() as PubsId + +const seed = createSeed({ + community: { + name: "jsonata query test", + slug: communitySlug, + }, + pubFields: { + Title: { schemaName: CoreSchemaType.String }, + Number: { schemaName: CoreSchemaType.Number }, + Boolean: { schemaName: CoreSchemaType.Boolean }, + Date: { schemaName: CoreSchemaType.DateTime }, + Array: { schemaName: CoreSchemaType.StringArray }, + }, + pubTypes: { + Article: { + Title: { isTitle: true }, + Number: { isTitle: false }, + Boolean: { isTitle: false }, + Date: { isTitle: false }, + Array: { isTitle: false }, + }, + Review: { + Title: { isTitle: true }, + Number: { isTitle: false }, + }, + }, + stages: {}, + pubs: [ + { + id: titlePubId, + pubType: "Article", + values: { + Title: "Test Article", + Number: 100, + Boolean: true, + }, + }, + { + id: title2PubId, + pubType: "Article", + values: { + Title: "Another Article", + Number: 50, + Boolean: false, + }, + }, + { + id: numberPubId, + pubType: "Review", + values: { + Title: "Some Review", + Number: 25, + }, + }, + { + id: number42PubId, + pubType: "Article", + values: { + Title: "The Answer", + Number: 42, + Boolean: true, + }, + }, + { + id: booleanPubId, + pubType: "Article", + values: { + Title: "Boolean Test", + Boolean: false, + }, + }, + { + id: arrayPubId, + pubType: "Article", + values: { + Title: "Array Test", + Array: ["geology", "biology", "chemistry"], + }, + }, + { + id: complexPubId, + pubType: "Article", + values: { + Title: "Important Document", + Number: 75, + Boolean: true, + Array: ["important", "featured"], + }, + }, + ], +}) + +const seedCommunity = async (trx = testDb) => { + const { seedCommunity } = await import("~/prisma/seed/seedCommunity") + return seedCommunity(seed, { randomSlug: false }, trx) +} + +const community = await seedCommunity() + +const slug = (field: string) => `${communitySlug}:${field.toLowerCase()}` + +// helper to run queries +const runQuery = async (expression: string, trx = getTrx()) => { + const query = compileJsonataQuery(expression) + const results = await trx + .selectFrom("pubs") + .selectAll() + .where("communityId", "=", community.community.id) + .where((eb) => applyJsonataFilter(eb, query, { communitySlug })) + .execute() + return results.map((r) => r.id) +} + +describe("parser", () => { + it("parses simple equality", () => { + const result = parseJsonataQuery('$.pub.values.title = "Test"') + expect(result.condition).toEqual({ + type: "comparison", + path: { kind: "value", fieldSlug: "title" }, + operator: "=", + value: "Test", + pathTransform: undefined, + }) + }) + + it("parses numeric comparison", () => { + const result = parseJsonataQuery("$.pub.values.number > 40") + expect(result.condition).toEqual({ + type: "comparison", + path: { kind: "value", fieldSlug: "number" }, + operator: ">", + value: 40, + pathTransform: undefined, + }) + }) + + it("parses logical and", () => { + const result = parseJsonataQuery('$.pub.values.title = "Test" and $.pub.values.number > 10') + expect(result.condition.type).toBe("logical") + if (result.condition.type === "logical") { + expect(result.condition.operator).toBe("and") + expect(result.condition.conditions).toHaveLength(2) + } + }) + + it("parses logical or", () => { + const result = parseJsonataQuery("$.pub.values.number < 30 or $.pub.values.number > 70") + expect(result.condition.type).toBe("logical") + if (result.condition.type === "logical") { + expect(result.condition.operator).toBe("or") + } + }) + + it("parses not function", () => { + const result = parseJsonataQuery('not($.pub.values.title = "Test")') + expect(result.condition.type).toBe("not") + }) + + it("parses contains function", () => { + const result = parseJsonataQuery('$contains($.pub.values.title, "important")') + expect(result.condition).toEqual({ + type: "function", + name: "contains", + path: { kind: "value", fieldSlug: "title" }, + arguments: ["important"], + }) + }) + + it("parses in operator with array", () => { + const result = parseJsonataQuery("$.pub.values.number in [10, 20, 30]") + expect(result.condition).toEqual({ + type: "comparison", + path: { kind: "value", fieldSlug: "number" }, + operator: "in", + value: [10, 20, 30], + pathTransform: undefined, + }) + }) + + it("parses builtin field id", () => { + const result = parseJsonataQuery('$.pub.id = "some-id"') + expect(result.condition).toEqual({ + type: "comparison", + path: { kind: "builtin", field: "id" }, + operator: "=", + value: "some-id", + pathTransform: undefined, + }) + }) + + it("parses builtin field createdAt", () => { + const result = parseJsonataQuery("$.pub.createdAt > 1704067200000") + expect(result.condition).toEqual({ + type: "comparison", + path: { kind: "builtin", field: "createdAt" }, + operator: ">", + value: 1704067200000, + pathTransform: undefined, + }) + }) + + it('parses "value in array" as contains', () => { + const result = parseJsonataQuery('"geology" in $.pub.values.keywords') + expect(result.condition).toEqual({ + type: "function", + name: "contains", + path: { kind: "value", fieldSlug: "keywords" }, + arguments: ["geology"], + }) + }) + + it("parses lowercase transform", () => { + const result = parseJsonataQuery('$lowercase($.pub.values.title) = "test"') + expect(result.condition).toEqual({ + type: "comparison", + path: { kind: "value", fieldSlug: "title" }, + operator: "=", + value: "test", + pathTransform: "lowercase", + }) + }) + + it("parses exists function", () => { + const result = parseJsonataQuery("$exists($.pub.values.title)") + expect(result.condition).toEqual({ + type: "function", + name: "exists", + path: { kind: "value", fieldSlug: "title" }, + arguments: [], + }) + }) + + it("parses complex nested expression", () => { + const result = parseJsonataQuery( + '($.pub.values.title = "Test" or $contains($.pub.values.title, "important")) and $.pub.values.number > 10' + ) + expect(result.condition.type).toBe("logical") + if (result.condition.type === "logical") { + expect(result.condition.operator).toBe("and") + expect(result.condition.conditions[0].type).toBe("logical") + } + }) +}) + +describe("sql generation", () => { + it("generates sql for simple equality", async () => { + const query = compileJsonataQuery(`$.pub.values.title = "Test Article"`) + const trx = getTrx() + const q = trx + .selectFrom("pubs") + .selectAll() + .where("communityId", "=", community.community.id) + .where((eb) => applyJsonataFilter(eb, query)) + + const compiled = q.compile() + expect(compiled.sql).toContain("pub_values") + expect(compiled.sql).toContain("pub_fields") + }) + + it("generates sql for numeric comparison", async () => { + const query = compileJsonataQuery("$.pub.values.number > 40") + const trx = getTrx() + const q = trx + .selectFrom("pubs") + .selectAll() + .where("communityId", "=", community.community.id) + .where((eb) => applyJsonataFilter(eb, query)) + + const compiled = q.compile() + expect(compiled.sql).toContain(">") + }) + + it("generates sql for logical and", async () => { + const query = compileJsonataQuery( + `$.pub.values.title = "Test" and $.pub.values.number > 10` + ) + const trx = getTrx() + const q = trx + .selectFrom("pubs") + .selectAll() + .where("communityId", "=", community.community.id) + .where((eb) => applyJsonataFilter(eb, query)) + + const compiled = q.compile() + expect(compiled.sql).toContain("and") + }) +}) + +describe("database queries", () => { + it("filters by simple string equality", async () => { + const ids = await runQuery(`$.pub.values.title = "Test Article"`) + expect(ids).toContain(titlePubId) + expect(ids).toHaveLength(1) + }) + + it("filters by numeric greater than", async () => { + const ids = await runQuery("$.pub.values.number > 50") + expect(ids).toContain(titlePubId) // 100 + expect(ids).toContain(complexPubId) // 75 + expect(ids).not.toContain(title2PubId) // 50 + }) + + it("filters by numeric less than or equal", async () => { + const ids = await runQuery("$.pub.values.number <= 42") + expect(ids).toContain(numberPubId) // 25 + expect(ids).toContain(number42PubId) // 42 + expect(ids).not.toContain(titlePubId) // 100 + }) + + it("filters with logical and", async () => { + const ids = await runQuery("$.pub.values.number > 40 and $.pub.values.boolean = true") + expect(ids).toContain(titlePubId) // 100, true + expect(ids).toContain(complexPubId) // 75, true + expect(ids).not.toContain(title2PubId) // 50, false + }) + + it("filters with logical or", async () => { + const ids = await runQuery("$.pub.values.number < 30 or $.pub.values.number > 90") + expect(ids).toContain(numberPubId) // 25 + expect(ids).toContain(titlePubId) // 100 + expect(ids).not.toContain(title2PubId) // 50 + }) + + it("filters with not", async () => { + const ids = await runQuery('not($.pub.values.title = "Test Article")') + expect(ids).not.toContain(titlePubId) + expect(ids.length).toBeGreaterThan(0) + }) + + it("filters with contains function", async () => { + const ids = await runQuery('$contains($.pub.values.title, "Article")') + expect(ids).toContain(titlePubId) + expect(ids).toContain(title2PubId) + expect(ids).not.toContain(numberPubId) // "Some Review" + }) + + it("filters with startsWith function", async () => { + const ids = await runQuery('$startsWith($.pub.values.title, "Test")') + expect(ids).toContain(titlePubId) + expect(ids).not.toContain(title2PubId) + }) + + it("filters with endsWith function", async () => { + const ids = await runQuery('$endsWith($.pub.values.title, "Review")') + expect(ids).toContain(numberPubId) + expect(ids).not.toContain(titlePubId) + }) + + it("filters with in operator", async () => { + const ids = await runQuery("$.pub.values.number in [25, 42, 100]") + expect(ids).toContain(numberPubId) // 25 + expect(ids).toContain(number42PubId) // 42 + expect(ids).toContain(titlePubId) // 100 + expect(ids).not.toContain(title2PubId) // 50 + }) + + it("filters with exists", async () => { + const ids = await runQuery("$exists($.pub.values.array)") + expect(ids).toContain(arrayPubId) + expect(ids).toContain(complexPubId) + expect(ids).not.toContain(titlePubId) + }) + + it("filters with complex nested expression", async () => { + const ids = await runQuery( + '($contains($.pub.values.title, "Article") or $contains($.pub.values.title, "Document")) and $.pub.values.number >= 50' + ) + expect(ids).toContain(titlePubId) // "Test Article", 100 + expect(ids).toContain(title2PubId) // "Another Article", 50 + expect(ids).toContain(complexPubId) // "Important Document", 75 + expect(ids).not.toContain(numberPubId) // "Some Review" + }) + + it("filters by builtin id field", async () => { + const ids = await runQuery(`$.pub.id = "${titlePubId}"`) + expect(ids).toEqual([titlePubId]) + }) + + it("filters with inequality", async () => { + const ids = await runQuery("$.pub.values.number != 42") + expect(ids).not.toContain(number42PubId) + expect(ids).toContain(titlePubId) + }) +}) + +describe("memory filtering", () => { + // create mock pubs for memory filtering + const mockPubs: ProcessedPub<{ withValues: true }>[] = [ + { + id: "pub-1" as PubsId, + communityId: "comm-1" as CommunitiesId, + pubTypeId: "type-1", + parentId: null, + createdAt: new Date("2024-01-01"), + updatedAt: new Date("2024-01-01"), + values: [ + { fieldSlug: "test:title", fieldId: "f1", value: "Test Article", relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, + { fieldSlug: "test:number", fieldId: "f2", value: 100, relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, + { fieldSlug: "test:boolean", fieldId: "f3", value: true, relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, + ], + }, + { + id: "pub-2" as PubsId, + communityId: "comm-1" as CommunitiesId, + pubTypeId: "type-1", + parentId: null, + createdAt: new Date("2024-02-01"), + updatedAt: new Date("2024-02-01"), + values: [ + { fieldSlug: "test:title", fieldId: "f1", value: "Another Article", relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, + { fieldSlug: "test:number", fieldId: "f2", value: 50, relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, + { fieldSlug: "test:boolean", fieldId: "f3", value: false, relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, + ], + }, + { + id: "pub-3" as PubsId, + communityId: "comm-1" as CommunitiesId, + pubTypeId: "type-2", + parentId: null, + createdAt: new Date("2024-03-01"), + updatedAt: new Date("2024-03-01"), + values: [ + { fieldSlug: "test:title", fieldId: "f1", value: "Important Document", relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, + { fieldSlug: "test:number", fieldId: "f2", value: 75, relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, + { fieldSlug: "test:array", fieldId: "f4", value: ["geology", "biology"], relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, + ], + }, + ] as any + + it("filters by string equality", () => { + const query = compileJsonataQuery('$.pub.values.title = "Test Article"') + const result = filterPubsWithJsonata(mockPubs, query) + expect(result).toHaveLength(1) + expect(result[0].id).toBe("pub-1") + }) + + it("filters by numeric comparison", () => { + const query = compileJsonataQuery("$.pub.values.number > 60") + const result = filterPubsWithJsonata(mockPubs, query) + expect(result).toHaveLength(2) + expect(result.map((p) => p.id)).toContain("pub-1") + expect(result.map((p) => p.id)).toContain("pub-3") + }) + + it("filters with logical and", () => { + const query = compileJsonataQuery( + "$.pub.values.number > 40 and $.pub.values.boolean = true" + ) + const result = filterPubsWithJsonata(mockPubs, query) + expect(result).toHaveLength(1) + expect(result[0].id).toBe("pub-1") + }) + + it("filters with logical or", () => { + const query = compileJsonataQuery( + "$.pub.values.number < 60 or $.pub.values.number > 90" + ) + const result = filterPubsWithJsonata(mockPubs, query) + expect(result).toHaveLength(2) + }) + + it("filters with not", () => { + const query = compileJsonataQuery('not($.pub.values.title = "Test Article")') + const result = filterPubsWithJsonata(mockPubs, query) + expect(result).toHaveLength(2) + expect(result.map((p) => p.id)).not.toContain("pub-1") + }) + + it("filters with contains on string", () => { + const query = compileJsonataQuery('$contains($.pub.values.title, "Article")') + const result = filterPubsWithJsonata(mockPubs, query) + expect(result).toHaveLength(2) + }) + + it("filters with contains on array", () => { + const query = compileJsonataQuery('"geology" in $.pub.values.array') + const result = filterPubsWithJsonata(mockPubs, query) + expect(result).toHaveLength(1) + expect(result[0].id).toBe("pub-3") + }) + + it("filters with startsWith", () => { + const query = compileJsonataQuery('$startsWith($.pub.values.title, "Test")') + const result = filterPubsWithJsonata(mockPubs, query) + expect(result).toHaveLength(1) + expect(result[0].id).toBe("pub-1") + }) + + it("filters with endsWith", () => { + const query = compileJsonataQuery('$endsWith($.pub.values.title, "Document")') + const result = filterPubsWithJsonata(mockPubs, query) + expect(result).toHaveLength(1) + expect(result[0].id).toBe("pub-3") + }) + + it("filters with exists", () => { + const query = compileJsonataQuery("$exists($.pub.values.array)") + const result = filterPubsWithJsonata(mockPubs, query) + expect(result).toHaveLength(1) + expect(result[0].id).toBe("pub-3") + }) + + it("filters by builtin id", () => { + const query = compileJsonataQuery('$.pub.id = "pub-2"') + const result = filterPubsWithJsonata(mockPubs, query) + expect(result).toHaveLength(1) + expect(result[0].id).toBe("pub-2") + }) + + it("filters with in operator", () => { + const query = compileJsonataQuery("$.pub.values.number in [50, 100]") + const result = filterPubsWithJsonata(mockPubs, query) + expect(result).toHaveLength(2) + }) + + it("single pub match check", () => { + const query = compileJsonataQuery("$.pub.values.number > 90") + expect(pubMatchesJsonataQuery(mockPubs[0], query)).toBe(true) + expect(pubMatchesJsonataQuery(mockPubs[1], query)).toBe(false) + }) +}) + +describe("comparison with original filter format", () => { + // these tests verify that jsonata queries produce equivalent results to the original filter syntax + + it("jsonata equality equals original filter equality", async () => { + const { getPubsWithRelatedValues } = await import("~/lib/server/pub") + const trx = getTrx() + + // original filter + const originalResult = await getPubsWithRelatedValues( + { communityId: community.community.id }, + { + trx, + filters: { + [slug("title")]: { $eq: "Test Article" }, + }, + } + ) + + // jsonata query + const jsonataIds = await runQuery(`$.pub.values.title = "Test Article"`) + + expect(jsonataIds).toEqual(originalResult.map((p) => p.id)) + }) + + it("jsonata numeric gt equals original filter gt", async () => { + const { getPubsWithRelatedValues } = await import("~/lib/server/pub") + const trx = getTrx() + + const originalResult = await getPubsWithRelatedValues( + { communityId: community.community.id }, + { + trx, + filters: { + [slug("number")]: { $gt: 50 }, + }, + } + ) + + const jsonataIds = await runQuery("$.pub.values.number > 50") + + expect(new Set(jsonataIds)).toEqual(new Set(originalResult.map((p) => p.id))) + }) + + it("jsonata and equals original filter and", async () => { + const { getPubsWithRelatedValues } = await import("~/lib/server/pub") + const trx = getTrx() + + const originalResult = await getPubsWithRelatedValues( + { communityId: community.community.id }, + { + trx, + filters: { + $and: [ + { [slug("number")]: { $gt: 40 } }, + { [slug("boolean")]: { $eq: true } }, + ], + }, + } + ) + + const jsonataIds = await runQuery("$.pub.values.number > 40 and $.pub.values.boolean = true") + + expect(new Set(jsonataIds)).toEqual(new Set(originalResult.map((p) => p.id))) + }) + + it("jsonata contains equals original filter contains", async () => { + const { getPubsWithRelatedValues } = await import("~/lib/server/pub") + const trx = getTrx() + + const originalResult = await getPubsWithRelatedValues( + { communityId: community.community.id }, + { + trx, + filters: { + [slug("title")]: { $contains: "Article" }, + }, + } + ) + + const jsonataIds = await runQuery('$contains($.pub.values.title, "Article")') + + expect(new Set(jsonataIds)).toEqual(new Set(originalResult.map((p) => p.id))) + }) + + it("jsonata in equals original filter in", async () => { + const { getPubsWithRelatedValues } = await import("~/lib/server/pub") + const trx = getTrx() + + const originalResult = await getPubsWithRelatedValues( + { communityId: community.community.id }, + { + trx, + filters: { + [slug("number")]: { $in: [25, 42, 100] }, + }, + } + ) + + const jsonataIds = await runQuery("$.pub.values.number in [25, 42, 100]") + + expect(new Set(jsonataIds)).toEqual(new Set(originalResult.map((p) => p.id))) + }) +}) diff --git a/core/lib/server/jsonata-query/memory-filter.ts b/core/lib/server/jsonata-query/memory-filter.ts new file mode 100644 index 000000000..cfa8ce505 --- /dev/null +++ b/core/lib/server/jsonata-query/memory-filter.ts @@ -0,0 +1,218 @@ +import type { ProcessedPub } from "contracts" +import type { CompiledQuery } from "./compiler" +import type { + ComparisonCondition, + FunctionCondition, + LogicalCondition, + NotCondition, + ParsedCondition, + PubFieldPath, +} from "./types" + +import { UnsupportedExpressionError } from "./errors" + +type AnyProcessedPub = ProcessedPub + +/** + * extracts a value from a pub based on a field path + */ +function getValueFromPath(pub: AnyProcessedPub, path: PubFieldPath): unknown { + if (path.kind === "builtin") { + switch (path.field) { + case "id": + return pub.id + case "createdAt": + return pub.createdAt + case "updatedAt": + return pub.updatedAt + case "pubTypeId": + return pub.pubTypeId + } + } + + if (path.kind === "pubType") { + const pubType = (pub as any).pubType + if (!pubType) { + return undefined + } + return pubType[path.field] + } + + // value field - find in values array + const value = pub.values.find((v) => { + // handle both full slug and short slug + const fieldSlug = v.fieldSlug + return fieldSlug === path.fieldSlug || fieldSlug.endsWith(`:${path.fieldSlug}`) + }) + + return value?.value +} + +/** + * applies a path transform to a value + */ +function applyTransform(value: unknown, transform?: string): unknown { + if (transform === "lowercase" && typeof value === "string") { + return value.toLowerCase() + } + if (transform === "uppercase" && typeof value === "string") { + return value.toUpperCase() + } + return value +} + +/** + * compares two values using the given operator + */ +function compareValues(left: unknown, operator: string, right: unknown): boolean { + // handle null comparisons + if (right === null) { + if (operator === "=") { + return left === null || left === undefined + } + if (operator === "!=") { + return left !== null && left !== undefined + } + } + + // handle "in" operator + if (operator === "in" && Array.isArray(right)) { + return right.includes(left) + } + + // convert dates for comparison + const normalizeValue = (v: unknown): unknown => { + if (v instanceof Date) { + return v.getTime() + } + if (typeof v === "string") { + const parsed = Date.parse(v) + if (!isNaN(parsed) && v.includes("-")) { + return parsed + } + } + return v + } + + const normalizedLeft = normalizeValue(left) + const normalizedRight = normalizeValue(right) + + switch (operator) { + case "=": + return normalizedLeft === normalizedRight + case "!=": + return normalizedLeft !== normalizedRight + case "<": + return (normalizedLeft as number) < (normalizedRight as number) + case "<=": + return (normalizedLeft as number) <= (normalizedRight as number) + case ">": + return (normalizedLeft as number) > (normalizedRight as number) + case ">=": + return (normalizedLeft as number) >= (normalizedRight as number) + default: + throw new UnsupportedExpressionError(`unsupported operator: ${operator}`) + } +} + +/** + * evaluates a comparison condition against a pub + */ +function evaluateComparison(pub: AnyProcessedPub, condition: ComparisonCondition): boolean { + let value = getValueFromPath(pub, condition.path) + value = applyTransform(value, condition.pathTransform) + return compareValues(value, condition.operator, condition.value) +} + +/** + * evaluates a function condition against a pub + */ +function evaluateFunction(pub: AnyProcessedPub, condition: FunctionCondition): boolean { + const value = getValueFromPath(pub, condition.path) + const args = condition.arguments + + switch (condition.name) { + case "contains": { + if (typeof value === "string") { + return value.includes(String(args[0])) + } + if (Array.isArray(value)) { + return value.includes(args[0]) + } + return false + } + case "startsWith": { + if (typeof value !== "string") { + return false + } + return value.startsWith(String(args[0])) + } + case "endsWith": { + if (typeof value !== "string") { + return false + } + return value.endsWith(String(args[0])) + } + case "exists": { + return value !== undefined && value !== null + } + default: + throw new UnsupportedExpressionError(`unsupported function: ${condition.name}`) + } +} + +/** + * evaluates a logical condition against a pub + */ +function evaluateLogical(pub: AnyProcessedPub, condition: LogicalCondition): boolean { + if (condition.operator === "and") { + return condition.conditions.every((c) => evaluateCondition(pub, c)) + } + return condition.conditions.some((c) => evaluateCondition(pub, c)) +} + +/** + * evaluates a not condition against a pub + */ +function evaluateNot(pub: AnyProcessedPub, condition: NotCondition): boolean { + return !evaluateCondition(pub, condition.condition) +} + +/** + * evaluates any condition against a pub + */ +function evaluateCondition(pub: AnyProcessedPub, condition: ParsedCondition): boolean { + switch (condition.type) { + case "comparison": + return evaluateComparison(pub, condition) + case "function": + return evaluateFunction(pub, condition) + case "logical": + return evaluateLogical(pub, condition) + case "not": + return evaluateNot(pub, condition) + } +} + +/** + * filters an array of pubs using a compiled jsonata query + * + * @example + * ```ts + * const query = compileJsonataQuery('$.pub.values.title = "Test" and $.pub.values.number > 10') + * const filtered = filterPubsWithJsonata(pubs, query) + * ``` + */ +export function filterPubsWithJsonata( + pubs: T[], + query: CompiledQuery +): T[] { + return pubs.filter((pub) => evaluateCondition(pub, query.condition)) +} + +/** + * tests if a single pub matches a compiled jsonata query + */ +export function pubMatchesJsonataQuery(pub: AnyProcessedPub, query: CompiledQuery): boolean { + return evaluateCondition(pub, query.condition) +} diff --git a/core/lib/server/jsonata-query/parser.ts b/core/lib/server/jsonata-query/parser.ts new file mode 100644 index 000000000..2c806a4f4 --- /dev/null +++ b/core/lib/server/jsonata-query/parser.ts @@ -0,0 +1,384 @@ +import type { + ComparisonCondition, + ComparisonOperator, + JsonataBinaryNode, + JsonataBlockNode, + JsonataFunctionNode, + JsonataNode, + JsonataNumberNode, + JsonataPathNode, + JsonataPathStep, + JsonataStringNode, + JsonataUnaryNode, + JsonataValueNode, + LiteralValue, + LogicalCondition, + LogicalOperator, + NotCondition, + ParsedCondition, + PubFieldPath, + StringFunction, +} from "./types" + +import jsonata from "jsonata" + +import { InvalidPathError, UnsupportedExpressionError } from "./errors" + +export type { ParsedCondition } + +export interface ParsedQuery { + condition: ParsedCondition + originalExpression: string +} + +const COMPARISON_OPS = new Set(["=", "!=", "<", "<=", ">", ">="]) as Set +const LOGICAL_OPS = new Set(["and", "or"]) as Set + +const SUPPORTED_FUNCTIONS = new Set([ + "contains", + "startsWith", + "endsWith", + "lowercase", + "uppercase", + "exists", + "not", +]) + +function isComparisonOp(op: string): op is ComparisonOperator { + return COMPARISON_OPS.has(op as ComparisonOperator) +} + +function isLogicalOp(op: string): op is LogicalOperator { + return LOGICAL_OPS.has(op as LogicalOperator) +} + +function isBinaryNode(node: JsonataNode): node is JsonataBinaryNode { + return node.type === "binary" +} + +function isPathNode(node: JsonataNode): node is JsonataPathNode { + return node.type === "path" +} + +function isFunctionNode(node: JsonataNode): node is JsonataFunctionNode { + return node.type === "function" +} + +function isBlockNode(node: JsonataNode): node is JsonataBlockNode { + return node.type === "block" +} + +function isUnaryNode(node: JsonataNode): node is JsonataUnaryNode { + return node.type === "unary" +} + +function isLiteralNode( + node: JsonataNode +): node is JsonataStringNode | JsonataNumberNode | JsonataValueNode { + return node.type === "string" || node.type === "number" || node.type === "value" +} + +/** + * extracts the pub field path from a jsonata path node + * + * expects paths like: + * - $.pub.values.fieldname + * - $.pub.id + * - $.pub.createdAt + * - $.pub.pubType.name + */ +function extractPubFieldPath(steps: JsonataPathStep[]): PubFieldPath { + if (steps.length < 3) { + throw new InvalidPathError("path too short, expected $.pub.something", stepsToString(steps)) + } + + // first step should be $ (empty variable) + if (steps[0].type !== "variable" || steps[0].value !== "") { + throw new InvalidPathError("path must start with $", stepsToString(steps)) + } + + // second step should be "pub" + if (steps[1].type !== "name" || steps[1].value !== "pub") { + throw new InvalidPathError("path must start with $.pub", stepsToString(steps)) + } + + const thirdStep = steps[2] + if (thirdStep.type !== "name") { + throw new InvalidPathError("expected name after $.pub", stepsToString(steps)) + } + + // handle builtin fields + if (["id", "createdAt", "updatedAt", "pubTypeId"].includes(thirdStep.value)) { + return { kind: "builtin", field: thirdStep.value as "id" | "createdAt" | "updatedAt" } + } + + // handle pubType.name or pubType.id + if (thirdStep.value === "pubType" && steps.length >= 4) { + const fourthStep = steps[3] + if (fourthStep.type === "name" && ["name", "id"].includes(fourthStep.value)) { + return { kind: "pubType", field: fourthStep.value as "name" | "id" } + } + throw new InvalidPathError("expected pubType.name or pubType.id", stepsToString(steps)) + } + + // handle values.fieldname + if (thirdStep.value === "values" && steps.length >= 4) { + const fourthStep = steps[3] + if (fourthStep.type === "name") { + return { kind: "value", fieldSlug: fourthStep.value } + } + throw new InvalidPathError("expected field name after $.pub.values", stepsToString(steps)) + } + + throw new InvalidPathError(`unsupported pub path: $.pub.${thirdStep.value}`, stepsToString(steps)) +} + +function stepsToString(steps: JsonataPathStep[]): string { + return steps.map((s) => (s.type === "variable" ? "$" : s.value)).join(".") +} + +/** + * extracts a literal value from a jsonata node + */ +function extractLiteral(node: JsonataNode): LiteralValue { + if (node.type === "string") { + return node.value + } + if (node.type === "number") { + return node.value + } + if (node.type === "value") { + return node.value + } + // handle array literals [1, 2, 3] + if (isUnaryNode(node) && node.value === "[" && node.expressions) { + return node.expressions.map(extractLiteral) + } + throw new UnsupportedExpressionError( + `expected literal value, got ${node.type}`, + node.type, + JSON.stringify(node) + ) +} + +/** + * gets the function name from a procedure + */ +function getFunctionName(procedure: JsonataFunctionNode["procedure"]): string { + if (procedure.type === "variable") { + return procedure.value + } + if (procedure.type === "path" && procedure.steps && procedure.steps.length > 0) { + return procedure.steps[0].value + } + throw new UnsupportedExpressionError("unexpected procedure type", procedure.type) +} + +/** + * parses a binary comparison node + */ +function parseComparison( + pathNode: JsonataPathNode | JsonataFunctionNode, + operator: ComparisonOperator, + valueNode: JsonataNode +): ComparisonCondition { + let path: PubFieldPath + let pathTransform: StringFunction | undefined + + if (isPathNode(pathNode)) { + path = extractPubFieldPath(pathNode.steps) + } else if (isFunctionNode(pathNode)) { + // handle things like $lowercase($.pub.values.title) = "hello" + const funcName = getFunctionName(pathNode.procedure) + if (!["lowercase", "uppercase"].includes(funcName)) { + throw new UnsupportedExpressionError( + `function ${funcName} cannot be used as path transform`, + funcName + ) + } + pathTransform = funcName as StringFunction + const arg = pathNode.arguments[0] + if (!isPathNode(arg)) { + throw new UnsupportedExpressionError("expected path as first argument to transform function") + } + path = extractPubFieldPath(arg.steps) + } else { + throw new UnsupportedExpressionError("expected path or function on left side of comparison") + } + + const value = extractLiteral(valueNode) + + return { type: "comparison", path, operator, value, pathTransform } +} + +/** + * parses a function call like $contains($.pub.values.title, "test") + */ +function parseFunctionCall(node: JsonataFunctionNode): ParsedCondition { + const funcName = getFunctionName(node.procedure) + + if (!SUPPORTED_FUNCTIONS.has(funcName)) { + throw new UnsupportedExpressionError(`unsupported function: ${funcName}`, funcName) + } + + // handle not() specially + if (funcName === "not") { + if (node.arguments.length !== 1) { + throw new UnsupportedExpressionError("not() expects exactly one argument") + } + const inner = parseNode(node.arguments[0]) + return { type: "not", condition: inner } satisfies NotCondition + } + + // handle exists() + if (funcName === "exists") { + if (node.arguments.length !== 1) { + throw new UnsupportedExpressionError("exists() expects exactly one argument") + } + const arg = node.arguments[0] + if (!isPathNode(arg)) { + throw new UnsupportedExpressionError("exists() expects a path argument") + } + const path = extractPubFieldPath(arg.steps) + return { type: "function", name: "exists", path, arguments: [] } + } + + // string functions: contains, startsWith, endsWith + if (["contains", "startsWith", "endsWith"].includes(funcName)) { + if (node.arguments.length !== 2) { + throw new UnsupportedExpressionError(`${funcName}() expects exactly two arguments`) + } + const pathArg = node.arguments[0] + const valueArg = node.arguments[1] + if (!isPathNode(pathArg)) { + throw new UnsupportedExpressionError(`${funcName}() expects a path as first argument`) + } + const path = extractPubFieldPath(pathArg.steps) + const value = extractLiteral(valueArg) + return { + type: "function", + name: funcName as StringFunction, + path, + arguments: [value], + } + } + + throw new UnsupportedExpressionError(`unhandled function: ${funcName}`, funcName) +} + +/** + * parses a binary node (comparison or logical) + */ +function parseBinary(node: JsonataBinaryNode): ParsedCondition { + const op = node.value + + // handle logical operators + if (isLogicalOp(op)) { + const left = parseNode(node.lhs) + const right = parseNode(node.rhs) + return { + type: "logical", + operator: op, + conditions: [left, right], + } satisfies LogicalCondition + } + + // handle "in" operator: $.pub.values.number in [42, 24, 54] + if (op === "in") { + // check if lhs is path and rhs is array + if (isPathNode(node.lhs)) { + const path = extractPubFieldPath(node.lhs.steps) + const value = extractLiteral(node.rhs) + return { type: "comparison", path, operator: "in", value } + } + // check if lhs is literal and rhs is path: "value" in $.pub.values.array + if (isLiteralNode(node.lhs) && isPathNode(node.rhs)) { + const path = extractPubFieldPath(node.rhs.steps) + const value = extractLiteral(node.lhs) + return { + type: "function", + name: "contains", + path, + arguments: [value], + } + } + throw new UnsupportedExpressionError("unsupported 'in' expression structure") + } + + // handle comparison operators + if (isComparisonOp(op)) { + // determine which side is the path and which is the value + if (isPathNode(node.lhs) || isFunctionNode(node.lhs)) { + return parseComparison(node.lhs, op, node.rhs) + } + if (isPathNode(node.rhs) || isFunctionNode(node.rhs)) { + // flip the operator for reversed comparison + const flippedOp = flipOperator(op) + return parseComparison(node.rhs, flippedOp, node.lhs) + } + throw new UnsupportedExpressionError("comparison must have at least one path") + } + + throw new UnsupportedExpressionError(`unsupported binary operator: ${op}`, "binary") +} + +function flipOperator(op: ComparisonOperator): ComparisonOperator { + switch (op) { + case "<": + return ">" + case ">": + return "<" + case "<=": + return ">=" + case ">=": + return "<=" + default: + return op + } +} + +/** + * parses any jsonata node into our condition format + */ +function parseNode(node: JsonataNode): ParsedCondition { + // unwrap block nodes (parentheses) + if (isBlockNode(node)) { + if (node.expressions.length !== 1) { + throw new UnsupportedExpressionError( + "block with multiple expressions not supported", + "block" + ) + } + return parseNode(node.expressions[0]) + } + + if (isBinaryNode(node)) { + return parseBinary(node) + } + + if (isFunctionNode(node)) { + return parseFunctionCall(node) + } + + throw new UnsupportedExpressionError(`unsupported node type: ${node.type}`, node.type) +} + +/** + * parses a jsonata expression string into our query format + * + * @example + * ```ts + * const query = parseJsonataQuery('$.pub.values.title = "Test" and $.pub.values.number > 10') + * // { condition: { type: 'logical', operator: 'and', conditions: [...] }, originalExpression: '...' } + * ``` + */ +export function parseJsonataQuery(expression: string): ParsedQuery { + const ast = jsonata(expression).ast() as JsonataNode + + const condition = parseNode(ast) + + return { + condition, + originalExpression: expression, + } +} diff --git a/core/lib/server/jsonata-query/sql-builder.ts b/core/lib/server/jsonata-query/sql-builder.ts new file mode 100644 index 000000000..225f46691 --- /dev/null +++ b/core/lib/server/jsonata-query/sql-builder.ts @@ -0,0 +1,304 @@ +import type { ExpressionBuilder, ExpressionWrapper } from "kysely" +import type { CompiledQuery } from "./compiler" +import type { + ComparisonCondition, + FunctionCondition, + LogicalCondition, + NotCondition, + ParsedCondition, + PubFieldPath, +} from "./types" + +import { sql } from "kysely" + +import { UnsupportedExpressionError } from "./errors" + +type AnyExpressionBuilder = ExpressionBuilder +type AnyExpressionWrapper = ExpressionWrapper + +export interface SqlBuilderOptions { + communitySlug?: string +} + +/** + * converts a pub field path to the appropriate sql column reference + */ +function pathToColumn( + path: PubFieldPath +): "value" | "pubs.createdAt" | "pubs.updatedAt" | "pubs.id" | "pubs.pubTypeId" { + if (path.kind === "builtin") { + return `pubs.${path.field}` as const + } + // for value fields, we'll handle them via subquery + return "value" +} + +/** + * resolves a field slug, optionally adding community prefix + */ +function resolveFieldSlug(fieldSlug: string, options?: SqlBuilderOptions): string { + if (!options?.communitySlug) { + return fieldSlug + } + // if already has a colon, assume it's already prefixed + if (fieldSlug.includes(":")) { + return fieldSlug + } + return `${options.communitySlug}:${fieldSlug}` +} + +/** + * builds a subquery that checks for a pub value matching certain conditions + */ +function buildValueExistsSubquery( + eb: AnyExpressionBuilder, + fieldSlug: string, + buildCondition: (innerEb: AnyExpressionBuilder) => AnyExpressionWrapper, + options?: SqlBuilderOptions +): AnyExpressionWrapper { + const resolvedSlug = resolveFieldSlug(fieldSlug, options) + return eb.exists( + eb + .selectFrom("pub_values") + .innerJoin("pub_fields", "pub_fields.id", "pub_values.fieldId") + .select(eb.lit(1).as("exists_check")) + .where("pub_values.pubId", "=", eb.ref("pubs.id")) + .where("pub_fields.slug", "=", resolvedSlug) + .where((innerEb) => buildCondition(innerEb)) + ) +} + +/** + * builds a subquery for pubType conditions + */ +function buildPubTypeSubquery( + eb: AnyExpressionBuilder, + field: "name" | "id", + buildCondition: (column: string) => AnyExpressionWrapper +): AnyExpressionWrapper { + if (field === "id") { + return buildCondition("pubs.pubTypeId") + } + return eb.exists( + eb + .selectFrom("pub_types") + .select(eb.lit(1).as("exists_check")) + .where("pub_types.id", "=", eb.ref("pubs.pubTypeId")) + .where((innerEb) => buildCondition("pub_types.name")) + ) +} + +/** + * wraps a value for json comparison if needed + */ +function wrapValue(value: unknown): unknown { + if (typeof value === "string") { + return JSON.stringify(value) + } + return value +} + +/** + * builds the sql condition for a comparison + */ +function buildComparisonCondition( + eb: AnyExpressionBuilder, + condition: ComparisonCondition, + options?: SqlBuilderOptions +): AnyExpressionWrapper { + const { path, operator, value, pathTransform } = condition + + // handle builtin fields directly on pubs table + // builtin fields are not json, so we don't wrap the value + if (path.kind === "builtin") { + const column = pathToColumn(path) + return buildOperatorCondition(eb, column, operator, value, pathTransform, false) + } + + // handle pubType fields (also not json) + if (path.kind === "pubType") { + return buildPubTypeSubquery(eb, path.field, (column) => + buildOperatorCondition(eb, column, operator, value, pathTransform, false) + ) + } + + // handle value fields via subquery (json values) + return buildValueExistsSubquery( + eb, + path.fieldSlug, + (innerEb) => buildOperatorCondition(innerEb, "value", operator, value, pathTransform, true), + options + ) +} + +/** + * builds an operator condition for a specific column + */ +function buildOperatorCondition( + eb: AnyExpressionBuilder, + column: string, + operator: string, + value: unknown, + pathTransform?: string, + isJsonValue = true +): AnyExpressionWrapper { + let col: ReturnType | string = column + + // apply path transform (lowercase, uppercase) + if (pathTransform === "lowercase") { + col = sql.raw(`lower(${column}::text)`) + } else if (pathTransform === "uppercase") { + col = sql.raw(`upper(${column}::text)`) + } + + const wrappedValue = isJsonValue ? wrapValue(value) : value + + switch (operator) { + case "=": + return eb(col, "=", wrappedValue) + case "!=": + return eb(col, "!=", wrappedValue) + case "<": + return eb(col, "<", wrappedValue) + case "<=": + return eb(col, "<=", wrappedValue) + case ">": + return eb(col, ">", wrappedValue) + case ">=": + return eb(col, ">=", wrappedValue) + case "in": + if (Array.isArray(value)) { + return eb(col, "in", isJsonValue ? value.map(wrapValue) : value) + } + return eb(col, "=", wrappedValue) + default: + throw new UnsupportedExpressionError(`unsupported operator: ${operator}`) + } +} + +/** + * builds the sql condition for a function call + */ +function buildFunctionCondition( + eb: AnyExpressionBuilder, + condition: FunctionCondition, + options?: SqlBuilderOptions +): AnyExpressionWrapper { + const { name, path, arguments: args } = condition + + // for value fields, strings are stored as JSON, so we need to account for quotes + const isValueField = path.kind === "value" + + const buildInner = (col: string) => { + const strArg = String(args[0]) + switch (name) { + case "contains": + return eb(sql.raw(`${col}::text`), "like", `%${strArg}%`) + case "startsWith": + // for json values, the string starts with a quote + if (isValueField) { + return eb(sql.raw(`${col}::text`), "like", `"${strArg}%`) + } + return eb(sql.raw(`${col}::text`), "like", `${strArg}%`) + case "endsWith": + // for json values, the string ends with a quote + if (isValueField) { + return eb(sql.raw(`${col}::text`), "like", `%${strArg}"`) + } + return eb(sql.raw(`${col}::text`), "like", `%${strArg}`) + case "exists": + return eb.lit(true) + default: + throw new UnsupportedExpressionError(`unsupported function: ${name}`) + } + } + + // handle builtin fields + if (path.kind === "builtin") { + const column = pathToColumn(path) + if (name === "exists") { + return eb(column, "is not", null) + } + return buildInner(column) + } + + // handle pubType fields + if (path.kind === "pubType") { + return buildPubTypeSubquery(eb, path.field, (column) => buildInner(column)) + } + + // handle value fields + if (name === "exists") { + return buildValueExistsSubquery(eb, path.fieldSlug, () => eb.lit(true), options) + } + + return buildValueExistsSubquery(eb, path.fieldSlug, () => buildInner("value"), options) +} + +/** + * builds the sql condition for a logical operation + */ +function buildLogicalCondition( + eb: AnyExpressionBuilder, + condition: LogicalCondition, + options?: SqlBuilderOptions +): AnyExpressionWrapper { + const conditions = condition.conditions.map((c) => buildCondition(eb, c, options)) + + if (condition.operator === "and") { + return eb.and(conditions) + } + return eb.or(conditions) +} + +/** + * builds the sql condition for a not operation + */ +function buildNotCondition( + eb: AnyExpressionBuilder, + condition: NotCondition, + options?: SqlBuilderOptions +): AnyExpressionWrapper { + return eb.not(buildCondition(eb, condition.condition, options)) +} + +/** + * builds the sql condition for any parsed condition + */ +function buildCondition( + eb: AnyExpressionBuilder, + condition: ParsedCondition, + options?: SqlBuilderOptions +): AnyExpressionWrapper { + switch (condition.type) { + case "comparison": + return buildComparisonCondition(eb, condition, options) + case "function": + return buildFunctionCondition(eb, condition, options) + case "logical": + return buildLogicalCondition(eb, condition, options) + case "not": + return buildNotCondition(eb, condition, options) + } +} + +/** + * applies a compiled jsonata query as a filter to a kysely query builder + * + * @example + * ```ts + * const query = compileJsonataQuery('$.pub.values.title = "Test"') + * const pubs = await db + * .selectFrom("pubs") + * .selectAll() + * .where((eb) => applyJsonataFilter(eb, query, { communitySlug: "my-community" })) + * .execute() + * ``` + */ +export function applyJsonataFilter( + eb: K, + query: CompiledQuery, + options?: SqlBuilderOptions +): AnyExpressionWrapper { + return buildCondition(eb, query.condition, options) +} diff --git a/core/lib/server/jsonata-query/types.ts b/core/lib/server/jsonata-query/types.ts new file mode 100644 index 000000000..d5f0a6c02 --- /dev/null +++ b/core/lib/server/jsonata-query/types.ts @@ -0,0 +1,131 @@ +// jsonata ast types (reverse-engineered from library) +export type JsonataNodeType = + | "binary" + | "unary" + | "path" + | "name" + | "variable" + | "string" + | "number" + | "value" // null, true, false + | "function" + | "block" + | "filter" + | "regex" + +export interface JsonataBaseNode { + type: JsonataNodeType + position: number + value?: unknown +} + +export interface JsonataBinaryNode extends JsonataBaseNode { + type: "binary" + value: "=" | "!=" | "<" | "<=" | ">" | ">=" | "and" | "or" | "in" | "&" + lhs: JsonataNode + rhs: JsonataNode +} + +export interface JsonataUnaryNode extends JsonataBaseNode { + type: "unary" + value: "[" | "-" // array literal or negation + expressions?: JsonataNode[] +} + +export interface JsonataPathStep { + type: "name" | "variable" + value: string + position: number + stages?: { type: "filter"; expr: JsonataNode; position: number }[] +} + +export interface JsonataPathNode extends JsonataBaseNode { + type: "path" + steps: JsonataPathStep[] +} + +export interface JsonataStringNode extends JsonataBaseNode { + type: "string" + value: string +} + +export interface JsonataNumberNode extends JsonataBaseNode { + type: "number" + value: number +} + +export interface JsonataValueNode extends JsonataBaseNode { + type: "value" + value: null | boolean +} + +export interface JsonataFunctionNode extends JsonataBaseNode { + type: "function" + value: "(" + arguments: JsonataNode[] + procedure: { type: "variable" | "path"; value: string; steps?: JsonataPathStep[] } +} + +export interface JsonataBlockNode extends JsonataBaseNode { + type: "block" + expressions: JsonataNode[] +} + +export type JsonataNode = + | JsonataBinaryNode + | JsonataUnaryNode + | JsonataPathNode + | JsonataStringNode + | JsonataNumberNode + | JsonataValueNode + | JsonataFunctionNode + | JsonataBlockNode + +// our internal representation +export type ComparisonOperator = "=" | "!=" | "<" | "<=" | ">" | ">=" | "in" +export type LogicalOperator = "and" | "or" +export type StringFunction = "contains" | "startsWith" | "endsWith" | "lowercase" | "uppercase" +export type BooleanFunction = "exists" | "not" + +export type PubFieldPath = + | { kind: "value"; fieldSlug: string } + | { kind: "builtin"; field: "id" | "createdAt" | "updatedAt" | "pubTypeId" } + | { kind: "pubType"; field: "name" | "id" } + +export type LiteralValue = string | number | boolean | null | LiteralValue[] + +export interface ComparisonCondition { + type: "comparison" + path: PubFieldPath + operator: ComparisonOperator + value: LiteralValue + // when we have function wrappers like $lowercase($.pub.values.title) + pathTransform?: StringFunction +} + +export interface FunctionCondition { + type: "function" + name: StringFunction | BooleanFunction + path: PubFieldPath + arguments: LiteralValue[] +} + +export interface LogicalCondition { + type: "logical" + operator: LogicalOperator + conditions: ParsedCondition[] +} + +export interface NotCondition { + type: "not" + condition: ParsedCondition +} + +export type ParsedCondition = + | ComparisonCondition + | FunctionCondition + | LogicalCondition + | NotCondition + +// re-export for convenience (also exported from parser.ts) +export type { ParsedCondition as Condition } diff --git a/core/package.json b/core/package.json index 83fd18980..80038d6d9 100644 --- a/core/package.json +++ b/core/package.json @@ -50,7 +50,10 @@ "storybook": "SKIP_VALIDATION=true PUBPUB_URL=http://localhost:6006 storybook dev -p 6006 --no-open", "build-storybook": "SKIP_VALIDATION=true storybook build" }, - "files": [".next", "public"], + "files": [ + ".next", + "public" + ], "prisma": { "__comment": "The #register-loader goes to the correct file based on the .imports setting below", "seed": "tsx --import #register-loader prisma/seed.ts" @@ -106,6 +109,7 @@ "hastscript": "^9.0.1", "import-in-the-middle": "1.14.2", "ioredis": "^5.7.0", + "jsonata": "^2.1.0", "jsonpath-plus": "^10.3.0", "jsonwebtoken": "^9.0.2", "katex": "catalog:", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 09b6b59c5..97c99e4ee 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,7 +56,7 @@ catalogs: version: 20.19.11 '@typescript/native-preview': specifier: latest - version: 7.0.0-dev.20260106.1 + version: 7.0.0-dev.20260119.1 '@vitejs/plugin-react': specifier: ^4.5.0 version: 4.7.0 @@ -370,6 +370,9 @@ importers: ioredis: specifier: ^5.7.0 version: 5.7.0 + jsonata: + specifier: ^2.1.0 + version: 2.1.0 jsonpath-plus: specifier: ^10.3.0 version: 10.3.0 @@ -643,7 +646,7 @@ importers: version: 9.0.8 '@typescript/native-preview': specifier: 'catalog:' - version: 7.0.0-dev.20260106.1 + version: 7.0.0-dev.20260119.1 '@vitejs/plugin-react': specifier: 'catalog:' version: 4.7.0(vite@6.3.5(@types/node@20.19.11)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.43.1)(tsx@4.20.5)(yaml@2.8.1)) @@ -758,7 +761,7 @@ importers: version: 19.1.7(@types/react@19.1.10) '@typescript/native-preview': specifier: 'catalog:' - version: 7.0.0-dev.20260106.1 + version: 7.0.0-dev.20260119.1 autoprefixer: specifier: 'catalog:' version: 10.4.21(postcss@8.5.6) @@ -819,7 +822,7 @@ importers: version: 20.19.11 '@typescript/native-preview': specifier: 'catalog:' - version: 7.0.0-dev.20260106.1 + version: 7.0.0-dev.20260119.1 dotenv-cli: specifier: ^7.4.4 version: 7.4.4 @@ -1078,7 +1081,7 @@ importers: version: 9.0.8 '@typescript/native-preview': specifier: 'catalog:' - version: 7.0.0-dev.20260106.1 + version: 7.0.0-dev.20260119.1 '@uiw/react-json-view': specifier: 2.0.0-alpha.27 version: 2.0.0-alpha.27(@babel/runtime@7.28.3)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) @@ -1127,7 +1130,7 @@ importers: version: 3.51.0(@types/node@20.19.11)(zod@3.25.76) '@typescript/native-preview': specifier: 'catalog:' - version: 7.0.0-dev.20260106.1 + version: 7.0.0-dev.20260119.1 tsconfig: specifier: workspace:* version: link:../../config/tsconfig @@ -1161,7 +1164,7 @@ importers: version: 8.15.5 '@typescript/native-preview': specifier: 'catalog:' - version: 7.0.0-dev.20260106.1 + version: 7.0.0-dev.20260119.1 dotenv-cli: specifier: ^7.4.4 version: 7.4.4 @@ -1197,7 +1200,7 @@ importers: version: 0.0.31(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@typescript/native-preview': specifier: 'catalog:' - version: 7.0.0-dev.20260106.1 + version: 7.0.0-dev.20260119.1 browserslist: specifier: ^4.25.3 version: 4.25.3 @@ -1241,7 +1244,7 @@ importers: version: 2.2.0 '@typescript/native-preview': specifier: 'catalog:' - version: 7.0.0-dev.20260106.1 + version: 7.0.0-dev.20260119.1 tsconfig: specifier: workspace:* version: link:../../config/tsconfig @@ -1263,7 +1266,7 @@ importers: version: 20.19.11 '@typescript/native-preview': specifier: 'catalog:' - version: 7.0.0-dev.20260106.1 + version: 7.0.0-dev.20260119.1 tsconfig: specifier: workspace:* version: link:../../config/tsconfig @@ -1288,7 +1291,7 @@ importers: devDependencies: '@typescript/native-preview': specifier: 'catalog:' - version: 7.0.0-dev.20260106.1 + version: 7.0.0-dev.20260119.1 react: specifier: catalog:react19 version: 19.2.3 @@ -1502,7 +1505,7 @@ importers: version: 19.1.10 '@typescript/native-preview': specifier: 'catalog:' - version: 7.0.0-dev.20260106.1 + version: 7.0.0-dev.20260119.1 react: specifier: catalog:react19 version: 19.2.3 @@ -1527,7 +1530,7 @@ importers: devDependencies: '@typescript/native-preview': specifier: 'catalog:' - version: 7.0.0-dev.20260106.1 + version: 7.0.0-dev.20260119.1 tsconfig: specifier: workspace:* version: link:../../config/tsconfig @@ -7483,43 +7486,43 @@ packages: '@types/yauzl@2.10.3': resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==} - '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260106.1': - resolution: {integrity: sha512-9n7HIVP3UMgWSK8Yi2H+23hrrEDNNfkW78mbkxfatGh/ghU4m2QuO8R6MdMdPsSmmKDvbWOtpLEuSZFKNzu7eQ==} + '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260119.1': + resolution: {integrity: sha512-siuRD9Shh5gVrgYG5HEWxFxG/dkZa4ndupGWKMfM4DwMG7zLeFayi6sB9yiwpD0d203ts01D7uTnTCALdiWXmQ==} cpu: [arm64] os: [darwin] - '@typescript/native-preview-darwin-x64@7.0.0-dev.20260106.1': - resolution: {integrity: sha512-zHJ1KsgQTpBnG3RbI1kjH/fD8juc5DTlQ9gbmJi23OhhaOgNF+PkqC2vAAWLFqdH99tAMvmJf9BJncbB7LwxXA==} + '@typescript/native-preview-darwin-x64@7.0.0-dev.20260119.1': + resolution: {integrity: sha512-ivqxCrbLXUqZU1OMojVRCnVx5gC/twgi7gKzBXMBLGOgfTkhajbHk/71J3OQhJwzR3T2ISG6FTfXKHhQMtgkkg==} cpu: [x64] os: [darwin] - '@typescript/native-preview-linux-arm64@7.0.0-dev.20260106.1': - resolution: {integrity: sha512-Mp7M7fgUsVW8MHadN58gjie1bzg06K1Id6vm2Aycnmk9rKgu8CxdaDayllr5giPo+iZLZOnw2FyGItrywd4fuA==} + '@typescript/native-preview-linux-arm64@7.0.0-dev.20260119.1': + resolution: {integrity: sha512-ttNri2Ui1CzlLnPJN0sQ4XBgrCMq4jjtxouitRGh7+YlToG561diLERjOwIhNfTzPDKRMS7XO090WoepbvzFpA==} cpu: [arm64] os: [linux] - '@typescript/native-preview-linux-arm@7.0.0-dev.20260106.1': - resolution: {integrity: sha512-z7hdUMOOhdiVg1RmWyRyrIG3IIki4eJF3/TUtVVSTRwGq3t+j2JnLUUoK5VgX7EiqlN6wuJ94/FpWtyf+X4fkw==} + '@typescript/native-preview-linux-arm@7.0.0-dev.20260119.1': + resolution: {integrity: sha512-Bev1d6NCgCmcGOgmdFG514tWRt2lNUSFjQ9RVnN86tSm+bl5p9Lv6TQjc38Ow9vY11J71IZs9HNN1AKWfBCj2Q==} cpu: [arm] os: [linux] - '@typescript/native-preview-linux-x64@7.0.0-dev.20260106.1': - resolution: {integrity: sha512-H+c7xgK0gItbntnPFvt9nVv+cjjjn0lTj2tIjBQcTbH92q9RgFkIfztgxrP5zD8MzJKDOyIw/iAUSsb57lyxjw==} + '@typescript/native-preview-linux-x64@7.0.0-dev.20260119.1': + resolution: {integrity: sha512-mwsjGZqUKju3SKPzlDuKhKgt9Ht8seA5OBhorvRZk2B5lwlH0gDsApGK4t50TcnzjpbWI85FVxI6wTq1T36dMg==} cpu: [x64] os: [linux] - '@typescript/native-preview-win32-arm64@7.0.0-dev.20260106.1': - resolution: {integrity: sha512-PS1FyYa+/sHQa5Va0yz21DxaBkGGwOYfjMyRSs6oHq01DzMnVIjtsdNAALP0+oqki8Adw0D2XtsdB5QapDbBJw==} + '@typescript/native-preview-win32-arm64@7.0.0-dev.20260119.1': + resolution: {integrity: sha512-463QnUaRCUhY/Flj/XinORTbBYuxoMthgJiBU1vu7mipLo2Yaipkkgn1ArGHkV9mjWBa7QIPCWg/V2KIEoVdcA==} cpu: [arm64] os: [win32] - '@typescript/native-preview-win32-x64@7.0.0-dev.20260106.1': - resolution: {integrity: sha512-AKVSTGcIE7d5KFtclhK3PVwUrsNnzziA7ZC/VDbMbvYCjLk7FE2GdNKaxQxLGHb53IUirgmltR5r4htn0WSM6A==} + '@typescript/native-preview-win32-x64@7.0.0-dev.20260119.1': + resolution: {integrity: sha512-039WAg5xJjqrRYVHMR9Y2y83dYSLofbyx/22Gc6ur3b/nR8u1wdErK9uwrguL3lxpKDo6qdhnkGlbX8FP0Bz+g==} cpu: [x64] os: [win32] - '@typescript/native-preview@7.0.0-dev.20260106.1': - resolution: {integrity: sha512-EeH81rQsgLjewxuVOBN0MnQWAyf5YNeHRP3+Et6wJyr4d7HuA7zFwfNaEdfX1k366kgpKOR5K6dakorBhKZGng==} + '@typescript/native-preview@7.0.0-dev.20260119.1': + resolution: {integrity: sha512-Tf74TdJVJlLRMN0W9VXK8jc0Gor9+wFRm40qTLt2JeHiPpSF5TEN/pHPjlf4Id1wDSJXH9p5/U1wFS3s5TS2PQ==} hasBin: true '@typescript/vfs@1.6.1': @@ -9109,8 +9112,8 @@ packages: resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} engines: {node: '>= 4'} - dompurify@3.2.6: - resolution: {integrity: sha512-/2GogDQlohXPZe6D6NOgQvXLPSYBqIWMnZ8zzOhn09REE4eyAzb+Hed3jhoM9OkuaJ8P6ZGTTVWQKAi8ieIzfQ==} + dompurify@3.2.7: + resolution: {integrity: sha512-WhL/YuveyGXJaerVlMYGWhvQswa7myDG17P7Vu65EWC05o8vfeNbvNf4d/BOvH99+ZW+LlQsc1GDKMa1vNK6dw==} domutils@3.2.2: resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==} @@ -10460,6 +10463,7 @@ packages: resolution: {integrity: sha512-t0etAxTUk1w5MYdNOkZBZ8rvYYN5iL+2dHCCx/DpkFm/bW28M6y5nUS83D4XdZiHy35Fpaw6LBb+F88fHZnVCw==} engines: {node: '>=8.17.0'} hasBin: true + bundledDependencies: [] jsonfile@6.2.0: resolution: {integrity: sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==} @@ -21508,36 +21512,36 @@ snapshots: '@types/node': 22.17.2 optional: true - '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260106.1': + '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260119.1': optional: true - '@typescript/native-preview-darwin-x64@7.0.0-dev.20260106.1': + '@typescript/native-preview-darwin-x64@7.0.0-dev.20260119.1': optional: true - '@typescript/native-preview-linux-arm64@7.0.0-dev.20260106.1': + '@typescript/native-preview-linux-arm64@7.0.0-dev.20260119.1': optional: true - '@typescript/native-preview-linux-arm@7.0.0-dev.20260106.1': + '@typescript/native-preview-linux-arm@7.0.0-dev.20260119.1': optional: true - '@typescript/native-preview-linux-x64@7.0.0-dev.20260106.1': + '@typescript/native-preview-linux-x64@7.0.0-dev.20260119.1': optional: true - '@typescript/native-preview-win32-arm64@7.0.0-dev.20260106.1': + '@typescript/native-preview-win32-arm64@7.0.0-dev.20260119.1': optional: true - '@typescript/native-preview-win32-x64@7.0.0-dev.20260106.1': + '@typescript/native-preview-win32-x64@7.0.0-dev.20260119.1': optional: true - '@typescript/native-preview@7.0.0-dev.20260106.1': + '@typescript/native-preview@7.0.0-dev.20260119.1': optionalDependencies: - '@typescript/native-preview-darwin-arm64': 7.0.0-dev.20260106.1 - '@typescript/native-preview-darwin-x64': 7.0.0-dev.20260106.1 - '@typescript/native-preview-linux-arm': 7.0.0-dev.20260106.1 - '@typescript/native-preview-linux-arm64': 7.0.0-dev.20260106.1 - '@typescript/native-preview-linux-x64': 7.0.0-dev.20260106.1 - '@typescript/native-preview-win32-arm64': 7.0.0-dev.20260106.1 - '@typescript/native-preview-win32-x64': 7.0.0-dev.20260106.1 + '@typescript/native-preview-darwin-arm64': 7.0.0-dev.20260119.1 + '@typescript/native-preview-darwin-x64': 7.0.0-dev.20260119.1 + '@typescript/native-preview-linux-arm': 7.0.0-dev.20260119.1 + '@typescript/native-preview-linux-arm64': 7.0.0-dev.20260119.1 + '@typescript/native-preview-linux-x64': 7.0.0-dev.20260119.1 + '@typescript/native-preview-win32-arm64': 7.0.0-dev.20260119.1 + '@typescript/native-preview-win32-x64': 7.0.0-dev.20260119.1 '@typescript/vfs@1.6.1(typescript@5.9.2)': dependencies: @@ -23451,7 +23455,7 @@ snapshots: dependencies: domelementtype: 2.3.0 - dompurify@3.2.6: + dompurify@3.2.7: optionalDependencies: '@types/trusted-types': 2.0.7 @@ -25838,7 +25842,7 @@ snapshots: d3-sankey: 0.12.3 dagre-d3-es: 7.0.11 dayjs: 1.11.13 - dompurify: 3.2.6 + dompurify: 3.2.7 katex: 0.16.22 khroma: 2.1.0 lodash-es: 4.17.21 From 7d97b8686b8b813c6c8f09aeeae8ef451fc1c1f0 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 19 Jan 2026 18:29:26 +0100 Subject: [PATCH 2/9] feat: add relation and search filtering --- core/lib/server/jsonata-query/index.ts | 9 + .../jsonata-query/jsonata-query.db.test.ts | 499 +++++++++++++++++- .../lib/server/jsonata-query/memory-filter.ts | 190 +++++++ core/lib/server/jsonata-query/parser.ts | 395 ++++++++++++++ core/lib/server/jsonata-query/sql-builder.ts | 332 +++++++++++- core/lib/server/jsonata-query/types.ts | 58 ++ 6 files changed, 1464 insertions(+), 19 deletions(-) diff --git a/core/lib/server/jsonata-query/index.ts b/core/lib/server/jsonata-query/index.ts index 9974e627d..03556328d 100644 --- a/core/lib/server/jsonata-query/index.ts +++ b/core/lib/server/jsonata-query/index.ts @@ -9,4 +9,13 @@ export type { FunctionCondition, LogicalCondition, NotCondition, + SearchCondition, + RelationCondition, + RelationDirection, + RelationContextPath, + RelationFilterCondition, + RelationComparisonCondition, + RelationFunctionCondition, + RelationLogicalCondition, + RelationNotCondition, } from "./types" diff --git a/core/lib/server/jsonata-query/jsonata-query.db.test.ts b/core/lib/server/jsonata-query/jsonata-query.db.test.ts index 944051256..03a4d6493 100644 --- a/core/lib/server/jsonata-query/jsonata-query.db.test.ts +++ b/core/lib/server/jsonata-query/jsonata-query.db.test.ts @@ -28,6 +28,13 @@ const number42PubId = crypto.randomUUID() as PubsId const booleanPubId = crypto.randomUUID() as PubsId const arrayPubId = crypto.randomUUID() as PubsId const complexPubId = crypto.randomUUID() as PubsId +// relation test pub ids +const author1Id = crypto.randomUUID() as PubsId +const author2Id = crypto.randomUUID() as PubsId +const author3Id = crypto.randomUUID() as PubsId +const bookPubId = crypto.randomUUID() as PubsId +const chapter1PubId = crypto.randomUUID() as PubsId +const chapter2PubId = crypto.randomUUID() as PubsId const seed = createSeed({ community: { @@ -40,6 +47,9 @@ const seed = createSeed({ Boolean: { schemaName: CoreSchemaType.Boolean }, Date: { schemaName: CoreSchemaType.DateTime }, Array: { schemaName: CoreSchemaType.StringArray }, + Institution: { schemaName: CoreSchemaType.String }, + Contributors: { schemaName: CoreSchemaType.String, relation: true }, + Chapters: { schemaName: CoreSchemaType.String, relation: true }, }, pubTypes: { Article: { @@ -48,6 +58,19 @@ const seed = createSeed({ Boolean: { isTitle: false }, Date: { isTitle: false }, Array: { isTitle: false }, + Contributors: { isTitle: false }, + }, + Author: { + Title: { isTitle: true }, + Institution: { isTitle: false }, + }, + Book: { + Title: { isTitle: true }, + Chapters: { isTitle: false }, + }, + Chapter: { + Title: { isTitle: true }, + Number: { isTitle: false }, }, Review: { Title: { isTitle: true }, @@ -56,6 +79,32 @@ const seed = createSeed({ }, stages: {}, pubs: [ + // first create authors (referenced later) + { + id: author1Id, + pubType: "Author", + values: { + Title: "John Smith", + Institution: "University of Groningen", + }, + }, + { + id: author2Id, + pubType: "Author", + values: { + Title: "Jane Doe", + Institution: "MIT", + }, + }, + { + id: author3Id, + pubType: "Author", + values: { + Title: "Bob Wilson", + Institution: "University of Groningen", + }, + }, + // articles with contributors { id: titlePubId, pubType: "Article", @@ -63,6 +112,10 @@ const seed = createSeed({ Title: "Test Article", Number: 100, Boolean: true, + Contributors: [ + { value: "Primary Author", relatedPubId: author1Id }, + { value: "Editor", relatedPubId: author2Id }, + ], }, }, { @@ -72,6 +125,7 @@ const seed = createSeed({ Title: "Another Article", Number: 50, Boolean: false, + Contributors: [{ value: "Primary Author", relatedPubId: author3Id }], }, }, { @@ -107,6 +161,35 @@ const seed = createSeed({ Array: ["geology", "biology", "chemistry"], }, }, + // chapters for the book (created first so book can reference them) + { + id: chapter1PubId, + pubType: "Chapter", + values: { + Title: "Chapter One", + Number: 1, + }, + }, + { + id: chapter2PubId, + pubType: "Chapter", + values: { + Title: "Chapter Two", + Number: 2, + }, + }, + // book with chapters + { + id: bookPubId, + pubType: "Book", + values: { + Title: "The Big Book", + Chapters: [ + { value: "Introduction", relatedPubId: chapter1PubId }, + { value: "Main Content", relatedPubId: chapter2PubId }, + ], + }, + }, { id: complexPubId, pubType: "Article", @@ -424,9 +507,33 @@ describe("memory filtering", () => { createdAt: new Date("2024-01-01"), updatedAt: new Date("2024-01-01"), values: [ - { fieldSlug: "test:title", fieldId: "f1", value: "Test Article", relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, - { fieldSlug: "test:number", fieldId: "f2", value: 100, relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, - { fieldSlug: "test:boolean", fieldId: "f3", value: true, relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, + { + fieldSlug: "test:title", + fieldId: "f1", + value: "Test Article", + relatedPubId: null, + createdAt: new Date(), + updatedAt: new Date(), + lastModifiedBy: "user", + }, + { + fieldSlug: "test:number", + fieldId: "f2", + value: 100, + relatedPubId: null, + createdAt: new Date(), + updatedAt: new Date(), + lastModifiedBy: "user", + }, + { + fieldSlug: "test:boolean", + fieldId: "f3", + value: true, + relatedPubId: null, + createdAt: new Date(), + updatedAt: new Date(), + lastModifiedBy: "user", + }, ], }, { @@ -437,9 +544,33 @@ describe("memory filtering", () => { createdAt: new Date("2024-02-01"), updatedAt: new Date("2024-02-01"), values: [ - { fieldSlug: "test:title", fieldId: "f1", value: "Another Article", relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, - { fieldSlug: "test:number", fieldId: "f2", value: 50, relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, - { fieldSlug: "test:boolean", fieldId: "f3", value: false, relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, + { + fieldSlug: "test:title", + fieldId: "f1", + value: "Another Article", + relatedPubId: null, + createdAt: new Date(), + updatedAt: new Date(), + lastModifiedBy: "user", + }, + { + fieldSlug: "test:number", + fieldId: "f2", + value: 50, + relatedPubId: null, + createdAt: new Date(), + updatedAt: new Date(), + lastModifiedBy: "user", + }, + { + fieldSlug: "test:boolean", + fieldId: "f3", + value: false, + relatedPubId: null, + createdAt: new Date(), + updatedAt: new Date(), + lastModifiedBy: "user", + }, ], }, { @@ -450,9 +581,33 @@ describe("memory filtering", () => { createdAt: new Date("2024-03-01"), updatedAt: new Date("2024-03-01"), values: [ - { fieldSlug: "test:title", fieldId: "f1", value: "Important Document", relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, - { fieldSlug: "test:number", fieldId: "f2", value: 75, relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, - { fieldSlug: "test:array", fieldId: "f4", value: ["geology", "biology"], relatedPubId: null, createdAt: new Date(), updatedAt: new Date(), lastModifiedBy: "user" }, + { + fieldSlug: "test:title", + fieldId: "f1", + value: "Important Document", + relatedPubId: null, + createdAt: new Date(), + updatedAt: new Date(), + lastModifiedBy: "user", + }, + { + fieldSlug: "test:number", + fieldId: "f2", + value: 75, + relatedPubId: null, + createdAt: new Date(), + updatedAt: new Date(), + lastModifiedBy: "user", + }, + { + fieldSlug: "test:array", + fieldId: "f4", + value: ["geology", "biology"], + relatedPubId: null, + createdAt: new Date(), + updatedAt: new Date(), + lastModifiedBy: "user", + }, ], }, ] as any @@ -482,9 +637,7 @@ describe("memory filtering", () => { }) it("filters with logical or", () => { - const query = compileJsonataQuery( - "$.pub.values.number < 60 or $.pub.values.number > 90" - ) + const query = compileJsonataQuery("$.pub.values.number < 60 or $.pub.values.number > 90") const result = filterPubsWithJsonata(mockPubs, query) expect(result).toHaveLength(2) }) @@ -602,15 +755,14 @@ describe("comparison with original filter format", () => { { trx, filters: { - $and: [ - { [slug("number")]: { $gt: 40 } }, - { [slug("boolean")]: { $eq: true } }, - ], + $and: [{ [slug("number")]: { $gt: 40 } }, { [slug("boolean")]: { $eq: true } }], }, } ) - const jsonataIds = await runQuery("$.pub.values.number > 40 and $.pub.values.boolean = true") + const jsonataIds = await runQuery( + "$.pub.values.number > 40 and $.pub.values.boolean = true" + ) expect(new Set(jsonataIds)).toEqual(new Set(originalResult.map((p) => p.id))) }) @@ -653,3 +805,316 @@ describe("comparison with original filter format", () => { expect(new Set(jsonataIds)).toEqual(new Set(originalResult.map((p) => p.id))) }) }) + +describe("search queries", () => { + it("parses $search function", () => { + const query = parseJsonataQuery('$search("test query")') + expect(query.condition).toEqual({ + type: "search", + query: "test query", + }) + }) + + it("searches for text in pub values", async () => { + const { db } = await import("~/kysely/database") + const _trx = getTrx() + + const results = await db + .selectFrom("pubs") + .select("pubs.id") + .innerJoin("communities", "communities.id", "pubs.communityId") + .where("communities.slug", "=", communitySlug) + .where((eb) => + applyJsonataFilter(eb, compileJsonataQuery('$search("Article")'), { + communitySlug, + }) + ) + .execute() + + // should find pubs that have "Article" in their values + expect(results.map((r) => r.id)).toContain(titlePubId) + expect(results.map((r) => r.id)).toContain(title2PubId) + }) + + it("searches for multiple words", async () => { + const { db } = await import("~/kysely/database") + const _trx = getTrx() + + const results = await db + .selectFrom("pubs") + .select("pubs.id") + .innerJoin("communities", "communities.id", "pubs.communityId") + .where("communities.slug", "=", communitySlug) + .where((eb) => + applyJsonataFilter(eb, compileJsonataQuery('$search("Test Article")'), { + communitySlug, + }) + ) + .execute() + + // should find pub with both "Test" and "Article" + expect(results.map((r) => r.id)).toContain(titlePubId) + }) + + it("combines search with other conditions", async () => { + const { db } = await import("~/kysely/database") + const _trx = getTrx() + + const results = await db + .selectFrom("pubs") + .select("pubs.id") + .innerJoin("communities", "communities.id", "pubs.communityId") + .where("communities.slug", "=", communitySlug) + .where((eb) => + applyJsonataFilter( + eb, + compileJsonataQuery('$search("Article") and $.pub.values.number > 60'), + { communitySlug } + ) + ) + .execute() + + // should find pubs with "Article" and number > 60 + expect(results.map((r) => r.id)).toContain(titlePubId) + expect(results.map((r) => r.id)).not.toContain(title2PubId) // number is 50 + }) +}) + +describe("relation queries", () => { + it("parses outgoing relation path", () => { + const query = parseJsonataQuery("$.pub.out.contributors") + expect(query.condition).toEqual({ + type: "relation", + direction: "out", + fieldSlug: "contributors", + filter: undefined, + }) + }) + + it("parses outgoing relation with value filter", () => { + const query = parseJsonataQuery('$.pub.out.contributors[$.value = "Editor"]') + expect(query.condition).toEqual({ + type: "relation", + direction: "out", + fieldSlug: "contributors", + filter: { + type: "relationComparison", + path: { kind: "relationValue" }, + operator: "=", + value: "Editor", + pathTransform: undefined, + }, + }) + }) + + it("parses outgoing relation with relatedPub value filter", () => { + const query = parseJsonataQuery( + '$.pub.out.contributors[$.relatedPub.values.institution = "MIT"]' + ) + expect(query.condition).toEqual({ + type: "relation", + direction: "out", + fieldSlug: "contributors", + filter: { + type: "relationComparison", + path: { kind: "relatedPubValue", fieldSlug: "institution" }, + operator: "=", + value: "MIT", + pathTransform: undefined, + }, + }) + }) + + it("parses incoming relation path", () => { + const query = parseJsonataQuery("$.pub.in.chapters") + expect(query.condition).toEqual({ + type: "relation", + direction: "in", + fieldSlug: "chapters", + filter: undefined, + }) + }) + + it("parses complex relation filter with and/or", () => { + const query = parseJsonataQuery( + '$.pub.out.contributors[$.value = "Editor" and $.relatedPub.values.institution = "MIT"]' + ) + expect(query.condition.type).toBe("relation") + const cond = query.condition as any + expect(cond.filter.type).toBe("relationLogical") + expect(cond.filter.operator).toBe("and") + }) + + it("finds pubs with outgoing relation", async () => { + const { db } = await import("~/kysely/database") + + const results = await db + .selectFrom("pubs") + .select("pubs.id") + .innerJoin("communities", "communities.id", "pubs.communityId") + .where("communities.slug", "=", communitySlug) + .where((eb) => + applyJsonataFilter(eb, compileJsonataQuery("$.pub.out.contributors"), { + communitySlug, + }) + ) + .execute() + + // should find articles that have contributors + expect(results.map((r) => r.id)).toContain(titlePubId) + expect(results.map((r) => r.id)).toContain(title2PubId) + // should not find authors (they don't have outgoing contributors) + expect(results.map((r) => r.id)).not.toContain(author1Id) + }) + + it("filters by relation value", async () => { + const { db } = await import("~/kysely/database") + + const results = await db + .selectFrom("pubs") + .select("pubs.id") + .innerJoin("communities", "communities.id", "pubs.communityId") + .where("communities.slug", "=", communitySlug) + .where((eb) => + applyJsonataFilter( + eb, + compileJsonataQuery('$.pub.out.contributors[$.value = "Editor"]'), + { communitySlug } + ) + ) + .execute() + + // titlePubId has an Editor contributor + expect(results.map((r) => r.id)).toContain(titlePubId) + // title2PubId only has "Primary Author" + expect(results.map((r) => r.id)).not.toContain(title2PubId) + }) + + it("filters by related pub field value", async () => { + const { db } = await import("~/kysely/database") + + const results = await db + .selectFrom("pubs") + .select("pubs.id") + .innerJoin("communities", "communities.id", "pubs.communityId") + .where("communities.slug", "=", communitySlug) + .where((eb) => + applyJsonataFilter( + eb, + compileJsonataQuery( + '$.pub.out.contributors[$.relatedPub.values.institution = "MIT"]' + ), + { communitySlug } + ) + ) + .execute() + + // titlePubId has Jane Doe from MIT as editor + expect(results.map((r) => r.id)).toContain(titlePubId) + // title2PubId has Bob Wilson from Groningen + expect(results.map((r) => r.id)).not.toContain(title2PubId) + }) + + it("filters by related pub contains function", async () => { + const { db } = await import("~/kysely/database") + + const results = await db + .selectFrom("pubs") + .select("pubs.id") + .innerJoin("communities", "communities.id", "pubs.communityId") + .where("communities.slug", "=", communitySlug) + .where((eb) => + applyJsonataFilter( + eb, + compileJsonataQuery( + '$.pub.out.contributors[$contains($.relatedPub.values.institution, "University")]' + ), + { communitySlug } + ) + ) + .execute() + + // titlePubId has John Smith from University of Groningen + expect(results.map((r) => r.id)).toContain(titlePubId) + // title2PubId has Bob Wilson from University of Groningen + expect(results.map((r) => r.id)).toContain(title2PubId) + }) + + it("combines relation filter with value and relatedPub conditions", async () => { + const { db } = await import("~/kysely/database") + + const results = await db + .selectFrom("pubs") + .select("pubs.id") + .innerJoin("communities", "communities.id", "pubs.communityId") + .where("communities.slug", "=", communitySlug) + .where((eb) => + applyJsonataFilter( + eb, + compileJsonataQuery( + '$.pub.out.contributors[$.value = "Primary Author" and $contains($.relatedPub.values.institution, "Groningen")]' + ), + { communitySlug } + ) + ) + .execute() + + // titlePubId has John Smith as Primary Author from Groningen + expect(results.map((r) => r.id)).toContain(titlePubId) + // title2PubId has Bob Wilson as Primary Author from Groningen + expect(results.map((r) => r.id)).toContain(title2PubId) + }) + + it("finds incoming relations", async () => { + const { db } = await import("~/kysely/database") + + const results = await db + .selectFrom("pubs") + .select("pubs.id") + .innerJoin("communities", "communities.id", "pubs.communityId") + .where("communities.slug", "=", communitySlug) + .where((eb) => + applyJsonataFilter(eb, compileJsonataQuery("$.pub.in.chapters"), { + communitySlug, + }) + ) + .execute() + + // should find chapters that are referenced by books + expect(results.map((r) => r.id)).toContain(chapter1PubId) + expect(results.map((r) => r.id)).toContain(chapter2PubId) + // book should not be found (nothing points to it via chapters) + expect(results.map((r) => r.id)).not.toContain(bookPubId) + }) + + it("filters incoming relations by source pub field", async () => { + const { db } = await import("~/kysely/database") + + const results = await db + .selectFrom("pubs") + .select("pubs.id") + .innerJoin("communities", "communities.id", "pubs.communityId") + .where("communities.slug", "=", communitySlug) + .where((eb) => + applyJsonataFilter( + eb, + compileJsonataQuery( + '$.pub.in.chapters[$contains($.relatedPub.values.title, "Big Book")]' + ), + { communitySlug } + ) + ) + .execute() + + // chapters that belong to "The Big Book" + expect(results.map((r) => r.id)).toContain(chapter1PubId) + expect(results.map((r) => r.id)).toContain(chapter2PubId) + }) + + it("enforces max relation depth", () => { + // this should fail because we nest too many relations + expect(() => { + parseJsonataQuery("$.pub.out.a[$.pub.out.b[$.pub.out.c[$.pub.out.d]]]") + }).toThrow() + }) +}) diff --git a/core/lib/server/jsonata-query/memory-filter.ts b/core/lib/server/jsonata-query/memory-filter.ts index cfa8ce505..d94764122 100644 --- a/core/lib/server/jsonata-query/memory-filter.ts +++ b/core/lib/server/jsonata-query/memory-filter.ts @@ -7,6 +7,10 @@ import type { NotCondition, ParsedCondition, PubFieldPath, + RelationCondition, + RelationContextPath, + RelationFilterCondition, + SearchCondition, } from "./types" import { UnsupportedExpressionError } from "./errors" @@ -178,6 +182,188 @@ function evaluateNot(pub: AnyProcessedPub, condition: NotCondition): boolean { return !evaluateCondition(pub, condition.condition) } +/** + * evaluates a search condition against a pub + * searches across all string values in the pub + */ +function evaluateSearch(pub: AnyProcessedPub, condition: SearchCondition): boolean { + const { query } = condition + const searchTerms = query.toLowerCase().split(/\s+/).filter(Boolean) + if (searchTerms.length === 0) { + return true + } + + // collect all searchable text from the pub + const searchableTexts: string[] = [] + + for (const v of pub.values) { + if (typeof v.value === "string") { + searchableTexts.push(v.value.toLowerCase()) + } else if (Array.isArray(v.value)) { + for (const item of v.value) { + if (typeof item === "string") { + searchableTexts.push(item.toLowerCase()) + } + } + } + } + + // all terms must match somewhere + return searchTerms.every((term) => searchableTexts.some((text) => text.includes(term))) +} + +// ============================================================================ +// relation filter evaluation +// ============================================================================ + +interface RelationContext { + relationValue: unknown + relatedPub: AnyProcessedPub +} + +/** + * extracts a value from relation context based on path + */ +function getRelationContextValue(ctx: RelationContext, path: RelationContextPath): unknown { + switch (path.kind) { + case "relationValue": + return ctx.relationValue + case "relatedPubValue": { + const value = ctx.relatedPub.values.find((v) => { + const fieldSlug = v.fieldSlug + return fieldSlug === path.fieldSlug || fieldSlug.endsWith(`:${path.fieldSlug}`) + }) + return value?.value + } + case "relatedPubBuiltin": + switch (path.field) { + case "id": + return ctx.relatedPub.id + case "createdAt": + return ctx.relatedPub.createdAt + case "updatedAt": + return ctx.relatedPub.updatedAt + case "pubTypeId": + return ctx.relatedPub.pubTypeId + } + break + case "relatedPubType": { + const pubType = (ctx.relatedPub as any).pubType + return pubType?.[path.field] + } + } + return undefined +} + +/** + * evaluates a relation filter condition against a relation context + */ +function evaluateRelationFilter(ctx: RelationContext, filter: RelationFilterCondition): boolean { + switch (filter.type) { + case "relationComparison": { + let value = getRelationContextValue(ctx, filter.path) + value = applyTransform(value, filter.pathTransform) + return compareValues(value, filter.operator, filter.value) + } + case "relationFunction": { + const value = getRelationContextValue(ctx, filter.path) + const args = filter.arguments + switch (filter.name) { + case "contains": + if (typeof value === "string") { + return value.includes(String(args[0])) + } + if (Array.isArray(value)) { + return value.includes(args[0]) + } + return false + case "startsWith": + return typeof value === "string" && value.startsWith(String(args[0])) + case "endsWith": + return typeof value === "string" && value.endsWith(String(args[0])) + case "exists": + return value !== undefined && value !== null + default: + throw new UnsupportedExpressionError( + `unsupported function in relation filter: ${filter.name}` + ) + } + } + case "relationLogical": + if (filter.operator === "and") { + return filter.conditions.every((c) => evaluateRelationFilter(ctx, c)) + } + return filter.conditions.some((c) => evaluateRelationFilter(ctx, c)) + case "relationNot": + return !evaluateRelationFilter(ctx, filter.condition) + } +} + +/** + * evaluates a relation condition against a pub + * + * for "out" relations: check if pub has any values pointing to related pubs matching the filter + * for "in" relations: check if any related pubs point to this pub and match the filter + */ +function evaluateRelation(pub: AnyProcessedPub, condition: RelationCondition): boolean { + const { direction, fieldSlug, filter } = condition + + if (direction === "out") { + // find relation values from this pub + const relationValues = pub.values.filter((v) => { + const matchesSlug = v.fieldSlug === fieldSlug || v.fieldSlug.endsWith(`:${fieldSlug}`) + return matchesSlug && v.relatedPub + }) + + if (relationValues.length === 0) { + return false + } + + // check if any related pub matches the filter + return relationValues.some((rv) => { + if (!filter) { + return true + } + const ctx: RelationContext = { + relationValue: rv.value, + relatedPub: rv.relatedPub as AnyProcessedPub, + } + return evaluateRelationFilter(ctx, filter) + }) + } + + // for "in" relations, we need to check children + // this requires the pub to have children loaded + const children = (pub as any).children as AnyProcessedPub[] | undefined + if (!children || children.length === 0) { + return false + } + + // find children that are connected via this field + return children.some((child) => { + const relationValues = child.values.filter((v) => { + const matchesSlug = v.fieldSlug === fieldSlug || v.fieldSlug.endsWith(`:${fieldSlug}`) + return matchesSlug && v.relatedPubId === pub.id + }) + + if (relationValues.length === 0) { + return false + } + + if (!filter) { + return true + } + + return relationValues.some((rv) => { + const ctx: RelationContext = { + relationValue: rv.value, + relatedPub: child, + } + return evaluateRelationFilter(ctx, filter) + }) + }) +} + /** * evaluates any condition against a pub */ @@ -191,6 +377,10 @@ function evaluateCondition(pub: AnyProcessedPub, condition: ParsedCondition): bo return evaluateLogical(pub, condition) case "not": return evaluateNot(pub, condition) + case "search": + return evaluateSearch(pub, condition) + case "relation": + return evaluateRelation(pub, condition) } } diff --git a/core/lib/server/jsonata-query/parser.ts b/core/lib/server/jsonata-query/parser.ts index 2c806a4f4..4ec69285e 100644 --- a/core/lib/server/jsonata-query/parser.ts +++ b/core/lib/server/jsonata-query/parser.ts @@ -17,6 +17,15 @@ import type { NotCondition, ParsedCondition, PubFieldPath, + RelationComparisonCondition, + RelationCondition, + RelationContextPath, + RelationDirection, + RelationFilterCondition, + RelationFunctionCondition, + RelationLogicalCondition, + RelationNotCondition, + SearchCondition, StringFunction, } from "./types" @@ -29,6 +38,8 @@ export type { ParsedCondition } export interface ParsedQuery { condition: ParsedCondition originalExpression: string + // track max relation depth for validation + maxRelationDepth: number } const COMPARISON_OPS = new Set(["=", "!=", "<", "<=", ">", ">="]) as Set @@ -42,8 +53,11 @@ const SUPPORTED_FUNCTIONS = new Set([ "uppercase", "exists", "not", + "search", ]) +const MAX_RELATION_DEPTH = 3 + function isComparisonOp(op: string): op is ComparisonOperator { return COMPARISON_OPS.has(op as ComparisonOperator) } @@ -133,6 +147,133 @@ function extractPubFieldPath(steps: JsonataPathStep[]): PubFieldPath { throw new InvalidPathError(`unsupported pub path: $.pub.${thirdStep.value}`, stepsToString(steps)) } +/** + * extracts a relation context path from steps inside a relation filter + * + * expects paths like: + * - $.value (the relation's own value) + * - $.relatedPub.values.fieldname + * - $.relatedPub.id + * - $.relatedPub.pubType.name + */ +function extractRelationContextPath(steps: JsonataPathStep[]): RelationContextPath { + if (steps.length < 2) { + throw new InvalidPathError( + "path too short, expected $.value or $.relatedPub.something", + stepsToString(steps) + ) + } + + // first step should be $ (empty variable) + if (steps[0].type !== "variable" || steps[0].value !== "") { + throw new InvalidPathError("path must start with $", stepsToString(steps)) + } + + const secondStep = steps[1] + if (secondStep.type !== "name") { + throw new InvalidPathError("expected name after $", stepsToString(steps)) + } + + // handle $.value - the relation's own value + if (secondStep.value === "value" && steps.length === 2) { + return { kind: "relationValue" } + } + + // handle $.relatedPub... + if (secondStep.value === "relatedPub") { + if (steps.length < 3) { + throw new InvalidPathError("expected field after $.relatedPub", stepsToString(steps)) + } + + const thirdStep = steps[2] + if (thirdStep.type !== "name") { + throw new InvalidPathError("expected name after $.relatedPub", stepsToString(steps)) + } + + // handle builtin fields on related pub + if (["id", "createdAt", "updatedAt", "pubTypeId"].includes(thirdStep.value)) { + return { + kind: "relatedPubBuiltin", + field: thirdStep.value as "id" | "createdAt" | "updatedAt" | "pubTypeId", + } + } + + // handle $.relatedPub.pubType.name or $.relatedPub.pubType.id + if (thirdStep.value === "pubType" && steps.length >= 4) { + const fourthStep = steps[3] + if (fourthStep.type === "name" && ["name", "id"].includes(fourthStep.value)) { + return { kind: "relatedPubType", field: fourthStep.value as "name" | "id" } + } + throw new InvalidPathError( + "expected pubType.name or pubType.id", + stepsToString(steps) + ) + } + + // handle $.relatedPub.values.fieldname + if (thirdStep.value === "values" && steps.length >= 4) { + const fourthStep = steps[3] + if (fourthStep.type === "name") { + return { kind: "relatedPubValue", fieldSlug: fourthStep.value } + } + throw new InvalidPathError( + "expected field name after $.relatedPub.values", + stepsToString(steps) + ) + } + + throw new InvalidPathError( + `unsupported relatedPub path: $.relatedPub.${thirdStep.value}`, + stepsToString(steps) + ) + } + + throw new InvalidPathError( + `unsupported relation context path: $.${secondStep.value}`, + stepsToString(steps) + ) +} + +/** + * checks if a path represents a relation query ($.pub.out.field or $.pub.in.field) + */ +function isRelationPath( + steps: JsonataPathStep[] +): { direction: RelationDirection; fieldSlug: string; filterExpr?: JsonataNode } | null { + if (steps.length < 4) { + return null + } + + // first step should be $ (empty variable) + if (steps[0].type !== "variable" || steps[0].value !== "") { + return null + } + + // second step should be "pub" + if (steps[1].type !== "name" || steps[1].value !== "pub") { + return null + } + + // third step should be "out" or "in" + const thirdStep = steps[2] + if (thirdStep.type !== "name" || !["out", "in"].includes(thirdStep.value)) { + return null + } + + const direction = thirdStep.value as RelationDirection + + // fourth step is the field name (with optional filter) + const fourthStep = steps[3] + if (fourthStep.type !== "name") { + return null + } + + const fieldSlug = fourthStep.value + const filterExpr = fourthStep.stages?.[0]?.expr + + return { direction, fieldSlug, filterExpr } +} + function stepsToString(steps: JsonataPathStep[]): string { return steps.map((s) => (s.type === "variable" ? "$" : s.value)).join(".") } @@ -221,6 +362,18 @@ function parseFunctionCall(node: JsonataFunctionNode): ParsedCondition { throw new UnsupportedExpressionError(`unsupported function: ${funcName}`, funcName) } + // handle $search() - full text search + if (funcName === "search") { + if (node.arguments.length !== 1) { + throw new UnsupportedExpressionError("search() expects exactly one argument") + } + const arg = node.arguments[0] + if (arg.type !== "string") { + throw new UnsupportedExpressionError("search() expects a string argument") + } + return { type: "search", query: arg.value } satisfies SearchCondition + } + // handle not() specially if (funcName === "not") { if (node.arguments.length !== 1) { @@ -266,6 +419,216 @@ function parseFunctionCall(node: JsonataFunctionNode): ParsedCondition { throw new UnsupportedExpressionError(`unhandled function: ${funcName}`, funcName) } +// ============================================================================ +// relation filter parsing (inside [...] of relation queries) +// ============================================================================ + +/** + * parses a comparison inside a relation filter context + */ +function parseRelationComparison( + pathNode: JsonataPathNode | JsonataFunctionNode, + operator: ComparisonOperator, + valueNode: JsonataNode +): RelationComparisonCondition { + let path: RelationContextPath + let pathTransform: StringFunction | undefined + + if (isPathNode(pathNode)) { + path = extractRelationContextPath(pathNode.steps) + } else if (isFunctionNode(pathNode)) { + const funcName = getFunctionName(pathNode.procedure) + if (!["lowercase", "uppercase"].includes(funcName)) { + throw new UnsupportedExpressionError( + `function ${funcName} cannot be used as path transform in relation filter`, + funcName + ) + } + pathTransform = funcName as StringFunction + const arg = pathNode.arguments[0] + if (!isPathNode(arg)) { + throw new UnsupportedExpressionError( + "expected path as first argument to transform function" + ) + } + path = extractRelationContextPath(arg.steps) + } else { + throw new UnsupportedExpressionError( + "expected path or function on left side of comparison in relation filter" + ) + } + + const value = extractLiteral(valueNode) + + return { type: "relationComparison", path, operator, value, pathTransform } +} + +/** + * parses a function call inside a relation filter context + */ +function parseRelationFunctionCall(node: JsonataFunctionNode): RelationFilterCondition { + const funcName = getFunctionName(node.procedure) + + // handle not() specially + if (funcName === "not") { + if (node.arguments.length !== 1) { + throw new UnsupportedExpressionError("not() expects exactly one argument") + } + const inner = parseRelationFilterNode(node.arguments[0]) + return { type: "relationNot", condition: inner } satisfies RelationNotCondition + } + + // handle exists() + if (funcName === "exists") { + if (node.arguments.length !== 1) { + throw new UnsupportedExpressionError("exists() expects exactly one argument") + } + const arg = node.arguments[0] + if (!isPathNode(arg)) { + throw new UnsupportedExpressionError("exists() expects a path argument") + } + const path = extractRelationContextPath(arg.steps) + return { type: "relationFunction", name: "exists", path, arguments: [] } + } + + // string functions: contains, startsWith, endsWith + if (["contains", "startsWith", "endsWith"].includes(funcName)) { + if (node.arguments.length !== 2) { + throw new UnsupportedExpressionError(`${funcName}() expects exactly two arguments`) + } + const pathArg = node.arguments[0] + const valueArg = node.arguments[1] + if (!isPathNode(pathArg)) { + throw new UnsupportedExpressionError(`${funcName}() expects a path as first argument`) + } + const path = extractRelationContextPath(pathArg.steps) + const value = extractLiteral(valueArg) + return { + type: "relationFunction", + name: funcName as StringFunction, + path, + arguments: [value], + } satisfies RelationFunctionCondition + } + + throw new UnsupportedExpressionError( + `unsupported function in relation filter: ${funcName}`, + funcName + ) +} + +/** + * parses a binary node inside a relation filter context + */ +function parseRelationBinary(node: JsonataBinaryNode): RelationFilterCondition { + const op = node.value + + // handle logical operators + if (isLogicalOp(op)) { + const left = parseRelationFilterNode(node.lhs) + const right = parseRelationFilterNode(node.rhs) + return { + type: "relationLogical", + operator: op, + conditions: [left, right], + } satisfies RelationLogicalCondition + } + + // handle "in" operator + if (op === "in") { + if (isPathNode(node.lhs)) { + const path = extractRelationContextPath(node.lhs.steps) + const value = extractLiteral(node.rhs) + return { type: "relationComparison", path, operator: "in", value } + } + if (isLiteralNode(node.lhs) && isPathNode(node.rhs)) { + const path = extractRelationContextPath(node.rhs.steps) + const value = extractLiteral(node.lhs) + return { + type: "relationFunction", + name: "contains", + path, + arguments: [value], + } + } + throw new UnsupportedExpressionError( + "unsupported 'in' expression structure in relation filter" + ) + } + + // handle comparison operators + if (isComparisonOp(op)) { + if (isPathNode(node.lhs) || isFunctionNode(node.lhs)) { + return parseRelationComparison(node.lhs, op, node.rhs) + } + if (isPathNode(node.rhs) || isFunctionNode(node.rhs)) { + const flippedOp = flipOperator(op) + return parseRelationComparison(node.rhs, flippedOp, node.lhs) + } + throw new UnsupportedExpressionError( + "comparison must have at least one path in relation filter" + ) + } + + throw new UnsupportedExpressionError( + `unsupported binary operator in relation filter: ${op}`, + "binary" + ) +} + +/** + * parses any node inside a relation filter context + */ +function parseRelationFilterNode(node: JsonataNode): RelationFilterCondition { + // unwrap block nodes (parentheses) + if (isBlockNode(node)) { + if (node.expressions.length !== 1) { + throw new UnsupportedExpressionError( + "block with multiple expressions not supported in relation filter", + "block" + ) + } + return parseRelationFilterNode(node.expressions[0]) + } + + if (isBinaryNode(node)) { + return parseRelationBinary(node) + } + + if (isFunctionNode(node)) { + return parseRelationFunctionCall(node) + } + + throw new UnsupportedExpressionError( + `unsupported node type in relation filter: ${node.type}`, + node.type + ) +} + +/** + * parses a relation path into a RelationCondition + */ +function parseRelationPath(pathNode: JsonataPathNode): RelationCondition { + const relation = isRelationPath(pathNode.steps) + if (!relation) { + throw new UnsupportedExpressionError("expected relation path") + } + + const { direction, fieldSlug, filterExpr } = relation + + let filter: RelationFilterCondition | undefined + if (filterExpr) { + filter = parseRelationFilterNode(filterExpr) + } + + return { + type: "relation", + direction, + fieldSlug, + filter, + } +} + /** * parses a binary node (comparison or logical) */ @@ -360,9 +723,33 @@ function parseNode(node: JsonataNode): ParsedCondition { return parseFunctionCall(node) } + // check if this is a relation path ($.pub.out.field or $.pub.in.field) + if (isPathNode(node)) { + const relation = isRelationPath(node.steps) + if (relation) { + return parseRelationPath(node) + } + } + throw new UnsupportedExpressionError(`unsupported node type: ${node.type}`, node.type) } +/** + * calculates the max relation depth in a condition + */ +function calculateRelationDepth(condition: ParsedCondition, currentDepth = 0): number { + switch (condition.type) { + case "relation": + return currentDepth + 1 + case "logical": + return Math.max(...condition.conditions.map((c) => calculateRelationDepth(c, currentDepth))) + case "not": + return calculateRelationDepth(condition.condition, currentDepth) + default: + return currentDepth + } +} + /** * parses a jsonata expression string into our query format * @@ -376,9 +763,17 @@ export function parseJsonataQuery(expression: string): ParsedQuery { const ast = jsonata(expression).ast() as JsonataNode const condition = parseNode(ast) + const maxRelationDepth = calculateRelationDepth(condition) + + if (maxRelationDepth > MAX_RELATION_DEPTH) { + throw new UnsupportedExpressionError( + `relation depth ${maxRelationDepth} exceeds maximum allowed depth of ${MAX_RELATION_DEPTH}` + ) + } return { condition, originalExpression: expression, + maxRelationDepth, } } diff --git a/core/lib/server/jsonata-query/sql-builder.ts b/core/lib/server/jsonata-query/sql-builder.ts index 225f46691..ad90d0590 100644 --- a/core/lib/server/jsonata-query/sql-builder.ts +++ b/core/lib/server/jsonata-query/sql-builder.ts @@ -1,4 +1,4 @@ -import type { ExpressionBuilder, ExpressionWrapper } from "kysely" +import type { ExpressionBuilder, ExpressionWrapper, RawBuilder } from "kysely" import type { CompiledQuery } from "./compiler" import type { ComparisonCondition, @@ -7,6 +7,12 @@ import type { NotCondition, ParsedCondition, PubFieldPath, + RelationComparisonCondition, + RelationCondition, + RelationContextPath, + RelationFilterCondition, + RelationFunctionCondition, + SearchCondition, } from "./types" import { sql } from "kysely" @@ -18,6 +24,8 @@ type AnyExpressionWrapper = ExpressionWrapper export interface SqlBuilderOptions { communitySlug?: string + // search config for full-text search + searchLanguage?: string } /** @@ -84,7 +92,7 @@ function buildPubTypeSubquery( .selectFrom("pub_types") .select(eb.lit(1).as("exists_check")) .where("pub_types.id", "=", eb.ref("pubs.pubTypeId")) - .where((innerEb) => buildCondition("pub_types.name")) + .where((_innerEb) => buildCondition("pub_types.name")) ) } @@ -262,6 +270,322 @@ function buildNotCondition( return eb.not(buildCondition(eb, condition.condition, options)) } +/** + * builds the sql condition for a full-text search + */ +function buildSearchCondition( + eb: AnyExpressionBuilder, + condition: SearchCondition, + options?: SqlBuilderOptions +): AnyExpressionWrapper { + const { query } = condition + const language = options?.searchLanguage ?? "english" + + // clean and prepare search terms + const cleanQuery = query.trim().replace(/[:@]/g, "") + if (cleanQuery.length < 2) { + return eb.lit(false) + } + + const terms = cleanQuery.split(/\s+/).filter((word) => word.length >= 2) + if (terms.length === 0) { + return eb.lit(false) + } + + // build tsquery with prefix matching for better UX + const prefixTerms = terms.map((term) => `${term}:*`).join(" & ") + + // searchVector is on pubs table + return sql`pubs."searchVector" @@ to_tsquery(${language}::regconfig, ${prefixTerms})` as unknown as AnyExpressionWrapper +} + +// ============================================================================ +// relation filter sql building +// ============================================================================ + +/** + * converts a relation context path to the appropriate column reference for subquery + */ +function relationPathToColumn( + path: RelationContextPath, + relatedPubAlias: string +): { column: string; isJsonValue: boolean } { + switch (path.kind) { + case "relationValue": + return { column: "pv.value", isJsonValue: true } + case "relatedPubValue": + // we'll handle this via another subquery + return { column: "rpv.value", isJsonValue: true } + case "relatedPubBuiltin": + return { column: `${relatedPubAlias}.${path.field}`, isJsonValue: false } + case "relatedPubType": + if (path.field === "id") { + return { column: `${relatedPubAlias}.pubTypeId`, isJsonValue: false } + } + // name requires a join to pub_types + return { column: "rpt.name", isJsonValue: false } + } +} + +/** + * builds a relation filter condition for use in a subquery + */ +function buildRelationFilterOperator( + eb: AnyExpressionBuilder, + column: string, + operator: string, + value: unknown, + pathTransform: string | undefined, + isJsonValue: boolean +): AnyExpressionWrapper { + let col: RawBuilder | string = column + + if (pathTransform === "lowercase") { + col = sql.raw(`lower(${column}::text)`) + } else if (pathTransform === "uppercase") { + col = sql.raw(`upper(${column}::text)`) + } + + const wrappedValue = isJsonValue ? wrapValue(value) : value + + switch (operator) { + case "=": + return eb(col, "=", wrappedValue) + case "!=": + return eb(col, "!=", wrappedValue) + case "<": + return eb(col, "<", wrappedValue) + case "<=": + return eb(col, "<=", wrappedValue) + case ">": + return eb(col, ">", wrappedValue) + case ">=": + return eb(col, ">=", wrappedValue) + case "in": + if (Array.isArray(value)) { + return eb(col, "in", isJsonValue ? value.map(wrapValue) : value) + } + return eb(col, "=", wrappedValue) + default: + throw new UnsupportedExpressionError( + `unsupported operator in relation filter: ${operator}` + ) + } +} + +/** + * builds a condition for a relation comparison + */ +function buildRelationComparisonCondition( + eb: AnyExpressionBuilder, + condition: RelationComparisonCondition, + relatedPubAlias: string, + options?: SqlBuilderOptions +): AnyExpressionWrapper { + const { path, operator, value, pathTransform } = condition + const { column, isJsonValue } = relationPathToColumn(path, relatedPubAlias) + + // for relatedPubValue, we need to join to the related pub's values + if (path.kind === "relatedPubValue") { + const resolvedSlug = resolveFieldSlug(path.fieldSlug, options) + return eb.exists( + eb + .selectFrom("pub_values as rpv") + .innerJoin("pub_fields as rpf", "rpf.id", "rpv.fieldId") + .select(eb.lit(1).as("rpv_check")) + .where("rpv.pubId", "=", eb.ref(`${relatedPubAlias}.id`)) + .where("rpf.slug", "=", resolvedSlug) + .where((innerEb) => + buildRelationFilterOperator( + innerEb, + "rpv.value", + operator, + value, + pathTransform, + true + ) + ) + ) + } + + // for relatedPubType name, need a subquery to pub_types + if (path.kind === "relatedPubType" && path.field === "name") { + return eb.exists( + eb + .selectFrom("pub_types as rpt") + .select(eb.lit(1).as("rpt_check")) + .where("rpt.id", "=", eb.ref(`${relatedPubAlias}.pubTypeId`)) + .where((innerEb) => + buildRelationFilterOperator( + innerEb, + "rpt.name", + operator, + value, + pathTransform, + false + ) + ) + ) + } + + return buildRelationFilterOperator(eb, column, operator, value, pathTransform, isJsonValue) +} + +/** + * builds a condition for a relation function call + */ +function buildRelationFunctionCondition( + eb: AnyExpressionBuilder, + condition: RelationFunctionCondition, + relatedPubAlias: string, + options?: SqlBuilderOptions +): AnyExpressionWrapper { + const { name, path, arguments: args } = condition + + const buildFunctionInner = (col: string, isJson: boolean) => { + const strArg = String(args[0]) + switch (name) { + case "contains": + return eb(sql.raw(`${col}::text`), "like", `%${strArg}%`) + case "startsWith": + if (isJson) { + return eb(sql.raw(`${col}::text`), "like", `"${strArg}%`) + } + return eb(sql.raw(`${col}::text`), "like", `${strArg}%`) + case "endsWith": + if (isJson) { + return eb(sql.raw(`${col}::text`), "like", `%${strArg}"`) + } + return eb(sql.raw(`${col}::text`), "like", `%${strArg}`) + case "exists": + return eb.lit(true) + default: + throw new UnsupportedExpressionError( + `unsupported function in relation filter: ${name}` + ) + } + } + + // handle relatedPubValue - need subquery + if (path.kind === "relatedPubValue") { + const resolvedSlug = resolveFieldSlug(path.fieldSlug, options) + if (name === "exists") { + return eb.exists( + eb + .selectFrom("pub_values as rpv") + .innerJoin("pub_fields as rpf", "rpf.id", "rpv.fieldId") + .select(eb.lit(1).as("rpv_check")) + .where("rpv.pubId", "=", eb.ref(`${relatedPubAlias}.id`)) + .where("rpf.slug", "=", resolvedSlug) + ) + } + return eb.exists( + eb + .selectFrom("pub_values as rpv") + .innerJoin("pub_fields as rpf", "rpf.id", "rpv.fieldId") + .select(eb.lit(1).as("rpv_check")) + .where("rpv.pubId", "=", eb.ref(`${relatedPubAlias}.id`)) + .where("rpf.slug", "=", resolvedSlug) + .where(() => buildFunctionInner("rpv.value", true)) + ) + } + + // handle relationValue ($.value) + if (path.kind === "relationValue") { + if (name === "exists") { + return eb("pv.value", "is not", null) + } + return buildFunctionInner("pv.value", true) + } + + // handle builtin fields + const { column, isJsonValue } = relationPathToColumn(path, relatedPubAlias) + if (name === "exists") { + return eb(column, "is not", null) + } + return buildFunctionInner(column, isJsonValue) +} + +/** + * builds a relation filter condition recursively + */ +function buildRelationFilter( + eb: AnyExpressionBuilder, + filter: RelationFilterCondition, + relatedPubAlias: string, + options?: SqlBuilderOptions +): AnyExpressionWrapper { + switch (filter.type) { + case "relationComparison": + return buildRelationComparisonCondition(eb, filter, relatedPubAlias, options) + case "relationFunction": + return buildRelationFunctionCondition(eb, filter, relatedPubAlias, options) + case "relationLogical": { + const conditions = filter.conditions.map((c) => + buildRelationFilter(eb, c, relatedPubAlias, options) + ) + return filter.operator === "and" ? eb.and(conditions) : eb.or(conditions) + } + case "relationNot": + return eb.not(buildRelationFilter(eb, filter.condition, relatedPubAlias, options)) + } +} + +/** + * builds the sql condition for a relation query + * + * for "out" relations: find pubs where there's a pub_value with relatedPubId pointing out + * for "in" relations: find pubs that are referenced by other pubs via the given field + */ +function buildRelationCondition( + eb: AnyExpressionBuilder, + condition: RelationCondition, + options?: SqlBuilderOptions +): AnyExpressionWrapper { + const { direction, fieldSlug, filter } = condition + const resolvedSlug = resolveFieldSlug(fieldSlug, options) + + if (direction === "out") { + // outgoing relation: this pub has a value that points to another pub + // pv.pubId = pubs.id and pv.relatedPubId = related_pub.id + let subquery = eb + .selectFrom("pub_values as pv") + .innerJoin("pub_fields as pf", "pf.id", "pv.fieldId") + .innerJoin("pubs as related_pub", "related_pub.id", "pv.relatedPubId") + .select(eb.lit(1).as("rel_check")) + .where("pv.pubId", "=", eb.ref("pubs.id")) + .where("pf.slug", "=", resolvedSlug) + .where("pv.relatedPubId", "is not", null) + + if (filter) { + subquery = subquery.where((innerEb) => + buildRelationFilter(innerEb, filter, "related_pub", options) + ) + } + + return eb.exists(subquery) + } + + // incoming relation: another pub has a value pointing to this pub + // pv.relatedPubId = pubs.id and pv.pubId = source_pub.id + let subquery = eb + .selectFrom("pub_values as pv") + .innerJoin("pub_fields as pf", "pf.id", "pv.fieldId") + .innerJoin("pubs as source_pub", "source_pub.id", "pv.pubId") + .select(eb.lit(1).as("rel_check")) + .where("pv.relatedPubId", "=", eb.ref("pubs.id")) + .where("pf.slug", "=", resolvedSlug) + + if (filter) { + // for incoming, the "relatedPub" in the filter context is the source_pub + subquery = subquery.where((innerEb) => + buildRelationFilter(innerEb, filter, "source_pub", options) + ) + } + + return eb.exists(subquery) +} + /** * builds the sql condition for any parsed condition */ @@ -279,6 +603,10 @@ function buildCondition( return buildLogicalCondition(eb, condition, options) case "not": return buildNotCondition(eb, condition, options) + case "search": + return buildSearchCondition(eb, condition, options) + case "relation": + return buildRelationCondition(eb, condition, options) } } diff --git a/core/lib/server/jsonata-query/types.ts b/core/lib/server/jsonata-query/types.ts index d5f0a6c02..240f1f35b 100644 --- a/core/lib/server/jsonata-query/types.ts +++ b/core/lib/server/jsonata-query/types.ts @@ -92,6 +92,13 @@ export type PubFieldPath = | { kind: "builtin"; field: "id" | "createdAt" | "updatedAt" | "pubTypeId" } | { kind: "pubType"; field: "name" | "id" } +// paths for use inside relation filters +export type RelationContextPath = + | { kind: "relationValue" } // $.value - the value of the relation itself + | { kind: "relatedPubValue"; fieldSlug: string } // $.relatedPub.values.fieldname + | { kind: "relatedPubBuiltin"; field: "id" | "createdAt" | "updatedAt" | "pubTypeId" } + | { kind: "relatedPubType"; field: "name" | "id" } + export type LiteralValue = string | number | boolean | null | LiteralValue[] export interface ComparisonCondition { @@ -121,11 +128,62 @@ export interface NotCondition { condition: ParsedCondition } +// full-text search condition +export interface SearchCondition { + type: "search" + query: string +} + +// condition used inside relation filters (different context) +export interface RelationComparisonCondition { + type: "relationComparison" + path: RelationContextPath + operator: ComparisonOperator + value: LiteralValue + pathTransform?: StringFunction +} + +export interface RelationFunctionCondition { + type: "relationFunction" + name: StringFunction | BooleanFunction + path: RelationContextPath + arguments: LiteralValue[] +} + +export interface RelationLogicalCondition { + type: "relationLogical" + operator: LogicalOperator + conditions: RelationFilterCondition[] +} + +export interface RelationNotCondition { + type: "relationNot" + condition: RelationFilterCondition +} + +export type RelationFilterCondition = + | RelationComparisonCondition + | RelationFunctionCondition + | RelationLogicalCondition + | RelationNotCondition + +// relation query condition: $.pub.out.fieldname[filter] or $.pub.in.fieldname[filter] +export type RelationDirection = "out" | "in" + +export interface RelationCondition { + type: "relation" + direction: RelationDirection + fieldSlug: string + filter?: RelationFilterCondition +} + export type ParsedCondition = | ComparisonCondition | FunctionCondition | LogicalCondition | NotCondition + | SearchCondition + | RelationCondition // re-export for convenience (also exported from parser.ts) export type { ParsedCondition as Condition } From abf76157cb778f8feb062e0b8180d6c73e3d4074 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 19 Jan 2026 19:51:56 +0100 Subject: [PATCH 3/9] feat: replace automation resolver with pub query --- core/actions/_lib/resolveAutomationInput.ts | 355 ++++++++++-------- .../StagePanelAutomationForm.tsx | 24 +- core/lib/server/jsonata-query/parser.ts | 66 ++-- core/lib/server/jsonata-query/sql-builder.ts | 3 +- core/lib/server/jsonata-query/types.ts | 24 +- core/lib/server/pub.ts | 10 + 6 files changed, 276 insertions(+), 206 deletions(-) diff --git a/core/actions/_lib/resolveAutomationInput.ts b/core/actions/_lib/resolveAutomationInput.ts index c031abe37..e2b9cda11 100644 --- a/core/actions/_lib/resolveAutomationInput.ts +++ b/core/actions/_lib/resolveAutomationInput.ts @@ -4,11 +4,16 @@ import type { FullAutomation, Json } from "db/types" import type { InterpolationContext } from "./interpolationContext" import { interpolate } from "@pubpub/json-interpolate" +import jsonata from "jsonata" import { logger } from "logger" import { tryCatch } from "utils/try-catch" -import { db } from "~/kysely/database" import { getPubsWithRelatedValues } from "~/lib/server" +import { + applyJsonataFilter, + compileJsonataQuery, + parseJsonataQuery, +} from "~/lib/server/jsonata-query" type ResolvedPub = ProcessedPub<{ withPubType: true @@ -22,154 +27,158 @@ export type ResolvedInput = | { type: "json"; json: Json } | { type: "unchanged" } -type ResolverExpressionType = - | { kind: "comparison"; leftPath: string; operator: string; rightPath: string } - | { kind: "transform" } - | { kind: "unknown" } - /** - * Parses a resolver expression to understand its structure. - * - * Supports comparison expressions like: - * - `$.json.some.id = $.pub.values.fieldname` - matches incoming json against pub field values - * - `$.pub.id = someExternalId` - matches pub by id - * - * Or transformation expressions that just transform the input. + * parses template string to find all {{ }} interpolation blocks */ -function parseResolverExpression(expression: string): ResolverExpressionType { - // Simple regex to detect comparison patterns - // This handles basic cases like `$.path.to.value = $.other.path` - const comparisonMatch = expression.match( - /^\s*(\$\.[^\s=!<>]+)\s*(=|!=|<|>|<=|>=)\s*(\$\.[^\s]+|\S+)\s*$/ - ) +function parseInterpolationBlocks( + template: string +): { expression: string; startIndex: number; endIndex: number }[] { + const blocks: { expression: string; startIndex: number; endIndex: number }[] = [] + let i = 0 - if (comparisonMatch) { - const [, leftPath, operator, rightPath] = comparisonMatch - return { - kind: "comparison", - leftPath, - operator, - rightPath, - } - } + while (i < template.length) { + if (template[i] === "{" && template[i + 1] === "{") { + const startIndex = i + i += 2 - // Check if it looks like a transform (returns an object or processes data) - if (expression.includes("{") || expression.includes("$map") || expression.includes("$filter")) { - return { kind: "transform" } - } + let braceDepth = 0 + let expression = "" + let foundClosing = false - return { kind: "unknown" } -} + while (i < template.length) { + const char = template[i] + const nextChar = template[i + 1] -/** - * Extracts the field slug from a pub values path like `$.pub.values.fieldname` - */ -function extractFieldSlugFromPath(path: string): string | null { - // Handle both formats: - // - $.pub.values.fieldname - // - $.pub.values["field-name"] or $.pub.values['field-name'] - const simpleMatch = path.match(/^\$\.pub\.values\.([a-zA-Z_][a-zA-Z0-9_-]*)$/) - if (simpleMatch) { - return simpleMatch[1] - } + if (char === "}" && nextChar === "}" && braceDepth === 0) { + foundClosing = true + blocks.push({ + expression: expression.trim(), + startIndex, + endIndex: i + 2, + }) + i += 2 + break + } + + if (char === "{") { + braceDepth++ + } else if (char === "}") { + braceDepth-- + } - const bracketMatch = path.match(/^\$\.pub\.values\[["']([^"']+)["']\]$/) - if (bracketMatch) { - return bracketMatch[1] + expression += char + i++ + } + + if (!foundClosing) { + throw new Error(`unclosed interpolation block starting at position ${startIndex}`) + } + } else { + i++ + } } - return null + return blocks } /** - * Extracts pubId from a path like `$.pub.id` + * converts a javascript value to a jsonata literal representation */ -function isPubIdPath(path: string): boolean { - return path === "$.pub.id" +function valueToJsonataLiteral(value: unknown): string { + if (value === null) { + return "null" + } + if (typeof value === "string") { + // escape quotes and wrap in quotes + return JSON.stringify(value) + } + if (typeof value === "number" || typeof value === "boolean") { + return String(value) + } + if (Array.isArray(value)) { + return `[${value.map(valueToJsonataLiteral).join(", ")}]` + } + if (typeof value === "object") { + const entries = Object.entries(value) + .map(([k, v]) => `"${k}": ${valueToJsonataLiteral(v)}`) + .join(", ") + return `{${entries}}` + } + return String(value) } /** - * Resolves a value from the interpolation context using a JSONata path. + * interpolates {{ }} blocks in a resolver expression, replacing them with literal values + * + * @example + * input: `$.pub.values.externalId = {{ $.json.body.articleId }}` + * context: { json: { body: { articleId: "abc123" } } } + * output: `$.pub.values.externalId = "abc123"` */ -async function resolvePathValue(path: string, context: InterpolationContext): Promise { - const [error, result] = await tryCatch(interpolate(path, context)) - if (error) { - logger.warn("Failed to resolve path value", { path, error: error.message }) - return undefined +async function interpolateResolverExpression( + expression: string, + context: InterpolationContext +): Promise { + const blocks = parseInterpolationBlocks(expression) + + if (blocks.length === 0) { + return expression } - return result -} -/** - * Finds a pub where a field value matches the given value. - */ -async function findPubByFieldValue( - communityId: CommunitiesId, - fieldSlug: string, - value: unknown, - communitySlug: string -): Promise { - const slugWithCommunitySlug = fieldSlug.startsWith(`${communitySlug}:`) - ? fieldSlug - : `${communitySlug}:${fieldSlug}` - - const pubs = (await getPubsWithRelatedValues( - { communityId }, - { - withPubType: true, - withRelatedPubs: true, - withStage: false, - withValues: true, - depth: 3, - filters: { - [slugWithCommunitySlug]: { $eq: value }, - }, - limit: 1, + let result = expression + + // process in reverse order to maintain correct indices + for (let i = blocks.length - 1; i >= 0; i--) { + const block = blocks[i] + const jsonataExpr = jsonata(block.expression) + const value = await jsonataExpr.evaluate(context) + + if (value === undefined) { + throw new Error( + `resolver interpolation '${block.expression}' returned undefined` + ) } - )) as ResolvedPub[] - if (pubs.length > 0) { - return pubs[0] + const literal = valueToJsonataLiteral(value) + result = result.slice(0, block.startIndex) + literal + result.slice(block.endIndex) } - logger.debug("No pub found with matching field value", { fieldSlug, value }) - return null + return result } -async function findPubById(communityId: CommunitiesId, pubId: PubsId): Promise { - const [error, pub] = await tryCatch( - getPubsWithRelatedValues( - { pubId, communityId }, - { - withPubType: true, - withRelatedPubs: true, - withStage: false, - withValues: true, - depth: 3, - } - ) - ) - - if (error) { - logger.warn("Failed to find pub by id", { pubId, error: error.message }) - return null +/** + * determines if expression is a query (for finding pubs) or a transform (returning json) + */ +function isQueryExpression(expression: string): boolean { + // queries start with $.pub and contain comparison operators or relation paths + if (!expression.includes("$.pub")) { + return false } - - return pub as ResolvedPub + // check for comparison operators or relation syntax + return /\s*(=|!=|<|>|<=|>=|in)\s*/.test(expression) || + expression.includes("$.pub.out.") || + expression.includes("$.pub.in.") || + expression.includes("$search(") || + expression.includes("$contains(") || + expression.includes("$startsWith(") || + expression.includes("$endsWith(") || + expression.includes("$exists(") } /** - * Resolves the automation input based on a resolver expression. + * resolves the automation input based on a resolver expression. * - * The resolver expression is a JSONata expression that can: - * 1. Resolve a different Pub using comparisons like `$.json.some.id = $.pub.values.fieldname` - * 2. Transform JSON input into a new structure for actions + * the resolver expression can be: + * 1. a query to find a pub, e.g. `$.pub.values.externalId = {{ $.json.body.id }}` + * 2. a transform to restructure the input, e.g. `{ "title": $.json.body.name }` * - * @param resolver - The JSONata resolver expression from the automation - * @param context - The interpolation context containing pub, json, community, etc. - * @param communityId - The community ID to search for pubs - * @param communitySlug - The community slug for field lookups - * @returns The resolved input (pub, json, or unchanged) + * use {{ expr }} syntax to interpolate values from the context into the expression. + * + * @param resolver - the resolver expression from the automation + * @param context - the interpolation context containing pub, json, community, etc. + * @param communityId - the community ID to search for pubs + * @param communitySlug - the community slug for field lookups + * @returns the resolved input (pub, json, or unchanged) */ export async function resolveAutomationInput( resolver: string, @@ -177,66 +186,92 @@ export async function resolveAutomationInput( communityId: CommunitiesId, communitySlug: string ): Promise { - const parsed = parseResolverExpression(resolver) + // first, interpolate any {{ }} blocks + const [interpolateError, interpolatedExpression] = await tryCatch( + interpolateResolverExpression(resolver, context) + ) - if (parsed.kind === "comparison") { - // For comparison expressions, we resolve the left side and search for a pub - // where the right side matches - const leftValue = await resolvePathValue(parsed.leftPath, context) + if (interpolateError) { + logger.warn("failed to interpolate resolver expression", { + resolver, + error: interpolateError.message, + }) + return { type: "unchanged" } + } - if (leftValue === undefined) { - logger.warn("Resolver left path resolved to undefined", { path: parsed.leftPath }) - return { type: "unchanged" } - } + // determine if this is a query (to find a pub) or a transform + if (isQueryExpression(interpolatedExpression)) { + // parse and compile the query + const [parseError, parsedQuery] = await tryCatch( + Promise.resolve(parseJsonataQuery(interpolatedExpression)) + ) - // Check if right side is a pub field value path - const fieldSlug = extractFieldSlugFromPath(parsed.rightPath) - if (fieldSlug) { - const pub = await findPubByFieldValue(communityId, fieldSlug, leftValue, communitySlug) - if (pub) { - return { type: "pub", pub } - } - logger.debug("No pub found matching resolver comparison", { - leftPath: parsed.leftPath, - leftValue, - fieldSlug, + if (parseError) { + logger.warn("failed to parse resolver as query", { + expression: interpolatedExpression, + error: parseError.message, }) - return { type: "unchanged" } - } + // fall through to transform mode + } else { + const compiled = compileJsonataQuery(interpolatedExpression) + + // execute the query to find a matching pub + const [queryError, pubs] = await tryCatch( + getPubsWithRelatedValues( + { communityId }, + { + withPubType: true, + withRelatedPubs: true, + withStage: false, + withValues: true, + depth: 3, + limit: 1, + customFilter: (eb) => + applyJsonataFilter(eb, compiled, { communitySlug }), + } + ) + ) + console.log("pubs", pubs) - // Check if right side is pub.id - if (isPubIdPath(parsed.rightPath)) { - // In this case, we're looking for a pub where id = leftValue - const pub = await findPubById(communityId, leftValue as PubsId) - if (pub) { - return { type: "pub", pub } + if (queryError) { + logger.warn("failed to execute resolver query", { + expression: interpolatedExpression, + error: queryError.message, + }) + return { type: "unchanged" } } + + if (pubs.length > 0) { + return { type: "pub", pub: pubs[0] as ResolvedPub } + } + + logger.debug("no pub found matching resolver query", { + expression: interpolatedExpression, + }) return { type: "unchanged" } } - - // If we can't parse the right side, try evaluating as a transform - logger.debug("Could not parse right side of comparison, treating as transform", { - rightPath: parsed.rightPath, - }) } - // For transform expressions or unknown patterns, evaluate the entire expression - const [error, result] = await tryCatch(interpolate(resolver, context)) + + // treat as a transform expression + const [transformError, result] = await tryCatch(interpolate(resolver, context)) - if (error) { - logger.error("Failed to evaluate resolver expression", { resolver, error: error.message }) + if (transformError) { + logger.error("failed to evaluate resolver transform", { + resolver, + error: transformError.message, + }) return { type: "unchanged" } } - // Otherwise, treat it as JSON return { type: "json", json: result as Json } } /** - * Checks if an automation has a resolver configured. + * checks if an automation has a resolver configured. */ -export function hasResolver(automation: FullAutomation): automation is FullAutomation & { - resolver: string -} { +export function hasResolver( + automation: FullAutomation +): automation is FullAutomation & { resolver: string } { return Boolean(automation.resolver && automation.resolver.trim().length > 0) } diff --git a/core/app/c/[communitySlug]/stages/manage/components/panel/automationsTab/StagePanelAutomationForm.tsx b/core/app/c/[communitySlug]/stages/manage/components/panel/automationsTab/StagePanelAutomationForm.tsx index 13b2b68de..0e3d808df 100644 --- a/core/app/c/[communitySlug]/stages/manage/components/panel/automationsTab/StagePanelAutomationForm.tsx +++ b/core/app/c/[communitySlug]/stages/manage/components/panel/automationsTab/StagePanelAutomationForm.tsx @@ -401,13 +401,21 @@ const ResolverFieldSection = memo( JSON input before actions run.

- Comparison expressions like{" "} - $.json.some.id = $.pub.values.fieldname will - find a Pub where the field matches the left side value. + Query expressions find a Pub matching + conditions. Use {"{{ expr }}"} to interpolate + values from incoming data:
+ + {"$.pub.values.externalId = {{ $.json.body.articleId }}"} +
- Transform expressions can restructure the - input data for the automation's actions. +
+ Transform expressions restructure input + data for actions: +
+ + {'{ "title": $.json.body.name }'} +

@@ -428,15 +436,15 @@ const ResolverFieldSection = memo( - Use a JSONata expression to resolve a Pub by comparing values, e.g.,{" "} - $.json.id = $.pub.values.externalId + Find a Pub by query, e.g.{" "} + {"$.pub.values.externalId = {{ $.json.body.id }}"} {fieldState.error && ( {fieldState.error.message} diff --git a/core/lib/server/jsonata-query/parser.ts b/core/lib/server/jsonata-query/parser.ts index 4ec69285e..83f581c55 100644 --- a/core/lib/server/jsonata-query/parser.ts +++ b/core/lib/server/jsonata-query/parser.ts @@ -1,32 +1,34 @@ -import type { - ComparisonCondition, - ComparisonOperator, - JsonataBinaryNode, - JsonataBlockNode, - JsonataFunctionNode, - JsonataNode, - JsonataNumberNode, - JsonataPathNode, - JsonataPathStep, - JsonataStringNode, - JsonataUnaryNode, - JsonataValueNode, - LiteralValue, - LogicalCondition, - LogicalOperator, - NotCondition, - ParsedCondition, - PubFieldPath, - RelationComparisonCondition, - RelationCondition, - RelationContextPath, - RelationDirection, - RelationFilterCondition, - RelationFunctionCondition, - RelationLogicalCondition, - RelationNotCondition, - SearchCondition, - StringFunction, +import { + BUILTIN_FIELDS, + type BuiltinField, + type ComparisonCondition, + type ComparisonOperator, + type JsonataBinaryNode, + type JsonataBlockNode, + type JsonataFunctionNode, + type JsonataNode, + type JsonataNumberNode, + type JsonataPathNode, + type JsonataPathStep, + type JsonataStringNode, + type JsonataUnaryNode, + type JsonataValueNode, + type LiteralValue, + type LogicalCondition, + type LogicalOperator, + type NotCondition, + type ParsedCondition, + type PubFieldPath, + type RelationComparisonCondition, + type RelationCondition, + type RelationContextPath, + type RelationDirection, + type RelationFilterCondition, + type RelationFunctionCondition, + type RelationLogicalCondition, + type RelationNotCondition, + type SearchCondition, + type StringFunction, } from "./types" import jsonata from "jsonata" @@ -92,6 +94,8 @@ function isLiteralNode( return node.type === "string" || node.type === "number" || node.type === "value" } + + /** * extracts the pub field path from a jsonata path node * @@ -122,8 +126,8 @@ function extractPubFieldPath(steps: JsonataPathStep[]): PubFieldPath { } // handle builtin fields - if (["id", "createdAt", "updatedAt", "pubTypeId"].includes(thirdStep.value)) { - return { kind: "builtin", field: thirdStep.value as "id" | "createdAt" | "updatedAt" } + if (BUILTIN_FIELDS.includes(thirdStep.value as BuiltinField)) { + return { kind: "builtin", field: thirdStep.value as BuiltinField } } // handle pubType.name or pubType.id diff --git a/core/lib/server/jsonata-query/sql-builder.ts b/core/lib/server/jsonata-query/sql-builder.ts index ad90d0590..8e0829bbd 100644 --- a/core/lib/server/jsonata-query/sql-builder.ts +++ b/core/lib/server/jsonata-query/sql-builder.ts @@ -1,6 +1,7 @@ import type { ExpressionBuilder, ExpressionWrapper, RawBuilder } from "kysely" import type { CompiledQuery } from "./compiler" import type { + BuiltinField, ComparisonCondition, FunctionCondition, LogicalCondition, @@ -33,7 +34,7 @@ export interface SqlBuilderOptions { */ function pathToColumn( path: PubFieldPath -): "value" | "pubs.createdAt" | "pubs.updatedAt" | "pubs.id" | "pubs.pubTypeId" { +): "value" | `pubs.${BuiltinField}` { if (path.kind === "builtin") { return `pubs.${path.field}` as const } diff --git a/core/lib/server/jsonata-query/types.ts b/core/lib/server/jsonata-query/types.ts index 240f1f35b..747db3a71 100644 --- a/core/lib/server/jsonata-query/types.ts +++ b/core/lib/server/jsonata-query/types.ts @@ -82,21 +82,33 @@ export type JsonataNode = | JsonataBlockNode // our internal representation -export type ComparisonOperator = "=" | "!=" | "<" | "<=" | ">" | ">=" | "in" -export type LogicalOperator = "and" | "or" -export type StringFunction = "contains" | "startsWith" | "endsWith" | "lowercase" | "uppercase" -export type BooleanFunction = "exists" | "not" +export const COMPARISON_OPS = ["=", "!=", "<", "<=", ">", ">=", "in"] as const +export type ComparisonOperator = (typeof COMPARISON_OPS)[number] + + +export const LOGICAL_OPS = ["and", "or"] as const +export type LogicalOperator = (typeof LOGICAL_OPS)[number] + +export const STRING_FUNCTIONS = ["contains", "startsWith", "endsWith", "lowercase", "uppercase"] as const +export type StringFunction = (typeof STRING_FUNCTIONS)[number] + +export const BOOLEAN_FUNCTIONS = ["exists", "not"] as const +export type BooleanFunction = (typeof BOOLEAN_FUNCTIONS)[number] + + +export const BUILTIN_FIELDS = ["id", "createdAt", "updatedAt", "pubTypeId", "title", "stageId"] as const +export type BuiltinField = (typeof BUILTIN_FIELDS)[number] export type PubFieldPath = | { kind: "value"; fieldSlug: string } - | { kind: "builtin"; field: "id" | "createdAt" | "updatedAt" | "pubTypeId" } + | { kind: "builtin"; field: BuiltinField } | { kind: "pubType"; field: "name" | "id" } // paths for use inside relation filters export type RelationContextPath = | { kind: "relationValue" } // $.value - the value of the relation itself | { kind: "relatedPubValue"; fieldSlug: string } // $.relatedPub.values.fieldname - | { kind: "relatedPubBuiltin"; field: "id" | "createdAt" | "updatedAt" | "pubTypeId" } + | { kind: "relatedPubBuiltin"; field: BuiltinField } | { kind: "relatedPubType"; field: "name" | "id" } export type LiteralValue = string | number | boolean | null | LiteralValue[] diff --git a/core/lib/server/pub.ts b/core/lib/server/pub.ts index afc9a6a56..55e591f17 100644 --- a/core/lib/server/pub.ts +++ b/core/lib/server/pub.ts @@ -31,6 +31,7 @@ import type { DefinitelyHas, MaybeHas, XOR } from "utils/types" import { type AliasedSelectQueryBuilder, type ExpressionBuilder, + type ExpressionWrapper, type Kysely, type ReferenceExpression, type SelectExpression, @@ -1237,6 +1238,11 @@ export interface GetPubsWithRelatedValuesOptions onlyTitles?: boolean trx?: typeof db filters?: Filter + /** + * A custom filter function that receives the expression builder and returns a filter condition. + * Useful for applying JSONata-based filters or other complex conditions. + */ + customFilter?: (eb: ExpressionBuilder) => ExpressionWrapper /** * Constraints on which pub types the user/token has access to. Will also filter related pubs. */ @@ -1732,6 +1738,10 @@ export async function getPubsWithRelatedValues qb.where((eb) => applyFilters(eb, options!.filters!)) ) + // custom filter (e.g. jsonata-based) + .$if(Boolean(options?.customFilter), (qb) => + qb.where((eb) => options!.customFilter!(eb)) + ) .$if(Boolean(orderBy), (qb) => qb.orderBy(orderBy!, orderDirection ?? "desc")) .$if(Boolean(limit), (qb) => qb.limit(limit!)) .$if(Boolean(offset), (qb) => qb.offset(offset!)) From d7c440976a86e428e564242a5536cfac33c8da62 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 19 Jan 2026 20:14:00 +0100 Subject: [PATCH 4/9] fix: last query problems --- core/actions/_lib/resolveAutomationInput.ts | 20 +- .../StagePanelAutomationForm.tsx | 112 +++++----- core/lib/server/jsonata-query/SYNTAX.md | 194 ++++++++++++++++++ core/lib/server/jsonata-query/compiler.ts | 2 +- core/lib/server/jsonata-query/index.ts | 19 +- .../lib/server/jsonata-query/memory-filter.ts | 39 +++- core/lib/server/jsonata-query/parser.ts | 92 +++++++-- core/lib/server/jsonata-query/sql-builder.ts | 71 +++++-- core/lib/server/jsonata-query/types.ts | 22 +- core/package.json | 5 +- 10 files changed, 449 insertions(+), 127 deletions(-) create mode 100644 core/lib/server/jsonata-query/SYNTAX.md diff --git a/core/actions/_lib/resolveAutomationInput.ts b/core/actions/_lib/resolveAutomationInput.ts index e2b9cda11..cdce995fb 100644 --- a/core/actions/_lib/resolveAutomationInput.ts +++ b/core/actions/_lib/resolveAutomationInput.ts @@ -1,10 +1,11 @@ import type { ProcessedPub } from "contracts" -import type { CommunitiesId, PubsId } from "db/public" +import type { CommunitiesId } from "db/public" import type { FullAutomation, Json } from "db/types" import type { InterpolationContext } from "./interpolationContext" -import { interpolate } from "@pubpub/json-interpolate" import jsonata from "jsonata" + +import { interpolate } from "@pubpub/json-interpolate" import { logger } from "logger" import { tryCatch } from "utils/try-catch" @@ -134,9 +135,7 @@ async function interpolateResolverExpression( const value = await jsonataExpr.evaluate(context) if (value === undefined) { - throw new Error( - `resolver interpolation '${block.expression}' returned undefined` - ) + throw new Error(`resolver interpolation '${block.expression}' returned undefined`) } const literal = valueToJsonataLiteral(value) @@ -155,7 +154,8 @@ function isQueryExpression(expression: string): boolean { return false } // check for comparison operators or relation syntax - return /\s*(=|!=|<|>|<=|>=|in)\s*/.test(expression) || + return ( + /\s*(=|!=|<|>|<=|>=|in)\s*/.test(expression) || expression.includes("$.pub.out.") || expression.includes("$.pub.in.") || expression.includes("$search(") || @@ -163,6 +163,7 @@ function isQueryExpression(expression: string): boolean { expression.includes("$startsWith(") || expression.includes("$endsWith(") || expression.includes("$exists(") + ) } /** @@ -202,7 +203,7 @@ export async function resolveAutomationInput( // determine if this is a query (to find a pub) or a transform if (isQueryExpression(interpolatedExpression)) { // parse and compile the query - const [parseError, parsedQuery] = await tryCatch( + const [parseError, _parsedQuery] = await tryCatch( Promise.resolve(parseJsonataQuery(interpolatedExpression)) ) @@ -226,12 +227,10 @@ export async function resolveAutomationInput( withValues: true, depth: 3, limit: 1, - customFilter: (eb) => - applyJsonataFilter(eb, compiled, { communitySlug }), + customFilter: (eb) => applyJsonataFilter(eb, compiled, { communitySlug }), } ) ) - console.log("pubs", pubs) if (queryError) { logger.warn("failed to execute resolver query", { @@ -252,7 +251,6 @@ export async function resolveAutomationInput( } } - // treat as a transform expression const [transformError, result] = await tryCatch(interpolate(resolver, context)) diff --git a/core/app/c/[communitySlug]/stages/manage/components/panel/automationsTab/StagePanelAutomationForm.tsx b/core/app/c/[communitySlug]/stages/manage/components/panel/automationsTab/StagePanelAutomationForm.tsx index 0e3d808df..295ab03a6 100644 --- a/core/app/c/[communitySlug]/stages/manage/components/panel/automationsTab/StagePanelAutomationForm.tsx +++ b/core/app/c/[communitySlug]/stages/manage/components/panel/automationsTab/StagePanelAutomationForm.tsx @@ -45,7 +45,9 @@ import { Input } from "ui/input" import { Item, ItemContent, ItemHeader } from "ui/item" import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "ui/select" import { FormSubmitButton } from "ui/submit-button" +import { Textarea } from "ui/textarea" import { type TokenContext, TokenProvider } from "ui/tokens" +import { Tooltip, TooltipContent, TooltipTrigger } from "ui/tooltip" import { cn } from "utils" import { ActionConfigBuilder } from "~/actions/_lib/ActionConfigBuilder" @@ -386,70 +388,82 @@ const ResolverFieldSection = memo( form: UseFormReturn resolver: string | null | undefined }) { + const hasResolver = props.resolver !== undefined && props.resolver !== null + + if (!hasResolver) { + return ( + + ) + } + return ( ( - -
- - Resolver (optional) - -

- A JSONata expression to resolve a different Pub or transform - JSON input before actions run. -
-
- Query expressions find a Pub matching - conditions. Use {"{{ expr }}"} to interpolate - values from incoming data: -
- - {"$.pub.values.externalId = {{ $.json.body.articleId }}"} - -
-
- Transform expressions restructure input - data for actions: -
- - {'{ "title": $.json.body.name }'} - -

- - - {props.resolver && ( - - )} +
+
+
+ + + + Resolver + + + +

+ A JSONata expression to resolve a different Pub or + transform JSON input before actions run. +

+

+ Query:{" "} + + {"$.pub.values.externalId = {{ $.json.body.id }}"} + +

+

+ Transform:{" "} + + {'{ "title": $.json.body.name }'} + +

+ + +
+
- - - Find a Pub by query, e.g.{" "} - {"$.pub.values.externalId = {{ $.json.body.id }}"} - {fieldState.error && ( - {fieldState.error.message} +

+ {fieldState.error.message} +

)} - +
)} /> ) diff --git a/core/lib/server/jsonata-query/SYNTAX.md b/core/lib/server/jsonata-query/SYNTAX.md new file mode 100644 index 000000000..70b4b031f --- /dev/null +++ b/core/lib/server/jsonata-query/SYNTAX.md @@ -0,0 +1,194 @@ +# JSONata Query Syntax + +Query language for filtering pubs. Based on JSONata with restrictions. + +## Paths + +### Pub field values + +``` +$.pub.values.title +$.pub.values.externalId +``` + +### Builtin fields + +``` +$.pub.id +$.pub.createdAt +$.pub.updatedAt +$.pub.pubTypeId +$.pub.title +$.pub.stageId +``` + +### Pub type + +``` +$.pub.pubType.name +$.pub.pubType.id +``` + +## Comparison operators + +### Equality + +``` +$.pub.values.title = "Test" +$.pub.values.count != 0 +``` + +### Numeric comparison + +``` +$.pub.values.count > 10 +$.pub.values.count >= 10 +$.pub.values.count < 100 +$.pub.values.count <= 100 +``` + +### In array + +``` +$.pub.values.status in ["draft", "published"] +``` + +## Logical operators + +### And + +``` +$.pub.values.status = "published" and $.pub.values.count > 0 +``` + +### Or + +``` +$.pub.values.status = "draft" or $.pub.values.status = "pending" +``` + +### Not + +``` +$not($.pub.values.archived = true) +``` + +## String functions + +### Contains + +``` +$contains($.pub.values.title, "chapter") +``` + +### Starts with + +``` +$startsWith($.pub.values.title, "Introduction") +``` + +### Ends with + +``` +$endsWith($.pub.values.filename, ".pdf") +``` + +## Case-insensitive matching + +Wrap the path in `$lowercase()` or `$uppercase()`. + +### Case-insensitive contains + +``` +$contains($lowercase($.pub.values.title), "snap") +``` + +### Case-insensitive equality + +``` +$lowercase($.pub.values.status) = "draft" +``` + +## Existence check + +``` +$exists($.pub.values.optionalField) +``` + +## Full-text search + +Searches across all pub values using PostgreSQL full-text search. + +``` +$search("climate change") +``` + +## Relations + +### Outgoing relations + +Find pubs that have outgoing relations via a field. + +``` +$.pub.out.contributors +``` + +### Outgoing relations with filter + +Filter by the relation value. + +``` +$.pub.out.contributors[$.value = "Editor"] +``` + +Filter by the related pub's field. + +``` +$.pub.out.contributors[$.relatedPub.values.institution = "MIT"] +``` + +Filter by the related pub's type. + +``` +$.pub.out.contributors[$.relatedPub.pubType.name = "Author"] +``` + +Combined filters. + +``` +$.pub.out.contributors[$.value = "Editor" and $contains($.relatedPub.values.name, "Smith")] +``` + +### Incoming relations + +Find pubs that are referenced by other pubs via a field. + +``` +$.pub.in.chapters +``` + +Filter by the source pub. + +``` +$.pub.in.chapters[$.relatedPub.values.title = "The Big Book"] +``` + +## Interpolation (resolver only) + +At the moment only used when configuring automations. + +Use `{{ }}` to interpolate values from the context when using as a resolver. + +``` +$.pub.values.externalId = {{ $.json.body.articleId }} +``` + +The expression inside `{{ }}` is evaluated against the automation context before the query runs. + +## Limits + +Maximum relation depth: 3 levels. + +## Unsupported + +Variable assignment, lambda functions, recursive descent, and other advanced JSONata features are not supported. diff --git a/core/lib/server/jsonata-query/compiler.ts b/core/lib/server/jsonata-query/compiler.ts index e3848c705..b8878acf2 100644 --- a/core/lib/server/jsonata-query/compiler.ts +++ b/core/lib/server/jsonata-query/compiler.ts @@ -1,4 +1,4 @@ -import type { ParsedCondition, ParsedQuery } from "./parser" +import type { ParsedCondition } from "./parser" import { parseJsonataQuery } from "./parser" diff --git a/core/lib/server/jsonata-query/index.ts b/core/lib/server/jsonata-query/index.ts index 03556328d..8a64a8473 100644 --- a/core/lib/server/jsonata-query/index.ts +++ b/core/lib/server/jsonata-query/index.ts @@ -1,21 +1,22 @@ -export { compileJsonataQuery, type CompiledQuery } from "./compiler" -export { applyJsonataFilter, type SqlBuilderOptions } from "./sql-builder" -export { filterPubsWithJsonata, pubMatchesJsonataQuery } from "./memory-filter" -export { parseJsonataQuery, type ParsedQuery, type ParsedCondition } from "./parser" -export { JsonataQueryError, UnsupportedExpressionError, InvalidPathError } from "./errors" export type { - PubFieldPath, ComparisonCondition, FunctionCondition, LogicalCondition, NotCondition, - SearchCondition, + PubFieldPath, + RelationComparisonCondition, RelationCondition, - RelationDirection, RelationContextPath, + RelationDirection, RelationFilterCondition, - RelationComparisonCondition, RelationFunctionCondition, RelationLogicalCondition, RelationNotCondition, + SearchCondition, } from "./types" + +export { type CompiledQuery, compileJsonataQuery } from "./compiler" +export { InvalidPathError, JsonataQueryError, UnsupportedExpressionError } from "./errors" +export { filterPubsWithJsonata, pubMatchesJsonataQuery } from "./memory-filter" +export { type ParsedCondition, type ParsedQuery, parseJsonataQuery } from "./parser" +export { applyJsonataFilter, type SqlBuilderOptions } from "./sql-builder" diff --git a/core/lib/server/jsonata-query/memory-filter.ts b/core/lib/server/jsonata-query/memory-filter.ts index d94764122..a27b10448 100644 --- a/core/lib/server/jsonata-query/memory-filter.ts +++ b/core/lib/server/jsonata-query/memory-filter.ts @@ -132,16 +132,25 @@ function evaluateComparison(pub: AnyProcessedPub, condition: ComparisonCondition * evaluates a function condition against a pub */ function evaluateFunction(pub: AnyProcessedPub, condition: FunctionCondition): boolean { - const value = getValueFromPath(pub, condition.path) + let value = getValueFromPath(pub, condition.path) const args = condition.arguments + // apply transform if present + value = applyTransform(value, condition.pathTransform) + + // also transform the search argument for consistency + let searchArg = args[0] + if (typeof searchArg === "string" && condition.pathTransform) { + searchArg = applyTransform(searchArg, condition.pathTransform) as string + } + switch (condition.name) { case "contains": { if (typeof value === "string") { - return value.includes(String(args[0])) + return value.includes(String(searchArg)) } if (Array.isArray(value)) { - return value.includes(args[0]) + return value.includes(searchArg) } return false } @@ -149,13 +158,13 @@ function evaluateFunction(pub: AnyProcessedPub, condition: FunctionCondition): b if (typeof value !== "string") { return false } - return value.startsWith(String(args[0])) + return value.startsWith(String(searchArg)) } case "endsWith": { if (typeof value !== "string") { return false } - return value.endsWith(String(args[0])) + return value.endsWith(String(searchArg)) } case "exists": { return value !== undefined && value !== null @@ -266,21 +275,31 @@ function evaluateRelationFilter(ctx: RelationContext, filter: RelationFilterCond return compareValues(value, filter.operator, filter.value) } case "relationFunction": { - const value = getRelationContextValue(ctx, filter.path) + let value = getRelationContextValue(ctx, filter.path) const args = filter.arguments + + // apply transform if present + value = applyTransform(value, filter.pathTransform) + + // also transform the search argument + let searchArg = args[0] + if (typeof searchArg === "string" && filter.pathTransform) { + searchArg = applyTransform(searchArg, filter.pathTransform) as string + } + switch (filter.name) { case "contains": if (typeof value === "string") { - return value.includes(String(args[0])) + return value.includes(String(searchArg)) } if (Array.isArray(value)) { - return value.includes(args[0]) + return value.includes(searchArg) } return false case "startsWith": - return typeof value === "string" && value.startsWith(String(args[0])) + return typeof value === "string" && value.startsWith(String(searchArg)) case "endsWith": - return typeof value === "string" && value.endsWith(String(args[0])) + return typeof value === "string" && value.endsWith(String(searchArg)) case "exists": return value !== undefined && value !== null default: diff --git a/core/lib/server/jsonata-query/parser.ts b/core/lib/server/jsonata-query/parser.ts index 83f581c55..3323f68c9 100644 --- a/core/lib/server/jsonata-query/parser.ts +++ b/core/lib/server/jsonata-query/parser.ts @@ -1,3 +1,6 @@ +import jsonata from "jsonata" + +import { InvalidPathError, UnsupportedExpressionError } from "./errors" import { BUILTIN_FIELDS, type BuiltinField, @@ -31,10 +34,6 @@ import { type StringFunction, } from "./types" -import jsonata from "jsonata" - -import { InvalidPathError, UnsupportedExpressionError } from "./errors" - export type { ParsedCondition } export interface ParsedQuery { @@ -94,8 +93,6 @@ function isLiteralNode( return node.type === "string" || node.type === "number" || node.type === "value" } - - /** * extracts the pub field path from a jsonata path node * @@ -148,7 +145,10 @@ function extractPubFieldPath(steps: JsonataPathStep[]): PubFieldPath { throw new InvalidPathError("expected field name after $.pub.values", stepsToString(steps)) } - throw new InvalidPathError(`unsupported pub path: $.pub.${thirdStep.value}`, stepsToString(steps)) + throw new InvalidPathError( + `unsupported pub path: $.pub.${thirdStep.value}`, + stepsToString(steps) + ) } /** @@ -208,10 +208,7 @@ function extractRelationContextPath(steps: JsonataPathStep[]): RelationContextPa if (fourthStep.type === "name" && ["name", "id"].includes(fourthStep.value)) { return { kind: "relatedPubType", field: fourthStep.value as "name" | "id" } } - throw new InvalidPathError( - "expected pubType.name or pubType.id", - stepsToString(steps) - ) + throw new InvalidPathError("expected pubType.name or pubType.id", stepsToString(steps)) } // handle $.relatedPub.values.fieldname @@ -344,7 +341,9 @@ function parseComparison( pathTransform = funcName as StringFunction const arg = pathNode.arguments[0] if (!isPathNode(arg)) { - throw new UnsupportedExpressionError("expected path as first argument to transform function") + throw new UnsupportedExpressionError( + "expected path as first argument to transform function" + ) } path = extractPubFieldPath(arg.steps) } else { @@ -401,22 +400,49 @@ function parseFunctionCall(node: JsonataFunctionNode): ParsedCondition { } // string functions: contains, startsWith, endsWith + // supports transforms like $contains($lowercase($.pub.values.title), "snap") if (["contains", "startsWith", "endsWith"].includes(funcName)) { if (node.arguments.length !== 2) { throw new UnsupportedExpressionError(`${funcName}() expects exactly two arguments`) } const pathArg = node.arguments[0] const valueArg = node.arguments[1] - if (!isPathNode(pathArg)) { - throw new UnsupportedExpressionError(`${funcName}() expects a path as first argument`) + + let path: PubFieldPath + let pathTransform: StringFunction | undefined + + if (isPathNode(pathArg)) { + path = extractPubFieldPath(pathArg.steps) + } else if (isFunctionNode(pathArg)) { + // handle transform wrapper like $lowercase($.pub.values.title) + const transformName = getFunctionName(pathArg.procedure) + if (!["lowercase", "uppercase"].includes(transformName)) { + throw new UnsupportedExpressionError( + `function ${transformName} cannot be used as path transform`, + transformName + ) + } + pathTransform = transformName as StringFunction + const innerArg = pathArg.arguments[0] + if (!isPathNode(innerArg)) { + throw new UnsupportedExpressionError( + "expected path as argument to transform function" + ) + } + path = extractPubFieldPath(innerArg.steps) + } else { + throw new UnsupportedExpressionError( + `${funcName}() expects a path or transform function as first argument` + ) } - const path = extractPubFieldPath(pathArg.steps) + const value = extractLiteral(valueArg) return { type: "function", name: funcName as StringFunction, path, arguments: [value], + pathTransform, } } @@ -496,22 +522,48 @@ function parseRelationFunctionCall(node: JsonataFunctionNode): RelationFilterCon } // string functions: contains, startsWith, endsWith + // supports transforms like $contains($lowercase($.relatedPub.values.title), "snap") if (["contains", "startsWith", "endsWith"].includes(funcName)) { if (node.arguments.length !== 2) { throw new UnsupportedExpressionError(`${funcName}() expects exactly two arguments`) } const pathArg = node.arguments[0] const valueArg = node.arguments[1] - if (!isPathNode(pathArg)) { - throw new UnsupportedExpressionError(`${funcName}() expects a path as first argument`) + + let path: RelationContextPath + let pathTransform: StringFunction | undefined + + if (isPathNode(pathArg)) { + path = extractRelationContextPath(pathArg.steps) + } else if (isFunctionNode(pathArg)) { + const transformName = getFunctionName(pathArg.procedure) + if (!["lowercase", "uppercase"].includes(transformName)) { + throw new UnsupportedExpressionError( + `function ${transformName} cannot be used as path transform in relation filter`, + transformName + ) + } + pathTransform = transformName as StringFunction + const innerArg = pathArg.arguments[0] + if (!isPathNode(innerArg)) { + throw new UnsupportedExpressionError( + "expected path as argument to transform function" + ) + } + path = extractRelationContextPath(innerArg.steps) + } else { + throw new UnsupportedExpressionError( + `${funcName}() expects a path or transform function as first argument` + ) } - const path = extractRelationContextPath(pathArg.steps) + const value = extractLiteral(valueArg) return { type: "relationFunction", name: funcName as StringFunction, path, arguments: [value], + pathTransform, } satisfies RelationFunctionCondition } @@ -746,7 +798,9 @@ function calculateRelationDepth(condition: ParsedCondition, currentDepth = 0): n case "relation": return currentDepth + 1 case "logical": - return Math.max(...condition.conditions.map((c) => calculateRelationDepth(c, currentDepth))) + return Math.max( + ...condition.conditions.map((c) => calculateRelationDepth(c, currentDepth)) + ) case "not": return calculateRelationDepth(condition.condition, currentDepth) default: diff --git a/core/lib/server/jsonata-query/sql-builder.ts b/core/lib/server/jsonata-query/sql-builder.ts index 8e0829bbd..a05108114 100644 --- a/core/lib/server/jsonata-query/sql-builder.ts +++ b/core/lib/server/jsonata-query/sql-builder.ts @@ -1,7 +1,7 @@ import type { ExpressionBuilder, ExpressionWrapper, RawBuilder } from "kysely" import type { CompiledQuery } from "./compiler" import type { - BuiltinField, + BuiltinField, ComparisonCondition, FunctionCondition, LogicalCondition, @@ -32,9 +32,7 @@ export interface SqlBuilderOptions { /** * converts a pub field path to the appropriate sql column reference */ -function pathToColumn( - path: PubFieldPath -): "value" | `pubs.${BuiltinField}` { +function pathToColumn(path: PubFieldPath): "value" | `pubs.${BuiltinField}` { if (path.kind === "builtin") { return `pubs.${path.field}` as const } @@ -193,28 +191,44 @@ function buildFunctionCondition( condition: FunctionCondition, options?: SqlBuilderOptions ): AnyExpressionWrapper { - const { name, path, arguments: args } = condition + const { name, path, arguments: args, pathTransform } = condition // for value fields, strings are stored as JSON, so we need to account for quotes const isValueField = path.kind === "value" const buildInner = (col: string) => { const strArg = String(args[0]) + // apply transform to column if present + let colExpr = `${col}::text` + if (pathTransform === "lowercase") { + colExpr = `lower(${col}::text)` + } else if (pathTransform === "uppercase") { + colExpr = `upper(${col}::text)` + } + + // when using transform, we need to also lowercase/uppercase the search arg + let searchArg = strArg + if (pathTransform === "lowercase") { + searchArg = strArg.toLowerCase() + } else if (pathTransform === "uppercase") { + searchArg = strArg.toUpperCase() + } + switch (name) { case "contains": - return eb(sql.raw(`${col}::text`), "like", `%${strArg}%`) + return eb(sql.raw(colExpr), "like", `%${searchArg}%`) case "startsWith": // for json values, the string starts with a quote - if (isValueField) { - return eb(sql.raw(`${col}::text`), "like", `"${strArg}%`) + if (isValueField && !pathTransform) { + return eb(sql.raw(colExpr), "like", `"${searchArg}%`) } - return eb(sql.raw(`${col}::text`), "like", `${strArg}%`) + return eb(sql.raw(colExpr), "like", `${searchArg}%`) case "endsWith": // for json values, the string ends with a quote - if (isValueField) { - return eb(sql.raw(`${col}::text`), "like", `%${strArg}"`) + if (isValueField && !pathTransform) { + return eb(sql.raw(colExpr), "like", `%${searchArg}"`) } - return eb(sql.raw(`${col}::text`), "like", `%${strArg}`) + return eb(sql.raw(colExpr), "like", `%${searchArg}`) case "exists": return eb.lit(true) default: @@ -441,23 +455,40 @@ function buildRelationFunctionCondition( relatedPubAlias: string, options?: SqlBuilderOptions ): AnyExpressionWrapper { - const { name, path, arguments: args } = condition + const { name, path, arguments: args, pathTransform } = condition const buildFunctionInner = (col: string, isJson: boolean) => { const strArg = String(args[0]) + + // apply transform to column if present + let colExpr = `${col}::text` + if (pathTransform === "lowercase") { + colExpr = `lower(${col}::text)` + } else if (pathTransform === "uppercase") { + colExpr = `upper(${col}::text)` + } + + // also transform the search argument + let searchArg = strArg + if (pathTransform === "lowercase") { + searchArg = strArg.toLowerCase() + } else if (pathTransform === "uppercase") { + searchArg = strArg.toUpperCase() + } + switch (name) { case "contains": - return eb(sql.raw(`${col}::text`), "like", `%${strArg}%`) + return eb(sql.raw(colExpr), "like", `%${searchArg}%`) case "startsWith": - if (isJson) { - return eb(sql.raw(`${col}::text`), "like", `"${strArg}%`) + if (isJson && !pathTransform) { + return eb(sql.raw(colExpr), "like", `"${searchArg}%`) } - return eb(sql.raw(`${col}::text`), "like", `${strArg}%`) + return eb(sql.raw(colExpr), "like", `${searchArg}%`) case "endsWith": - if (isJson) { - return eb(sql.raw(`${col}::text`), "like", `%${strArg}"`) + if (isJson && !pathTransform) { + return eb(sql.raw(colExpr), "like", `%${searchArg}"`) } - return eb(sql.raw(`${col}::text`), "like", `%${strArg}`) + return eb(sql.raw(colExpr), "like", `%${searchArg}`) case "exists": return eb.lit(true) default: diff --git a/core/lib/server/jsonata-query/types.ts b/core/lib/server/jsonata-query/types.ts index 747db3a71..d9a2491df 100644 --- a/core/lib/server/jsonata-query/types.ts +++ b/core/lib/server/jsonata-query/types.ts @@ -85,18 +85,29 @@ export type JsonataNode = export const COMPARISON_OPS = ["=", "!=", "<", "<=", ">", ">=", "in"] as const export type ComparisonOperator = (typeof COMPARISON_OPS)[number] - export const LOGICAL_OPS = ["and", "or"] as const export type LogicalOperator = (typeof LOGICAL_OPS)[number] -export const STRING_FUNCTIONS = ["contains", "startsWith", "endsWith", "lowercase", "uppercase"] as const +export const STRING_FUNCTIONS = [ + "contains", + "startsWith", + "endsWith", + "lowercase", + "uppercase", +] as const export type StringFunction = (typeof STRING_FUNCTIONS)[number] export const BOOLEAN_FUNCTIONS = ["exists", "not"] as const export type BooleanFunction = (typeof BOOLEAN_FUNCTIONS)[number] - -export const BUILTIN_FIELDS = ["id", "createdAt", "updatedAt", "pubTypeId", "title", "stageId"] as const +export const BUILTIN_FIELDS = [ + "id", + "createdAt", + "updatedAt", + "pubTypeId", + "title", + "stageId", +] as const export type BuiltinField = (typeof BUILTIN_FIELDS)[number] export type PubFieldPath = @@ -127,6 +138,8 @@ export interface FunctionCondition { name: StringFunction | BooleanFunction path: PubFieldPath arguments: LiteralValue[] + // optional transform on the path, e.g. $contains($lowercase($.pub.values.title), "snap") + pathTransform?: StringFunction } export interface LogicalCondition { @@ -160,6 +173,7 @@ export interface RelationFunctionCondition { name: StringFunction | BooleanFunction path: RelationContextPath arguments: LiteralValue[] + pathTransform?: StringFunction } export interface RelationLogicalCondition { diff --git a/core/package.json b/core/package.json index 80038d6d9..9f35e8ad2 100644 --- a/core/package.json +++ b/core/package.json @@ -50,10 +50,7 @@ "storybook": "SKIP_VALIDATION=true PUBPUB_URL=http://localhost:6006 storybook dev -p 6006 --no-open", "build-storybook": "SKIP_VALIDATION=true storybook build" }, - "files": [ - ".next", - "public" - ], + "files": [".next", "public"], "prisma": { "__comment": "The #register-loader goes to the correct file based on the .imports setting below", "seed": "tsx --import #register-loader prisma/seed.ts" From c84459326452f5a662f6c4db2cbe898a34c75c3c Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Thu, 22 Jan 2026 17:27:58 +0100 Subject: [PATCH 5/9] fix: types --- core/lib/server/jsonata-query/memory-filter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lib/server/jsonata-query/memory-filter.ts b/core/lib/server/jsonata-query/memory-filter.ts index a27b10448..9aa9271b3 100644 --- a/core/lib/server/jsonata-query/memory-filter.ts +++ b/core/lib/server/jsonata-query/memory-filter.ts @@ -46,7 +46,7 @@ function getValueFromPath(pub: AnyProcessedPub, path: PubFieldPath): unknown { const value = pub.values.find((v) => { // handle both full slug and short slug const fieldSlug = v.fieldSlug - return fieldSlug === path.fieldSlug || fieldSlug.endsWith(`:${path.fieldSlug}`) + return path.kind === "value" && (fieldSlug === path.fieldSlug || fieldSlug.endsWith(`:${path.fieldSlug}`)) }) return value?.value From afc873a6102746037574d397cc8b05ae378354e9 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Thu, 22 Jan 2026 17:35:19 +0100 Subject: [PATCH 6/9] fix: lint --- core/lib/server/jsonata-query/memory-filter.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/lib/server/jsonata-query/memory-filter.ts b/core/lib/server/jsonata-query/memory-filter.ts index 9aa9271b3..fe20377a0 100644 --- a/core/lib/server/jsonata-query/memory-filter.ts +++ b/core/lib/server/jsonata-query/memory-filter.ts @@ -46,7 +46,10 @@ function getValueFromPath(pub: AnyProcessedPub, path: PubFieldPath): unknown { const value = pub.values.find((v) => { // handle both full slug and short slug const fieldSlug = v.fieldSlug - return path.kind === "value" && (fieldSlug === path.fieldSlug || fieldSlug.endsWith(`:${path.fieldSlug}`)) + return ( + path.kind === "value" && + (fieldSlug === path.fieldSlug || fieldSlug.endsWith(`:${path.fieldSlug}`)) + ) }) return value?.value From 8e335eff648b8f22fd10da4189ae54b3fdf7b687 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 26 Jan 2026 12:53:33 +0100 Subject: [PATCH 7/9] fix: make slightly less redundant --- .../lib/server/jsonata-query/memory-filter.ts | 39 ++-- core/lib/server/jsonata-query/parser.ts | 2 - core/lib/server/jsonata-query/sql-builder.ts | 172 ++++++------------ 3 files changed, 69 insertions(+), 144 deletions(-) diff --git a/core/lib/server/jsonata-query/memory-filter.ts b/core/lib/server/jsonata-query/memory-filter.ts index fe20377a0..3f41494e4 100644 --- a/core/lib/server/jsonata-query/memory-filter.ts +++ b/core/lib/server/jsonata-query/memory-filter.ts @@ -42,9 +42,7 @@ function getValueFromPath(pub: AnyProcessedPub, path: PubFieldPath): unknown { return pubType[path.field] } - // value field - find in values array const value = pub.values.find((v) => { - // handle both full slug and short slug const fieldSlug = v.fieldSlug return ( path.kind === "value" && @@ -56,7 +54,7 @@ function getValueFromPath(pub: AnyProcessedPub, path: PubFieldPath): unknown { } /** - * applies a path transform to a value + * applies a path transform to a value, eg lowercase, uppercase */ function applyTransform(value: unknown, transform?: string): unknown { if (transform === "lowercase" && typeof value === "string") { @@ -72,7 +70,6 @@ function applyTransform(value: unknown, transform?: string): unknown { * compares two values using the given operator */ function compareValues(left: unknown, operator: string, right: unknown): boolean { - // handle null comparisons if (right === null) { if (operator === "=") { return left === null || left === undefined @@ -82,19 +79,17 @@ function compareValues(left: unknown, operator: string, right: unknown): boolean } } - // handle "in" operator if (operator === "in" && Array.isArray(right)) { return right.includes(left) } - // convert dates for comparison const normalizeValue = (v: unknown): unknown => { if (v instanceof Date) { return v.getTime() } if (typeof v === "string") { const parsed = Date.parse(v) - if (!isNaN(parsed) && v.includes("-")) { + if (!Number.isNaN(parsed)) { return parsed } } @@ -123,7 +118,8 @@ function compareValues(left: unknown, operator: string, right: unknown): boolean } /** - * evaluates a comparison condition against a pub + * evaluates a comparison condition against a pub, + * eg $.pub.values.title = "Test" */ function evaluateComparison(pub: AnyProcessedPub, condition: ComparisonCondition): boolean { let value = getValueFromPath(pub, condition.path) @@ -132,16 +128,14 @@ function evaluateComparison(pub: AnyProcessedPub, condition: ComparisonCondition } /** - * evaluates a function condition against a pub + * evaluates a function condition against a pub, eg $contains($.pub.values.title, "Test") */ function evaluateFunction(pub: AnyProcessedPub, condition: FunctionCondition): boolean { let value = getValueFromPath(pub, condition.path) const args = condition.arguments - // apply transform if present value = applyTransform(value, condition.pathTransform) - // also transform the search argument for consistency let searchArg = args[0] if (typeof searchArg === "string" && condition.pathTransform) { searchArg = applyTransform(searchArg, condition.pathTransform) as string @@ -178,7 +172,7 @@ function evaluateFunction(pub: AnyProcessedPub, condition: FunctionCondition): b } /** - * evaluates a logical condition against a pub + * evaluates a logical condition against a pub, eg $.pub.values.title = "Test" and $.pub.values.count > 10 */ function evaluateLogical(pub: AnyProcessedPub, condition: LogicalCondition): boolean { if (condition.operator === "and") { @@ -188,7 +182,7 @@ function evaluateLogical(pub: AnyProcessedPub, condition: LogicalCondition): boo } /** - * evaluates a not condition against a pub + * evaluates a not condition against a pub, eg $not($.pub.values.title = "Test") */ function evaluateNot(pub: AnyProcessedPub, condition: NotCondition): boolean { return !evaluateCondition(pub, condition.condition) @@ -205,7 +199,6 @@ function evaluateSearch(pub: AnyProcessedPub, condition: SearchCondition): boole return true } - // collect all searchable text from the pub const searchableTexts: string[] = [] for (const v of pub.values) { @@ -220,14 +213,9 @@ function evaluateSearch(pub: AnyProcessedPub, condition: SearchCondition): boole } } - // all terms must match somewhere return searchTerms.every((term) => searchableTexts.some((text) => text.includes(term))) } -// ============================================================================ -// relation filter evaluation -// ============================================================================ - interface RelationContext { relationValue: unknown relatedPub: AnyProcessedPub @@ -257,8 +245,17 @@ function getRelationContextValue(ctx: RelationContext, path: RelationContextPath return ctx.relatedPub.updatedAt case "pubTypeId": return ctx.relatedPub.pubTypeId + case "title": + return ctx.relatedPub.title + case "stageId": + return ctx.relatedPub.stageId + default: { + const _exhaustiveCheck: never = path + throw new UnsupportedExpressionError( + `unsupported related pub builtin field: ${(path as any)?.field}` + ) + } } - break case "relatedPubType": { const pubType = (ctx.relatedPub as any).pubType return pubType?.[path.field] @@ -281,10 +278,8 @@ function evaluateRelationFilter(ctx: RelationContext, filter: RelationFilterCond let value = getRelationContextValue(ctx, filter.path) const args = filter.arguments - // apply transform if present value = applyTransform(value, filter.pathTransform) - // also transform the search argument let searchArg = args[0] if (typeof searchArg === "string" && filter.pathTransform) { searchArg = applyTransform(searchArg, filter.pathTransform) as string diff --git a/core/lib/server/jsonata-query/parser.ts b/core/lib/server/jsonata-query/parser.ts index 3323f68c9..a28817ca8 100644 --- a/core/lib/server/jsonata-query/parser.ts +++ b/core/lib/server/jsonata-query/parser.ts @@ -702,7 +702,6 @@ function parseBinary(node: JsonataBinaryNode): ParsedCondition { } satisfies LogicalCondition } - // handle "in" operator: $.pub.values.number in [42, 24, 54] if (op === "in") { // check if lhs is path and rhs is array if (isPathNode(node.lhs)) { @@ -710,7 +709,6 @@ function parseBinary(node: JsonataBinaryNode): ParsedCondition { const value = extractLiteral(node.rhs) return { type: "comparison", path, operator: "in", value } } - // check if lhs is literal and rhs is path: "value" in $.pub.values.array if (isLiteralNode(node.lhs) && isPathNode(node.rhs)) { const path = extractPubFieldPath(node.rhs.steps) const value = extractLiteral(node.lhs) diff --git a/core/lib/server/jsonata-query/sql-builder.ts b/core/lib/server/jsonata-query/sql-builder.ts index a05108114..766c7807b 100644 --- a/core/lib/server/jsonata-query/sql-builder.ts +++ b/core/lib/server/jsonata-query/sql-builder.ts @@ -14,6 +14,7 @@ import type { RelationFilterCondition, RelationFunctionCondition, SearchCondition, + StringFunction, } from "./types" import { sql } from "kysely" @@ -119,13 +120,13 @@ function buildComparisonCondition( // builtin fields are not json, so we don't wrap the value if (path.kind === "builtin") { const column = pathToColumn(path) - return buildOperatorCondition(eb, column, operator, value, pathTransform, false) + return buildOperatorCondition(eb, column, operator, value, false, pathTransform) } // handle pubType fields (also not json) if (path.kind === "pubType") { return buildPubTypeSubquery(eb, path.field, (column) => - buildOperatorCondition(eb, column, operator, value, pathTransform, false) + buildOperatorCondition(eb, column, operator, value, false, pathTransform) ) } @@ -133,7 +134,7 @@ function buildComparisonCondition( return buildValueExistsSubquery( eb, path.fieldSlug, - (innerEb) => buildOperatorCondition(innerEb, "value", operator, value, pathTransform, true), + (innerEb) => buildOperatorCondition(innerEb, "value", operator, value, true, pathTransform), options ) } @@ -146,38 +147,31 @@ function buildOperatorCondition( column: string, operator: string, value: unknown, - pathTransform?: string, - isJsonValue = true + isJsonValue = true, + pathTransform?: StringFunction ): AnyExpressionWrapper { - let col: ReturnType | string = column - - // apply path transform (lowercase, uppercase) - if (pathTransform === "lowercase") { - col = sql.raw(`lower(${column}::text)`) - } else if (pathTransform === "uppercase") { - col = sql.raw(`upper(${column}::text)`) - } + const colExpr = applyTransform(column, pathTransform) const wrappedValue = isJsonValue ? wrapValue(value) : value switch (operator) { case "=": - return eb(col, "=", wrappedValue) + return eb(colExpr, "=", wrappedValue) case "!=": - return eb(col, "!=", wrappedValue) + return eb(colExpr, "!=", wrappedValue) case "<": - return eb(col, "<", wrappedValue) + return eb(colExpr, "<", wrappedValue) case "<=": - return eb(col, "<=", wrappedValue) + return eb(colExpr, "<=", wrappedValue) case ">": - return eb(col, ">", wrappedValue) + return eb(colExpr, ">", wrappedValue) case ">=": - return eb(col, ">=", wrappedValue) + return eb(colExpr, ">=", wrappedValue) case "in": if (Array.isArray(value)) { - return eb(col, "in", isJsonValue ? value.map(wrapValue) : value) + return eb(colExpr, "in", isJsonValue ? value.map(wrapValue) : value) } - return eb(col, "=", wrappedValue) + return eb(colExpr, "=", wrappedValue) default: throw new UnsupportedExpressionError(`unsupported operator: ${operator}`) } @@ -193,18 +187,12 @@ function buildFunctionCondition( ): AnyExpressionWrapper { const { name, path, arguments: args, pathTransform } = condition - // for value fields, strings are stored as JSON, so we need to account for quotes const isValueField = path.kind === "value" const buildInner = (col: string) => { const strArg = String(args[0]) - // apply transform to column if present - let colExpr = `${col}::text` - if (pathTransform === "lowercase") { - colExpr = `lower(${col}::text)` - } else if (pathTransform === "uppercase") { - colExpr = `upper(${col}::text)` - } + + const colExpr = applyTransform(col, pathTransform) // when using transform, we need to also lowercase/uppercase the search arg let searchArg = strArg @@ -216,19 +204,19 @@ function buildFunctionCondition( switch (name) { case "contains": - return eb(sql.raw(colExpr), "like", `%${searchArg}%`) + return eb(colExpr, "like", `%${searchArg}%`) case "startsWith": // for json values, the string starts with a quote if (isValueField && !pathTransform) { - return eb(sql.raw(colExpr), "like", `"${searchArg}%`) + return eb(colExpr, "like", `"${searchArg}%`) } - return eb(sql.raw(colExpr), "like", `${searchArg}%`) + return eb(colExpr, "like", `${searchArg}%`) case "endsWith": // for json values, the string ends with a quote if (isValueField && !pathTransform) { - return eb(sql.raw(colExpr), "like", `%${searchArg}"`) + return eb(colExpr, "like", `%${searchArg}"`) } - return eb(sql.raw(colExpr), "like", `%${searchArg}`) + return eb(colExpr, "like", `%${searchArg}`) case "exists": return eb.lit(true) default: @@ -236,21 +224,19 @@ function buildFunctionCondition( } } - // handle builtin fields if (path.kind === "builtin") { const column = pathToColumn(path) + // like, when would you use this, but whatever if (name === "exists") { return eb(column, "is not", null) } return buildInner(column) } - // handle pubType fields if (path.kind === "pubType") { return buildPubTypeSubquery(eb, path.field, (column) => buildInner(column)) } - // handle value fields if (name === "exists") { return buildValueExistsSubquery(eb, path.fieldSlug, () => eb.lit(true), options) } @@ -258,9 +244,6 @@ function buildFunctionCondition( return buildValueExistsSubquery(eb, path.fieldSlug, () => buildInner("value"), options) } -/** - * builds the sql condition for a logical operation - */ function buildLogicalCondition( eb: AnyExpressionBuilder, condition: LogicalCondition, @@ -274,9 +257,6 @@ function buildLogicalCondition( return eb.or(conditions) } -/** - * builds the sql condition for a not operation - */ function buildNotCondition( eb: AnyExpressionBuilder, condition: NotCondition, @@ -285,9 +265,6 @@ function buildNotCondition( return eb.not(buildCondition(eb, condition.condition, options)) } -/** - * builds the sql condition for a full-text search - */ function buildSearchCondition( eb: AnyExpressionBuilder, condition: SearchCondition, @@ -296,7 +273,6 @@ function buildSearchCondition( const { query } = condition const language = options?.searchLanguage ?? "english" - // clean and prepare search terms const cleanQuery = query.trim().replace(/[:@]/g, "") if (cleanQuery.length < 2) { return eb.lit(false) @@ -307,17 +283,11 @@ function buildSearchCondition( return eb.lit(false) } - // build tsquery with prefix matching for better UX const prefixTerms = terms.map((term) => `${term}:*`).join(" & ") - // searchVector is on pubs table return sql`pubs."searchVector" @@ to_tsquery(${language}::regconfig, ${prefixTerms})` as unknown as AnyExpressionWrapper } -// ============================================================================ -// relation filter sql building -// ============================================================================ - /** * converts a relation context path to the appropriate column reference for subquery */ @@ -339,52 +309,12 @@ function relationPathToColumn( } // name requires a join to pub_types return { column: "rpt.name", isJsonValue: false } - } -} - -/** - * builds a relation filter condition for use in a subquery - */ -function buildRelationFilterOperator( - eb: AnyExpressionBuilder, - column: string, - operator: string, - value: unknown, - pathTransform: string | undefined, - isJsonValue: boolean -): AnyExpressionWrapper { - let col: RawBuilder | string = column - - if (pathTransform === "lowercase") { - col = sql.raw(`lower(${column}::text)`) - } else if (pathTransform === "uppercase") { - col = sql.raw(`upper(${column}::text)`) - } - - const wrappedValue = isJsonValue ? wrapValue(value) : value - - switch (operator) { - case "=": - return eb(col, "=", wrappedValue) - case "!=": - return eb(col, "!=", wrappedValue) - case "<": - return eb(col, "<", wrappedValue) - case "<=": - return eb(col, "<=", wrappedValue) - case ">": - return eb(col, ">", wrappedValue) - case ">=": - return eb(col, ">=", wrappedValue) - case "in": - if (Array.isArray(value)) { - return eb(col, "in", isJsonValue ? value.map(wrapValue) : value) - } - return eb(col, "=", wrappedValue) - default: + default: { + const _exhaustiveCheck: never = path throw new UnsupportedExpressionError( - `unsupported operator in relation filter: ${operator}` + `unsupported relation context path: ${(path as any)?.kind}` ) + } } } @@ -400,7 +330,6 @@ function buildRelationComparisonCondition( const { path, operator, value, pathTransform } = condition const { column, isJsonValue } = relationPathToColumn(path, relatedPubAlias) - // for relatedPubValue, we need to join to the related pub's values if (path.kind === "relatedPubValue") { const resolvedSlug = resolveFieldSlug(path.fieldSlug, options) return eb.exists( @@ -411,19 +340,18 @@ function buildRelationComparisonCondition( .where("rpv.pubId", "=", eb.ref(`${relatedPubAlias}.id`)) .where("rpf.slug", "=", resolvedSlug) .where((innerEb) => - buildRelationFilterOperator( + buildOperatorCondition( innerEb, "rpv.value", operator, value, - pathTransform, - true + true, + pathTransform ) ) ) } - // for relatedPubType name, need a subquery to pub_types if (path.kind === "relatedPubType" && path.field === "name") { return eb.exists( eb @@ -431,19 +359,19 @@ function buildRelationComparisonCondition( .select(eb.lit(1).as("rpt_check")) .where("rpt.id", "=", eb.ref(`${relatedPubAlias}.pubTypeId`)) .where((innerEb) => - buildRelationFilterOperator( + buildOperatorCondition( innerEb, "rpt.name", operator, value, - pathTransform, - false + false, + pathTransform ) ) ) } - return buildRelationFilterOperator(eb, column, operator, value, pathTransform, isJsonValue) + return buildOperatorCondition(eb, column, operator, value, isJsonValue, pathTransform) } /** @@ -460,15 +388,8 @@ function buildRelationFunctionCondition( const buildFunctionInner = (col: string, isJson: boolean) => { const strArg = String(args[0]) - // apply transform to column if present - let colExpr = `${col}::text` - if (pathTransform === "lowercase") { - colExpr = `lower(${col}::text)` - } else if (pathTransform === "uppercase") { - colExpr = `upper(${col}::text)` - } + const colExpr = applyTransform(col, pathTransform) - // also transform the search argument let searchArg = strArg if (pathTransform === "lowercase") { searchArg = strArg.toLowerCase() @@ -478,17 +399,17 @@ function buildRelationFunctionCondition( switch (name) { case "contains": - return eb(sql.raw(colExpr), "like", `%${searchArg}%`) + return eb(colExpr, "like", `%${searchArg}%`) case "startsWith": if (isJson && !pathTransform) { - return eb(sql.raw(colExpr), "like", `"${searchArg}%`) + return eb(colExpr, "like", `"${searchArg}%`) } - return eb(sql.raw(colExpr), "like", `${searchArg}%`) + return eb(colExpr, "like", `${searchArg}%`) case "endsWith": if (isJson && !pathTransform) { - return eb(sql.raw(colExpr), "like", `%${searchArg}"`) + return eb(colExpr, "like", `%${searchArg}"`) } - return eb(sql.raw(colExpr), "like", `%${searchArg}`) + return eb(colExpr, "like", `%${searchArg}`) case "exists": return eb.lit(true) default: @@ -498,7 +419,6 @@ function buildRelationFunctionCondition( } } - // handle relatedPubValue - need subquery if (path.kind === "relatedPubValue") { const resolvedSlug = resolveFieldSlug(path.fieldSlug, options) if (name === "exists") { @@ -662,3 +582,15 @@ export function applyJsonataFilter( ): AnyExpressionWrapper { return buildCondition(eb, query.condition, options) } + +function applyTransform(col: string, pathTransform?: StringFunction): RawBuilder { + switch (pathTransform) { + case "lowercase": + return sql`lower(${col}::text)` + case "uppercase": + return sql`upper(${col}::text)` + default: { + return sql`${col}::text` + } + } +} From fad31864145d4aa80ea7014a1d7a668366da7134 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 26 Jan 2026 13:06:03 +0100 Subject: [PATCH 8/9] fix: clean up a bit --- .../lib/server/jsonata-query/memory-filter.ts | 392 ++++------ core/lib/server/jsonata-query/operators.ts | 272 +++++++ core/lib/server/jsonata-query/parser.ts | 704 ++++++++---------- core/lib/server/jsonata-query/sql-builder.ts | 567 ++++++-------- core/lib/server/jsonata-query/types.ts | 59 +- 5 files changed, 962 insertions(+), 1032 deletions(-) create mode 100644 core/lib/server/jsonata-query/operators.ts diff --git a/core/lib/server/jsonata-query/memory-filter.ts b/core/lib/server/jsonata-query/memory-filter.ts index 3f41494e4..8c1aa68ad 100644 --- a/core/lib/server/jsonata-query/memory-filter.ts +++ b/core/lib/server/jsonata-query/memory-filter.ts @@ -10,170 +10,119 @@ import type { RelationCondition, RelationContextPath, RelationFilterCondition, - SearchCondition, + StringFunction, + TransformFunction, } from "./types" import { UnsupportedExpressionError } from "./errors" +import { + applyMemoryTransform, + evaluateMemoryComparison, + evaluateMemoryExists, + evaluateMemoryStringFunction, +} from "./operators" type AnyProcessedPub = ProcessedPub -/** - * extracts a value from a pub based on a field path - */ -function getValueFromPath(pub: AnyProcessedPub, path: PubFieldPath): unknown { - if (path.kind === "builtin") { - switch (path.field) { - case "id": - return pub.id - case "createdAt": - return pub.createdAt - case "updatedAt": - return pub.updatedAt - case "pubTypeId": - return pub.pubTypeId - } - } +// value extraction - if (path.kind === "pubType") { - const pubType = (pub as any).pubType - if (!pubType) { - return undefined +function getValueFromPubPath(pub: AnyProcessedPub, path: PubFieldPath): unknown { + switch (path.kind) { + case "builtin": + switch (path.field) { + case "id": + return pub.id + case "createdAt": + return pub.createdAt + case "updatedAt": + return pub.updatedAt + case "pubTypeId": + return pub.pubTypeId + case "title": + return pub.title + case "stageId": + return pub.stageId + } + break + case "pubType": { + const pubType = (pub as any).pubType + if (!pubType) { + return undefined + } + return pubType[path.field] + } + case "value": { + const value = pub.values.find((v) => { + const fieldSlug = v.fieldSlug + return fieldSlug === path.fieldSlug || fieldSlug.endsWith(`:${path.fieldSlug}`) + }) + return value?.value } - return pubType[path.field] } - - const value = pub.values.find((v) => { - const fieldSlug = v.fieldSlug - return ( - path.kind === "value" && - (fieldSlug === path.fieldSlug || fieldSlug.endsWith(`:${path.fieldSlug}`)) - ) - }) - - return value?.value } -/** - * applies a path transform to a value, eg lowercase, uppercase - */ -function applyTransform(value: unknown, transform?: string): unknown { - if (transform === "lowercase" && typeof value === "string") { - return value.toLowerCase() - } - if (transform === "uppercase" && typeof value === "string") { - return value.toUpperCase() - } - return value +interface RelationContext { + relationValue: unknown + relatedPub: AnyProcessedPub } -/** - * compares two values using the given operator - */ -function compareValues(left: unknown, operator: string, right: unknown): boolean { - if (right === null) { - if (operator === "=") { - return left === null || left === undefined - } - if (operator === "!=") { - return left !== null && left !== undefined - } - } - - if (operator === "in" && Array.isArray(right)) { - return right.includes(left) - } - - const normalizeValue = (v: unknown): unknown => { - if (v instanceof Date) { - return v.getTime() +function getValueFromRelationPath(ctx: RelationContext, path: RelationContextPath): unknown { + switch (path.kind) { + case "relationValue": + return ctx.relationValue + case "relatedPubValue": { + const value = ctx.relatedPub.values.find((v) => { + const fieldSlug = v.fieldSlug + return fieldSlug === path.fieldSlug || fieldSlug.endsWith(`:${path.fieldSlug}`) + }) + return value?.value } - if (typeof v === "string") { - const parsed = Date.parse(v) - if (!Number.isNaN(parsed)) { - return parsed + case "relatedPubBuiltin": + switch (path.field) { + case "id": + return ctx.relatedPub.id + case "createdAt": + return ctx.relatedPub.createdAt + case "updatedAt": + return ctx.relatedPub.updatedAt + case "pubTypeId": + return ctx.relatedPub.pubTypeId + case "title": + return ctx.relatedPub.title + case "stageId": + return ctx.relatedPub.stageId } + break + case "relatedPubType": { + const pubType = (ctx.relatedPub as any).pubType + return pubType?.[path.field] } - return v - } - - const normalizedLeft = normalizeValue(left) - const normalizedRight = normalizeValue(right) - - switch (operator) { - case "=": - return normalizedLeft === normalizedRight - case "!=": - return normalizedLeft !== normalizedRight - case "<": - return (normalizedLeft as number) < (normalizedRight as number) - case "<=": - return (normalizedLeft as number) <= (normalizedRight as number) - case ">": - return (normalizedLeft as number) > (normalizedRight as number) - case ">=": - return (normalizedLeft as number) >= (normalizedRight as number) - default: - throw new UnsupportedExpressionError(`unsupported operator: ${operator}`) } } -/** - * evaluates a comparison condition against a pub, - * eg $.pub.values.title = "Test" - */ +// condition evaluation (uses shared operators) + function evaluateComparison(pub: AnyProcessedPub, condition: ComparisonCondition): boolean { - let value = getValueFromPath(pub, condition.path) - value = applyTransform(value, condition.pathTransform) - return compareValues(value, condition.operator, condition.value) + let value = getValueFromPubPath(pub, condition.path) + value = applyMemoryTransform(value, condition.pathTransform) + return evaluateMemoryComparison(value, condition.operator, condition.value) } -/** - * evaluates a function condition against a pub, eg $contains($.pub.values.title, "Test") - */ function evaluateFunction(pub: AnyProcessedPub, condition: FunctionCondition): boolean { - let value = getValueFromPath(pub, condition.path) - const args = condition.arguments - - value = applyTransform(value, condition.pathTransform) + const value = getValueFromPubPath(pub, condition.path) - let searchArg = args[0] - if (typeof searchArg === "string" && condition.pathTransform) { - searchArg = applyTransform(searchArg, condition.pathTransform) as string + if (condition.name === "exists") { + return evaluateMemoryExists(value) } - switch (condition.name) { - case "contains": { - if (typeof value === "string") { - return value.includes(String(searchArg)) - } - if (Array.isArray(value)) { - return value.includes(searchArg) - } - return false - } - case "startsWith": { - if (typeof value !== "string") { - return false - } - return value.startsWith(String(searchArg)) - } - case "endsWith": { - if (typeof value !== "string") { - return false - } - return value.endsWith(String(searchArg)) - } - case "exists": { - return value !== undefined && value !== null - } - default: - throw new UnsupportedExpressionError(`unsupported function: ${condition.name}`) - } + return evaluateMemoryStringFunction( + condition.name as StringFunction, + value, + condition.arguments[0], + condition.pathTransform + ) } -/** - * evaluates a logical condition against a pub, eg $.pub.values.title = "Test" and $.pub.values.count > 10 - */ function evaluateLogical(pub: AnyProcessedPub, condition: LogicalCondition): boolean { if (condition.operator === "and") { return condition.conditions.every((c) => evaluateCondition(pub, c)) @@ -181,19 +130,11 @@ function evaluateLogical(pub: AnyProcessedPub, condition: LogicalCondition): boo return condition.conditions.some((c) => evaluateCondition(pub, c)) } -/** - * evaluates a not condition against a pub, eg $not($.pub.values.title = "Test") - */ function evaluateNot(pub: AnyProcessedPub, condition: NotCondition): boolean { return !evaluateCondition(pub, condition.condition) } -/** - * evaluates a search condition against a pub - * searches across all string values in the pub - */ -function evaluateSearch(pub: AnyProcessedPub, condition: SearchCondition): boolean { - const { query } = condition +function evaluateSearch(pub: AnyProcessedPub, query: string): boolean { const searchTerms = query.toLowerCase().split(/\s+/).filter(Boolean) if (searchTerms.length === 0) { return true @@ -216,96 +157,59 @@ function evaluateSearch(pub: AnyProcessedPub, condition: SearchCondition): boole return searchTerms.every((term) => searchableTexts.some((text) => text.includes(term))) } -interface RelationContext { - relationValue: unknown - relatedPub: AnyProcessedPub +// relation filter evaluation + +function evaluateRelationComparison( + ctx: RelationContext, + path: RelationContextPath, + operator: string, + value: unknown, + transform?: TransformFunction +): boolean { + let extractedValue = getValueFromRelationPath(ctx, path) + extractedValue = applyMemoryTransform(extractedValue, transform) + return evaluateMemoryComparison(extractedValue, operator as any, value) } -/** - * extracts a value from relation context based on path - */ -function getRelationContextValue(ctx: RelationContext, path: RelationContextPath): unknown { - switch (path.kind) { - case "relationValue": - return ctx.relationValue - case "relatedPubValue": { - const value = ctx.relatedPub.values.find((v) => { - const fieldSlug = v.fieldSlug - return fieldSlug === path.fieldSlug || fieldSlug.endsWith(`:${path.fieldSlug}`) - }) - return value?.value - } - case "relatedPubBuiltin": - switch (path.field) { - case "id": - return ctx.relatedPub.id - case "createdAt": - return ctx.relatedPub.createdAt - case "updatedAt": - return ctx.relatedPub.updatedAt - case "pubTypeId": - return ctx.relatedPub.pubTypeId - case "title": - return ctx.relatedPub.title - case "stageId": - return ctx.relatedPub.stageId - default: { - const _exhaustiveCheck: never = path - throw new UnsupportedExpressionError( - `unsupported related pub builtin field: ${(path as any)?.field}` - ) - } - } - case "relatedPubType": { - const pubType = (ctx.relatedPub as any).pubType - return pubType?.[path.field] - } +function evaluateRelationFunction( + ctx: RelationContext, + path: RelationContextPath, + funcName: string, + args: unknown[], + transform?: TransformFunction +): boolean { + const value = getValueFromRelationPath(ctx, path) + + if (funcName === "exists") { + return evaluateMemoryExists(value) } - return undefined + + return evaluateMemoryStringFunction( + funcName as StringFunction, + value, + args[0], + transform + ) } -/** - * evaluates a relation filter condition against a relation context - */ function evaluateRelationFilter(ctx: RelationContext, filter: RelationFilterCondition): boolean { switch (filter.type) { - case "relationComparison": { - let value = getRelationContextValue(ctx, filter.path) - value = applyTransform(value, filter.pathTransform) - return compareValues(value, filter.operator, filter.value) - } - case "relationFunction": { - let value = getRelationContextValue(ctx, filter.path) - const args = filter.arguments - - value = applyTransform(value, filter.pathTransform) - - let searchArg = args[0] - if (typeof searchArg === "string" && filter.pathTransform) { - searchArg = applyTransform(searchArg, filter.pathTransform) as string - } - - switch (filter.name) { - case "contains": - if (typeof value === "string") { - return value.includes(String(searchArg)) - } - if (Array.isArray(value)) { - return value.includes(searchArg) - } - return false - case "startsWith": - return typeof value === "string" && value.startsWith(String(searchArg)) - case "endsWith": - return typeof value === "string" && value.endsWith(String(searchArg)) - case "exists": - return value !== undefined && value !== null - default: - throw new UnsupportedExpressionError( - `unsupported function in relation filter: ${filter.name}` - ) - } - } + case "relationComparison": + return evaluateRelationComparison( + ctx, + filter.path, + filter.operator, + filter.value, + filter.pathTransform + ) + case "relationFunction": + return evaluateRelationFunction( + ctx, + filter.path, + filter.name, + filter.arguments, + filter.pathTransform + ) case "relationLogical": if (filter.operator === "and") { return filter.conditions.every((c) => evaluateRelationFilter(ctx, c)) @@ -316,17 +220,10 @@ function evaluateRelationFilter(ctx: RelationContext, filter: RelationFilterCond } } -/** - * evaluates a relation condition against a pub - * - * for "out" relations: check if pub has any values pointing to related pubs matching the filter - * for "in" relations: check if any related pubs point to this pub and match the filter - */ function evaluateRelation(pub: AnyProcessedPub, condition: RelationCondition): boolean { const { direction, fieldSlug, filter } = condition if (direction === "out") { - // find relation values from this pub const relationValues = pub.values.filter((v) => { const matchesSlug = v.fieldSlug === fieldSlug || v.fieldSlug.endsWith(`:${fieldSlug}`) return matchesSlug && v.relatedPub @@ -336,7 +233,6 @@ function evaluateRelation(pub: AnyProcessedPub, condition: RelationCondition): b return false } - // check if any related pub matches the filter return relationValues.some((rv) => { if (!filter) { return true @@ -349,14 +245,11 @@ function evaluateRelation(pub: AnyProcessedPub, condition: RelationCondition): b }) } - // for "in" relations, we need to check children - // this requires the pub to have children loaded const children = (pub as any).children as AnyProcessedPub[] | undefined if (!children || children.length === 0) { return false } - // find children that are connected via this field return children.some((child) => { const relationValues = child.values.filter((v) => { const matchesSlug = v.fieldSlug === fieldSlug || v.fieldSlug.endsWith(`:${fieldSlug}`) @@ -381,9 +274,8 @@ function evaluateRelation(pub: AnyProcessedPub, condition: RelationCondition): b }) } -/** - * evaluates any condition against a pub - */ +// main dispatcher + function evaluateCondition(pub: AnyProcessedPub, condition: ParsedCondition): boolean { switch (condition.type) { case "comparison": @@ -395,21 +287,14 @@ function evaluateCondition(pub: AnyProcessedPub, condition: ParsedCondition): bo case "not": return evaluateNot(pub, condition) case "search": - return evaluateSearch(pub, condition) + return evaluateSearch(pub, condition.query) case "relation": return evaluateRelation(pub, condition) } } -/** - * filters an array of pubs using a compiled jsonata query - * - * @example - * ```ts - * const query = compileJsonataQuery('$.pub.values.title = "Test" and $.pub.values.number > 10') - * const filtered = filterPubsWithJsonata(pubs, query) - * ``` - */ +// public api + export function filterPubsWithJsonata( pubs: T[], query: CompiledQuery @@ -417,9 +302,6 @@ export function filterPubsWithJsonata( return pubs.filter((pub) => evaluateCondition(pub, query.condition)) } -/** - * tests if a single pub matches a compiled jsonata query - */ export function pubMatchesJsonataQuery(pub: AnyProcessedPub, query: CompiledQuery): boolean { return evaluateCondition(pub, query.condition) } diff --git a/core/lib/server/jsonata-query/operators.ts b/core/lib/server/jsonata-query/operators.ts new file mode 100644 index 000000000..e00fca272 --- /dev/null +++ b/core/lib/server/jsonata-query/operators.ts @@ -0,0 +1,272 @@ +import type { ExpressionBuilder, ExpressionWrapper, RawBuilder } from "kysely" +import type { ComparisonOperator, StringFunction, TransformFunction } from "./types" + +import { sql } from "kysely" + +import { UnsupportedExpressionError } from "./errors" + +type AnyExpressionBuilder = ExpressionBuilder +type AnyExpressionWrapper = ExpressionWrapper + +// sql operator building + +export function wrapJsonValue(value: unknown): unknown { + if (typeof value === "string") { + return JSON.stringify(value) + } + return value +} + +export function applyColumnTransformForText( + column: string, + transform?: TransformFunction +): RawBuilder { + switch (transform) { + case "lowercase": + return sql.raw(`lower(${column}::text)`) + case "uppercase": + return sql.raw(`upper(${column}::text)`) + default: + return sql.raw(`${column}::text`) + } +} + +export function applySearchArgTransform(arg: string, transform?: TransformFunction): string { + switch (transform) { + case "lowercase": + return arg.toLowerCase() + case "uppercase": + return arg.toUpperCase() + default: + return arg + } +} + +type SqlComparisonBuilder = ( + eb: AnyExpressionBuilder, + column: string, + value: unknown +) => AnyExpressionWrapper + +const sqlComparisonMap = { + "=": (eb, col, value) => eb(col, "=", value), + "!=": (eb, col, value) => eb(col, "!=", value), + "<": (eb, col, value) => eb(col, "<", value), + "<=": (eb, col, value) => eb(col, "<=", value), + ">": (eb, col, value) => eb(col, ">", value), + ">=": (eb, col, value) => eb(col, ">=", value), + in: (eb, col, value) => { + if (Array.isArray(value)) { + return eb(col, "in", value) + } + return eb(col, "=", value) + }, +} as const satisfies Record + +export function buildSqlComparison( + eb: AnyExpressionBuilder, + column: string, + operator: ComparisonOperator, + value: unknown, + isJsonValue: boolean, + transform?: TransformFunction +): AnyExpressionWrapper { + // for transforms (lowercase/uppercase), we need to cast to text and also wrap string values + if (transform) { + const colExpr = applyColumnTransformForText(column, transform) + const wrappedValue = isJsonValue ? wrapJsonValue(value) : value + const builder = sqlComparisonMap[operator] + if (!builder) { + throw new UnsupportedExpressionError(`unsupported operator: ${operator}`) + } + // transform function needs text comparison + if (operator === "in" && Array.isArray(wrappedValue)) { + return eb(colExpr, "in", isJsonValue ? wrappedValue.map(wrapJsonValue) : wrappedValue) + } + return builder(eb, colExpr as any, wrappedValue) + } + + // no transform - use column directly, only wrap strings for json + const wrappedValue = isJsonValue ? wrapJsonValue(value) : value + const finalValue = + operator === "in" && Array.isArray(wrappedValue) && isJsonValue + ? wrappedValue.map(wrapJsonValue) + : wrappedValue + + const builder = sqlComparisonMap[operator] + if (!builder) { + throw new UnsupportedExpressionError(`unsupported operator: ${operator}`) + } + return builder(eb, column, finalValue) +} + +type SqlStringFunctionBuilder = ( + eb: AnyExpressionBuilder, + colExpr: RawBuilder, + searchArg: string, + isJsonValue: boolean, + hasTransform: boolean +) => AnyExpressionWrapper + +const sqlStringFunctionMap = { + contains: (eb, col, arg) => eb(col, "like", `%${arg}%`), + startsWith: (eb, col, arg, isJson, hasTransform) => { + // json strings start with a quote when no transform is applied + if (isJson && !hasTransform) { + return eb(col, "like", `"${arg}%`) + } + return eb(col, "like", `${arg}%`) + }, + endsWith: (eb, col, arg, isJson, hasTransform) => { + // json strings end with a quote when no transform is applied + if (isJson && !hasTransform) { + return eb(col, "like", `%${arg}"`) + } + return eb(col, "like", `%${arg}`) + }, +} as const satisfies Record + +export function buildSqlStringFunction( + eb: AnyExpressionBuilder, + column: string, + funcName: StringFunction, + arg: string, + isJsonValue: boolean, + transform?: TransformFunction +): AnyExpressionWrapper { + const colExpr = applyColumnTransformForText(column, transform) + const searchArg = applySearchArgTransform(arg, transform) + + const builder = sqlStringFunctionMap[funcName] + if (!builder) { + throw new UnsupportedExpressionError(`unsupported string function: ${funcName}`) + } + return builder(eb, colExpr, searchArg, isJsonValue, !!transform) +} + +export function buildSqlExists( + eb: AnyExpressionBuilder, + column: string, + isNullCheck: boolean +): AnyExpressionWrapper { + if (isNullCheck) { + return eb(column, "is not", null) + } + return eb.lit(true) +} + +export function applyMemoryTransform(value: unknown, transform?: TransformFunction): unknown { + if (typeof value !== "string") { + return value + } + switch (transform) { + case "lowercase": + return value.toLowerCase() + case "uppercase": + return value.toUpperCase() + default: + return value + } +} + +function normalizeForComparison(v: unknown): unknown { + if (v instanceof Date) { + return v.getTime() + } + if (typeof v === "string") { + const parsed = Date.parse(v) + if (!Number.isNaN(parsed)) { + return parsed + } + } + return v +} + +type MemoryComparisonFn = (left: unknown, right: unknown) => boolean + +const memoryComparisonMap = { + "=": (left, right) => { + if (right === null) { + return left === null || left === undefined + } + return normalizeForComparison(left) === normalizeForComparison(right) + }, + "!=": (left, right) => { + if (right === null) { + return left !== null && left !== undefined + } + return normalizeForComparison(left) !== normalizeForComparison(right) + }, + "<": (left, right) => + (normalizeForComparison(left) as number) < (normalizeForComparison(right) as number), + "<=": (left, right) => + (normalizeForComparison(left) as number) <= (normalizeForComparison(right) as number), + ">": (left, right) => + (normalizeForComparison(left) as number) > (normalizeForComparison(right) as number), + ">=": (left, right) => + (normalizeForComparison(left) as number) >= (normalizeForComparison(right) as number), + in: (left, right) => { + if (Array.isArray(right)) { + return right.includes(left) + } + return left === right + }, +} as const satisfies Record + +export function evaluateMemoryComparison( + left: unknown, + operator: ComparisonOperator, + right: unknown +): boolean { + const compareFn = memoryComparisonMap[operator] + if (!compareFn) { + throw new UnsupportedExpressionError(`unsupported operator: ${operator}`) + } + return compareFn(left, right) +} + +type MemoryStringFunctionFn = (value: unknown, arg: unknown) => boolean + +const memoryStringFunctionMap = { + contains: (value, arg) => { + if (typeof value === "string") { + return value.includes(String(arg)) + } + if (Array.isArray(value)) { + return value.includes(arg) + } + return false + }, + startsWith: (value, arg) => { + if (typeof value !== "string") { + return false + } + return value.startsWith(String(arg)) + }, + endsWith: (value, arg) => { + if (typeof value !== "string") { + return false + } + return value.endsWith(String(arg)) + }, +} as const satisfies Record + +export function evaluateMemoryStringFunction( + funcName: StringFunction, + value: unknown, + arg: unknown, + transform?: TransformFunction +): boolean { + const transformedValue = applyMemoryTransform(value, transform) + const transformedArg = typeof arg === "string" ? applyMemoryTransform(arg, transform) : arg + + const evalFn = memoryStringFunctionMap[funcName] + if (!evalFn) { + throw new UnsupportedExpressionError(`unsupported string function: ${funcName}`) + } + return evalFn(transformedValue, transformedArg) +} + +export function evaluateMemoryExists(value: unknown): boolean { + return value !== undefined && value !== null +} diff --git a/core/lib/server/jsonata-query/parser.ts b/core/lib/server/jsonata-query/parser.ts index a28817ca8..7ef45e4bf 100644 --- a/core/lib/server/jsonata-query/parser.ts +++ b/core/lib/server/jsonata-query/parser.ts @@ -6,6 +6,12 @@ import { type BuiltinField, type ComparisonCondition, type ComparisonOperator, + type FunctionCondition, + isBooleanFunction, + isComparisonOp, + isLogicalOp, + isStringFunction, + isTransformFunction, type JsonataBinaryNode, type JsonataBlockNode, type JsonataFunctionNode, @@ -18,7 +24,6 @@ import { type JsonataValueNode, type LiteralValue, type LogicalCondition, - type LogicalOperator, type NotCondition, type ParsedCondition, type PubFieldPath, @@ -32,6 +37,7 @@ import { type RelationNotCondition, type SearchCondition, type StringFunction, + type TransformFunction, } from "./types" export type { ParsedCondition } @@ -39,13 +45,9 @@ export type { ParsedCondition } export interface ParsedQuery { condition: ParsedCondition originalExpression: string - // track max relation depth for validation maxRelationDepth: number } -const COMPARISON_OPS = new Set(["=", "!=", "<", "<=", ">", ">="]) as Set -const LOGICAL_OPS = new Set(["and", "or"]) as Set - const SUPPORTED_FUNCTIONS = new Set([ "contains", "startsWith", @@ -59,13 +61,7 @@ const SUPPORTED_FUNCTIONS = new Set([ const MAX_RELATION_DEPTH = 3 -function isComparisonOp(op: string): op is ComparisonOperator { - return COMPARISON_OPS.has(op as ComparisonOperator) -} - -function isLogicalOp(op: string): op is LogicalOperator { - return LOGICAL_OPS.has(op as LogicalOperator) -} +// node type guards function isBinaryNode(node: JsonataNode): node is JsonataBinaryNode { return node.type === "binary" @@ -93,26 +89,68 @@ function isLiteralNode( return node.type === "string" || node.type === "number" || node.type === "value" } -/** - * extracts the pub field path from a jsonata path node - * - * expects paths like: - * - $.pub.values.fieldname - * - $.pub.id - * - $.pub.createdAt - * - $.pub.pubType.name - */ +// utility functions + +function stepsToString(steps: JsonataPathStep[]): string { + return steps.map((s) => (s.type === "variable" ? "$" : s.value)).join(".") +} + +function extractLiteral(node: JsonataNode): LiteralValue { + if (node.type === "string") { + return node.value + } + if (node.type === "number") { + return node.value + } + if (node.type === "value") { + return node.value + } + if (isUnaryNode(node) && node.value === "[" && node.expressions) { + return node.expressions.map(extractLiteral) + } + throw new UnsupportedExpressionError( + `expected literal value, got ${node.type}`, + node.type, + JSON.stringify(node) + ) +} + +function getFunctionName(procedure: JsonataFunctionNode["procedure"]): string { + if (procedure.type === "variable") { + return procedure.value + } + if (procedure.type === "path" && procedure.steps && procedure.steps.length > 0) { + return procedure.steps[0].value + } + throw new UnsupportedExpressionError("unexpected procedure type", procedure.type) +} + +function flipOperator(op: ComparisonOperator): ComparisonOperator { + switch (op) { + case "<": + return ">" + case ">": + return "<" + case "<=": + return ">=" + case ">=": + return "<=" + default: + return op + } +} + +// path extraction + function extractPubFieldPath(steps: JsonataPathStep[]): PubFieldPath { if (steps.length < 3) { throw new InvalidPathError("path too short, expected $.pub.something", stepsToString(steps)) } - // first step should be $ (empty variable) if (steps[0].type !== "variable" || steps[0].value !== "") { throw new InvalidPathError("path must start with $", stepsToString(steps)) } - // second step should be "pub" if (steps[1].type !== "name" || steps[1].value !== "pub") { throw new InvalidPathError("path must start with $.pub", stepsToString(steps)) } @@ -122,12 +160,10 @@ function extractPubFieldPath(steps: JsonataPathStep[]): PubFieldPath { throw new InvalidPathError("expected name after $.pub", stepsToString(steps)) } - // handle builtin fields if (BUILTIN_FIELDS.includes(thirdStep.value as BuiltinField)) { return { kind: "builtin", field: thirdStep.value as BuiltinField } } - // handle pubType.name or pubType.id if (thirdStep.value === "pubType" && steps.length >= 4) { const fourthStep = steps[3] if (fourthStep.type === "name" && ["name", "id"].includes(fourthStep.value)) { @@ -136,7 +172,6 @@ function extractPubFieldPath(steps: JsonataPathStep[]): PubFieldPath { throw new InvalidPathError("expected pubType.name or pubType.id", stepsToString(steps)) } - // handle values.fieldname if (thirdStep.value === "values" && steps.length >= 4) { const fourthStep = steps[3] if (fourthStep.type === "name") { @@ -151,15 +186,6 @@ function extractPubFieldPath(steps: JsonataPathStep[]): PubFieldPath { ) } -/** - * extracts a relation context path from steps inside a relation filter - * - * expects paths like: - * - $.value (the relation's own value) - * - $.relatedPub.values.fieldname - * - $.relatedPub.id - * - $.relatedPub.pubType.name - */ function extractRelationContextPath(steps: JsonataPathStep[]): RelationContextPath { if (steps.length < 2) { throw new InvalidPathError( @@ -168,7 +194,6 @@ function extractRelationContextPath(steps: JsonataPathStep[]): RelationContextPa ) } - // first step should be $ (empty variable) if (steps[0].type !== "variable" || steps[0].value !== "") { throw new InvalidPathError("path must start with $", stepsToString(steps)) } @@ -178,12 +203,10 @@ function extractRelationContextPath(steps: JsonataPathStep[]): RelationContextPa throw new InvalidPathError("expected name after $", stepsToString(steps)) } - // handle $.value - the relation's own value if (secondStep.value === "value" && steps.length === 2) { return { kind: "relationValue" } } - // handle $.relatedPub... if (secondStep.value === "relatedPub") { if (steps.length < 3) { throw new InvalidPathError("expected field after $.relatedPub", stepsToString(steps)) @@ -194,7 +217,6 @@ function extractRelationContextPath(steps: JsonataPathStep[]): RelationContextPa throw new InvalidPathError("expected name after $.relatedPub", stepsToString(steps)) } - // handle builtin fields on related pub if (["id", "createdAt", "updatedAt", "pubTypeId"].includes(thirdStep.value)) { return { kind: "relatedPubBuiltin", @@ -202,7 +224,6 @@ function extractRelationContextPath(steps: JsonataPathStep[]): RelationContextPa } } - // handle $.relatedPub.pubType.name or $.relatedPub.pubType.id if (thirdStep.value === "pubType" && steps.length >= 4) { const fourthStep = steps[3] if (fourthStep.type === "name" && ["name", "id"].includes(fourthStep.value)) { @@ -211,7 +232,6 @@ function extractRelationContextPath(steps: JsonataPathStep[]): RelationContextPa throw new InvalidPathError("expected pubType.name or pubType.id", stepsToString(steps)) } - // handle $.relatedPub.values.fieldname if (thirdStep.value === "values" && steps.length >= 4) { const fourthStep = steps[3] if (fourthStep.type === "name") { @@ -235,9 +255,8 @@ function extractRelationContextPath(steps: JsonataPathStep[]): RelationContextPa ) } -/** - * checks if a path represents a relation query ($.pub.out.field or $.pub.in.field) - */ +// relation path detection + function isRelationPath( steps: JsonataPathStep[] ): { direction: RelationDirection; fieldSlug: string; filterExpr?: JsonataNode } | null { @@ -245,119 +264,140 @@ function isRelationPath( return null } - // first step should be $ (empty variable) if (steps[0].type !== "variable" || steps[0].value !== "") { return null } - // second step should be "pub" if (steps[1].type !== "name" || steps[1].value !== "pub") { return null } - // third step should be "out" or "in" const thirdStep = steps[2] if (thirdStep.type !== "name" || !["out", "in"].includes(thirdStep.value)) { return null } const direction = thirdStep.value as RelationDirection - - // fourth step is the field name (with optional filter) const fourthStep = steps[3] if (fourthStep.type !== "name") { return null } - const fieldSlug = fourthStep.value - const filterExpr = fourthStep.stages?.[0]?.expr - - return { direction, fieldSlug, filterExpr } -} - -function stepsToString(steps: JsonataPathStep[]): string { - return steps.map((s) => (s.type === "variable" ? "$" : s.value)).join(".") -} - -/** - * extracts a literal value from a jsonata node - */ -function extractLiteral(node: JsonataNode): LiteralValue { - if (node.type === "string") { - return node.value - } - if (node.type === "number") { - return node.value - } - if (node.type === "value") { - return node.value - } - // handle array literals [1, 2, 3] - if (isUnaryNode(node) && node.value === "[" && node.expressions) { - return node.expressions.map(extractLiteral) + return { + direction, + fieldSlug: fourthStep.value, + filterExpr: fourthStep.stages?.[0]?.expr, } - throw new UnsupportedExpressionError( - `expected literal value, got ${node.type}`, - node.type, - JSON.stringify(node) - ) } -/** - * gets the function name from a procedure - */ -function getFunctionName(procedure: JsonataFunctionNode["procedure"]): string { - if (procedure.type === "variable") { - return procedure.value - } - if (procedure.type === "path" && procedure.steps && procedure.steps.length > 0) { - return procedure.steps[0].value - } - throw new UnsupportedExpressionError("unexpected procedure type", procedure.type) +// generic comparison parsing (works for both pub and relation contexts) + +interface ParsedComparisonBase

{ + path: P + operator: ComparisonOperator + value: LiteralValue + pathTransform?: TransformFunction } -/** - * parses a binary comparison node - */ -function parseComparison( +function parseComparisonGeneric

( pathNode: JsonataPathNode | JsonataFunctionNode, operator: ComparisonOperator, - valueNode: JsonataNode -): ComparisonCondition { - let path: PubFieldPath - let pathTransform: StringFunction | undefined + valueNode: JsonataNode, + extractPath: (steps: JsonataPathStep[]) => P, + contextName: string +): ParsedComparisonBase

{ + let path: P + let pathTransform: TransformFunction | undefined if (isPathNode(pathNode)) { - path = extractPubFieldPath(pathNode.steps) + path = extractPath(pathNode.steps) } else if (isFunctionNode(pathNode)) { - // handle things like $lowercase($.pub.values.title) = "hello" const funcName = getFunctionName(pathNode.procedure) - if (!["lowercase", "uppercase"].includes(funcName)) { + if (!isTransformFunction(funcName)) { throw new UnsupportedExpressionError( - `function ${funcName} cannot be used as path transform`, + `function ${funcName} cannot be used as path transform${contextName ? ` in ${contextName}` : ""}`, funcName ) } - pathTransform = funcName as StringFunction + pathTransform = funcName const arg = pathNode.arguments[0] if (!isPathNode(arg)) { throw new UnsupportedExpressionError( "expected path as first argument to transform function" ) } - path = extractPubFieldPath(arg.steps) + path = extractPath(arg.steps) } else { - throw new UnsupportedExpressionError("expected path or function on left side of comparison") + throw new UnsupportedExpressionError( + `expected path or function on left side of comparison${contextName ? ` in ${contextName}` : ""}` + ) } - const value = extractLiteral(valueNode) + return { + path, + operator, + value: extractLiteral(valueNode), + pathTransform, + } +} + +// generic string function parsing - return { type: "comparison", path, operator, value, pathTransform } +interface ParsedFunctionBase

{ + name: StringFunction + path: P + arguments: LiteralValue[] + pathTransform?: TransformFunction } -/** - * parses a function call like $contains($.pub.values.title, "test") - */ +function parseStringFunctionGeneric

( + funcName: string, + node: JsonataFunctionNode, + extractPath: (steps: JsonataPathStep[]) => P, + contextName: string +): ParsedFunctionBase

{ + if (node.arguments.length !== 2) { + throw new UnsupportedExpressionError(`${funcName}() expects exactly two arguments`) + } + + const pathArg = node.arguments[0] + const valueArg = node.arguments[1] + + let path: P + let pathTransform: TransformFunction | undefined + + if (isPathNode(pathArg)) { + path = extractPath(pathArg.steps) + } else if (isFunctionNode(pathArg)) { + const transformName = getFunctionName(pathArg.procedure) + if (!isTransformFunction(transformName)) { + throw new UnsupportedExpressionError( + `function ${transformName} cannot be used as path transform${contextName ? ` in ${contextName}` : ""}`, + transformName + ) + } + pathTransform = transformName + const innerArg = pathArg.arguments[0] + if (!isPathNode(innerArg)) { + throw new UnsupportedExpressionError("expected path as argument to transform function") + } + path = extractPath(innerArg.steps) + } else { + throw new UnsupportedExpressionError( + `${funcName}() expects a path or transform function as first argument` + ) + } + + return { + name: funcName as StringFunction, + path, + arguments: [extractLiteral(valueArg)], + pathTransform, + } +} + +// top-level parsing + function parseFunctionCall(node: JsonataFunctionNode): ParsedCondition { const funcName = getFunctionName(node.procedure) @@ -365,7 +405,6 @@ function parseFunctionCall(node: JsonataFunctionNode): ParsedCondition { throw new UnsupportedExpressionError(`unsupported function: ${funcName}`, funcName) } - // handle $search() - full text search if (funcName === "search") { if (node.arguments.length !== 1) { throw new UnsupportedExpressionError("search() expects exactly one argument") @@ -377,7 +416,6 @@ function parseFunctionCall(node: JsonataFunctionNode): ParsedCondition { return { type: "search", query: arg.value } satisfies SearchCondition } - // handle not() specially if (funcName === "not") { if (node.arguments.length !== 1) { throw new UnsupportedExpressionError("not() expects exactly one argument") @@ -386,120 +424,141 @@ function parseFunctionCall(node: JsonataFunctionNode): ParsedCondition { return { type: "not", condition: inner } satisfies NotCondition } - // handle exists() - if (funcName === "exists") { + if (isBooleanFunction(funcName)) { if (node.arguments.length !== 1) { - throw new UnsupportedExpressionError("exists() expects exactly one argument") + throw new UnsupportedExpressionError(`${funcName}() expects exactly one argument`) } const arg = node.arguments[0] if (!isPathNode(arg)) { - throw new UnsupportedExpressionError("exists() expects a path argument") + throw new UnsupportedExpressionError(`${funcName}() expects a path argument`) } - const path = extractPubFieldPath(arg.steps) - return { type: "function", name: "exists", path, arguments: [] } + return { + type: "function", + name: funcName, + path: extractPubFieldPath(arg.steps), + arguments: [], + } satisfies FunctionCondition } - // string functions: contains, startsWith, endsWith - // supports transforms like $contains($lowercase($.pub.values.title), "snap") - if (["contains", "startsWith", "endsWith"].includes(funcName)) { - if (node.arguments.length !== 2) { - throw new UnsupportedExpressionError(`${funcName}() expects exactly two arguments`) - } - const pathArg = node.arguments[0] - const valueArg = node.arguments[1] - - let path: PubFieldPath - let pathTransform: StringFunction | undefined - - if (isPathNode(pathArg)) { - path = extractPubFieldPath(pathArg.steps) - } else if (isFunctionNode(pathArg)) { - // handle transform wrapper like $lowercase($.pub.values.title) - const transformName = getFunctionName(pathArg.procedure) - if (!["lowercase", "uppercase"].includes(transformName)) { - throw new UnsupportedExpressionError( - `function ${transformName} cannot be used as path transform`, - transformName - ) - } - pathTransform = transformName as StringFunction - const innerArg = pathArg.arguments[0] - if (!isPathNode(innerArg)) { - throw new UnsupportedExpressionError( - "expected path as argument to transform function" - ) - } - path = extractPubFieldPath(innerArg.steps) - } else { - throw new UnsupportedExpressionError( - `${funcName}() expects a path or transform function as first argument` - ) - } - - const value = extractLiteral(valueArg) + if (isStringFunction(funcName)) { + const parsed = parseStringFunctionGeneric(funcName, node, extractPubFieldPath, "") return { type: "function", - name: funcName as StringFunction, - path, - arguments: [value], - pathTransform, - } + ...parsed, + } satisfies FunctionCondition } throw new UnsupportedExpressionError(`unhandled function: ${funcName}`, funcName) } -// ============================================================================ -// relation filter parsing (inside [...] of relation queries) -// ============================================================================ +function parseBinary(node: JsonataBinaryNode): ParsedCondition { + const op = node.value -/** - * parses a comparison inside a relation filter context - */ -function parseRelationComparison( - pathNode: JsonataPathNode | JsonataFunctionNode, - operator: ComparisonOperator, - valueNode: JsonataNode -): RelationComparisonCondition { - let path: RelationContextPath - let pathTransform: StringFunction | undefined + if (isLogicalOp(op)) { + const left = parseNode(node.lhs) + const right = parseNode(node.rhs) + return { + type: "logical", + operator: op, + conditions: [left, right], + } satisfies LogicalCondition + } - if (isPathNode(pathNode)) { - path = extractRelationContextPath(pathNode.steps) - } else if (isFunctionNode(pathNode)) { - const funcName = getFunctionName(pathNode.procedure) - if (!["lowercase", "uppercase"].includes(funcName)) { - throw new UnsupportedExpressionError( - `function ${funcName} cannot be used as path transform in relation filter`, - funcName + if (op === "in") { + if (isPathNode(node.lhs)) { + const path = extractPubFieldPath(node.lhs.steps) + const value = extractLiteral(node.rhs) + return { type: "comparison", path, operator: "in", value } satisfies ComparisonCondition + } + if (isLiteralNode(node.lhs) && isPathNode(node.rhs)) { + const path = extractPubFieldPath(node.rhs.steps) + const value = extractLiteral(node.lhs) + return { + type: "function", + name: "contains", + path, + arguments: [value], + } satisfies FunctionCondition + } + throw new UnsupportedExpressionError("unsupported 'in' expression structure") + } + + if (isComparisonOp(op)) { + if (isPathNode(node.lhs) || isFunctionNode(node.lhs)) { + const parsed = parseComparisonGeneric(node.lhs, op, node.rhs, extractPubFieldPath, "") + return { type: "comparison", ...parsed } satisfies ComparisonCondition + } + if (isPathNode(node.rhs) || isFunctionNode(node.rhs)) { + const parsed = parseComparisonGeneric( + node.rhs, + flipOperator(op), + node.lhs, + extractPubFieldPath, + "" ) + return { type: "comparison", ...parsed } satisfies ComparisonCondition } - pathTransform = funcName as StringFunction - const arg = pathNode.arguments[0] - if (!isPathNode(arg)) { + throw new UnsupportedExpressionError("comparison must have at least one path") + } + + throw new UnsupportedExpressionError(`unsupported binary operator: ${op}`, "binary") +} + +function parseRelationPath(pathNode: JsonataPathNode): RelationCondition { + const relation = isRelationPath(pathNode.steps) + if (!relation) { + throw new UnsupportedExpressionError("expected relation path") + } + + const { direction, fieldSlug, filterExpr } = relation + + let filter: RelationFilterCondition | undefined + if (filterExpr) { + filter = parseRelationFilterNode(filterExpr) + } + + return { + type: "relation", + direction, + fieldSlug, + filter, + } +} + +function parseNode(node: JsonataNode): ParsedCondition { + if (isBlockNode(node)) { + if (node.expressions.length !== 1) { throw new UnsupportedExpressionError( - "expected path as first argument to transform function" + "block with multiple expressions not supported", + "block" ) } - path = extractRelationContextPath(arg.steps) - } else { - throw new UnsupportedExpressionError( - "expected path or function on left side of comparison in relation filter" - ) + return parseNode(node.expressions[0]) } - const value = extractLiteral(valueNode) + if (isBinaryNode(node)) { + return parseBinary(node) + } + + if (isFunctionNode(node)) { + return parseFunctionCall(node) + } + + if (isPathNode(node)) { + const relation = isRelationPath(node.steps) + if (relation) { + return parseRelationPath(node) + } + } - return { type: "relationComparison", path, operator, value, pathTransform } + throw new UnsupportedExpressionError(`unsupported node type: ${node.type}`, node.type) } -/** - * parses a function call inside a relation filter context - */ +// relation filter parsing + function parseRelationFunctionCall(node: JsonataFunctionNode): RelationFilterCondition { const funcName = getFunctionName(node.procedure) - // handle not() specially if (funcName === "not") { if (node.arguments.length !== 1) { throw new UnsupportedExpressionError("not() expects exactly one argument") @@ -508,62 +567,32 @@ function parseRelationFunctionCall(node: JsonataFunctionNode): RelationFilterCon return { type: "relationNot", condition: inner } satisfies RelationNotCondition } - // handle exists() - if (funcName === "exists") { + if (isBooleanFunction(funcName)) { if (node.arguments.length !== 1) { - throw new UnsupportedExpressionError("exists() expects exactly one argument") + throw new UnsupportedExpressionError(`${funcName}() expects exactly one argument`) } const arg = node.arguments[0] if (!isPathNode(arg)) { - throw new UnsupportedExpressionError("exists() expects a path argument") + throw new UnsupportedExpressionError(`${funcName}() expects a path argument`) } - const path = extractRelationContextPath(arg.steps) - return { type: "relationFunction", name: "exists", path, arguments: [] } + return { + type: "relationFunction", + name: funcName, + path: extractRelationContextPath(arg.steps), + arguments: [], + } satisfies RelationFunctionCondition } - // string functions: contains, startsWith, endsWith - // supports transforms like $contains($lowercase($.relatedPub.values.title), "snap") - if (["contains", "startsWith", "endsWith"].includes(funcName)) { - if (node.arguments.length !== 2) { - throw new UnsupportedExpressionError(`${funcName}() expects exactly two arguments`) - } - const pathArg = node.arguments[0] - const valueArg = node.arguments[1] - - let path: RelationContextPath - let pathTransform: StringFunction | undefined - - if (isPathNode(pathArg)) { - path = extractRelationContextPath(pathArg.steps) - } else if (isFunctionNode(pathArg)) { - const transformName = getFunctionName(pathArg.procedure) - if (!["lowercase", "uppercase"].includes(transformName)) { - throw new UnsupportedExpressionError( - `function ${transformName} cannot be used as path transform in relation filter`, - transformName - ) - } - pathTransform = transformName as StringFunction - const innerArg = pathArg.arguments[0] - if (!isPathNode(innerArg)) { - throw new UnsupportedExpressionError( - "expected path as argument to transform function" - ) - } - path = extractRelationContextPath(innerArg.steps) - } else { - throw new UnsupportedExpressionError( - `${funcName}() expects a path or transform function as first argument` - ) - } - - const value = extractLiteral(valueArg) + if (isStringFunction(funcName)) { + const parsed = parseStringFunctionGeneric( + funcName, + node, + extractRelationContextPath, + "relation filter" + ) return { type: "relationFunction", - name: funcName as StringFunction, - path, - arguments: [value], - pathTransform, + ...parsed, } satisfies RelationFunctionCondition } @@ -573,13 +602,9 @@ function parseRelationFunctionCall(node: JsonataFunctionNode): RelationFilterCon ) } -/** - * parses a binary node inside a relation filter context - */ function parseRelationBinary(node: JsonataBinaryNode): RelationFilterCondition { const op = node.value - // handle logical operators if (isLogicalOp(op)) { const left = parseRelationFilterNode(node.lhs) const right = parseRelationFilterNode(node.rhs) @@ -590,12 +615,16 @@ function parseRelationBinary(node: JsonataBinaryNode): RelationFilterCondition { } satisfies RelationLogicalCondition } - // handle "in" operator if (op === "in") { if (isPathNode(node.lhs)) { const path = extractRelationContextPath(node.lhs.steps) const value = extractLiteral(node.rhs) - return { type: "relationComparison", path, operator: "in", value } + return { + type: "relationComparison", + path, + operator: "in", + value, + } satisfies RelationComparisonCondition } if (isLiteralNode(node.lhs) && isPathNode(node.rhs)) { const path = extractRelationContextPath(node.rhs.steps) @@ -605,21 +634,39 @@ function parseRelationBinary(node: JsonataBinaryNode): RelationFilterCondition { name: "contains", path, arguments: [value], - } + } satisfies RelationFunctionCondition } throw new UnsupportedExpressionError( "unsupported 'in' expression structure in relation filter" ) } - // handle comparison operators if (isComparisonOp(op)) { if (isPathNode(node.lhs) || isFunctionNode(node.lhs)) { - return parseRelationComparison(node.lhs, op, node.rhs) + const parsed = parseComparisonGeneric( + node.lhs, + op, + node.rhs, + extractRelationContextPath, + "relation filter" + ) + return { + type: "relationComparison", + ...parsed, + } satisfies RelationComparisonCondition } if (isPathNode(node.rhs) || isFunctionNode(node.rhs)) { - const flippedOp = flipOperator(op) - return parseRelationComparison(node.rhs, flippedOp, node.lhs) + const parsed = parseComparisonGeneric( + node.rhs, + flipOperator(op), + node.lhs, + extractRelationContextPath, + "relation filter" + ) + return { + type: "relationComparison", + ...parsed, + } satisfies RelationComparisonCondition } throw new UnsupportedExpressionError( "comparison must have at least one path in relation filter" @@ -632,11 +679,7 @@ function parseRelationBinary(node: JsonataBinaryNode): RelationFilterCondition { ) } -/** - * parses any node inside a relation filter context - */ function parseRelationFilterNode(node: JsonataNode): RelationFilterCondition { - // unwrap block nodes (parentheses) if (isBlockNode(node)) { if (node.expressions.length !== 1) { throw new UnsupportedExpressionError( @@ -661,136 +704,8 @@ function parseRelationFilterNode(node: JsonataNode): RelationFilterCondition { ) } -/** - * parses a relation path into a RelationCondition - */ -function parseRelationPath(pathNode: JsonataPathNode): RelationCondition { - const relation = isRelationPath(pathNode.steps) - if (!relation) { - throw new UnsupportedExpressionError("expected relation path") - } - - const { direction, fieldSlug, filterExpr } = relation - - let filter: RelationFilterCondition | undefined - if (filterExpr) { - filter = parseRelationFilterNode(filterExpr) - } - - return { - type: "relation", - direction, - fieldSlug, - filter, - } -} - -/** - * parses a binary node (comparison or logical) - */ -function parseBinary(node: JsonataBinaryNode): ParsedCondition { - const op = node.value - - // handle logical operators - if (isLogicalOp(op)) { - const left = parseNode(node.lhs) - const right = parseNode(node.rhs) - return { - type: "logical", - operator: op, - conditions: [left, right], - } satisfies LogicalCondition - } - - if (op === "in") { - // check if lhs is path and rhs is array - if (isPathNode(node.lhs)) { - const path = extractPubFieldPath(node.lhs.steps) - const value = extractLiteral(node.rhs) - return { type: "comparison", path, operator: "in", value } - } - if (isLiteralNode(node.lhs) && isPathNode(node.rhs)) { - const path = extractPubFieldPath(node.rhs.steps) - const value = extractLiteral(node.lhs) - return { - type: "function", - name: "contains", - path, - arguments: [value], - } - } - throw new UnsupportedExpressionError("unsupported 'in' expression structure") - } - - // handle comparison operators - if (isComparisonOp(op)) { - // determine which side is the path and which is the value - if (isPathNode(node.lhs) || isFunctionNode(node.lhs)) { - return parseComparison(node.lhs, op, node.rhs) - } - if (isPathNode(node.rhs) || isFunctionNode(node.rhs)) { - // flip the operator for reversed comparison - const flippedOp = flipOperator(op) - return parseComparison(node.rhs, flippedOp, node.lhs) - } - throw new UnsupportedExpressionError("comparison must have at least one path") - } - - throw new UnsupportedExpressionError(`unsupported binary operator: ${op}`, "binary") -} - -function flipOperator(op: ComparisonOperator): ComparisonOperator { - switch (op) { - case "<": - return ">" - case ">": - return "<" - case "<=": - return ">=" - case ">=": - return "<=" - default: - return op - } -} +// depth calculation -/** - * parses any jsonata node into our condition format - */ -function parseNode(node: JsonataNode): ParsedCondition { - // unwrap block nodes (parentheses) - if (isBlockNode(node)) { - if (node.expressions.length !== 1) { - throw new UnsupportedExpressionError( - "block with multiple expressions not supported", - "block" - ) - } - return parseNode(node.expressions[0]) - } - - if (isBinaryNode(node)) { - return parseBinary(node) - } - - if (isFunctionNode(node)) { - return parseFunctionCall(node) - } - - // check if this is a relation path ($.pub.out.field or $.pub.in.field) - if (isPathNode(node)) { - const relation = isRelationPath(node.steps) - if (relation) { - return parseRelationPath(node) - } - } - - throw new UnsupportedExpressionError(`unsupported node type: ${node.type}`, node.type) -} - -/** - * calculates the max relation depth in a condition - */ function calculateRelationDepth(condition: ParsedCondition, currentDepth = 0): number { switch (condition.type) { case "relation": @@ -806,15 +721,8 @@ function calculateRelationDepth(condition: ParsedCondition, currentDepth = 0): n } } -/** - * parses a jsonata expression string into our query format - * - * @example - * ```ts - * const query = parseJsonataQuery('$.pub.values.title = "Test" and $.pub.values.number > 10') - * // { condition: { type: 'logical', operator: 'and', conditions: [...] }, originalExpression: '...' } - * ``` - */ +// public api + export function parseJsonataQuery(expression: string): ParsedQuery { const ast = jsonata(expression).ast() as JsonataNode diff --git a/core/lib/server/jsonata-query/sql-builder.ts b/core/lib/server/jsonata-query/sql-builder.ts index 766c7807b..aa5b56ba1 100644 --- a/core/lib/server/jsonata-query/sql-builder.ts +++ b/core/lib/server/jsonata-query/sql-builder.ts @@ -1,8 +1,8 @@ -import type { ExpressionBuilder, ExpressionWrapper, RawBuilder } from "kysely" +import type { ExpressionBuilder, ExpressionWrapper } from "kysely" import type { CompiledQuery } from "./compiler" import type { - BuiltinField, ComparisonCondition, + ComparisonOperator, FunctionCondition, LogicalCondition, NotCondition, @@ -15,49 +15,33 @@ import type { RelationFunctionCondition, SearchCondition, StringFunction, + TransformFunction, } from "./types" import { sql } from "kysely" -import { UnsupportedExpressionError } from "./errors" +import { buildSqlComparison, buildSqlExists, buildSqlStringFunction } from "./operators" type AnyExpressionBuilder = ExpressionBuilder type AnyExpressionWrapper = ExpressionWrapper export interface SqlBuilderOptions { communitySlug?: string - // search config for full-text search searchLanguage?: string } -/** - * converts a pub field path to the appropriate sql column reference - */ -function pathToColumn(path: PubFieldPath): "value" | `pubs.${BuiltinField}` { - if (path.kind === "builtin") { - return `pubs.${path.field}` as const - } - // for value fields, we'll handle them via subquery - return "value" -} - -/** - * resolves a field slug, optionally adding community prefix - */ function resolveFieldSlug(fieldSlug: string, options?: SqlBuilderOptions): string { if (!options?.communitySlug) { return fieldSlug } - // if already has a colon, assume it's already prefixed if (fieldSlug.includes(":")) { return fieldSlug } return `${options.communitySlug}:${fieldSlug}` } -/** - * builds a subquery that checks for a pub value matching certain conditions - */ +// subquery builders + function buildValueExistsSubquery( eb: AnyExpressionBuilder, fieldSlug: string, @@ -76,9 +60,6 @@ function buildValueExistsSubquery( ) } -/** - * builds a subquery for pubType conditions - */ function buildPubTypeSubquery( eb: AnyExpressionBuilder, field: "name" | "id", @@ -92,195 +73,176 @@ function buildPubTypeSubquery( .selectFrom("pub_types") .select(eb.lit(1).as("exists_check")) .where("pub_types.id", "=", eb.ref("pubs.pubTypeId")) - .where((_innerEb) => buildCondition("pub_types.name")) + .where(() => buildCondition("pub_types.name")) ) } -/** - * wraps a value for json comparison if needed - */ -function wrapValue(value: unknown): unknown { - if (typeof value === "string") { - return JSON.stringify(value) +// column resolution for different path types + +interface ColumnInfo { + column: string + isJsonValue: boolean +} + +function pubFieldPathToColumn(path: PubFieldPath): ColumnInfo { + switch (path.kind) { + case "builtin": + return { column: `pubs.${path.field}`, isJsonValue: false } + case "pubType": + return { + column: path.field === "id" ? "pubs.pubTypeId" : "pub_types.name", + isJsonValue: false, + } + case "value": + return { column: "value", isJsonValue: true } } - return value } -/** - * builds the sql condition for a comparison - */ -function buildComparisonCondition( - eb: AnyExpressionBuilder, - condition: ComparisonCondition, +function relationPathToColumn(path: RelationContextPath, relatedPubAlias: string): ColumnInfo { + switch (path.kind) { + case "relationValue": + return { column: "pv.value", isJsonValue: true } + case "relatedPubValue": + return { column: "rpv.value", isJsonValue: true } + case "relatedPubBuiltin": + return { column: `${relatedPubAlias}.${path.field}`, isJsonValue: false } + case "relatedPubType": + if (path.field === "id") { + return { column: `${relatedPubAlias}.pubTypeId`, isJsonValue: false } + } + return { column: "rpt.name", isJsonValue: false } + } +} + +// generic condition building + +interface ConditionBuilderContext { + eb: AnyExpressionBuilder options?: SqlBuilderOptions +} + +function buildComparisonForColumn( + ctx: ConditionBuilderContext, + column: string, + operator: ComparisonOperator, + value: unknown, + isJsonValue: boolean, + transform?: TransformFunction +): AnyExpressionWrapper { + return buildSqlComparison(ctx.eb, column, operator, value, isJsonValue, transform) +} + +function buildFunctionForColumn( + ctx: ConditionBuilderContext, + column: string, + funcName: StringFunction | "exists", + args: unknown[], + isJsonValue: boolean, + transform?: TransformFunction +): AnyExpressionWrapper { + if (funcName === "exists") { + return buildSqlExists(ctx.eb, column, true) + } + return buildSqlStringFunction(ctx.eb, column, funcName, String(args[0]), isJsonValue, transform) +} + +// top-level condition builders + +function buildComparisonCondition( + ctx: ConditionBuilderContext, + condition: ComparisonCondition ): AnyExpressionWrapper { const { path, operator, value, pathTransform } = condition - // handle builtin fields directly on pubs table - // builtin fields are not json, so we don't wrap the value if (path.kind === "builtin") { - const column = pathToColumn(path) - return buildOperatorCondition(eb, column, operator, value, false, pathTransform) + const { column, isJsonValue } = pubFieldPathToColumn(path) + return buildComparisonForColumn(ctx, column, operator, value, isJsonValue, pathTransform) } - // handle pubType fields (also not json) if (path.kind === "pubType") { - return buildPubTypeSubquery(eb, path.field, (column) => - buildOperatorCondition(eb, column, operator, value, false, pathTransform) + return buildPubTypeSubquery(ctx.eb, path.field, (column) => + buildComparisonForColumn(ctx, column, operator, value, false, pathTransform) ) } - // handle value fields via subquery (json values) return buildValueExistsSubquery( - eb, + ctx.eb, path.fieldSlug, - (innerEb) => buildOperatorCondition(innerEb, "value", operator, value, true, pathTransform), - options + (innerEb) => + buildComparisonForColumn( + { eb: innerEb, options: ctx.options }, + "value", + operator, + value, + true, + pathTransform + ), + ctx.options ) } -/** - * builds an operator condition for a specific column - */ -function buildOperatorCondition( - eb: AnyExpressionBuilder, - column: string, - operator: string, - value: unknown, - isJsonValue = true, - pathTransform?: StringFunction -): AnyExpressionWrapper { - const colExpr = applyTransform(column, pathTransform) - - const wrappedValue = isJsonValue ? wrapValue(value) : value - - switch (operator) { - case "=": - return eb(colExpr, "=", wrappedValue) - case "!=": - return eb(colExpr, "!=", wrappedValue) - case "<": - return eb(colExpr, "<", wrappedValue) - case "<=": - return eb(colExpr, "<=", wrappedValue) - case ">": - return eb(colExpr, ">", wrappedValue) - case ">=": - return eb(colExpr, ">=", wrappedValue) - case "in": - if (Array.isArray(value)) { - return eb(colExpr, "in", isJsonValue ? value.map(wrapValue) : value) - } - return eb(colExpr, "=", wrappedValue) - default: - throw new UnsupportedExpressionError(`unsupported operator: ${operator}`) - } -} - -/** - * builds the sql condition for a function call - */ function buildFunctionCondition( - eb: AnyExpressionBuilder, - condition: FunctionCondition, - options?: SqlBuilderOptions + ctx: ConditionBuilderContext, + condition: FunctionCondition ): AnyExpressionWrapper { const { name, path, arguments: args, pathTransform } = condition - const isValueField = path.kind === "value" - - const buildInner = (col: string) => { - const strArg = String(args[0]) - - const colExpr = applyTransform(col, pathTransform) - - // when using transform, we need to also lowercase/uppercase the search arg - let searchArg = strArg - if (pathTransform === "lowercase") { - searchArg = strArg.toLowerCase() - } else if (pathTransform === "uppercase") { - searchArg = strArg.toUpperCase() - } - - switch (name) { - case "contains": - return eb(colExpr, "like", `%${searchArg}%`) - case "startsWith": - // for json values, the string starts with a quote - if (isValueField && !pathTransform) { - return eb(colExpr, "like", `"${searchArg}%`) - } - return eb(colExpr, "like", `${searchArg}%`) - case "endsWith": - // for json values, the string ends with a quote - if (isValueField && !pathTransform) { - return eb(colExpr, "like", `%${searchArg}"`) - } - return eb(colExpr, "like", `%${searchArg}`) - case "exists": - return eb.lit(true) - default: - throw new UnsupportedExpressionError(`unsupported function: ${name}`) - } - } - if (path.kind === "builtin") { - const column = pathToColumn(path) - // like, when would you use this, but whatever + const { column, isJsonValue } = pubFieldPathToColumn(path) if (name === "exists") { - return eb(column, "is not", null) + return ctx.eb(column, "is not", null) } - return buildInner(column) + return buildFunctionForColumn(ctx, column, name, args, isJsonValue, pathTransform) } if (path.kind === "pubType") { - return buildPubTypeSubquery(eb, path.field, (column) => buildInner(column)) + return buildPubTypeSubquery(ctx.eb, path.field, (column) => + buildFunctionForColumn(ctx, column, name, args, false, pathTransform) + ) } if (name === "exists") { - return buildValueExistsSubquery(eb, path.fieldSlug, () => eb.lit(true), options) + return buildValueExistsSubquery(ctx.eb, path.fieldSlug, () => ctx.eb.lit(true), ctx.options) } - return buildValueExistsSubquery(eb, path.fieldSlug, () => buildInner("value"), options) + return buildValueExistsSubquery( + ctx.eb, + path.fieldSlug, + () => buildFunctionForColumn(ctx, "value", name, args, true, pathTransform), + ctx.options + ) } function buildLogicalCondition( - eb: AnyExpressionBuilder, - condition: LogicalCondition, - options?: SqlBuilderOptions + ctx: ConditionBuilderContext, + condition: LogicalCondition ): AnyExpressionWrapper { - const conditions = condition.conditions.map((c) => buildCondition(eb, c, options)) - - if (condition.operator === "and") { - return eb.and(conditions) - } - return eb.or(conditions) + const conditions = condition.conditions.map((c) => buildCondition(ctx, c)) + return condition.operator === "and" ? ctx.eb.and(conditions) : ctx.eb.or(conditions) } function buildNotCondition( - eb: AnyExpressionBuilder, - condition: NotCondition, - options?: SqlBuilderOptions + ctx: ConditionBuilderContext, + condition: NotCondition ): AnyExpressionWrapper { - return eb.not(buildCondition(eb, condition.condition, options)) + return ctx.eb.not(buildCondition(ctx, condition.condition)) } function buildSearchCondition( - eb: AnyExpressionBuilder, - condition: SearchCondition, - options?: SqlBuilderOptions + ctx: ConditionBuilderContext, + condition: SearchCondition ): AnyExpressionWrapper { const { query } = condition - const language = options?.searchLanguage ?? "english" + const language = ctx.options?.searchLanguage ?? "english" const cleanQuery = query.trim().replace(/[:@]/g, "") if (cleanQuery.length < 2) { - return eb.lit(false) + return ctx.eb.lit(false) } const terms = cleanQuery.split(/\s+/).filter((word) => word.length >= 2) if (terms.length === 0) { - return eb.lit(false) + return ctx.eb.lit(false) } const prefixTerms = terms.map((term) => `${term}:*`).join(" & ") @@ -288,309 +250,200 @@ function buildSearchCondition( return sql`pubs."searchVector" @@ to_tsquery(${language}::regconfig, ${prefixTerms})` as unknown as AnyExpressionWrapper } -/** - * converts a relation context path to the appropriate column reference for subquery - */ -function relationPathToColumn( - path: RelationContextPath, +// relation filter builders + +interface RelationFilterContext { + eb: AnyExpressionBuilder relatedPubAlias: string -): { column: string; isJsonValue: boolean } { - switch (path.kind) { - case "relationValue": - return { column: "pv.value", isJsonValue: true } - case "relatedPubValue": - // we'll handle this via another subquery - return { column: "rpv.value", isJsonValue: true } - case "relatedPubBuiltin": - return { column: `${relatedPubAlias}.${path.field}`, isJsonValue: false } - case "relatedPubType": - if (path.field === "id") { - return { column: `${relatedPubAlias}.pubTypeId`, isJsonValue: false } - } - // name requires a join to pub_types - return { column: "rpt.name", isJsonValue: false } - default: { - const _exhaustiveCheck: never = path - throw new UnsupportedExpressionError( - `unsupported relation context path: ${(path as any)?.kind}` - ) - } - } + options?: SqlBuilderOptions } -/** - * builds a condition for a relation comparison - */ function buildRelationComparisonCondition( - eb: AnyExpressionBuilder, - condition: RelationComparisonCondition, - relatedPubAlias: string, - options?: SqlBuilderOptions + ctx: RelationFilterContext, + condition: RelationComparisonCondition ): AnyExpressionWrapper { const { path, operator, value, pathTransform } = condition - const { column, isJsonValue } = relationPathToColumn(path, relatedPubAlias) + const { column, isJsonValue } = relationPathToColumn(path, ctx.relatedPubAlias) if (path.kind === "relatedPubValue") { - const resolvedSlug = resolveFieldSlug(path.fieldSlug, options) - return eb.exists( - eb + const resolvedSlug = resolveFieldSlug(path.fieldSlug, ctx.options) + return ctx.eb.exists( + ctx.eb .selectFrom("pub_values as rpv") .innerJoin("pub_fields as rpf", "rpf.id", "rpv.fieldId") - .select(eb.lit(1).as("rpv_check")) - .where("rpv.pubId", "=", eb.ref(`${relatedPubAlias}.id`)) + .select(ctx.eb.lit(1).as("rpv_check")) + .where("rpv.pubId", "=", ctx.eb.ref(`${ctx.relatedPubAlias}.id`)) .where("rpf.slug", "=", resolvedSlug) .where((innerEb) => - buildOperatorCondition( - innerEb, - "rpv.value", - operator, - value, - true, - pathTransform - ) + buildSqlComparison(innerEb, "rpv.value", operator, value, true, pathTransform) ) ) } if (path.kind === "relatedPubType" && path.field === "name") { - return eb.exists( - eb + return ctx.eb.exists( + ctx.eb .selectFrom("pub_types as rpt") - .select(eb.lit(1).as("rpt_check")) - .where("rpt.id", "=", eb.ref(`${relatedPubAlias}.pubTypeId`)) + .select(ctx.eb.lit(1).as("rpt_check")) + .where("rpt.id", "=", ctx.eb.ref(`${ctx.relatedPubAlias}.pubTypeId`)) .where((innerEb) => - buildOperatorCondition( - innerEb, - "rpt.name", - operator, - value, - false, - pathTransform - ) + buildSqlComparison(innerEb, "rpt.name", operator, value, false, pathTransform) ) ) } - return buildOperatorCondition(eb, column, operator, value, isJsonValue, pathTransform) + return buildSqlComparison(ctx.eb, column, operator, value, isJsonValue, pathTransform) } -/** - * builds a condition for a relation function call - */ function buildRelationFunctionCondition( - eb: AnyExpressionBuilder, - condition: RelationFunctionCondition, - relatedPubAlias: string, - options?: SqlBuilderOptions + ctx: RelationFilterContext, + condition: RelationFunctionCondition ): AnyExpressionWrapper { const { name, path, arguments: args, pathTransform } = condition - const buildFunctionInner = (col: string, isJson: boolean) => { - const strArg = String(args[0]) - - const colExpr = applyTransform(col, pathTransform) - - let searchArg = strArg - if (pathTransform === "lowercase") { - searchArg = strArg.toLowerCase() - } else if (pathTransform === "uppercase") { - searchArg = strArg.toUpperCase() - } - - switch (name) { - case "contains": - return eb(colExpr, "like", `%${searchArg}%`) - case "startsWith": - if (isJson && !pathTransform) { - return eb(colExpr, "like", `"${searchArg}%`) - } - return eb(colExpr, "like", `${searchArg}%`) - case "endsWith": - if (isJson && !pathTransform) { - return eb(colExpr, "like", `%${searchArg}"`) - } - return eb(colExpr, "like", `%${searchArg}`) - case "exists": - return eb.lit(true) - default: - throw new UnsupportedExpressionError( - `unsupported function in relation filter: ${name}` - ) - } - } - if (path.kind === "relatedPubValue") { - const resolvedSlug = resolveFieldSlug(path.fieldSlug, options) + const resolvedSlug = resolveFieldSlug(path.fieldSlug, ctx.options) if (name === "exists") { - return eb.exists( - eb + return ctx.eb.exists( + ctx.eb .selectFrom("pub_values as rpv") .innerJoin("pub_fields as rpf", "rpf.id", "rpv.fieldId") - .select(eb.lit(1).as("rpv_check")) - .where("rpv.pubId", "=", eb.ref(`${relatedPubAlias}.id`)) + .select(ctx.eb.lit(1).as("rpv_check")) + .where("rpv.pubId", "=", ctx.eb.ref(`${ctx.relatedPubAlias}.id`)) .where("rpf.slug", "=", resolvedSlug) ) } - return eb.exists( - eb + return ctx.eb.exists( + ctx.eb .selectFrom("pub_values as rpv") .innerJoin("pub_fields as rpf", "rpf.id", "rpv.fieldId") - .select(eb.lit(1).as("rpv_check")) - .where("rpv.pubId", "=", eb.ref(`${relatedPubAlias}.id`)) + .select(ctx.eb.lit(1).as("rpv_check")) + .where("rpv.pubId", "=", ctx.eb.ref(`${ctx.relatedPubAlias}.id`)) .where("rpf.slug", "=", resolvedSlug) - .where(() => buildFunctionInner("rpv.value", true)) + .where(() => + buildSqlStringFunction( + ctx.eb, + "rpv.value", + name as StringFunction, + String(args[0]), + true, + pathTransform + ) + ) ) } - // handle relationValue ($.value) - if (path.kind === "relationValue") { - if (name === "exists") { - return eb("pv.value", "is not", null) - } - return buildFunctionInner("pv.value", true) - } + const { column, isJsonValue } = relationPathToColumn(path, ctx.relatedPubAlias) - // handle builtin fields - const { column, isJsonValue } = relationPathToColumn(path, relatedPubAlias) if (name === "exists") { - return eb(column, "is not", null) + return ctx.eb(column, "is not", null) } - return buildFunctionInner(column, isJsonValue) + + return buildSqlStringFunction( + ctx.eb, + column, + name as StringFunction, + String(args[0]), + isJsonValue, + pathTransform + ) } -/** - * builds a relation filter condition recursively - */ function buildRelationFilter( - eb: AnyExpressionBuilder, - filter: RelationFilterCondition, - relatedPubAlias: string, - options?: SqlBuilderOptions + ctx: RelationFilterContext, + filter: RelationFilterCondition ): AnyExpressionWrapper { switch (filter.type) { case "relationComparison": - return buildRelationComparisonCondition(eb, filter, relatedPubAlias, options) + return buildRelationComparisonCondition(ctx, filter) case "relationFunction": - return buildRelationFunctionCondition(eb, filter, relatedPubAlias, options) + return buildRelationFunctionCondition(ctx, filter) case "relationLogical": { - const conditions = filter.conditions.map((c) => - buildRelationFilter(eb, c, relatedPubAlias, options) - ) - return filter.operator === "and" ? eb.and(conditions) : eb.or(conditions) + const conditions = filter.conditions.map((c) => buildRelationFilter(ctx, c)) + return filter.operator === "and" ? ctx.eb.and(conditions) : ctx.eb.or(conditions) } case "relationNot": - return eb.not(buildRelationFilter(eb, filter.condition, relatedPubAlias, options)) + return ctx.eb.not(buildRelationFilter(ctx, filter.condition)) } } -/** - * builds the sql condition for a relation query - * - * for "out" relations: find pubs where there's a pub_value with relatedPubId pointing out - * for "in" relations: find pubs that are referenced by other pubs via the given field - */ function buildRelationCondition( - eb: AnyExpressionBuilder, - condition: RelationCondition, - options?: SqlBuilderOptions + ctx: ConditionBuilderContext, + condition: RelationCondition ): AnyExpressionWrapper { const { direction, fieldSlug, filter } = condition - const resolvedSlug = resolveFieldSlug(fieldSlug, options) + const resolvedSlug = resolveFieldSlug(fieldSlug, ctx.options) if (direction === "out") { - // outgoing relation: this pub has a value that points to another pub - // pv.pubId = pubs.id and pv.relatedPubId = related_pub.id - let subquery = eb + let subquery = ctx.eb .selectFrom("pub_values as pv") .innerJoin("pub_fields as pf", "pf.id", "pv.fieldId") .innerJoin("pubs as related_pub", "related_pub.id", "pv.relatedPubId") - .select(eb.lit(1).as("rel_check")) - .where("pv.pubId", "=", eb.ref("pubs.id")) + .select(ctx.eb.lit(1).as("rel_check")) + .where("pv.pubId", "=", ctx.eb.ref("pubs.id")) .where("pf.slug", "=", resolvedSlug) .where("pv.relatedPubId", "is not", null) if (filter) { subquery = subquery.where((innerEb) => - buildRelationFilter(innerEb, filter, "related_pub", options) + buildRelationFilter( + { eb: innerEb, relatedPubAlias: "related_pub", options: ctx.options }, + filter + ) ) } - return eb.exists(subquery) + return ctx.eb.exists(subquery) } - // incoming relation: another pub has a value pointing to this pub - // pv.relatedPubId = pubs.id and pv.pubId = source_pub.id - let subquery = eb + let subquery = ctx.eb .selectFrom("pub_values as pv") .innerJoin("pub_fields as pf", "pf.id", "pv.fieldId") .innerJoin("pubs as source_pub", "source_pub.id", "pv.pubId") - .select(eb.lit(1).as("rel_check")) - .where("pv.relatedPubId", "=", eb.ref("pubs.id")) + .select(ctx.eb.lit(1).as("rel_check")) + .where("pv.relatedPubId", "=", ctx.eb.ref("pubs.id")) .where("pf.slug", "=", resolvedSlug) if (filter) { - // for incoming, the "relatedPub" in the filter context is the source_pub subquery = subquery.where((innerEb) => - buildRelationFilter(innerEb, filter, "source_pub", options) + buildRelationFilter( + { eb: innerEb, relatedPubAlias: "source_pub", options: ctx.options }, + filter + ) ) } - return eb.exists(subquery) + return ctx.eb.exists(subquery) } -/** - * builds the sql condition for any parsed condition - */ +// main condition dispatcher + function buildCondition( - eb: AnyExpressionBuilder, - condition: ParsedCondition, - options?: SqlBuilderOptions + ctx: ConditionBuilderContext, + condition: ParsedCondition ): AnyExpressionWrapper { switch (condition.type) { case "comparison": - return buildComparisonCondition(eb, condition, options) + return buildComparisonCondition(ctx, condition) case "function": - return buildFunctionCondition(eb, condition, options) + return buildFunctionCondition(ctx, condition) case "logical": - return buildLogicalCondition(eb, condition, options) + return buildLogicalCondition(ctx, condition) case "not": - return buildNotCondition(eb, condition, options) + return buildNotCondition(ctx, condition) case "search": - return buildSearchCondition(eb, condition, options) + return buildSearchCondition(ctx, condition) case "relation": - return buildRelationCondition(eb, condition, options) + return buildRelationCondition(ctx, condition) } } -/** - * applies a compiled jsonata query as a filter to a kysely query builder - * - * @example - * ```ts - * const query = compileJsonataQuery('$.pub.values.title = "Test"') - * const pubs = await db - * .selectFrom("pubs") - * .selectAll() - * .where((eb) => applyJsonataFilter(eb, query, { communitySlug: "my-community" })) - * .execute() - * ``` - */ +// public api + export function applyJsonataFilter( eb: K, query: CompiledQuery, options?: SqlBuilderOptions ): AnyExpressionWrapper { - return buildCondition(eb, query.condition, options) -} - -function applyTransform(col: string, pathTransform?: StringFunction): RawBuilder { - switch (pathTransform) { - case "lowercase": - return sql`lower(${col}::text)` - case "uppercase": - return sql`upper(${col}::text)` - default: { - return sql`${col}::text` - } - } + return buildCondition({ eb, options }, query.condition) } diff --git a/core/lib/server/jsonata-query/types.ts b/core/lib/server/jsonata-query/types.ts index d9a2491df..5c4848285 100644 --- a/core/lib/server/jsonata-query/types.ts +++ b/core/lib/server/jsonata-query/types.ts @@ -81,23 +81,20 @@ export type JsonataNode = | JsonataFunctionNode | JsonataBlockNode -// our internal representation +// operator and function definitions export const COMPARISON_OPS = ["=", "!=", "<", "<=", ">", ">=", "in"] as const export type ComparisonOperator = (typeof COMPARISON_OPS)[number] export const LOGICAL_OPS = ["and", "or"] as const export type LogicalOperator = (typeof LOGICAL_OPS)[number] -export const STRING_FUNCTIONS = [ - "contains", - "startsWith", - "endsWith", - "lowercase", - "uppercase", -] as const +export const STRING_FUNCTIONS = ["contains", "startsWith", "endsWith"] as const export type StringFunction = (typeof STRING_FUNCTIONS)[number] -export const BOOLEAN_FUNCTIONS = ["exists", "not"] as const +export const TRANSFORM_FUNCTIONS = ["lowercase", "uppercase"] as const +export type TransformFunction = (typeof TRANSFORM_FUNCTIONS)[number] + +export const BOOLEAN_FUNCTIONS = ["exists"] as const export type BooleanFunction = (typeof BOOLEAN_FUNCTIONS)[number] export const BUILTIN_FIELDS = [ @@ -110,27 +107,27 @@ export const BUILTIN_FIELDS = [ ] as const export type BuiltinField = (typeof BUILTIN_FIELDS)[number] +// path types export type PubFieldPath = | { kind: "value"; fieldSlug: string } | { kind: "builtin"; field: BuiltinField } | { kind: "pubType"; field: "name" | "id" } -// paths for use inside relation filters export type RelationContextPath = - | { kind: "relationValue" } // $.value - the value of the relation itself - | { kind: "relatedPubValue"; fieldSlug: string } // $.relatedPub.values.fieldname + | { kind: "relationValue" } + | { kind: "relatedPubValue"; fieldSlug: string } | { kind: "relatedPubBuiltin"; field: BuiltinField } | { kind: "relatedPubType"; field: "name" | "id" } export type LiteralValue = string | number | boolean | null | LiteralValue[] +// top-level condition types (for pub queries) export interface ComparisonCondition { type: "comparison" path: PubFieldPath operator: ComparisonOperator value: LiteralValue - // when we have function wrappers like $lowercase($.pub.values.title) - pathTransform?: StringFunction + pathTransform?: TransformFunction } export interface FunctionCondition { @@ -138,8 +135,7 @@ export interface FunctionCondition { name: StringFunction | BooleanFunction path: PubFieldPath arguments: LiteralValue[] - // optional transform on the path, e.g. $contains($lowercase($.pub.values.title), "snap") - pathTransform?: StringFunction + pathTransform?: TransformFunction } export interface LogicalCondition { @@ -153,19 +149,18 @@ export interface NotCondition { condition: ParsedCondition } -// full-text search condition export interface SearchCondition { type: "search" query: string } -// condition used inside relation filters (different context) +// relation context condition types (for filters inside relation queries) export interface RelationComparisonCondition { type: "relationComparison" path: RelationContextPath operator: ComparisonOperator value: LiteralValue - pathTransform?: StringFunction + pathTransform?: TransformFunction } export interface RelationFunctionCondition { @@ -173,7 +168,7 @@ export interface RelationFunctionCondition { name: StringFunction | BooleanFunction path: RelationContextPath arguments: LiteralValue[] - pathTransform?: StringFunction + pathTransform?: TransformFunction } export interface RelationLogicalCondition { @@ -193,7 +188,6 @@ export type RelationFilterCondition = | RelationLogicalCondition | RelationNotCondition -// relation query condition: $.pub.out.fieldname[filter] or $.pub.in.fieldname[filter] export type RelationDirection = "out" | "in" export interface RelationCondition { @@ -211,5 +205,26 @@ export type ParsedCondition = | SearchCondition | RelationCondition -// re-export for convenience (also exported from parser.ts) +// type guards +export function isComparisonOp(op: string): op is ComparisonOperator { + return (COMPARISON_OPS as readonly string[]).includes(op) +} + +export function isLogicalOp(op: string): op is LogicalOperator { + return (LOGICAL_OPS as readonly string[]).includes(op) +} + +export function isStringFunction(name: string): name is StringFunction { + return (STRING_FUNCTIONS as readonly string[]).includes(name) +} + +export function isTransformFunction(name: string): name is TransformFunction { + return (TRANSFORM_FUNCTIONS as readonly string[]).includes(name) +} + +export function isBooleanFunction(name: string): name is BooleanFunction { + return (BOOLEAN_FUNCTIONS as readonly string[]).includes(name) +} + +// re-export for convenience export type { ParsedCondition as Condition } From cdce0991b39c908bfdd0ad33364a4da2715a1674 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Mon, 26 Jan 2026 13:10:08 +0100 Subject: [PATCH 9/9] fix: lint --- core/lib/server/jsonata-query/memory-filter.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/core/lib/server/jsonata-query/memory-filter.ts b/core/lib/server/jsonata-query/memory-filter.ts index 8c1aa68ad..1ef8da47e 100644 --- a/core/lib/server/jsonata-query/memory-filter.ts +++ b/core/lib/server/jsonata-query/memory-filter.ts @@ -14,7 +14,6 @@ import type { TransformFunction, } from "./types" -import { UnsupportedExpressionError } from "./errors" import { applyMemoryTransform, evaluateMemoryComparison, @@ -184,12 +183,7 @@ function evaluateRelationFunction( return evaluateMemoryExists(value) } - return evaluateMemoryStringFunction( - funcName as StringFunction, - value, - args[0], - transform - ) + return evaluateMemoryStringFunction(funcName as StringFunction, value, args[0], transform) } function evaluateRelationFilter(ctx: RelationContext, filter: RelationFilterCondition): boolean {