From 2a60560add2c60fb788b41d0ca3497666e5d30a7 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 13 Jan 2026 16:50:23 +1100 Subject: [PATCH 1/2] chore(deps): update cipherstash-client to local path for memory leak fix Point cipherstash-client and cts-common dependencies to local cipherstash-suite checkout on fix/memory-leak-0.31.2 branch for testing the memory leak patch before release. --- Cargo.lock | 125 ++++++++++++++++++++++++++++++++--------------------- Cargo.toml | 4 +- 2 files changed, 77 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5524e4d9..dd39ebc7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -779,9 +779,7 @@ dependencies = [ [[package]] name = "cipherstash-client" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f3b35ded7920c83e600ee0044102183af5cd307aff4d7070061e6a2d403e956" +version = "0.32.0" dependencies = [ "aes-gcm-siv", "anyhow", @@ -793,7 +791,7 @@ dependencies = [ "blake3", "cfg-if", "chrono", - "cipherstash-config", + "cipherstash-config 0.2.4", "cipherstash-core", "cllw-ore", "cts-common", @@ -812,7 +810,7 @@ dependencies = [ "percent-encoding", "rand 0.8.5", "rand_chacha 0.3.1", - "recipher", + "recipher 0.1.3", "reqwest", "reqwest-middleware", "reqwest-retry", @@ -837,6 +835,14 @@ dependencies = [ "zerokms-protocol", ] +[[package]] +name = "cipherstash-config" +version = "0.2.4" +dependencies = [ + "serde", + "thiserror 1.0.69", +] + [[package]] name = "cipherstash-config" version = "0.2.4" @@ -850,8 +856,6 @@ dependencies = [ [[package]] name = "cipherstash-core" version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd56dfac0a35146968ef6696fb822b22f70a664a8739874385876d5452844b7a" dependencies = [ "hmac", "lazy_static", @@ -888,7 +892,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand 0.9.0", - "recipher", + "recipher 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "regex", "rust_decimal", "rustls", @@ -908,7 +912,7 @@ dependencies = [ "tracing", "tracing-subscriber", "uuid", - "vitaminc-protected", + "vitaminc-protected 0.1.0-pre3", "x509-parser", ] @@ -919,7 +923,7 @@ dependencies = [ "bytes", "chrono", "cipherstash-client", - "cipherstash-config", + "cipherstash-config 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "cipherstash-proxy", "fake 4.2.0", "hex", @@ -991,8 +995,6 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cllw-ore" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61894ceda17ee0b19d2255a7813c878d283ef55c35ca6c6f64e7220cacc1f72f" dependencies = [ "bit-vec", "bitvec", @@ -1182,8 +1184,6 @@ dependencies = [ [[package]] name = "cts-common" version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ab8098fcbe24a615b214857f9a06c48932438c060ecc32ea9e41481adea9faf" dependencies = [ "arrayvec", "axum", @@ -3042,6 +3042,16 @@ dependencies = [ "unarray", ] +[[package]] +name = "protected-derive" +version = "0.1.0-pre2" +source = "git+https://github.com/cipherstash/vitaminc?branch=timing-safe-eq#5edd32e990dff79c765adeecf1cd9ac9d2945a71" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "psm" version = "0.1.26" @@ -3249,6 +3259,25 @@ dependencies = [ "bitflags 2.9.0", ] +[[package]] +name = "recipher" +version = "0.1.3" +dependencies = [ + "aes", + "async-trait", + "cmac", + "hex", + "hex-literal", + "opaque-debug", + "rand 0.8.5", + "rand_chacha 0.3.1", + "serde", + "serde_cbor", + "sha2", + "thiserror 1.0.69", + "zeroize", +] + [[package]] name = "recipher" version = "0.1.3" @@ -4711,42 +4740,54 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vitaminc" -version = "0.1.0-pre4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f800bb9d02311571a8de172d63a4c9c10e07869089e3ddd84a8b05f3bef2d93" +version = "0.1.0-pre2" +source = "git+https://github.com/cipherstash/vitaminc?branch=timing-safe-eq#5edd32e990dff79c765adeecf1cd9ac9d2945a71" dependencies = [ "vitaminc-encrypt", - "vitaminc-protected", + "vitaminc-protected 0.1.0-pre2", "vitaminc-random", "vitaminc-traits", ] [[package]] name = "vitaminc-aead" -version = "0.1.0-pre4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2b9f64cfbcc0c8d781e2a7d33beb183fd8f58735bed8c662286548042de820" +version = "0.1.0-pre2" +source = "git+https://github.com/cipherstash/vitaminc?branch=timing-safe-eq#5edd32e990dff79c765adeecf1cd9ac9d2945a71" dependencies = [ "bytes", "serde", - "vitaminc-protected", + "vitaminc-protected 0.1.0-pre2", "vitaminc-random", "zeroize", ] [[package]] name = "vitaminc-encrypt" -version = "0.1.0-pre4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc871c8b8c61e5d2e4e097526e9cf7bc99913a45c23f31c2f6d5f1f2d0b8eec7" +version = "0.1.0-pre2" +source = "git+https://github.com/cipherstash/vitaminc?branch=timing-safe-eq#5edd32e990dff79c765adeecf1cd9ac9d2945a71" dependencies = [ "aws-lc-rs", "vitaminc-aead", - "vitaminc-protected", + "vitaminc-protected 0.1.0-pre2", "vitaminc-random", "zeroize", ] +[[package]] +name = "vitaminc-protected" +version = "0.1.0-pre2" +source = "git+https://github.com/cipherstash/vitaminc?branch=timing-safe-eq#5edd32e990dff79c765adeecf1cd9ac9d2945a71" +dependencies = [ + "bitvec", + "digest", + "opaque-debug", + "protected-derive", + "serde", + "serde_bytes", + "subtle", + "zeroize", +] + [[package]] name = "vitaminc-protected" version = "0.1.0-pre3" @@ -4776,41 +4817,27 @@ dependencies = [ [[package]] name = "vitaminc-random" -version = "0.1.0-pre4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ed841b58f676dc55a49d65284c91a2da2b71e57508c646a77c91f4dd31eb96" +version = "0.1.0-pre2" +source = "git+https://github.com/cipherstash/vitaminc?branch=timing-safe-eq#5edd32e990dff79c765adeecf1cd9ac9d2945a71" dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "thiserror 1.0.69", - "vitaminc-protected", - "vitaminc-random-derives", + "vitaminc-protected 0.1.0-pre2", "zeroize", ] -[[package]] -name = "vitaminc-random-derives" -version = "0.1.0-pre4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "172cdde4c52be52584990097655de39d835e9bcca5593c2b3517c8978ac87e89" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.101", -] - [[package]] name = "vitaminc-traits" -version = "0.1.0-pre4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb63364ce6b2a33176d2a1aba75d6b77d982518654e84192419d18b8c97f36b5" +version = "0.1.0-pre2" +source = "git+https://github.com/cipherstash/vitaminc?branch=timing-safe-eq#5edd32e990dff79c765adeecf1cd9ac9d2945a71" dependencies = [ "anyhow", "bytes", "rmp-serde", "serde", "thiserror 1.0.69", - "vitaminc-protected", + "vitaminc-protected 0.1.0-pre2", "vitaminc-random", "zeroize", ] @@ -5612,13 +5639,11 @@ dependencies = [ [[package]] name = "zerokms-protocol" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d250d0934f3b3071c9d7a91dd26b389c45d5b1f8653aa6660e4efee09bf3063a" +version = "0.9.0" dependencies = [ "async-trait", "base64", - "cipherstash-config", + "cipherstash-config 0.2.4", "const-hex", "cts-common", "fake 2.10.0", diff --git a/Cargo.toml b/Cargo.toml index 4693bdd9..28409d36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,8 +43,8 @@ debug = true [workspace.dependencies] sqltk = { version = "0.10.0" } -cipherstash-client = { version = "0.31.1" } -cts-common = { version = "0.4.0" } +cipherstash-client = { path = "../cipherstash-suite/packages/cipherstash-client" } +cts-common = { path = "../cipherstash-suite/packages/cts-common" } thiserror = "2.0.9" tokio = { version = "1.44.2", features = ["full"] } From ce7cd5c78dc93eb4128c564cc0d80c44d2221873 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 13 Jan 2026 16:50:39 +1100 Subject: [PATCH 2/2] feat(ste-vec): add term_filters support with integration tests Expose term_filters configuration for SteVec indexes, enabling case-insensitive JSON field matching using the downcase filter. Changes: - Add term_filters field to SteVecIndexOpts struct with serde default - Update IndexType::SteVec conversion to pass term_filters - Add encrypted_jsonb_filtered column to test schema with downcase filter - Add helper functions for filtered JSON fixtures (common.rs) - Add 7 integration tests covering case-insensitive queries Note: Term filters apply during encryption, so decrypted data is also transformed (e.g., "Alice" stored/returned as "alice"). --- .../src/common.rs | 76 +++++++++ .../src/select/jsonb_term_filter.rs | 155 ++++++++++++++++++ .../src/select/mod.rs | 1 + .../src/proxy/encrypt_config/config.rs | 12 +- tests/sql/schema.sql | 18 ++ 5 files changed, 259 insertions(+), 3 deletions(-) create mode 100644 packages/cipherstash-proxy-integration/src/select/jsonb_term_filter.rs diff --git a/packages/cipherstash-proxy-integration/src/common.rs b/packages/cipherstash-proxy-integration/src/common.rs index c4e085ad..4c45b875 100644 --- a/packages/cipherstash-proxy-integration/src/common.rs +++ b/packages/cipherstash-proxy-integration/src/common.rs @@ -370,6 +370,82 @@ pub async fn assert_encrypted_jsonb(id: i64, plaintext: &Value) { } } +/// Insert a JSON value into the encrypted_jsonb_filtered column (with downcase term filter). +pub async fn insert_jsonb_filtered() -> (i64, Value) { + let id = random_id(); + + let encrypted_jsonb = serde_json::json!({ + "id": id, + "name": "John", + "city": "Melbourne", + "nested": { + "title": "Engineer", + "department": "Technology", + }, + "tags": ["Hello", "World"], + }); + + let sql = "INSERT INTO encrypted (id, encrypted_jsonb_filtered) VALUES ($1, $2)".to_string(); + + insert(&sql, &[&id, &encrypted_jsonb]).await; + + // Verify encryption actually occurred + assert_encrypted_jsonb_filtered(id, &encrypted_jsonb).await; + + (id, encrypted_jsonb) +} + +/// Insert multiple JSON values for term filter search testing. +/// Creates rows with mixed case strings that should match when queried with lowercase. +pub async fn insert_jsonb_filtered_for_search() -> Vec<(i64, Value)> { + let test_data = vec![ + serde_json::json!({"name": "Alice", "number": 1}), + serde_json::json!({"name": "BOB", "number": 2}), + serde_json::json!({"name": "Charlie", "number": 3}), + serde_json::json!({"name": "DIANA", "number": 4}), + serde_json::json!({"name": "Eve", "number": 5}), + ]; + + let mut results = Vec::new(); + + for encrypted_jsonb in test_data { + let id = random_id(); + + let sql = "INSERT INTO encrypted (id, encrypted_jsonb_filtered) VALUES ($1, $2)"; + insert(sql, &[&id, &encrypted_jsonb]).await; + + // Verify encryption actually occurred + assert_encrypted_jsonb_filtered(id, &encrypted_jsonb).await; + + results.push((id, encrypted_jsonb)); + } + + results +} + +/// Verifies that a JSON value in encrypted_jsonb_filtered was actually encrypted. +pub async fn assert_encrypted_jsonb_filtered(id: i64, plaintext: &Value) { + let sql = "SELECT encrypted_jsonb_filtered::text FROM encrypted WHERE id = $1"; + let stored: Vec = query_direct_by(sql, &id).await; + + assert_eq!(stored.len(), 1, "Expected exactly one row"); + let stored_text = &stored[0]; + + let plaintext_str = plaintext.to_string(); + assert_ne!( + stored_text, &plaintext_str, + "ENCRYPTION FAILED for encrypted_jsonb_filtered: Stored value matches plaintext! Data was not encrypted." + ); + + // Additional verification: the encrypted format should be different structure + if let Ok(stored_json) = serde_json::from_str::(stored_text) { + assert_ne!( + stored_json, *plaintext, + "ENCRYPTION FAILED for encrypted_jsonb_filtered: Stored JSON structure matches plaintext!" + ); + } +} + /// Verifies that a numeric value was actually encrypted in the database. /// Queries directly (bypassing proxy) and asserts stored value differs from plaintext. pub async fn assert_encrypted_numeric(id: i64, column: &str, plaintext: T) diff --git a/packages/cipherstash-proxy-integration/src/select/jsonb_term_filter.rs b/packages/cipherstash-proxy-integration/src/select/jsonb_term_filter.rs new file mode 100644 index 00000000..2ca6bd1f --- /dev/null +++ b/packages/cipherstash-proxy-integration/src/select/jsonb_term_filter.rs @@ -0,0 +1,155 @@ +//! Tests for term filters on SteVec indexes. +//! +//! The `encrypted_jsonb_filtered` column has a downcase term filter configured, +//! meaning all string values are lowercased before encryption. This enables +//! case-insensitive queries - but note that the decrypted data is also lowercased. + +#[cfg(test)] +mod tests { + use crate::common::{ + clear, insert_jsonb_filtered, insert_jsonb_filtered_for_search, query_by_params, + simple_query, trace, + }; + use crate::support::json_path::JsonPath; + use serde_json::Value; + + /// Test case-insensitive equality matching with the downcase term filter. + /// Data is inserted with mixed case ("Alice", "BOB") but stored/returned as lowercase. + #[tokio::test] + async fn select_jsonb_filtered_case_insensitive_eq() { + trace(); + clear().await; + insert_jsonb_filtered_for_search().await; + + // Query with lowercase "alice" should match the row originally inserted as "Alice" + let selector = "name"; + let value = Value::from("alice"); + + // Extended protocol + let sql = + "SELECT encrypted_jsonb_filtered FROM encrypted WHERE encrypted_jsonb_filtered -> $1 = $2"; + let actual = query_by_params::(sql, &[&selector, &value]).await; + + // Term filter lowercases during encryption, so returned value is lowercase + assert_eq!(actual.len(), 1); + assert_eq!(actual[0]["name"], "alice"); + assert_eq!(actual[0]["number"], 1); + } + + /// Test that data inserted with uppercase is stored and returned as lowercase + #[tokio::test] + async fn select_jsonb_filtered_uppercase_query_matches() { + trace(); + clear().await; + insert_jsonb_filtered_for_search().await; + + // Query with "bob" should match the row originally inserted as "BOB" + let selector = "name"; + let value = Value::from("bob"); + + let sql = + "SELECT encrypted_jsonb_filtered FROM encrypted WHERE encrypted_jsonb_filtered -> $1 = $2"; + let actual = query_by_params::(sql, &[&selector, &value]).await; + + // Both stored and queried values are lowercased + assert_eq!(actual.len(), 1); + assert_eq!(actual[0]["name"], "bob"); + assert_eq!(actual[0]["number"], 2); + } + + /// Test simple protocol with case-insensitive matching + #[tokio::test] + async fn select_jsonb_filtered_simple_protocol() { + trace(); + clear().await; + insert_jsonb_filtered_for_search().await; + + // Simple protocol query - value is lowercased on both sides + let sql = + "SELECT encrypted_jsonb_filtered FROM encrypted WHERE encrypted_jsonb_filtered -> 'name' = '\"charlie\"'"; + let actual = simple_query::(sql).await; + + assert_eq!(actual.len(), 1); + assert_eq!(actual[0]["name"], "charlie"); + assert_eq!(actual[0]["number"], 3); + } + + /// Test that numbers are not affected by the downcase filter + #[tokio::test] + async fn select_jsonb_filtered_numbers_unchanged() { + trace(); + clear().await; + insert_jsonb_filtered_for_search().await; + + let selector = "number"; + let value = Value::from(4); + + let sql = + "SELECT encrypted_jsonb_filtered FROM encrypted WHERE encrypted_jsonb_filtered -> $1 = $2"; + let actual = query_by_params::(sql, &[&selector, &value]).await; + + assert_eq!(actual.len(), 1); + // Name is lowercased by term filter + assert_eq!(actual[0]["name"], "diana"); + assert_eq!(actual[0]["number"], 4); + } + + /// Test case-insensitive matching using jsonb_path_query_first + #[tokio::test] + async fn select_jsonb_filtered_path_query_case_insensitive() { + trace(); + clear().await; + insert_jsonb_filtered_for_search().await; + + let json_path_selector = JsonPath::new("name"); + let value = Value::from("eve"); + + let sql = + "SELECT encrypted_jsonb_filtered FROM encrypted WHERE jsonb_path_query_first(encrypted_jsonb_filtered, $1) = $2"; + let actual = query_by_params::(sql, &[&json_path_selector, &value]).await; + + assert_eq!(actual.len(), 1); + assert_eq!(actual[0]["name"], "eve"); + assert_eq!(actual[0]["number"], 5); + } + + /// Test nested field access with term filter + #[tokio::test] + async fn select_jsonb_filtered_nested_case_insensitive() { + trace(); + clear().await; + let (_id, _) = insert_jsonb_filtered().await; + + // The fixture has nested.title = "Engineer" which gets lowercased + // Query with lowercase should match + let json_path_selector = JsonPath::new("nested.title"); + let value = Value::from("engineer"); + + let sql = + "SELECT encrypted_jsonb_filtered FROM encrypted WHERE jsonb_path_query_first(encrypted_jsonb_filtered, $1) = $2"; + let actual = query_by_params::(sql, &[&json_path_selector, &value]).await; + + assert_eq!(actual.len(), 1); + assert_eq!(actual[0]["nested"]["title"], "engineer"); + } + + /// Test that original fixture data is correctly inserted and queryable + #[tokio::test] + async fn select_jsonb_filtered_fixture_data() { + trace(); + clear().await; + let (_id, _expected) = insert_jsonb_filtered().await; + + // Query by name field - both query and stored data are lowercased + let selector = "name"; + let value = Value::from("john"); + + let sql = + "SELECT encrypted_jsonb_filtered FROM encrypted WHERE encrypted_jsonb_filtered -> $1 = $2"; + let actual = query_by_params::(sql, &[&selector, &value]).await; + + assert_eq!(actual.len(), 1); + assert_eq!(actual[0]["name"], "john"); + assert_eq!(actual[0]["city"], "melbourne"); + } +} diff --git a/packages/cipherstash-proxy-integration/src/select/mod.rs b/packages/cipherstash-proxy-integration/src/select/mod.rs index ad2b02a4..de6f9fca 100644 --- a/packages/cipherstash-proxy-integration/src/select/mod.rs +++ b/packages/cipherstash-proxy-integration/src/select/mod.rs @@ -9,6 +9,7 @@ mod jsonb_get_field_as_ciphertext; mod jsonb_path_exists; mod jsonb_path_query; mod jsonb_path_query_first; +mod jsonb_term_filter; mod order_by; mod order_by_with_null; mod pg_catalog; diff --git a/packages/cipherstash-proxy/src/proxy/encrypt_config/config.rs b/packages/cipherstash-proxy/src/proxy/encrypt_config/config.rs index 3c2d493c..4375d0c4 100644 --- a/packages/cipherstash-proxy/src/proxy/encrypt_config/config.rs +++ b/packages/cipherstash-proxy/src/proxy/encrypt_config/config.rs @@ -96,6 +96,8 @@ pub struct MatchIndexOpts { #[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] pub struct SteVecIndexOpts { prefix: String, + #[serde(default)] + term_filters: Vec, } fn default_tokenizer() -> Tokenizer { @@ -182,8 +184,11 @@ impl Column { })) } - if let Some(SteVecIndexOpts { prefix }) = self.indexes.ste_vec_index { - config = config.add_index(Index::new(IndexType::SteVec { prefix })) + if let Some(SteVecIndexOpts { prefix, term_filters }) = self.indexes.ste_vec_index { + config = config.add_index(Index::new(IndexType::SteVec { + prefix, + term_filters, + })) } config @@ -463,7 +468,8 @@ mod tests { assert_eq!( column.indexes[0].index_type, IndexType::SteVec { - prefix: "event-data".into() + prefix: "event-data".into(), + term_filters: vec![], }, ); } diff --git a/tests/sql/schema.sql b/tests/sql/schema.sql index 64447652..69a6ec5e 100644 --- a/tests/sql/schema.sql +++ b/tests/sql/schema.sql @@ -33,6 +33,7 @@ CREATE TABLE encrypted ( encrypted_float8 eql_v2_encrypted, encrypted_date eql_v2_encrypted, encrypted_jsonb eql_v2_encrypted, + encrypted_jsonb_filtered eql_v2_encrypted, PRIMARY KEY(id) ); @@ -157,6 +158,14 @@ SELECT eql_v2.add_search_config( '{"prefix": "encrypted/encrypted_jsonb"}' ); +SELECT eql_v2.add_search_config( + 'encrypted', + 'encrypted_jsonb_filtered', + 'ste_vec', + 'jsonb', + '{"prefix": "encrypted/encrypted_jsonb_filtered", "term_filters": [{"kind": "downcase"}]}' +); + SELECT eql_v2.add_encrypted_constraint('encrypted', 'encrypted_text'); @@ -177,6 +186,7 @@ CREATE TABLE encrypted_elixir ( encrypted_float8 eql_v2_encrypted, encrypted_date eql_v2_encrypted, encrypted_jsonb eql_v2_encrypted, + encrypted_jsonb_filtered eql_v2_encrypted, PRIMARY KEY(id) ); @@ -301,5 +311,13 @@ SELECT eql_v2.add_search_config( '{"prefix": "encrypted/encrypted_jsonb"}' ); +SELECT eql_v2.add_search_config( + 'encrypted_elixir', + 'encrypted_jsonb_filtered', + 'ste_vec', + 'jsonb', + '{"prefix": "encrypted/encrypted_jsonb_filtered", "term_filters": [{"kind": "downcase"}]}' +); + SELECT eql_v2.add_encrypted_constraint('encrypted_elixir', 'encrypted_text');