From 579431b9bfbdd6b681ccb29afc464ea807fe48a4 Mon Sep 17 00:00:00 2001 From: James Swirhun Date: Tue, 6 Jan 2026 04:40:49 +0000 Subject: [PATCH 1/2] Fix publisher connections documentation - Fix connections format from object to array syntax - Clarify that local parquet files don't need connection config - Add DuckDB attached databases examples (BigQuery, Snowflake, PostgreSQL) - Update all database connection examples with correct format Signed-off-by: James Swirhun --- .../publishing/connections.malloynb | 237 ++++++++++++++---- 1 file changed, 194 insertions(+), 43 deletions(-) diff --git a/src/documentation/user_guides/publishing/connections.malloynb b/src/documentation/user_guides/publishing/connections.malloynb index d139a12e..5acf13b1 100644 --- a/src/documentation/user_guides/publishing/connections.malloynb +++ b/src/documentation/user_guides/publishing/connections.malloynb @@ -18,12 +18,13 @@ Publisher uses `publisher.config.json` for database connections. This file lives "projects": [ { "name": "default", - "connections": { - "connection_name": { + "connections": [ + { + "name": "connection_name", "type": "backend_type", ...connection options... } - }, + ], "packages": [ { "name": "my-package", @@ -35,23 +36,19 @@ Publisher uses `publisher.config.json` for database connections. This file lives } ``` +**Note:** For packages with local parquet/CSV files, you don't need a connections config at all — Publisher automatically uses DuckDB. + --- -## DuckDB +## DuckDB (Local Parquet Files) -DuckDB can read data files bundled in your published package: +For packages with embedded parquet/CSV files, **no connection configuration is needed**. Publisher automatically uses DuckDB for local data files: ```json { "projects": [ { "name": "default", - "connections": { - "duckdb": { - "type": "duckdb", - "workingDirectory": "./data" - } - }, "packages": [ { "name": "my-analytics", @@ -63,7 +60,151 @@ DuckDB can read data files bundled in your published package: } ``` -The `workingDirectory` sets the base path for file references. If your Malloy code references `'flights.parquet'`, it will look in the `./data` directory. +In your Malloy model, reference files relative to the package root: + +```malloy +source: flights is duckdb.table('data/flights.parquet') +``` + +**Folder structure:** +``` +my-analytics/ +├── publisher.json +├── model.malloy +└── data/ + └── flights.parquet +``` + +--- + +## DuckDB with Attached Databases + +DuckDB can federate queries to external databases (BigQuery, Snowflake, PostgreSQL) using attached databases. This lets you query cloud data warehouses through DuckDB. + +**Attach BigQuery:** + +```json +{ + "projects": [ + { + "name": "default", + "connections": [ + { + "name": "duckdb", + "type": "duckdb", + "duckdbConnection": { + "attachedDatabases": [ + { + "name": "my_bq", + "type": "bigquery", + "bigqueryConnection": { + "defaultProjectId": "my-gcp-project", + "serviceAccountKeyJson": "{ \"type\": \"service_account\", ... }" + } + } + ] + } + } + ], + "packages": [...] + } + ] +} +``` + +In your Malloy model: +```malloy +source: events is duckdb.table('my_bq.my_dataset.events') +``` + +**Attach Snowflake:** + +```json +{ + "projects": [ + { + "name": "default", + "connections": [ + { + "name": "duckdb", + "type": "duckdb", + "duckdbConnection": { + "attachedDatabases": [ + { + "name": "my_sf", + "type": "snowflake", + "snowflakeConnection": { + "account": "myorg-myaccount", + "username": "my_user", + "password": "my_password", + "database": "analytics", + "warehouse": "compute_wh" + } + } + ] + } + } + ], + "packages": [...] + } + ] +} +``` + +**Attach PostgreSQL:** + +```json +{ + "projects": [ + { + "name": "default", + "connections": [ + { + "name": "duckdb", + "type": "duckdb", + "duckdbConnection": { + "attachedDatabases": [ + { + "name": "my_pg", + "type": "postgres", + "postgresConnection": { + "host": "db.example.com", + "port": 5432, + "databaseName": "analytics", + "userName": "readonly_user", + "password": "my_password" + } + } + ] + } + } + ], + "packages": [...] + } + ] +} +``` + +**Multiple attached databases:** + +```json +{ + "duckdbConnection": { + "attachedDatabases": [ + { + "name": "warehouse", + "type": "bigquery", + "bigqueryConnection": { ... } + }, + { + "name": "app_db", + "type": "postgres", + "postgresConnection": { ... } + } + ] + } +} +``` --- @@ -74,13 +215,14 @@ The `workingDirectory` sets the base path for file references. If your Malloy co "projects": [ { "name": "default", - "connections": { - "md": { + "connections": [ + { + "name": "md", "type": "motherduck", "token": "your_motherduck_token_here", "database": "my_database" } - }, + ], "packages": [...] } ] @@ -100,14 +242,15 @@ Embed the service account JSON directly in the config: "projects": [ { "name": "default", - "connections": { - "bigquery": { + "connections": [ + { + "name": "bigquery", "type": "bigquery", "defaultProjectId": "my-gcp-project", "location": "US", - "serviceAccountKeyJson": "{\n \"type\": \"service_account\",\n \"project_id\": \"my-gcp-project\",\n \"private_key_id\": \"abc123def456\",\n \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvgIBAD...your-key-here...\\n-----END PRIVATE KEY-----\\n\",\n \"client_email\": \"malloy-publisher@my-gcp-project.iam.gserviceaccount.com\",\n \"client_id\": \"123456789\",\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://oauth2.googleapis.com/token\",\n \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/malloy-publisher%40my-gcp-project.iam.gserviceaccount.com\",\n \"universe_domain\": \"googleapis.com\"\n}" + "serviceAccountKeyJson": "{\n \"type\": \"service_account\",\n \"project_id\": \"my-gcp-project\",\n ...rest of service account JSON...\n}" } - }, + ], "packages": [...] } ] @@ -121,13 +264,14 @@ Embed the service account JSON directly in the config: "projects": [ { "name": "default", - "connections": { - "bigquery": { + "connections": [ + { + "name": "bigquery", "type": "bigquery", "defaultProjectId": "my-gcp-project", "location": "US" } - }, + ], "packages": [...] } ] @@ -145,8 +289,9 @@ Embed the service account JSON directly in the config: "projects": [ { "name": "default", - "connections": { - "snowflake": { + "connections": [ + { + "name": "snowflake", "type": "snowflake", "account": "myorg-myaccount", "username": "publisher_user", @@ -155,7 +300,7 @@ Embed the service account JSON directly in the config: "database": "analytics", "schema": "public" } - }, + ], "packages": [...] } ] @@ -169,8 +314,9 @@ Embed the service account JSON directly in the config: "projects": [ { "name": "default", - "connections": { - "snowflake": { + "connections": [ + { + "name": "snowflake", "type": "snowflake", "account": "myorg-myaccount", "username": "publisher_user", @@ -179,7 +325,7 @@ Embed the service account JSON directly in the config: "warehouse": "compute_wh", "database": "analytics" } - }, + ], "packages": [...] } ] @@ -195,8 +341,9 @@ Embed the service account JSON directly in the config: "projects": [ { "name": "default", - "connections": { - "postgres": { + "connections": [ + { + "name": "postgres", "type": "postgres", "host": "db.example.com", "port": 5432, @@ -204,7 +351,7 @@ Embed the service account JSON directly in the config: "username": "publisher_readonly", "password": "your_password_here" } - }, + ], "packages": [...] } ] @@ -220,8 +367,9 @@ Embed the service account JSON directly in the config: "projects": [ { "name": "default", - "connections": { - "mysql": { + "connections": [ + { + "name": "mysql", "type": "mysql", "host": "db.example.com", "port": 3306, @@ -229,7 +377,7 @@ Embed the service account JSON directly in the config: "user": "publisher_readonly", "password": "your_password_here" } - }, + ], "packages": [...] } ] @@ -245,13 +393,14 @@ Embed the service account JSON directly in the config: "projects": [ { "name": "default", - "connections": { - "trino": { + "connections": [ + { + "name": "trino", "type": "trino", "server": "https://trino.example.com", "port": 8443 } - }, + ], "packages": [...] } ] @@ -271,8 +420,9 @@ You can configure multiple projects for different environments: "projects": [ { "name": "staging", - "connections": { - "postgres": { + "connections": [ + { + "name": "postgres", "type": "postgres", "host": "staging-db.example.com", "port": 5432, @@ -280,7 +430,7 @@ You can configure multiple projects for different environments: "username": "staging_user", "password": "staging_password" } - }, + ], "packages": [ { "name": "analytics", @@ -290,8 +440,9 @@ You can configure multiple projects for different environments: }, { "name": "production", - "connections": { - "postgres": { + "connections": [ + { + "name": "postgres", "type": "postgres", "host": "prod-db.example.com", "port": 5432, @@ -299,7 +450,7 @@ You can configure multiple projects for different environments: "username": "prod_readonly", "password": "prod_password" } - }, + ], "packages": [ { "name": "analytics", From a6123d93ce18f7f13c8cab5b31fc4097b44e31f9 Mon Sep 17 00:00:00 2001 From: James Swirhun Date: Tue, 6 Jan 2026 16:36:21 -0700 Subject: [PATCH 2/2] docs: fix publisher config format in connections and publishing guides - Use array format for connections with `name` field inside objects - Add type-specific nested configs (postgresConnection, bigqueryConnection, etc.) - Add DuckDB attached databases documentation - Clarify --init flag usage and state persistence behavior Signed-off-by: James Swirhun Signed-off-by: James Swirhun --- .../publishing/connections.malloynb | 111 +++++++++++------- .../publishing/publishing.malloynb | 39 ++++-- 2 files changed, 97 insertions(+), 53 deletions(-) diff --git a/src/documentation/user_guides/publishing/connections.malloynb b/src/documentation/user_guides/publishing/connections.malloynb index 5acf13b1..46041033 100644 --- a/src/documentation/user_guides/publishing/connections.malloynb +++ b/src/documentation/user_guides/publishing/connections.malloynb @@ -22,7 +22,9 @@ Publisher uses `publisher.config.json` for database connections. This file lives { "name": "connection_name", "type": "backend_type", - ...connection options... + "Connection": { + ...connection options... + } } ], "packages": [ @@ -36,6 +38,11 @@ Publisher uses `publisher.config.json` for database connections. This file lives } ``` +Each connection object requires: +- `name`: The connection name used in your Malloy code +- `type`: The backend type (`duckdb`, `postgres`, `bigquery`, etc.) +- Type-specific options nested under `Connection` (e.g., `postgresConnection`, `bigqueryConnection`) + **Note:** For packages with local parquet/CSV files, you don't need a connections config at all — Publisher automatically uses DuckDB. --- @@ -219,8 +226,10 @@ source: events is duckdb.table('my_bq.my_dataset.events') { "name": "md", "type": "motherduck", - "token": "your_motherduck_token_here", - "database": "my_database" + "motherduckConnection": { + "accessToken": "your_motherduck_token_here", + "database": "my_database" + } } ], "packages": [...] @@ -246,9 +255,11 @@ Embed the service account JSON directly in the config: { "name": "bigquery", "type": "bigquery", - "defaultProjectId": "my-gcp-project", - "location": "US", - "serviceAccountKeyJson": "{\n \"type\": \"service_account\",\n \"project_id\": \"my-gcp-project\",\n ...rest of service account JSON...\n}" + "bigqueryConnection": { + "defaultProjectId": "my-gcp-project", + "location": "US", + "serviceAccountKeyJson": "{\n \"type\": \"service_account\",\n \"project_id\": \"my-gcp-project\",\n ...rest of service account JSON...\n}" + } } ], "packages": [...] @@ -268,8 +279,10 @@ Embed the service account JSON directly in the config: { "name": "bigquery", "type": "bigquery", - "defaultProjectId": "my-gcp-project", - "location": "US" + "bigqueryConnection": { + "defaultProjectId": "my-gcp-project", + "location": "US" + } } ], "packages": [...] @@ -293,12 +306,14 @@ Embed the service account JSON directly in the config: { "name": "snowflake", "type": "snowflake", - "account": "myorg-myaccount", - "username": "publisher_user", - "password": "your_password_here", - "warehouse": "compute_wh", - "database": "analytics", - "schema": "public" + "snowflakeConnection": { + "account": "myorg-myaccount", + "username": "publisher_user", + "password": "your_password_here", + "warehouse": "compute_wh", + "database": "analytics", + "schema": "public" + } } ], "packages": [...] @@ -318,12 +333,14 @@ Embed the service account JSON directly in the config: { "name": "snowflake", "type": "snowflake", - "account": "myorg-myaccount", - "username": "publisher_user", - "privateKey": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBAD...your-key-here...\n-----END PRIVATE KEY-----", - "privateKeyPassphrase": "your_passphrase_if_encrypted", - "warehouse": "compute_wh", - "database": "analytics" + "snowflakeConnection": { + "account": "myorg-myaccount", + "username": "publisher_user", + "privateKey": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBAD...your-key-here...\n-----END PRIVATE KEY-----", + "privateKeyPass": "your_passphrase_if_encrypted", + "warehouse": "compute_wh", + "database": "analytics" + } } ], "packages": [...] @@ -345,11 +362,13 @@ Embed the service account JSON directly in the config: { "name": "postgres", "type": "postgres", - "host": "db.example.com", - "port": 5432, - "database": "analytics", - "username": "publisher_readonly", - "password": "your_password_here" + "postgresConnection": { + "host": "db.example.com", + "port": 5432, + "databaseName": "analytics", + "userName": "publisher_readonly", + "password": "your_password_here" + } } ], "packages": [...] @@ -371,11 +390,13 @@ Embed the service account JSON directly in the config: { "name": "mysql", "type": "mysql", - "host": "db.example.com", - "port": 3306, - "database": "analytics", - "user": "publisher_readonly", - "password": "your_password_here" + "mysqlConnection": { + "host": "db.example.com", + "port": 3306, + "database": "analytics", + "user": "publisher_readonly", + "password": "your_password_here" + } } ], "packages": [...] @@ -397,8 +418,10 @@ Embed the service account JSON directly in the config: { "name": "trino", "type": "trino", - "server": "https://trino.example.com", - "port": 8443 + "trinoConnection": { + "server": "https://trino.example.com", + "port": 8443 + } } ], "packages": [...] @@ -424,11 +447,13 @@ You can configure multiple projects for different environments: { "name": "postgres", "type": "postgres", - "host": "staging-db.example.com", - "port": 5432, - "database": "analytics", - "username": "staging_user", - "password": "staging_password" + "postgresConnection": { + "host": "staging-db.example.com", + "port": 5432, + "databaseName": "analytics", + "userName": "staging_user", + "password": "staging_password" + } } ], "packages": [ @@ -444,11 +469,13 @@ You can configure multiple projects for different environments: { "name": "postgres", "type": "postgres", - "host": "prod-db.example.com", - "port": 5432, - "database": "analytics", - "username": "prod_readonly", - "password": "prod_password" + "postgresConnection": { + "host": "prod-db.example.com", + "port": 5432, + "databaseName": "analytics", + "userName": "prod_readonly", + "password": "prod_password" + } } ], "packages": [ diff --git a/src/documentation/user_guides/publishing/publishing.malloynb b/src/documentation/user_guides/publishing/publishing.malloynb index a743c1d4..e230b0b6 100644 --- a/src/documentation/user_guides/publishing/publishing.malloynb +++ b/src/documentation/user_guides/publishing/publishing.malloynb @@ -69,24 +69,38 @@ For alternative deployment methods (Docker, build from source), see [Deployment Go to `http://localhost:4000` to browse and query your models. -That's it for local files. If your models use DuckDB with `.parquet`, `.csv`, or `.json` files, you're done. +That's it for local files. If your models use DuckDB with `.parquet`, `.csv`, or `.json` files, you're done—no config file needed. -**Note:** If you edit a `.malloy` file while Publisher is running, you'll need to restart it to pick up changes. +**Note:** If you edit a `.malloy` file while Publisher is running, restart it to pick up changes. **Want to connect to a database?** Create `publisher.config.json` in the same directory: ```json { - "connections": { - "my_postgres": { - "type": "postgres", - "host": "localhost", - "port": 5432, - "database": "analytics", - "user": "malloy", - "password": "" + "projects": [ + { + "name": "default", + "connections": [ + { + "name": "my_postgres", + "type": "postgres", + "postgresConnection": { + "host": "localhost", + "port": 5432, + "databaseName": "analytics", + "userName": "malloy", + "password": "" + } + } + ], + "packages": [ + { + "name": "my-analytics", + "location": "./my-analytics" + } + ] } - } + ] } ``` @@ -206,10 +220,13 @@ npx @malloy-publisher/server --init --server_root . ``` Use `--init` when: +- **After upgrading** Publisher (schema may have changed) - You've updated `publisher.config.json` and want those changes applied - You want to reset to the original configuration - You're troubleshooting configuration issues +**Note:** On first run, Publisher automatically creates the database and syncs from `publisher.config.json`—no `--init` needed. + ### Mutable vs Frozen Configuration By default, Publisher allows configuration changes via the API. To lock the configuration (e.g., in production):