From 11e9dcff680296e36426c6e9a148bf36c1dce899 Mon Sep 17 00:00:00 2001 From: Rory McKinley Date: Mon, 19 Jan 2026 11:05:38 +0200 Subject: [PATCH 1/4] Instrument all the things * As suggested by Claude --- packages/ws-worker/src/api/claim.ts | 6 ++++++ packages/ws-worker/src/server.ts | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/packages/ws-worker/src/api/claim.ts b/packages/ws-worker/src/api/claim.ts index 19c215538..8fbca71a1 100644 --- a/packages/ws-worker/src/api/claim.ts +++ b/packages/ws-worker/src/api/claim.ts @@ -163,6 +163,12 @@ const claim = ( delete app.openClaims[claimId]; logger.error('TIMEOUT on claim. Runs may be lost.'); reject(new Error('timeout')); + }) + .receive('*', (response) => { + delete app.openClaims[claimId]; + logger.error(`[Claim ${claimId}] Received UNEXPECTED response status. Full response:`, JSON.stringify(response, null, 2)); + logger.error(`[Claim ${claimId}] Channel state:`, app.queueChannel?.state); + reject(new Error('unexpected response status')); }); }); }; diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 88c09bdb4..b4cd2c8e3 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -114,6 +114,35 @@ function connect(app: ServerApp, logger: Logger, options: ServerOptions = {}) { app.socket = socket; app.queueChannel = channel; + // Add channel event listeners for debugging + channel.onError((error) => { + logger.error('Channel error event:', error); + logger.error('Channel state:', channel.state); + logger.error('Open claims:', Object.keys(app.openClaims)); + }); + + channel.onClose((event) => { + logger.error('Channel close event:', event); + logger.error('Channel state:', channel.state); + logger.error('Open claims:', Object.keys(app.openClaims)); + }); + logger.debug('Channel event listeners attached (onError, onClose)'); + + // Wrap the channel's push method to log all WebSocket messages (Patch 5) + const originalPush = channel.push.bind(channel); + channel.push = function(event, payload) { + logger.debug('[Channel Push] Event:', event); + logger.debug('[Channel Push] Payload:', JSON.stringify(payload, null, 2)); + logger.debug('[Channel Push] Channel state:', channel.state); + + const result = originalPush(event, payload); + + logger.debug('[Channel Push] Push returned, ReceiveHook attached'); + + return result; + }; + logger.debug('Channel push method wrapped for debugging'); + // trigger the workloop if (options.noLoop) { // @ts-ignore From 6dfd98436f6f6d71dfa44ad71d33702a41e394b6 Mon Sep 17 00:00:00 2001 From: Rory McKinley Date: Mon, 19 Jan 2026 14:16:17 +0200 Subject: [PATCH 2/4] Even more instrumentation --- packages/ws-worker/src/server.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index b4cd2c8e3..5d188f01b 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -116,7 +116,21 @@ function connect(app: ServerApp, logger: Logger, options: ServerOptions = {}) { // Add channel event listeners for debugging channel.onError((error) => { - logger.error('Channel error event:', error); + logger.error('Channel error event (raw):', error); + logger.error('Channel error type:', typeof error); + logger.error('Channel error constructor:', error?.constructor?.name); + logger.error('Channel error keys:', Object.keys(error || {})); + logger.error('Channel error stringified:', JSON.stringify(error, null, 2)); + // Try to extract common error properties + if (error) { + logger.error('Error details:', { + message: error.message, + reason: error.reason, + response: error.response, + status: error.status, + stack: error.stack, + }); + } logger.error('Channel state:', channel.state); logger.error('Open claims:', Object.keys(app.openClaims)); }); From b5a1e2aa00e21c01530d7bc120d720344ad641f1 Mon Sep 17 00:00:00 2001 From: Rory McKinley Date: Mon, 19 Jan 2026 15:06:37 +0200 Subject: [PATCH 3/4] More instrumentation --- packages/ws-worker/src/channels/worker-queue.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/ws-worker/src/channels/worker-queue.ts b/packages/ws-worker/src/channels/worker-queue.ts index 3a081c42a..93e80a574 100644 --- a/packages/ws-worker/src/channels/worker-queue.ts +++ b/packages/ws-worker/src/channels/worker-queue.ts @@ -106,6 +106,13 @@ const connectToWorkerQueue = ( // if we fail to connect, the socket will try to reconnect // forever (?) with backoff - see reconnectAfterMs socket.onError((e: any) => { + logger.error('Socket error event:', e); + logger.error('Socket error type:', typeof e); + logger.error('Socket error details:', { + message: e?.message, + code: e?.code, + reason: e?.reason, + }); Sentry.addBreadcrumb({ category: 'lifecycle', message: 'Error in web socket connection', From 0aba7703919d245331b9139de26bd6abaed9b947 Mon Sep 17 00:00:00 2001 From: Rory McKinley Date: Mon, 19 Jan 2026 15:21:42 +0200 Subject: [PATCH 4/4] instrumentation --- packages/ws-worker/src/server.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 5d188f01b..5555a4a4b 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -114,6 +114,24 @@ function connect(app: ServerApp, logger: Logger, options: ServerOptions = {}) { app.socket = socket; app.queueChannel = channel; + // Intercept raw channel messages to see what Lightning is sending + const originalOnMessage = channel.onMessage.bind(channel); + channel.onMessage = function(event, payload, ref) { + logger.debug('─────────────────────────────────────'); + logger.debug('[Channel onMessage] Received raw message'); + logger.debug('[Channel onMessage] Event:', event); + logger.debug('[Channel onMessage] Payload:', JSON.stringify(payload, null, 2)); + logger.debug('[Channel onMessage] Ref:', ref); + logger.debug('[Channel onMessage] Channel state before processing:', channel.state); + logger.debug('─────────────────────────────────────'); + + const result = originalOnMessage(event, payload, ref); + + logger.debug('[Channel onMessage] After processing, channel state:', channel.state); + + return result; + }; + // Add channel event listeners for debugging channel.onError((error) => { logger.error('Channel error event (raw):', error);