Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions lib/command/run-workers.js
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,22 @@ export default async function (workerCount, selectedRuns, options) {

output.print(`CodeceptJS v${Codecept.version()} ${output.standWithUkraine()}`)
output.print(`Running tests in ${output.styles.bold(numberOfWorkers)} workers...`)
output.print()
store.hasWorkers = true

const workers = new Workers(numberOfWorkers, config)
workers.overrideConfig(overrideConfigs)

// Show test distribution after workers are initialized
await workers.bootstrapAll()

const workerObjects = workers.getWorkers()
output.print()
output.print('Test distribution:')
workerObjects.forEach((worker, index) => {
const testCount = worker.tests.length
output.print(` Worker ${index + 1}: ${testCount} test${testCount !== 1 ? 's' : ''}`)
})
output.print()

workers.on(event.test.failed, test => {
output.test.failed(test)
Expand All @@ -68,7 +79,6 @@ export default async function (workerCount, selectedRuns, options) {
if (options.verbose) {
await getMachineInfo()
}
await workers.bootstrapAll()
await workers.run()
} catch (err) {
output.error(err)
Expand Down
88 changes: 82 additions & 6 deletions lib/command/workers/runTests.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,45 @@ const stderr = ''

const { options, tests, testRoot, workerIndex, poolMode } = workerData

// Global error handlers to catch critical errors but not test failures
process.on('uncaughtException', (err) => {
// Log to stderr to bypass stdout suppression
process.stderr.write(`[Worker ${workerIndex}] UNCAUGHT EXCEPTION: ${err.message}\n`)
process.stderr.write(`${err.stack}\n`)

// Don't exit on test assertion errors - those are handled by mocha
if (err.name === 'AssertionError' || err.message?.includes('expected')) {
return
}
process.exit(1)
})

process.on('unhandledRejection', (reason, promise) => {
// Log to stderr to bypass stdout suppression
const msg = reason?.message || String(reason)
process.stderr.write(`[Worker ${workerIndex}] UNHANDLED REJECTION: ${msg}\n`)
if (reason?.stack) {
process.stderr.write(`${reason.stack}\n`)
}

// Don't exit on test-related rejections
if (msg.includes('expected') || msg.includes('AssertionError')) {
return
}
process.exit(1)
})

// hide worker output
// In pool mode, only suppress output if debug is NOT enabled
// In regular mode, hide result output but allow step output in verbose/debug
if (poolMode && !options.debug) {
// In pool mode without debug, allow test names and important output but suppress verbose details
const originalWrite = process.stdout.write
process.stdout.write = string => {
// Always allow Worker logs
if (string.includes('[Worker')) {
return originalWrite.call(process.stdout, string)
}
// Allow test names (✔ or ✖), Scenario Steps, failures, and important markers
if (
string.includes('✔') ||
Expand All @@ -45,7 +77,12 @@ if (poolMode && !options.debug) {
return originalWrite.call(process.stdout, string)
}
} else if (!poolMode && !options.debug && !options.verbose) {
const originalWrite = process.stdout.write
process.stdout.write = string => {
// Always allow Worker logs
if (string.includes('[Worker')) {
return originalWrite.call(process.stdout, string)
}
stdout += string
return true
}
Expand Down Expand Up @@ -82,6 +119,13 @@ let config
// Load test and run
initPromise = (async function () {
try {
// Add staggered delay at the very start to prevent resource conflicts
// Longer delay for browser initialization conflicts
const delay = (workerIndex - 1) * 2000 // 0ms, 2s, 4s, etc.
if (delay > 0) {
await new Promise(resolve => setTimeout(resolve, delay))
}

// Import modules dynamically to avoid ES Module loader race conditions in Node 22.x
const eventModule = await import('../../event.js')
const containerModule = await import('../../container.js')
Expand All @@ -98,16 +142,32 @@ initPromise = (async function () {

const overrideConfigs = tryOrDefault(() => JSON.parse(options.override), {})

// IMPORTANT: await is required here since getConfig is async
const baseConfig = await getConfig(options.config || testRoot)
let baseConfig
try {
// IMPORTANT: await is required here since getConfig is async
baseConfig = await getConfig(options.config || testRoot)
} catch (configErr) {
process.stderr.write(`[Worker ${workerIndex}] FAILED loading config: ${configErr.message}\n`)
process.stderr.write(`${configErr.stack}\n`)
await new Promise(resolve => setTimeout(resolve, 100))
process.exit(1)
}

// important deep merge so dynamic things e.g. functions on config are not overridden
config = deepMerge(baseConfig, overrideConfigs)

// Pass workerIndex as child option for output.process() to display worker prefix
const optsWithChild = { ...options, child: workerIndex }
codecept = new Codecept(config, optsWithChild)
await codecept.init(testRoot)

try {
await codecept.init(testRoot)
} catch (initErr) {
process.stderr.write(`[Worker ${workerIndex}] FAILED during codecept.init(): ${initErr.message}\n`)
process.stderr.write(`${initErr.stack}\n`)
process.exit(1)
}

codecept.loadTests()
mocha = container.mocha()

Expand All @@ -126,10 +186,12 @@ initPromise = (async function () {
await runTests()
} else {
// No tests to run, close the worker
console.error(`[Worker ${workerIndex}] ERROR: No tests found after filtering! Assigned ${tests.length} UIDs but none matched.`)
parentPort?.close()
}
} catch (err) {
console.error('Error in worker initialization:', err)
process.stderr.write(`[Worker ${workerIndex}] FATAL ERROR: ${err.message}\n`)
process.stderr.write(`${err.stack}\n`)
process.exit(1)
}
})()
Expand All @@ -147,8 +209,14 @@ async function runTests() {
disablePause()
try {
await codecept.run()
} catch (err) {
throw err
} finally {
await codecept.teardown()
try {
await codecept.teardown()
} catch (err) {
// Ignore teardown errors
}
}
}

Expand Down Expand Up @@ -336,8 +404,16 @@ function filterTests() {
mocha.files = files
mocha.loadFiles()

for (const suite of mocha.suite.suites) {
// Recursively filter tests in all suites (including nested ones)
const filterSuiteTests = (suite) => {
suite.tests = suite.tests.filter(test => tests.indexOf(test.uid) >= 0)
for (const childSuite of suite.suites) {
filterSuiteTests(childSuite)
}
}

for (const suite of mocha.suite.suites) {
filterSuiteTests(suite)
}
}

Expand Down
2 changes: 1 addition & 1 deletion lib/helper/Playwright.js
Original file line number Diff line number Diff line change
Expand Up @@ -912,7 +912,7 @@ class Playwright extends Helper {
}

async _finishTest() {
if ((restartsSession() || restartsContext() || restartsBrowser()) && this.isRunning) {
if (this.isRunning) {
try {
await Promise.race([this._stopBrowser(), new Promise((_, reject) => setTimeout(() => reject(new Error('Test finish timeout')), 10000))])
} catch (e) {
Expand Down
16 changes: 2 additions & 14 deletions lib/listener/helpers.js
Original file line number Diff line number Diff line change
Expand Up @@ -73,30 +73,18 @@ export default function () {
})

event.dispatcher.on(event.all.result, () => {
// Skip _finishTest for all helpers if any browser helper restarts to avoid double cleanup
const hasBrowserRestart = Object.values(helpers).some(helper =>
(helper.config && (helper.config.restart === 'browser' || helper.config.restart === 'context' || helper.config.restart === true)) ||
(helper.options && (helper.options.restart === 'browser' || helper.options.restart === 'context' || helper.options.restart === true))
)

Object.keys(helpers).forEach(key => {
const helper = helpers[key]
if (helper._finishTest && !hasBrowserRestart) {
if (helper._finishTest) {
recorder.add(`hook ${key}._finishTest()`, () => helper._finishTest(), true, false)
}
})
})

event.dispatcher.on(event.all.after, () => {
// Skip _cleanup for all helpers if any browser helper restarts to avoid double cleanup
const hasBrowserRestart = Object.values(helpers).some(helper =>
(helper.config && (helper.config.restart === 'browser' || helper.config.restart === 'context' || helper.config.restart === true)) ||
(helper.options && (helper.options.restart === 'browser' || helper.options.restart === 'context' || helper.options.restart === true))
)

Object.keys(helpers).forEach(key => {
const helper = helpers[key]
if (helper._cleanup && !hasBrowserRestart) {
if (helper._cleanup) {
recorder.add(`hook ${key}._cleanup()`, () => helper._cleanup(), true, false)
}
})
Expand Down
4 changes: 2 additions & 2 deletions lib/utils/typescript.js
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ const __dirname = __dirname_fn(__filename);
let jsContent = transpileTS(filePath)

// Find all relative TypeScript imports in this file
const importRegex = /from\s+['"](\..+?)(?:\.ts)?['"]/g
const importRegex = /from\s+['"](\.[^'"]+?)(?:\.ts)?['"]/g
let match
const imports = []

Expand Down Expand Up @@ -170,7 +170,7 @@ const __dirname = __dirname_fn(__filename);

// After all dependencies are transpiled, rewrite imports in this file
jsContent = jsContent.replace(
/from\s+['"](\..+?)(?:\.ts)?['"]/g,
/from\s+['"](\.[^'"]+?)(?:\.ts)?['"]/g,
(match, importPath) => {
let resolvedPath = path.resolve(fileBaseDir, importPath)
const originalExt = path.extname(importPath)
Expand Down
74 changes: 71 additions & 3 deletions lib/workers.js
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,9 @@ class Workers extends EventEmitter {
// If Codecept isn't initialized yet, return empty groups as a safe fallback
if (!this.codecept) return populateGroups(numberOfWorkers)
const files = this.codecept.testFiles

// Create a fresh mocha instance to avoid state pollution
Container.createMocha(this.codecept.config.mocha || {}, this.options)
const mocha = Container.mocha()
mocha.files = files
mocha.loadFiles()
Expand All @@ -384,6 +387,10 @@ class Workers extends EventEmitter {
groupCounter++
}
})

// Clean up after collecting test UIDs
mocha.unloadFiles()

return groups
}

Expand Down Expand Up @@ -452,9 +459,12 @@ class Workers extends EventEmitter {
const files = this.codecept.testFiles
const groups = populateGroups(numberOfWorkers)

// Create a fresh mocha instance to avoid state pollution
Container.createMocha(this.codecept.config.mocha || {}, this.options)
const mocha = Container.mocha()
mocha.files = files
mocha.loadFiles()

mocha.suite.suites.forEach(suite => {
const i = indexOfSmallestElement(groups)
suite.tests.forEach(test => {
Expand All @@ -463,6 +473,10 @@ class Workers extends EventEmitter {
}
})
})

// Clean up after collecting test UIDs
mocha.unloadFiles()

return groups
}

Expand Down Expand Up @@ -504,8 +518,24 @@ class Workers extends EventEmitter {
// Workers are already running, this is just a placeholder step
})

// Add overall timeout to prevent infinite hanging
const overallTimeout = setTimeout(() => {
console.error('[Main] Overall timeout reached (10 minutes). Force terminating remaining workers...')
workerThreads.forEach(w => {
try {
w.terminate()
} catch (e) {
// ignore
}
})
this._finishRun()
}, 600000) // 10 minutes

return new Promise(resolve => {
this.on('end', resolve)
this.on('end', () => {
clearTimeout(overallTimeout)
resolve()
})
})
}

Expand All @@ -528,8 +558,32 @@ class Workers extends EventEmitter {
if (this.isPoolMode) {
this.activeWorkers.set(worker, { available: true, workerIndex: null })
}

// Track last activity time to detect hanging workers
let lastActivity = Date.now()
let currentTest = null
const workerTimeout = 300000 // 5 minutes

const timeoutChecker = setInterval(() => {
const elapsed = Date.now() - lastActivity
if (elapsed > workerTimeout) {
console.error(`[Main] Worker appears to be hanging (no activity for ${Math.floor(elapsed/1000)}s). Terminating...`)
if (currentTest) {
console.error(`[Main] Last test: ${currentTest}`)
}
clearInterval(timeoutChecker)
worker.terminate()
}
}, 30000) // Check every 30 seconds

worker.on('message', message => {
lastActivity = Date.now() // Update activity timestamp

// Track current test
if (message.event === event.test.started && message.data) {
currentTest = message.data.title || message.data.fullTitle
}

output.process(message.workerIndex)

// Handle test requests for pool mode
Expand Down Expand Up @@ -646,11 +700,25 @@ class Workers extends EventEmitter {
})

worker.on('error', err => {
console.error(`[Main] Worker error:`, err.message || err)
if (currentTest) {
console.error(`[Main] Failed during test: ${currentTest}`)
}
this.errors.push(err)
})

worker.on('exit', () => {
worker.on('exit', (code) => {
clearInterval(timeoutChecker)
this.closedWorkers += 1

if (code !== 0) {
console.error(`[Main] Worker exited with code ${code}`)
if (currentTest) {
console.error(`[Main] Last test running: ${currentTest}`)
}
// Mark as failed
process.exitCode = 1
}

if (this.isPoolMode) {
// Pool mode: finish when all workers have exited and no more tests
Expand All @@ -666,7 +734,7 @@ class Workers extends EventEmitter {

_finishRun() {
event.dispatcher.emit(event.workers.after, { tests: this.workers.map(worker => worker.tests) })
if (Container.result().hasFailed) {
if (Container.result().hasFailed || this.errors.length > 0) {
process.exitCode = 1
} else {
process.exitCode = 0
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "codeceptjs",
"version": "4.0.1-beta.9",
"version": "4.0.2-beta.17",
"type": "module",
"description": "Supercharged End 2 End Testing Framework for NodeJS",
"keywords": [
Expand Down
Loading