From 4ba35ca9db5575df98132e2b3038df0b8653ae51 Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Wed, 11 Feb 2026 13:37:52 +0000 Subject: [PATCH 01/19] feat(types): add llms-txt type definitions Add AgentFile and LlmsTxtResult interfaces for llms.txt generation and export from the shared types barrel. --- shared/types/index.ts | 1 + shared/types/llms-txt.ts | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 shared/types/llms-txt.ts diff --git a/shared/types/index.ts b/shared/types/index.ts index 88e28afe0..e378738e1 100644 --- a/shared/types/index.ts +++ b/shared/types/index.ts @@ -9,3 +9,4 @@ export * from './i18n-status' export * from './comparison' export * from './skills' export * from './version-downloads' +export * from './llms-txt' diff --git a/shared/types/llms-txt.ts b/shared/types/llms-txt.ts new file mode 100644 index 000000000..85bca8758 --- /dev/null +++ b/shared/types/llms-txt.ts @@ -0,0 +1,31 @@ +/** + * Agent instruction file discovered in a package + */ +export interface AgentFile { + /** Relative path within the package (e.g., "CLAUDE.md", ".github/copilot-instructions.md") */ + path: string + /** File content */ + content: string + /** Human-readable display name (e.g., "Claude Code", "GitHub Copilot") */ + displayName: string +} + +/** + * Result of gathering all data needed to generate llms.txt + */ +export interface LlmsTxtResult { + /** Package name (e.g., "nuxt" or "@nuxt/kit") */ + packageName: string + /** Resolved version (e.g., "3.12.0") */ + version: string + /** Package description from packument */ + description?: string + /** Homepage URL */ + homepage?: string + /** Repository URL */ + repositoryUrl?: string + /** README content (raw markdown) */ + readme?: string + /** Discovered agent instruction files */ + agentFiles: AgentFile[] +} From b7b4a1df75656125696bc89a866caf903a0f9c3c Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Wed, 11 Feb 2026 13:38:01 +0000 Subject: [PATCH 02/19] feat(llms-txt): add core utility functions Add discoverAgentFiles, fetchAgentFiles, generateLlmsTxt, and handleLlmsTxt orchestrator for llms.txt generation from npm packages. --- server/utils/llms-txt.ts | 246 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 server/utils/llms-txt.ts diff --git a/server/utils/llms-txt.ts b/server/utils/llms-txt.ts new file mode 100644 index 000000000..aeb38016e --- /dev/null +++ b/server/utils/llms-txt.ts @@ -0,0 +1,246 @@ +import type { Packument } from '@npm/types' +import type { JsDelivrFileNode, AgentFile, LlmsTxtResult } from '#shared/types' +import { NPM_MISSING_README_SENTINEL } from '#shared/utils/constants' + +/** Well-known agent instruction files at the package root */ +const ROOT_AGENT_FILES: Record = { + 'CLAUDE.md': 'Claude Code', + 'AGENTS.md': 'Agent Instructions', + 'AGENT.md': 'Agent Instructions', + '.cursorrules': 'Cursor Rules', + '.windsurfrules': 'Windsurf Rules', + '.clinerules': 'Cline Rules', +} + +/** Well-known agent files inside specific directories */ +const DIRECTORY_AGENT_FILES: Record = { + '.github/copilot-instructions.md': 'GitHub Copilot', +} + +/** Directories containing rule files (match *.md inside) */ +const RULE_DIRECTORIES: Record = { + '.cursor/rules': 'Cursor Rules', + '.windsurf/rules': 'Windsurf Rules', +} + +/** + * Discover agent instruction file paths from a jsDelivr file tree. + * Scans root-level files, known subdirectory files, and rule directories. + */ +export function discoverAgentFiles(files: JsDelivrFileNode[]): string[] { + const discovered: string[] = [] + + for (const node of files) { + // Root-level well-known files + if (node.type === 'file' && node.name in ROOT_AGENT_FILES) { + discovered.push(node.name) + } + + // Directory-based files + if (node.type === 'directory') { + // .github/copilot-instructions.md + if (node.name === '.github' && node.files) { + for (const child of node.files) { + const fullPath = `.github/${child.name}` + if (child.type === 'file' && fullPath in DIRECTORY_AGENT_FILES) { + discovered.push(fullPath) + } + } + } + + // .cursor/rules/*.md and .windsurf/rules/*.md + for (const dirPath of Object.keys(RULE_DIRECTORIES)) { + const [topDir, subDir] = dirPath.split('/') + if (node.name === topDir && node.files) { + const rulesDir = node.files.find(f => f.type === 'directory' && f.name === subDir) + if (rulesDir?.files) { + for (const ruleFile of rulesDir.files) { + if (ruleFile.type === 'file' && ruleFile.name.endsWith('.md')) { + discovered.push(`${dirPath}/${ruleFile.name}`) + } + } + } + } + } + } + } + + return discovered +} + +/** + * Get the display name for an agent file path. + */ +function getDisplayName(filePath: string): string { + if (filePath in ROOT_AGENT_FILES) return ROOT_AGENT_FILES[filePath] + if (filePath in DIRECTORY_AGENT_FILES) return DIRECTORY_AGENT_FILES[filePath] + + for (const [dirPath, displayName] of Object.entries(RULE_DIRECTORIES)) { + if (filePath.startsWith(`${dirPath}/`)) return `${displayName}: ${filePath.split('/').pop()}` + } + + return filePath +} + +/** + * Fetch agent instruction files from jsDelivr CDN. + * Fetches in parallel, gracefully skipping failures. + */ +export async function fetchAgentFiles( + packageName: string, + version: string, + filePaths: string[], +): Promise { + const results = await Promise.all( + filePaths.map(async (path): Promise => { + try { + const url = `https://cdn.jsdelivr.net/npm/${packageName}@${version}/${path}` + const response = await fetch(url) + if (!response.ok) return null + const content = await response.text() + return { path, content, displayName: getDisplayName(path) } + } catch { + return null + } + }), + ) + + return results.filter((r): r is AgentFile => r !== null) +} + +/** + * Generate llms.txt markdown content per the llmstxt.org spec. + * + * Structure: + * - H1 title with package name and version + * - Blockquote description (if available) + * - Metadata list (homepage, repository, npm) + * - README section + * - Agent Instructions section (one sub-heading per file) + */ +export function generateLlmsTxt(result: LlmsTxtResult): string { + const lines: string[] = [] + + // Title + lines.push(`# ${result.packageName}@${result.version}`) + lines.push('') + + // Description blockquote + if (result.description) { + lines.push(`> ${result.description}`) + lines.push('') + } + + // Metadata + const meta: string[] = [] + if (result.homepage) meta.push(`- Homepage: ${result.homepage}`) + if (result.repositoryUrl) meta.push(`- Repository: ${result.repositoryUrl}`) + meta.push(`- npm: https://www.npmjs.com/package/${result.packageName}/v/${result.version}`) + lines.push(...meta) + lines.push('') + + // README + if (result.readme) { + lines.push('## README') + lines.push('') + lines.push(result.readme) + lines.push('') + } + + // Agent instructions + if (result.agentFiles.length > 0) { + lines.push('## Agent Instructions') + lines.push('') + + for (const file of result.agentFiles) { + lines.push(`### ${file.displayName} (\`${file.path}\`)`) + lines.push('') + lines.push(file.content) + lines.push('') + } + } + + return lines.join('\n').trimEnd() + '\n' +} + +/** Standard README filenames to try from jsDelivr CDN */ +const README_FILENAMES = ['README.md', 'readme.md', 'Readme.md'] + +/** Fetch README from jsDelivr CDN as fallback */ +async function fetchReadmeFromCdn(packageName: string, version: string): Promise { + for (const filename of README_FILENAMES) { + try { + const url = `https://cdn.jsdelivr.net/npm/${packageName}@${version}/${filename}` + const response = await fetch(url) + if (response.ok) return await response.text() + } catch { + // Try next + } + } + return null +} + +/** Extract README from packument data */ +function getReadmeFromPackument(packageData: Packument, requestedVersion?: string): string | null { + const readme = requestedVersion + ? packageData.versions[requestedVersion]?.readme + : packageData.readme + + if (readme && readme !== NPM_MISSING_README_SENTINEL) { + return readme + } + return null +} + +/** Extract a clean repository URL from packument repository field */ +function parseRepoUrl( + repository?: { type?: string; url?: string; directory?: string } | string, +): string | undefined { + if (!repository) return undefined + const url = typeof repository === 'string' ? repository : repository.url + if (!url) return undefined + return url.replace(/^git\+/, '').replace(/\.git$/, '') +} + +/** + * Orchestrates fetching all data and generating llms.txt for a package. + * Shared by both versioned and unversioned route handlers. + */ +export async function handleLlmsTxt( + packageName: string, + requestedVersion?: string, +): Promise { + const packageData = await fetchNpmPackage(packageName) + const resolvedVersion = requestedVersion ?? packageData['dist-tags']?.latest + + if (!resolvedVersion) { + throw createError({ statusCode: 404, message: 'Could not resolve package version.' }) + } + + // Extract README from packument (sync) + const readmeFromPackument = getReadmeFromPackument(packageData, requestedVersion) + + // Fetch file tree (and README from CDN if packument didn't have one) + const [fileTreeData, cdnReadme] = await Promise.all([ + fetchFileTree(packageName, resolvedVersion), + readmeFromPackument ? null : fetchReadmeFromCdn(packageName, resolvedVersion), + ]) + + const readme = readmeFromPackument ?? cdnReadme ?? undefined + + // Discover and fetch agent files + const agentFilePaths = discoverAgentFiles(fileTreeData.files) + const agentFiles = await fetchAgentFiles(packageName, resolvedVersion, agentFilePaths) + + const result: LlmsTxtResult = { + packageName, + version: resolvedVersion, + description: packageData.description, + homepage: packageData.homepage, + repositoryUrl: parseRepoUrl(packageData.repository), + readme, + agentFiles, + } + + return generateLlmsTxt(result) +} From 25341d74601c3133cd9584fc082db071c606a2be Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Wed, 11 Feb 2026 13:38:08 +0000 Subject: [PATCH 03/19] feat(llms-txt): add API route for llms.txt generation Serve llms.txt at /api/registry/llms-txt/[...pkg] following existing registry API patterns with cached event handler and SWR. --- server/api/registry/llms-txt/[...pkg].get.ts | 50 ++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 server/api/registry/llms-txt/[...pkg].get.ts diff --git a/server/api/registry/llms-txt/[...pkg].get.ts b/server/api/registry/llms-txt/[...pkg].get.ts new file mode 100644 index 000000000..79e58f4bd --- /dev/null +++ b/server/api/registry/llms-txt/[...pkg].get.ts @@ -0,0 +1,50 @@ +import * as v from 'valibot' +import { PackageRouteParamsSchema } from '#shared/schemas/package' +import { CACHE_MAX_AGE_ONE_HOUR } from '#shared/utils/constants' +import { handleApiError } from '#server/utils/error-handler' +import { handleLlmsTxt } from '#server/utils/llms-txt' + +/** + * Serves llms.txt for an npm package — a single LLM-friendly markdown document + * aggregating README and agent instruction files (CLAUDE.md, AGENTS.md, etc.). + * + * URL patterns: + * - /api/registry/llms-txt/nuxt → latest version + * - /api/registry/llms-txt/@nuxt/kit → scoped, latest + * - /api/registry/llms-txt/nuxt/v/3.12.0 → specific version + * - /api/registry/llms-txt/@nuxt/kit/v/1.0.0 → scoped, specific version + */ +export default defineCachedEventHandler( + async event => { + const pkgParamSegments = getRouterParam(event, 'pkg')?.split('/') ?? [] + if (pkgParamSegments.length === 0) { + throw createError({ statusCode: 404, message: 'Package name is required.' }) + } + + const { rawPackageName, rawVersion } = parsePackageParams(pkgParamSegments) + + try { + const { packageName, version } = v.parse(PackageRouteParamsSchema, { + packageName: rawPackageName, + version: rawVersion, + }) + + const content = await handleLlmsTxt(packageName, version) + setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') + return content + } catch (error: unknown) { + handleApiError(error, { + statusCode: 502, + message: 'Failed to generate llms.txt.', + }) + } + }, + { + maxAge: CACHE_MAX_AGE_ONE_HOUR, + swr: true, + getKey: event => { + const pkg = getRouterParam(event, 'pkg') ?? '' + return `llms-txt:${pkg.replace(/\/+$/, '').trim()}` + }, + }, +) From c6be6cf580d3d6ccf98b758a7a82bf8456c00dd1 Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Wed, 11 Feb 2026 13:38:16 +0000 Subject: [PATCH 04/19] test(llms-txt): add unit tests for utility functions Cover discoverAgentFiles, fetchAgentFiles, and generateLlmsTxt including root files, directory scanning, graceful failures, scoped packages, and full/minimal output generation. --- test/unit/server/utils/llms-txt.spec.ts | 301 ++++++++++++++++++++++++ 1 file changed, 301 insertions(+) create mode 100644 test/unit/server/utils/llms-txt.spec.ts diff --git a/test/unit/server/utils/llms-txt.spec.ts b/test/unit/server/utils/llms-txt.spec.ts new file mode 100644 index 000000000..af326694d --- /dev/null +++ b/test/unit/server/utils/llms-txt.spec.ts @@ -0,0 +1,301 @@ +import { describe, expect, it, vi } from 'vitest' +import type { JsDelivrFileNode, LlmsTxtResult } from '../../../../shared/types' +import { + discoverAgentFiles, + fetchAgentFiles, + generateLlmsTxt, +} from '../../../../server/utils/llms-txt' + +describe('discoverAgentFiles', () => { + it('discovers root-level agent files', () => { + const files: JsDelivrFileNode[] = [ + { type: 'file', name: 'CLAUDE.md', size: 100 }, + { type: 'file', name: 'AGENTS.md', size: 200 }, + { type: 'file', name: 'AGENT.md', size: 50 }, + { type: 'file', name: '.cursorrules', size: 80 }, + { type: 'file', name: '.windsurfrules', size: 60 }, + { type: 'file', name: '.clinerules', size: 40 }, + { type: 'file', name: 'package.json', size: 500 }, + { type: 'file', name: 'README.md', size: 3000 }, + ] + + const result = discoverAgentFiles(files) + + expect(result).toContain('CLAUDE.md') + expect(result).toContain('AGENTS.md') + expect(result).toContain('AGENT.md') + expect(result).toContain('.cursorrules') + expect(result).toContain('.windsurfrules') + expect(result).toContain('.clinerules') + expect(result).not.toContain('package.json') + expect(result).not.toContain('README.md') + expect(result).toHaveLength(6) + }) + + it('discovers .github/copilot-instructions.md', () => { + const files: JsDelivrFileNode[] = [ + { + type: 'directory', + name: '.github', + files: [ + { type: 'file', name: 'copilot-instructions.md', size: 150 }, + { type: 'file', name: 'FUNDING.yml', size: 30 }, + ], + }, + ] + + const result = discoverAgentFiles(files) + + expect(result).toEqual(['.github/copilot-instructions.md']) + }) + + it('discovers .cursor/rules/*.md files', () => { + const files: JsDelivrFileNode[] = [ + { + type: 'directory', + name: '.cursor', + files: [ + { + type: 'directory', + name: 'rules', + files: [ + { type: 'file', name: 'coding-style.md', size: 100 }, + { type: 'file', name: 'testing.md', size: 80 }, + { type: 'file', name: 'config.json', size: 50 }, + ], + }, + ], + }, + ] + + const result = discoverAgentFiles(files) + + expect(result).toContain('.cursor/rules/coding-style.md') + expect(result).toContain('.cursor/rules/testing.md') + expect(result).not.toContain('.cursor/rules/config.json') + expect(result).toHaveLength(2) + }) + + it('discovers .windsurf/rules/*.md files', () => { + const files: JsDelivrFileNode[] = [ + { + type: 'directory', + name: '.windsurf', + files: [ + { + type: 'directory', + name: 'rules', + files: [{ type: 'file', name: 'project.md', size: 200 }], + }, + ], + }, + ] + + const result = discoverAgentFiles(files) + + expect(result).toEqual(['.windsurf/rules/project.md']) + }) + + it('returns empty array for empty file tree', () => { + expect(discoverAgentFiles([])).toEqual([]) + }) + + it('returns empty array when no agent files exist', () => { + const files: JsDelivrFileNode[] = [ + { type: 'file', name: 'package.json', size: 500 }, + { type: 'file', name: 'index.js', size: 1000 }, + { + type: 'directory', + name: 'src', + files: [{ type: 'file', name: 'main.ts', size: 200 }], + }, + ] + + expect(discoverAgentFiles(files)).toEqual([]) + }) +}) + +describe('fetchAgentFiles', () => { + it('fetches files in parallel and returns results', async () => { + const fetchMock = vi.fn().mockImplementation((url: string) => { + if (url.includes('CLAUDE.md')) { + return Promise.resolve({ ok: true, text: () => Promise.resolve('# Claude instructions') }) + } + if (url.includes('AGENTS.md')) { + return Promise.resolve({ ok: true, text: () => Promise.resolve('# Agent config') }) + } + return Promise.resolve({ ok: false }) + }) + vi.stubGlobal('fetch', fetchMock) + + try { + const result = await fetchAgentFiles('test-pkg', '1.0.0', ['CLAUDE.md', 'AGENTS.md']) + + expect(result).toHaveLength(2) + expect(result[0]).toMatchObject({ + path: 'CLAUDE.md', + content: '# Claude instructions', + displayName: 'Claude Code', + }) + expect(result[1]).toMatchObject({ + path: 'AGENTS.md', + content: '# Agent config', + displayName: 'Agent Instructions', + }) + expect(fetchMock).toHaveBeenCalledTimes(2) + } finally { + vi.unstubAllGlobals() + } + }) + + it('gracefully skips failed fetches', async () => { + const fetchMock = vi.fn().mockImplementation((url: string) => { + if (url.includes('CLAUDE.md')) { + return Promise.resolve({ ok: true, text: () => Promise.resolve('# Claude') }) + } + return Promise.resolve({ ok: false }) + }) + vi.stubGlobal('fetch', fetchMock) + + try { + const result = await fetchAgentFiles('test-pkg', '1.0.0', ['CLAUDE.md', 'missing.md']) + + expect(result).toHaveLength(1) + expect(result[0]?.path).toBe('CLAUDE.md') + } finally { + vi.unstubAllGlobals() + } + }) + + it('gracefully handles network errors', async () => { + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('Network error'))) + + try { + const result = await fetchAgentFiles('test-pkg', '1.0.0', ['CLAUDE.md']) + expect(result).toEqual([]) + } finally { + vi.unstubAllGlobals() + } + }) + + it('returns empty array for empty file paths', async () => { + const result = await fetchAgentFiles('test-pkg', '1.0.0', []) + expect(result).toEqual([]) + }) + + it('constructs correct CDN URLs for scoped packages', async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve('content'), + }) + vi.stubGlobal('fetch', fetchMock) + + try { + await fetchAgentFiles('@nuxt/kit', '1.0.0', ['CLAUDE.md']) + expect(fetchMock).toHaveBeenCalledWith( + 'https://cdn.jsdelivr.net/npm/@nuxt/kit@1.0.0/CLAUDE.md', + ) + } finally { + vi.unstubAllGlobals() + } + }) +}) + +describe('generateLlmsTxt', () => { + it('generates full output with all fields', () => { + const result: LlmsTxtResult = { + packageName: 'nuxt', + version: '3.12.0', + description: 'The Intuitive Vue Framework', + homepage: 'https://nuxt.com', + repositoryUrl: 'https://github.com/nuxt/nuxt', + readme: '# Nuxt\n\nThe Intuitive Vue Framework.', + agentFiles: [ + { + path: 'CLAUDE.md', + content: '# Claude\n\nUse Nuxt conventions.', + displayName: 'Claude Code', + }, + { path: '.cursorrules', content: 'Use composition API.', displayName: 'Cursor Rules' }, + ], + } + + const output = generateLlmsTxt(result) + + expect(output).toContain('# nuxt@3.12.0') + expect(output).toContain('> The Intuitive Vue Framework') + expect(output).toContain('- Homepage: https://nuxt.com') + expect(output).toContain('- Repository: https://github.com/nuxt/nuxt') + expect(output).toContain('- npm: https://www.npmjs.com/package/nuxt/v/3.12.0') + expect(output).toContain('## README') + expect(output).toContain('# Nuxt') + expect(output).toContain('## Agent Instructions') + expect(output).toContain('### Claude Code (`CLAUDE.md`)') + expect(output).toContain('Use Nuxt conventions.') + expect(output).toContain('### Cursor Rules (`.cursorrules`)') + expect(output).toContain('Use composition API.') + expect(output.endsWith('\n')).toBe(true) + }) + + it('generates minimal output with no optional fields', () => { + const result: LlmsTxtResult = { + packageName: 'tiny-pkg', + version: '0.1.0', + agentFiles: [], + } + + const output = generateLlmsTxt(result) + + expect(output).toContain('# tiny-pkg@0.1.0') + expect(output).toContain('- npm: https://www.npmjs.com/package/tiny-pkg/v/0.1.0') + expect(output).not.toContain('>') + expect(output).not.toContain('Homepage') + expect(output).not.toContain('Repository') + expect(output).not.toContain('## README') + expect(output).not.toContain('## Agent Instructions') + }) + + it('omits Agent Instructions section when no agent files exist', () => { + const result: LlmsTxtResult = { + packageName: 'test-pkg', + version: '1.0.0', + description: 'A test package', + readme: '# Test\n\nHello world.', + agentFiles: [], + } + + const output = generateLlmsTxt(result) + + expect(output).toContain('## README') + expect(output).not.toContain('## Agent Instructions') + }) + + it('omits README section when no readme provided', () => { + const result: LlmsTxtResult = { + packageName: 'no-readme', + version: '1.0.0', + agentFiles: [ + { path: 'AGENTS.md', content: 'Agent rules here.', displayName: 'Agent Instructions' }, + ], + } + + const output = generateLlmsTxt(result) + + expect(output).not.toContain('## README') + expect(output).toContain('## Agent Instructions') + expect(output).toContain('### Agent Instructions (`AGENTS.md`)') + }) + + it('handles scoped package names in npm URL', () => { + const result: LlmsTxtResult = { + packageName: '@nuxt/kit', + version: '1.0.0', + agentFiles: [], + } + + const output = generateLlmsTxt(result) + + expect(output).toContain('# @nuxt/kit@1.0.0') + expect(output).toContain('- npm: https://www.npmjs.com/package/@nuxt/kit/v/1.0.0') + }) +}) From 594b42dbfa526b9c2087a6324070793384cea2be Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Wed, 11 Feb 2026 15:25:21 +0000 Subject: [PATCH 05/19] feat(llms-txt): replace API route with file-based server routes Use Nitro server routes at /package/.../llms.txt instead of an API route with middleware rewriting. Single handler re-exported across four route files for unscoped, scoped, and versioned URL patterns. --- nuxt.config.ts | 1 + .../package/[name]/llms.txt.get.ts} | 28 +++++++++++-------- .../[name]/v/[version]/llms.txt.get.ts | 1 + .../package/[org]/[name]/llms.txt.get.ts | 1 + .../[org]/[name]/v/[version]/llms.txt.get.ts | 1 + 5 files changed, 20 insertions(+), 12 deletions(-) rename server/{api/registry/llms-txt/[...pkg].get.ts => routes/package/[name]/llms.txt.get.ts} (56%) create mode 100644 server/routes/package/[name]/v/[version]/llms.txt.get.ts create mode 100644 server/routes/package/[org]/[name]/llms.txt.get.ts create mode 100644 server/routes/package/[org]/[name]/v/[version]/llms.txt.get.ts diff --git a/nuxt.config.ts b/nuxt.config.ts index 5965c351e..b4468bcc0 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -112,6 +112,7 @@ export default defineNuxtConfig({ '/api/registry/file/**': { isr: true, cache: { maxAge: 365 * 24 * 60 * 60 } }, '/api/registry/provenance/**': { isr: true, cache: { maxAge: 365 * 24 * 60 * 60 } }, '/api/registry/files/**': { isr: true, cache: { maxAge: 365 * 24 * 60 * 60 } }, + '/package/**/llms.txt': { isr: 3600 }, '/api/registry/package-meta/**': { isr: 300 }, '/:pkg/.well-known/skills/**': { isr: 3600 }, '/:scope/:pkg/.well-known/skills/**': { isr: 3600 }, diff --git a/server/api/registry/llms-txt/[...pkg].get.ts b/server/routes/package/[name]/llms.txt.get.ts similarity index 56% rename from server/api/registry/llms-txt/[...pkg].get.ts rename to server/routes/package/[name]/llms.txt.get.ts index 79e58f4bd..89510289f 100644 --- a/server/api/registry/llms-txt/[...pkg].get.ts +++ b/server/routes/package/[name]/llms.txt.get.ts @@ -5,23 +5,24 @@ import { handleApiError } from '#server/utils/error-handler' import { handleLlmsTxt } from '#server/utils/llms-txt' /** - * Serves llms.txt for an npm package — a single LLM-friendly markdown document - * aggregating README and agent instruction files (CLAUDE.md, AGENTS.md, etc.). + * Serves llms.txt for an npm package. * - * URL patterns: - * - /api/registry/llms-txt/nuxt → latest version - * - /api/registry/llms-txt/@nuxt/kit → scoped, latest - * - /api/registry/llms-txt/nuxt/v/3.12.0 → specific version - * - /api/registry/llms-txt/@nuxt/kit/v/1.0.0 → scoped, specific version + * Handles all URL shapes via re-exports: + * - /package/:name/llms.txt + * - /package/:org/:name/llms.txt + * - /package/:name/v/:version/llms.txt + * - /package/:org/:name/v/:version/llms.txt */ export default defineCachedEventHandler( async event => { - const pkgParamSegments = getRouterParam(event, 'pkg')?.split('/') ?? [] - if (pkgParamSegments.length === 0) { + const org = getRouterParam(event, 'org') + const name = getRouterParam(event, 'name') + const rawVersion = getRouterParam(event, 'version') + if (!name) { throw createError({ statusCode: 404, message: 'Package name is required.' }) } - const { rawPackageName, rawVersion } = parsePackageParams(pkgParamSegments) + const rawPackageName = org ? `${org}/${name}` : name try { const { packageName, version } = v.parse(PackageRouteParamsSchema, { @@ -43,8 +44,11 @@ export default defineCachedEventHandler( maxAge: CACHE_MAX_AGE_ONE_HOUR, swr: true, getKey: event => { - const pkg = getRouterParam(event, 'pkg') ?? '' - return `llms-txt:${pkg.replace(/\/+$/, '').trim()}` + const org = getRouterParam(event, 'org') + const name = getRouterParam(event, 'name') + const version = getRouterParam(event, 'version') + const pkg = org ? `${org}/${name}` : name + return version ? `llms-txt:${pkg}@${version}` : `llms-txt:${pkg}` }, }, ) diff --git a/server/routes/package/[name]/v/[version]/llms.txt.get.ts b/server/routes/package/[name]/v/[version]/llms.txt.get.ts new file mode 100644 index 000000000..f227b344b --- /dev/null +++ b/server/routes/package/[name]/v/[version]/llms.txt.get.ts @@ -0,0 +1 @@ +export { default } from '../../llms.txt.get' diff --git a/server/routes/package/[org]/[name]/llms.txt.get.ts b/server/routes/package/[org]/[name]/llms.txt.get.ts new file mode 100644 index 000000000..552f2e647 --- /dev/null +++ b/server/routes/package/[org]/[name]/llms.txt.get.ts @@ -0,0 +1 @@ +export { default } from '../../[name]/llms.txt.get' diff --git a/server/routes/package/[org]/[name]/v/[version]/llms.txt.get.ts b/server/routes/package/[org]/[name]/v/[version]/llms.txt.get.ts new file mode 100644 index 000000000..88644280a --- /dev/null +++ b/server/routes/package/[org]/[name]/v/[version]/llms.txt.get.ts @@ -0,0 +1 @@ +export { default } from '../../../../[name]/llms.txt.get' From d2e0a305f99f498c8a84728bef51eb323f6e1b00 Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Thu, 12 Feb 2026 11:32:36 +0000 Subject: [PATCH 06/19] refactor(llms-txt): add handler factory and split llms.txt/llms_full.txt content Add createPackageLlmsTxtHandler factory for DRY route creation. handleLlmsTxt now accepts includeAgentFiles option to control whether agent instruction files are included (llms_full.txt) or omitted (llms.txt). Add handleOrgLlmsTxt for org-level package listings and generateRootLlmsTxt for the root /llms.txt discovery page. Simplify route handlers to single-line factory calls. --- server/routes/package/[name]/llms.txt.get.ts | 55 +---- .../package/[org]/[name]/llms.txt.get.ts | 4 +- server/utils/llms-txt.ts | 200 ++++++++++++++++-- 3 files changed, 193 insertions(+), 66 deletions(-) diff --git a/server/routes/package/[name]/llms.txt.get.ts b/server/routes/package/[name]/llms.txt.get.ts index 89510289f..b36f6b43d 100644 --- a/server/routes/package/[name]/llms.txt.get.ts +++ b/server/routes/package/[name]/llms.txt.get.ts @@ -1,54 +1,3 @@ -import * as v from 'valibot' -import { PackageRouteParamsSchema } from '#shared/schemas/package' -import { CACHE_MAX_AGE_ONE_HOUR } from '#shared/utils/constants' -import { handleApiError } from '#server/utils/error-handler' -import { handleLlmsTxt } from '#server/utils/llms-txt' +import { createPackageLlmsTxtHandler } from '#server/utils/llms-txt' -/** - * Serves llms.txt for an npm package. - * - * Handles all URL shapes via re-exports: - * - /package/:name/llms.txt - * - /package/:org/:name/llms.txt - * - /package/:name/v/:version/llms.txt - * - /package/:org/:name/v/:version/llms.txt - */ -export default defineCachedEventHandler( - async event => { - const org = getRouterParam(event, 'org') - const name = getRouterParam(event, 'name') - const rawVersion = getRouterParam(event, 'version') - if (!name) { - throw createError({ statusCode: 404, message: 'Package name is required.' }) - } - - const rawPackageName = org ? `${org}/${name}` : name - - try { - const { packageName, version } = v.parse(PackageRouteParamsSchema, { - packageName: rawPackageName, - version: rawVersion, - }) - - const content = await handleLlmsTxt(packageName, version) - setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') - return content - } catch (error: unknown) { - handleApiError(error, { - statusCode: 502, - message: 'Failed to generate llms.txt.', - }) - } - }, - { - maxAge: CACHE_MAX_AGE_ONE_HOUR, - swr: true, - getKey: event => { - const org = getRouterParam(event, 'org') - const name = getRouterParam(event, 'name') - const version = getRouterParam(event, 'version') - const pkg = org ? `${org}/${name}` : name - return version ? `llms-txt:${pkg}@${version}` : `llms-txt:${pkg}` - }, - }, -) +export default createPackageLlmsTxtHandler() diff --git a/server/routes/package/[org]/[name]/llms.txt.get.ts b/server/routes/package/[org]/[name]/llms.txt.get.ts index 552f2e647..b36f6b43d 100644 --- a/server/routes/package/[org]/[name]/llms.txt.get.ts +++ b/server/routes/package/[org]/[name]/llms.txt.get.ts @@ -1 +1,3 @@ -export { default } from '../../[name]/llms.txt.get' +import { createPackageLlmsTxtHandler } from '#server/utils/llms-txt' + +export default createPackageLlmsTxtHandler() diff --git a/server/utils/llms-txt.ts b/server/utils/llms-txt.ts index aeb38016e..4e6eafd38 100644 --- a/server/utils/llms-txt.ts +++ b/server/utils/llms-txt.ts @@ -1,6 +1,13 @@ +import * as v from 'valibot' import type { Packument } from '@npm/types' import type { JsDelivrFileNode, AgentFile, LlmsTxtResult } from '#shared/types' -import { NPM_MISSING_README_SENTINEL } from '#shared/utils/constants' +import { PackageRouteParamsSchema } from '#shared/schemas/package' +import { + NPM_MISSING_README_SENTINEL, + NPM_REGISTRY, + CACHE_MAX_AGE_ONE_HOUR, +} from '#shared/utils/constants' +import { handleApiError } from '#server/utils/error-handler' /** Well-known agent instruction files at the package root */ const ROOT_AGENT_FILES: Record = { @@ -116,7 +123,7 @@ export async function fetchAgentFiles( * - Blockquote description (if available) * - Metadata list (homepage, repository, npm) * - README section - * - Agent Instructions section (one sub-heading per file) + * - Agent Instructions section (one sub-heading per file, full mode only) */ export function generateLlmsTxt(result: LlmsTxtResult): string { const lines: string[] = [] @@ -204,12 +211,18 @@ function parseRepoUrl( /** * Orchestrates fetching all data and generating llms.txt for a package. - * Shared by both versioned and unversioned route handlers. + * + * When `includeAgentFiles` is false (default, for llms.txt), skips the file tree + * fetch and agent file discovery entirely — only returns README + metadata. + * When true (for llms_full.txt), includes agent instruction files. */ export async function handleLlmsTxt( packageName: string, requestedVersion?: string, + options?: { includeAgentFiles?: boolean }, ): Promise { + const includeAgentFiles = options?.includeAgentFiles ?? false + const packageData = await fetchNpmPackage(packageName) const resolvedVersion = requestedVersion ?? packageData['dist-tags']?.latest @@ -220,18 +233,25 @@ export async function handleLlmsTxt( // Extract README from packument (sync) const readmeFromPackument = getReadmeFromPackument(packageData, requestedVersion) - // Fetch file tree (and README from CDN if packument didn't have one) - const [fileTreeData, cdnReadme] = await Promise.all([ - fetchFileTree(packageName, resolvedVersion), - readmeFromPackument ? null : fetchReadmeFromCdn(packageName, resolvedVersion), - ]) + let agentFiles: AgentFile[] = [] + let cdnReadme: string | null = null + + if (includeAgentFiles) { + // Full mode: fetch file tree for agent discovery + README fallback in parallel + const [fileTreeData, readme] = await Promise.all([ + fetchFileTree(packageName, resolvedVersion), + readmeFromPackument ? null : fetchReadmeFromCdn(packageName, resolvedVersion), + ]) + cdnReadme = readme + const agentFilePaths = discoverAgentFiles(fileTreeData.files) + agentFiles = await fetchAgentFiles(packageName, resolvedVersion, agentFilePaths) + } else if (!readmeFromPackument) { + // Standard mode: only fetch README from CDN if packument lacks it + cdnReadme = await fetchReadmeFromCdn(packageName, resolvedVersion) + } const readme = readmeFromPackument ?? cdnReadme ?? undefined - // Discover and fetch agent files - const agentFilePaths = discoverAgentFiles(fileTreeData.files) - const agentFiles = await fetchAgentFiles(packageName, resolvedVersion, agentFilePaths) - const result: LlmsTxtResult = { packageName, version: resolvedVersion, @@ -244,3 +264,159 @@ export async function handleLlmsTxt( return generateLlmsTxt(result) } + +// Validation for org names (matches server/api/registry/org/[org]/packages.get.ts) +const NPM_ORG_NAME_RE = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/i + +/** + * Generate llms.txt for an npm organization/scope. + * Lists all packages in the org with links to their llms.txt pages. + */ +export async function handleOrgLlmsTxt(orgName: string, baseUrl: string): Promise { + if (!orgName || orgName.length > 50 || !NPM_ORG_NAME_RE.test(orgName)) { + throw createError({ statusCode: 404, message: `Invalid org name: ${orgName}` }) + } + + const data = await $fetch>( + `${NPM_REGISTRY}/-/org/${encodeURIComponent(orgName)}/package`, + ) + + const packages = Object.keys(data).sort() + + if (packages.length === 0) { + throw createError({ statusCode: 404, message: `No packages found for @${orgName}` }) + } + + const lines: string[] = [] + + lines.push(`# @${orgName}`) + lines.push('') + lines.push(`> npm packages published under the @${orgName} scope`) + lines.push('') + lines.push(`- npm: https://www.npmjs.com/org/${orgName}`) + lines.push('') + lines.push('## Packages') + lines.push('') + + for (const pkg of packages) { + const encodedPkg = pkg.replace('/', '/') + lines.push(`- [${pkg}](${baseUrl}/package/${encodedPkg}/llms.txt)`) + } + + lines.push('') + + return lines.join('\n').trimEnd() + '\n' +} + +/** + * Generate the root /llms.txt explaining available routes. + */ +export function generateRootLlmsTxt(baseUrl: string): string { + const lines: string[] = [] + + lines.push('# npmx.dev') + lines.push('') + lines.push('> A fast, modern browser for the npm registry') + lines.push('') + lines.push('This site provides LLM-friendly documentation for npm packages.') + lines.push('') + lines.push('## Available Routes') + lines.push('') + lines.push('### Package Documentation (llms.txt)') + lines.push('') + lines.push('README and package metadata in markdown format.') + lines.push('') + lines.push(`- \`${baseUrl}/package//llms.txt\` — unscoped package (latest version)`) + lines.push( + `- \`${baseUrl}/package//v//llms.txt\` — unscoped package (specific version)`, + ) + lines.push(`- \`${baseUrl}/package/@//llms.txt\` — scoped package (latest version)`) + lines.push( + `- \`${baseUrl}/package/@//v//llms.txt\` — scoped package (specific version)`, + ) + lines.push('') + lines.push('### Full Package Documentation (llms_full.txt)') + lines.push('') + lines.push( + 'README, package metadata, and agent instruction files (CLAUDE.md, .cursorrules, etc.).', + ) + lines.push('') + lines.push(`- \`${baseUrl}/package//llms_full.txt\` — unscoped package (latest version)`) + lines.push( + `- \`${baseUrl}/package//v//llms_full.txt\` — unscoped package (specific version)`, + ) + lines.push( + `- \`${baseUrl}/package/@//llms_full.txt\` — scoped package (latest version)`, + ) + lines.push( + `- \`${baseUrl}/package/@//v//llms_full.txt\` — scoped package (specific version)`, + ) + lines.push('') + lines.push('### Organization Packages (llms.txt)') + lines.push('') + lines.push('List of all packages under an npm scope with links to their documentation.') + lines.push('') + lines.push(`- \`${baseUrl}/package/@/llms.txt\` — organization package listing`) + lines.push('') + lines.push('## Examples') + lines.push('') + lines.push(`- [nuxt llms.txt](${baseUrl}/package/nuxt/llms.txt)`) + lines.push(`- [nuxt llms_full.txt](${baseUrl}/package/nuxt/llms_full.txt)`) + lines.push(`- [@nuxt/kit llms.txt](${baseUrl}/package/@nuxt/kit/llms.txt)`) + lines.push(`- [@nuxt org packages](${baseUrl}/package/@nuxt/llms.txt)`) + lines.push('') + + return lines.join('\n').trimEnd() + '\n' +} + +/** + * Create a cached event handler for package-level llms.txt or llms_full.txt. + * + * Each route file should call this factory and `export default` the result. + * This avoids the re-export pattern that Nitro doesn't register as routes. + */ +export function createPackageLlmsTxtHandler(options?: { full?: boolean }) { + const full = options?.full ?? false + + return defineCachedEventHandler( + async event => { + const org = getRouterParam(event, 'org') + const name = getRouterParam(event, 'name') + const rawVersion = getRouterParam(event, 'version') + + if (!name) { + throw createError({ statusCode: 404, message: 'Package name is required.' }) + } + + const rawPackageName = org ? `${org}/${name}` : name + + try { + const { packageName, version } = v.parse(PackageRouteParamsSchema, { + packageName: rawPackageName, + version: rawVersion, + }) + + const content = await handleLlmsTxt(packageName, version, { includeAgentFiles: full }) + setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') + return content + } catch (error: unknown) { + handleApiError(error, { + statusCode: 502, + message: `Failed to generate ${full ? 'llms_full.txt' : 'llms.txt'}.`, + }) + } + }, + { + maxAge: CACHE_MAX_AGE_ONE_HOUR, + swr: true, + getKey: event => { + const org = getRouterParam(event, 'org') + const name = getRouterParam(event, 'name') + const version = getRouterParam(event, 'version') + const pkg = org ? `${org}/${name}` : name + const prefix = full ? 'llms-full-txt' : 'llms-txt' + return version ? `${prefix}:${pkg}@${version}` : `${prefix}:${pkg}` + }, + }, + ) +} From fcd8ca19043b18b3347a5665f019a4f38f4eceaf Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Thu, 12 Feb 2026 11:32:45 +0000 Subject: [PATCH 07/19] feat(llms-txt): add middleware for versioned, org, and root routes Add server middleware to handle llms.txt routes that Nitro's radix3 file-based router cannot resolve (parameterized intermediate segments don't match literal children). Handles versioned package paths, org-level package listings, and root /llms.txt discovery page. Remove broken versioned route files and add llms_full.txt routes. --- server/middleware/llms-txt.ts | 94 +++++++++++++++++++ .../package/[name]/llms_full.txt.get.ts | 3 + .../[name]/v/[version]/llms.txt.get.ts | 1 - .../package/[org]/[name]/llms_full.txt.get.ts | 3 + .../[org]/[name]/v/[version]/llms.txt.get.ts | 1 - 5 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 server/middleware/llms-txt.ts create mode 100644 server/routes/package/[name]/llms_full.txt.get.ts delete mode 100644 server/routes/package/[name]/v/[version]/llms.txt.get.ts create mode 100644 server/routes/package/[org]/[name]/llms_full.txt.get.ts delete mode 100644 server/routes/package/[org]/[name]/v/[version]/llms.txt.get.ts diff --git a/server/middleware/llms-txt.ts b/server/middleware/llms-txt.ts new file mode 100644 index 000000000..4e7b384f8 --- /dev/null +++ b/server/middleware/llms-txt.ts @@ -0,0 +1,94 @@ +import * as v from 'valibot' +import { PackageRouteParamsSchema } from '#shared/schemas/package' +import { handleApiError } from '#server/utils/error-handler' +import { handleLlmsTxt, handleOrgLlmsTxt, generateRootLlmsTxt } from '#server/utils/llms-txt' + +/** + * Middleware to handle llms.txt / llms_full.txt routes that can't be served + * by Nitro's file-based routing (versioned paths hit a radix3 limitation + * where parameterized intermediate segments don't resolve literal children). + * + * Handles: + * - /llms.txt (root — file-based route is blocked by canonical-redirects) + * - /package/:name/v/:version/llms.txt + * - /package/:name/v/:version/llms_full.txt + * - /package/@:org/:name/v/:version/llms.txt + * - /package/@:org/:name/v/:version/llms_full.txt + * - /package/@:org/llms.txt (org listing) + * + * Non-versioned package routes are left to file-based handlers. + */ +export default defineEventHandler(async event => { + const path = event.path.split('?')[0] + + if (!path.endsWith('/llms.txt') && !path.endsWith('/llms_full.txt')) return + + const full = path.endsWith('/llms_full.txt') + const suffix = full ? '/llms_full.txt' : '/llms.txt' + + // Root /llms.txt + if (path === '/llms.txt') { + const url = getRequestURL(event) + const baseUrl = `${url.protocol}//${url.host}` + setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') + setHeader(event, 'Cache-Control', 's-maxage=3600, stale-while-revalidate=86400') + return generateRootLlmsTxt(baseUrl) + } + + if (!path.startsWith('/package/')) return + + // Strip /package/ prefix and /llms[_full].txt suffix + const inner = path.slice('/package/'.length, -suffix.length) + + // Org-level: /package/@org/llms.txt (inner = "@org") + if (!full && inner.startsWith('@') && !inner.includes('/')) { + const orgName = inner.slice(1) + try { + const url = getRequestURL(event) + const baseUrl = `${url.protocol}//${url.host}` + const content = await handleOrgLlmsTxt(orgName, baseUrl) + setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') + setHeader(event, 'Cache-Control', 's-maxage=3600, stale-while-revalidate=86400') + return content + } catch (error: unknown) { + handleApiError(error, { statusCode: 502, message: 'Failed to generate org llms.txt.' }) + } + } + + // Versioned paths — only handle if /v/ is present (non-versioned are handled by file routes) + if (!inner.includes('/v/')) return + + let rawPackageName: string + let rawVersion: string + + if (inner.startsWith('@')) { + // Scoped: @org/name/v/version + const match = inner.match(/^(@[^/]+\/[^/]+)\/v\/(.+)$/) + if (!match) return + rawPackageName = match[1] + rawVersion = match[2] + } else { + // Unscoped: name/v/version + const match = inner.match(/^([^/]+)\/v\/(.+)$/) + if (!match) return + rawPackageName = match[1] + rawVersion = match[2] + } + + try { + const { packageName, version } = v.parse(PackageRouteParamsSchema, { + packageName: rawPackageName, + version: rawVersion, + }) + + const content = await handleLlmsTxt(packageName, version, { includeAgentFiles: full }) + setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') + setHeader(event, 'Cache-Control', 's-maxage=3600, stale-while-revalidate=86400') + return content + } catch (error: unknown) { + handleApiError(error, { + statusCode: 502, + message: `Failed to generate ${full ? 'llms_full.txt' : 'llms.txt'}.`, + }) + } +}) diff --git a/server/routes/package/[name]/llms_full.txt.get.ts b/server/routes/package/[name]/llms_full.txt.get.ts new file mode 100644 index 000000000..da9be4399 --- /dev/null +++ b/server/routes/package/[name]/llms_full.txt.get.ts @@ -0,0 +1,3 @@ +import { createPackageLlmsTxtHandler } from '#server/utils/llms-txt' + +export default createPackageLlmsTxtHandler({ full: true }) diff --git a/server/routes/package/[name]/v/[version]/llms.txt.get.ts b/server/routes/package/[name]/v/[version]/llms.txt.get.ts deleted file mode 100644 index f227b344b..000000000 --- a/server/routes/package/[name]/v/[version]/llms.txt.get.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from '../../llms.txt.get' diff --git a/server/routes/package/[org]/[name]/llms_full.txt.get.ts b/server/routes/package/[org]/[name]/llms_full.txt.get.ts new file mode 100644 index 000000000..da9be4399 --- /dev/null +++ b/server/routes/package/[org]/[name]/llms_full.txt.get.ts @@ -0,0 +1,3 @@ +import { createPackageLlmsTxtHandler } from '#server/utils/llms-txt' + +export default createPackageLlmsTxtHandler({ full: true }) diff --git a/server/routes/package/[org]/[name]/v/[version]/llms.txt.get.ts b/server/routes/package/[org]/[name]/v/[version]/llms.txt.get.ts deleted file mode 100644 index 88644280a..000000000 --- a/server/routes/package/[org]/[name]/v/[version]/llms.txt.get.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from '../../../../[name]/llms.txt.get' From 8d4f5a9eff09d7417af7f5358ee377828bae6d0d Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Thu, 12 Feb 2026 11:32:52 +0000 Subject: [PATCH 08/19] feat(llms-txt): support shorthand URL redirects for llms.txt paths Extend canonical redirect regexes with optional /llms.txt and /llms_full.txt suffix capture groups so shorthand URLs like /nuxt/llms.txt redirect to /package/nuxt/llms.txt. Add explicit /llms.txt root path skip to prevent it matching as a package name. --- .../middleware/canonical-redirects.global.ts | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/server/middleware/canonical-redirects.global.ts b/server/middleware/canonical-redirects.global.ts index 230ab3443..86e47fd96 100644 --- a/server/middleware/canonical-redirects.global.ts +++ b/server/middleware/canonical-redirects.global.ts @@ -46,26 +46,42 @@ export default defineEventHandler(async event => { return } + // /llms.txt at root is handled by the llms-txt middleware + if (path === '/llms.txt') { + return + } + // /@org/pkg or /pkg → /package/org/pkg or /package/pkg - let pkgMatch = path.match(/^\/(?:(?@[^/]+)\/)?(?[^/@]+)$/) + // Also handles trailing /llms.txt or /llms_full.txt suffixes + let pkgMatch = path.match( + /^\/(?:(?@[^/]+)\/)?(?[^/@]+)(?\/(?:llms\.txt|llms_full\.txt))?$/, + ) if (pkgMatch?.groups) { const args = [pkgMatch.groups.org, pkgMatch.groups.name].filter(Boolean).join('/') + const suffix = pkgMatch.groups.suffix ?? '' setHeader(event, 'cache-control', cacheControl) - return sendRedirect(event, `/package/${args}` + (query ? '?' + query : ''), 301) + return sendRedirect(event, `/package/${args}${suffix}` + (query ? '?' + query : ''), 301) } // /@org/pkg/v/version or /@org/pkg@version → /package/org/pkg/v/version // /pkg/v/version or /pkg@version → /package/pkg/v/version + // Also handles trailing /llms.txt or /llms_full.txt suffixes const pkgVersionMatch = - path.match(/^\/(?:(?@[^/]+)\/)?(?[^/@]+)\/v\/(?[^/]+)$/) || - path.match(/^\/(?:(?@[^/]+)\/)?(?[^/@]+)@(?[^/]+)$/) + path.match( + /^\/(?:(?@[^/]+)\/)?(?[^/@]+)\/v\/(?[^/]+)(?\/(?:llms\.txt|llms_full\.txt))?$/, + ) || + path.match( + /^\/(?:(?@[^/]+)\/)?(?[^/@]+)@(?[^/]+)(?\/(?:llms\.txt|llms_full\.txt))?$/, + ) if (pkgVersionMatch?.groups) { const args = [pkgVersionMatch.groups.org, pkgVersionMatch.groups.name].filter(Boolean).join('/') + const versionSuffix = pkgVersionMatch.groups.suffix ?? '' setHeader(event, 'cache-control', cacheControl) return sendRedirect( event, - `/package/${args}/v/${pkgVersionMatch.groups.version}` + (query ? '?' + query : ''), + `/package/${args}/v/${pkgVersionMatch.groups.version}${versionSuffix}` + + (query ? '?' + query : ''), 301, ) } From fb11b66d89dade004a824dfa6ab77c21c37bd4ff Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Thu, 12 Feb 2026 11:32:58 +0000 Subject: [PATCH 09/19] chore(llms-txt): add ISR cache rules and vitest server alias Add ISR rules for llms_full.txt and root /llms.txt routes in nuxt.config.ts. Add #server alias to vitest config for resolving server utility imports in unit tests. --- nuxt.config.ts | 2 ++ vitest.config.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/nuxt.config.ts b/nuxt.config.ts index b4468bcc0..28064817c 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -113,6 +113,8 @@ export default defineNuxtConfig({ '/api/registry/provenance/**': { isr: true, cache: { maxAge: 365 * 24 * 60 * 60 } }, '/api/registry/files/**': { isr: true, cache: { maxAge: 365 * 24 * 60 * 60 } }, '/package/**/llms.txt': { isr: 3600 }, + '/package/**/llms_full.txt': { isr: 3600 }, + '/llms.txt': { isr: 3600 }, '/api/registry/package-meta/**': { isr: 300 }, '/:pkg/.well-known/skills/**': { isr: 3600 }, '/:scope/:pkg/.well-known/skills/**': { isr: 3600 }, diff --git a/vitest.config.ts b/vitest.config.ts index 9b96a1dbe..3be3e6f21 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -12,6 +12,7 @@ export default defineConfig({ resolve: { alias: { '#shared': `${rootDir}/shared`, + '#server': `${rootDir}/server`, }, }, test: { From 8911b38835507766794bc3d1a7c0d6a51db2a6e5 Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Thu, 12 Feb 2026 11:33:04 +0000 Subject: [PATCH 10/19] test(llms-txt): add generateRootLlmsTxt unit tests Test route pattern inclusion, example links, base URL substitution, and trailing newline for the root /llms.txt discovery page output. --- test/unit/server/utils/llms-txt.spec.ts | 34 +++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/unit/server/utils/llms-txt.spec.ts b/test/unit/server/utils/llms-txt.spec.ts index af326694d..fb3eafd8a 100644 --- a/test/unit/server/utils/llms-txt.spec.ts +++ b/test/unit/server/utils/llms-txt.spec.ts @@ -4,6 +4,7 @@ import { discoverAgentFiles, fetchAgentFiles, generateLlmsTxt, + generateRootLlmsTxt, } from '../../../../server/utils/llms-txt' describe('discoverAgentFiles', () => { @@ -299,3 +300,36 @@ describe('generateLlmsTxt', () => { expect(output).toContain('- npm: https://www.npmjs.com/package/@nuxt/kit/v/1.0.0') }) }) + +describe('generateRootLlmsTxt', () => { + it('includes all route patterns', () => { + const output = generateRootLlmsTxt('https://npmx.dev') + + expect(output).toContain('# npmx.dev') + expect(output).toContain('https://npmx.dev/package//llms.txt') + expect(output).toContain('https://npmx.dev/package//v//llms.txt') + expect(output).toContain('https://npmx.dev/package/@//llms.txt') + expect(output).toContain('https://npmx.dev/package/@//v//llms.txt') + expect(output).toContain('https://npmx.dev/package//llms_full.txt') + expect(output).toContain('https://npmx.dev/package/@/llms.txt') + }) + + it('includes example links', () => { + const output = generateRootLlmsTxt('https://npmx.dev') + + expect(output).toContain('[nuxt llms.txt](https://npmx.dev/package/nuxt/llms.txt)') + expect(output).toContain('[@nuxt org packages](https://npmx.dev/package/@nuxt/llms.txt)') + }) + + it('uses provided base URL', () => { + const output = generateRootLlmsTxt('http://localhost:3000') + + expect(output).toContain('http://localhost:3000/package//llms.txt') + expect(output).not.toContain('https://npmx.dev') + }) + + it('ends with newline', () => { + const output = generateRootLlmsTxt('https://npmx.dev') + expect(output.endsWith('\n')).toBe(true) + }) +}) From dd07537746ef98ac0da33e82827c0ebf6315100d Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Thu, 12 Feb 2026 13:30:40 +0000 Subject: [PATCH 11/19] fix(llms-txt): move all handling to middleware for Vercel compatibility Vercel ISR glob rules (/package/**/llms.txt) create catch-all serverless functions that intercept requests before Nitro's file-based routes can resolve them, breaking scoped packages and versioned paths. Move all llms.txt/llms_full.txt handling into the middleware, remove ISR route rules, and delete file-based route files. --- nuxt.config.ts | 3 - server/middleware/llms-txt.ts | 69 +++++++++++-------- server/routes/package/[name]/llms.txt.get.ts | 3 - .../package/[name]/llms_full.txt.get.ts | 3 - .../package/[org]/[name]/llms.txt.get.ts | 3 - .../package/[org]/[name]/llms_full.txt.get.ts | 3 - server/utils/llms-txt.ts | 61 +--------------- 7 files changed, 41 insertions(+), 104 deletions(-) delete mode 100644 server/routes/package/[name]/llms.txt.get.ts delete mode 100644 server/routes/package/[name]/llms_full.txt.get.ts delete mode 100644 server/routes/package/[org]/[name]/llms.txt.get.ts delete mode 100644 server/routes/package/[org]/[name]/llms_full.txt.get.ts diff --git a/nuxt.config.ts b/nuxt.config.ts index 28064817c..5965c351e 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -112,9 +112,6 @@ export default defineNuxtConfig({ '/api/registry/file/**': { isr: true, cache: { maxAge: 365 * 24 * 60 * 60 } }, '/api/registry/provenance/**': { isr: true, cache: { maxAge: 365 * 24 * 60 * 60 } }, '/api/registry/files/**': { isr: true, cache: { maxAge: 365 * 24 * 60 * 60 } }, - '/package/**/llms.txt': { isr: 3600 }, - '/package/**/llms_full.txt': { isr: 3600 }, - '/llms.txt': { isr: 3600 }, '/api/registry/package-meta/**': { isr: 300 }, '/:pkg/.well-known/skills/**': { isr: 3600 }, '/:scope/:pkg/.well-known/skills/**': { isr: 3600 }, diff --git a/server/middleware/llms-txt.ts b/server/middleware/llms-txt.ts index 4e7b384f8..aad852b87 100644 --- a/server/middleware/llms-txt.ts +++ b/server/middleware/llms-txt.ts @@ -3,20 +3,27 @@ import { PackageRouteParamsSchema } from '#shared/schemas/package' import { handleApiError } from '#server/utils/error-handler' import { handleLlmsTxt, handleOrgLlmsTxt, generateRootLlmsTxt } from '#server/utils/llms-txt' +const CACHE_HEADER = 's-maxage=3600, stale-while-revalidate=86400' + /** - * Middleware to handle llms.txt / llms_full.txt routes that can't be served - * by Nitro's file-based routing (versioned paths hit a radix3 limitation - * where parameterized intermediate segments don't resolve literal children). + * Middleware to handle ALL llms.txt / llms_full.txt routes. * - * Handles: - * - /llms.txt (root — file-based route is blocked by canonical-redirects) - * - /package/:name/v/:version/llms.txt - * - /package/:name/v/:version/llms_full.txt - * - /package/@:org/:name/v/:version/llms.txt - * - /package/@:org/:name/v/:version/llms_full.txt - * - /package/@:org/llms.txt (org listing) + * All llms.txt handling lives here rather than in file-based routes because + * Vercel's ISR route rules with glob patterns (e.g. `/package/ ** /llms.txt`) + * create catch-all serverless functions that interfere with Nitro's file-based + * route resolution — scoped packages and versioned paths fail to match. * - * Non-versioned package routes are left to file-based handlers. + * Handles: + * - /llms.txt (root discovery page) + * - /package/@:org/llms.txt (org package listing) + * - /package/:name/llms.txt (unscoped, latest) + * - /package/:name/llms_full.txt (unscoped, latest, full) + * - /package/@:org/:name/llms.txt (scoped, latest) + * - /package/@:org/:name/llms_full.txt (scoped, latest, full) + * - /package/:name/v/:version/llms.txt (unscoped, versioned) + * - /package/:name/v/:version/llms_full.txt (unscoped, versioned, full) + * - /package/@:org/:name/v/:version/llms.txt (scoped, versioned) + * - /package/@:org/:name/v/:version/llms_full.txt (scoped, versioned, full) */ export default defineEventHandler(async event => { const path = event.path.split('?')[0] @@ -31,7 +38,7 @@ export default defineEventHandler(async event => { const url = getRequestURL(event) const baseUrl = `${url.protocol}//${url.host}` setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') - setHeader(event, 'Cache-Control', 's-maxage=3600, stale-while-revalidate=86400') + setHeader(event, 'Cache-Control', CACHE_HEADER) return generateRootLlmsTxt(baseUrl) } @@ -48,33 +55,37 @@ export default defineEventHandler(async event => { const baseUrl = `${url.protocol}//${url.host}` const content = await handleOrgLlmsTxt(orgName, baseUrl) setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') - setHeader(event, 'Cache-Control', 's-maxage=3600, stale-while-revalidate=86400') + setHeader(event, 'Cache-Control', CACHE_HEADER) return content } catch (error: unknown) { handleApiError(error, { statusCode: 502, message: 'Failed to generate org llms.txt.' }) } } - // Versioned paths — only handle if /v/ is present (non-versioned are handled by file routes) - if (!inner.includes('/v/')) return - + // Parse package name and optional version from inner path let rawPackageName: string - let rawVersion: string + let rawVersion: string | undefined - if (inner.startsWith('@')) { - // Scoped: @org/name/v/version - const match = inner.match(/^(@[^/]+\/[^/]+)\/v\/(.+)$/) - if (!match) return - rawPackageName = match[1] - rawVersion = match[2] + if (inner.includes('/v/')) { + // Versioned path + if (inner.startsWith('@')) { + const match = inner.match(/^(@[^/]+\/[^/]+)\/v\/(.+)$/) + if (!match) return + rawPackageName = match[1] + rawVersion = match[2] + } else { + const match = inner.match(/^([^/]+)\/v\/(.+)$/) + if (!match) return + rawPackageName = match[1] + rawVersion = match[2] + } } else { - // Unscoped: name/v/version - const match = inner.match(/^([^/]+)\/v\/(.+)$/) - if (!match) return - rawPackageName = match[1] - rawVersion = match[2] + // Latest version — inner is just the package name + rawPackageName = inner } + if (!rawPackageName) return + try { const { packageName, version } = v.parse(PackageRouteParamsSchema, { packageName: rawPackageName, @@ -83,7 +94,7 @@ export default defineEventHandler(async event => { const content = await handleLlmsTxt(packageName, version, { includeAgentFiles: full }) setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') - setHeader(event, 'Cache-Control', 's-maxage=3600, stale-while-revalidate=86400') + setHeader(event, 'Cache-Control', CACHE_HEADER) return content } catch (error: unknown) { handleApiError(error, { diff --git a/server/routes/package/[name]/llms.txt.get.ts b/server/routes/package/[name]/llms.txt.get.ts deleted file mode 100644 index b36f6b43d..000000000 --- a/server/routes/package/[name]/llms.txt.get.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { createPackageLlmsTxtHandler } from '#server/utils/llms-txt' - -export default createPackageLlmsTxtHandler() diff --git a/server/routes/package/[name]/llms_full.txt.get.ts b/server/routes/package/[name]/llms_full.txt.get.ts deleted file mode 100644 index da9be4399..000000000 --- a/server/routes/package/[name]/llms_full.txt.get.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { createPackageLlmsTxtHandler } from '#server/utils/llms-txt' - -export default createPackageLlmsTxtHandler({ full: true }) diff --git a/server/routes/package/[org]/[name]/llms.txt.get.ts b/server/routes/package/[org]/[name]/llms.txt.get.ts deleted file mode 100644 index b36f6b43d..000000000 --- a/server/routes/package/[org]/[name]/llms.txt.get.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { createPackageLlmsTxtHandler } from '#server/utils/llms-txt' - -export default createPackageLlmsTxtHandler() diff --git a/server/routes/package/[org]/[name]/llms_full.txt.get.ts b/server/routes/package/[org]/[name]/llms_full.txt.get.ts deleted file mode 100644 index da9be4399..000000000 --- a/server/routes/package/[org]/[name]/llms_full.txt.get.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { createPackageLlmsTxtHandler } from '#server/utils/llms-txt' - -export default createPackageLlmsTxtHandler({ full: true }) diff --git a/server/utils/llms-txt.ts b/server/utils/llms-txt.ts index 4e6eafd38..5c6941443 100644 --- a/server/utils/llms-txt.ts +++ b/server/utils/llms-txt.ts @@ -1,13 +1,6 @@ -import * as v from 'valibot' import type { Packument } from '@npm/types' import type { JsDelivrFileNode, AgentFile, LlmsTxtResult } from '#shared/types' -import { PackageRouteParamsSchema } from '#shared/schemas/package' -import { - NPM_MISSING_README_SENTINEL, - NPM_REGISTRY, - CACHE_MAX_AGE_ONE_HOUR, -} from '#shared/utils/constants' -import { handleApiError } from '#server/utils/error-handler' +import { NPM_MISSING_README_SENTINEL, NPM_REGISTRY } from '#shared/utils/constants' /** Well-known agent instruction files at the package root */ const ROOT_AGENT_FILES: Record = { @@ -368,55 +361,3 @@ export function generateRootLlmsTxt(baseUrl: string): string { return lines.join('\n').trimEnd() + '\n' } - -/** - * Create a cached event handler for package-level llms.txt or llms_full.txt. - * - * Each route file should call this factory and `export default` the result. - * This avoids the re-export pattern that Nitro doesn't register as routes. - */ -export function createPackageLlmsTxtHandler(options?: { full?: boolean }) { - const full = options?.full ?? false - - return defineCachedEventHandler( - async event => { - const org = getRouterParam(event, 'org') - const name = getRouterParam(event, 'name') - const rawVersion = getRouterParam(event, 'version') - - if (!name) { - throw createError({ statusCode: 404, message: 'Package name is required.' }) - } - - const rawPackageName = org ? `${org}/${name}` : name - - try { - const { packageName, version } = v.parse(PackageRouteParamsSchema, { - packageName: rawPackageName, - version: rawVersion, - }) - - const content = await handleLlmsTxt(packageName, version, { includeAgentFiles: full }) - setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') - return content - } catch (error: unknown) { - handleApiError(error, { - statusCode: 502, - message: `Failed to generate ${full ? 'llms_full.txt' : 'llms.txt'}.`, - }) - } - }, - { - maxAge: CACHE_MAX_AGE_ONE_HOUR, - swr: true, - getKey: event => { - const org = getRouterParam(event, 'org') - const name = getRouterParam(event, 'name') - const version = getRouterParam(event, 'version') - const pkg = org ? `${org}/${name}` : name - const prefix = full ? 'llms-full-txt' : 'llms-txt' - return version ? `${prefix}:${pkg}@${version}` : `${prefix}:${pkg}` - }, - }, - ) -} From f5c8dd85d9c6022ca616b25a4b1a447385eb9d2a Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Thu, 12 Feb 2026 14:07:57 +0000 Subject: [PATCH 12/19] fix(llms-txt): resolve type errors in middleware and utils Fix strict TypeScript errors: add fallback for split()[0] possibly undefined, narrow regex match group types, use non-null assertions for Record lookups after in-check, and use Nuxt's auto-generated Packument type instead of @npm/types import. --- server/middleware/llms-txt.ts | 6 +++--- server/utils/llms-txt.ts | 13 ++++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/server/middleware/llms-txt.ts b/server/middleware/llms-txt.ts index aad852b87..0ca86eb92 100644 --- a/server/middleware/llms-txt.ts +++ b/server/middleware/llms-txt.ts @@ -26,7 +26,7 @@ const CACHE_HEADER = 's-maxage=3600, stale-while-revalidate=86400' * - /package/@:org/:name/v/:version/llms_full.txt (scoped, versioned, full) */ export default defineEventHandler(async event => { - const path = event.path.split('?')[0] + const path = event.path.split('?')[0] ?? '/' if (!path.endsWith('/llms.txt') && !path.endsWith('/llms_full.txt')) return @@ -70,12 +70,12 @@ export default defineEventHandler(async event => { // Versioned path if (inner.startsWith('@')) { const match = inner.match(/^(@[^/]+\/[^/]+)\/v\/(.+)$/) - if (!match) return + if (!match?.[1] || !match[2]) return rawPackageName = match[1] rawVersion = match[2] } else { const match = inner.match(/^([^/]+)\/v\/(.+)$/) - if (!match) return + if (!match?.[1] || !match[2]) return rawPackageName = match[1] rawVersion = match[2] } diff --git a/server/utils/llms-txt.ts b/server/utils/llms-txt.ts index 5c6941443..861ea6888 100644 --- a/server/utils/llms-txt.ts +++ b/server/utils/llms-txt.ts @@ -1,4 +1,3 @@ -import type { Packument } from '@npm/types' import type { JsDelivrFileNode, AgentFile, LlmsTxtResult } from '#shared/types' import { NPM_MISSING_README_SENTINEL, NPM_REGISTRY } from '#shared/utils/constants' @@ -72,11 +71,12 @@ export function discoverAgentFiles(files: JsDelivrFileNode[]): string[] { * Get the display name for an agent file path. */ function getDisplayName(filePath: string): string { - if (filePath in ROOT_AGENT_FILES) return ROOT_AGENT_FILES[filePath] - if (filePath in DIRECTORY_AGENT_FILES) return DIRECTORY_AGENT_FILES[filePath] + if (filePath in ROOT_AGENT_FILES) return ROOT_AGENT_FILES[filePath]! + if (filePath in DIRECTORY_AGENT_FILES) return DIRECTORY_AGENT_FILES[filePath]! for (const [dirPath, displayName] of Object.entries(RULE_DIRECTORIES)) { - if (filePath.startsWith(`${dirPath}/`)) return `${displayName}: ${filePath.split('/').pop()}` + if (filePath.startsWith(`${dirPath}/`)) + return `${displayName}: ${filePath.split('/').pop() ?? filePath}` } return filePath @@ -181,7 +181,10 @@ async function fetchReadmeFromCdn(packageName: string, version: string): Promise } /** Extract README from packument data */ -function getReadmeFromPackument(packageData: Packument, requestedVersion?: string): string | null { +function getReadmeFromPackument( + packageData: Awaited>, + requestedVersion?: string, +): string | null { const readme = requestedVersion ? packageData.versions[requestedVersion]?.readme : packageData.readme From afc4de311c4218a742165da378d035eb2b2e0973 Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Thu, 12 Feb 2026 23:50:27 +0000 Subject: [PATCH 13/19] refactor(llm-docs): rename llms-txt files to llm-docs --- server/middleware/canonical-redirects.global.ts | 2 +- server/middleware/{llms-txt.ts => llm-docs.ts} | 2 +- server/utils/{llms-txt.ts => llm-docs.ts} | 0 shared/types/index.ts | 2 +- shared/types/{llms-txt.ts => llm-docs.ts} | 0 test/unit/server/utils/{llms-txt.spec.ts => llm-docs.spec.ts} | 2 +- 6 files changed, 4 insertions(+), 4 deletions(-) rename server/middleware/{llms-txt.ts => llm-docs.ts} (99%) rename server/utils/{llms-txt.ts => llm-docs.ts} (100%) rename shared/types/{llms-txt.ts => llm-docs.ts} (100%) rename test/unit/server/utils/{llms-txt.spec.ts => llm-docs.spec.ts} (99%) diff --git a/server/middleware/canonical-redirects.global.ts b/server/middleware/canonical-redirects.global.ts index 86e47fd96..27772c930 100644 --- a/server/middleware/canonical-redirects.global.ts +++ b/server/middleware/canonical-redirects.global.ts @@ -46,7 +46,7 @@ export default defineEventHandler(async event => { return } - // /llms.txt at root is handled by the llms-txt middleware + // /llms.txt at root is handled by the llm-docs middleware if (path === '/llms.txt') { return } diff --git a/server/middleware/llms-txt.ts b/server/middleware/llm-docs.ts similarity index 99% rename from server/middleware/llms-txt.ts rename to server/middleware/llm-docs.ts index 0ca86eb92..eebc4ebbd 100644 --- a/server/middleware/llms-txt.ts +++ b/server/middleware/llm-docs.ts @@ -1,7 +1,7 @@ import * as v from 'valibot' import { PackageRouteParamsSchema } from '#shared/schemas/package' import { handleApiError } from '#server/utils/error-handler' -import { handleLlmsTxt, handleOrgLlmsTxt, generateRootLlmsTxt } from '#server/utils/llms-txt' +import { handleLlmsTxt, handleOrgLlmsTxt, generateRootLlmsTxt } from '#server/utils/llm-docs' const CACHE_HEADER = 's-maxage=3600, stale-while-revalidate=86400' diff --git a/server/utils/llms-txt.ts b/server/utils/llm-docs.ts similarity index 100% rename from server/utils/llms-txt.ts rename to server/utils/llm-docs.ts diff --git a/shared/types/index.ts b/shared/types/index.ts index e378738e1..25daaa0f6 100644 --- a/shared/types/index.ts +++ b/shared/types/index.ts @@ -9,4 +9,4 @@ export * from './i18n-status' export * from './comparison' export * from './skills' export * from './version-downloads' -export * from './llms-txt' +export * from './llm-docs' diff --git a/shared/types/llms-txt.ts b/shared/types/llm-docs.ts similarity index 100% rename from shared/types/llms-txt.ts rename to shared/types/llm-docs.ts diff --git a/test/unit/server/utils/llms-txt.spec.ts b/test/unit/server/utils/llm-docs.spec.ts similarity index 99% rename from test/unit/server/utils/llms-txt.spec.ts rename to test/unit/server/utils/llm-docs.spec.ts index fb3eafd8a..962a64967 100644 --- a/test/unit/server/utils/llms-txt.spec.ts +++ b/test/unit/server/utils/llm-docs.spec.ts @@ -5,7 +5,7 @@ import { fetchAgentFiles, generateLlmsTxt, generateRootLlmsTxt, -} from '../../../../server/utils/llms-txt' +} from '../../../../server/utils/llm-docs' describe('discoverAgentFiles', () => { it('discovers root-level agent files', () => { From 30d5c14c4577d9fe4d8b7550ffa23f6fd2676ec0 Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Thu, 12 Feb 2026 23:56:57 +0000 Subject: [PATCH 14/19] test(llm-docs): add failing tests for .md routes in generateRootLlmsTxt --- test/unit/server/utils/llm-docs.spec.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/unit/server/utils/llm-docs.spec.ts b/test/unit/server/utils/llm-docs.spec.ts index 962a64967..9ae9e83d0 100644 --- a/test/unit/server/utils/llm-docs.spec.ts +++ b/test/unit/server/utils/llm-docs.spec.ts @@ -332,4 +332,19 @@ describe('generateRootLlmsTxt', () => { const output = generateRootLlmsTxt('https://npmx.dev') expect(output.endsWith('\n')).toBe(true) }) + + it('includes .md route patterns', () => { + const output = generateRootLlmsTxt('https://npmx.dev') + + expect(output).toContain('https://npmx.dev/package/.md') + expect(output).toContain('https://npmx.dev/package//v/.md') + expect(output).toContain('https://npmx.dev/package/@/.md') + expect(output).toContain('https://npmx.dev/package/@//v/.md') + }) + + it('includes .md example links', () => { + const output = generateRootLlmsTxt('https://npmx.dev') + + expect(output).toContain('[nuxt README](https://npmx.dev/package/nuxt.md)') + }) }) From 05b7bec4d9d9a825ccdc9f72c8a24b819378b3d2 Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Thu, 12 Feb 2026 23:58:58 +0000 Subject: [PATCH 15/19] feat(llm-docs): add handlePackageMd for raw README .md routes --- server/utils/llm-docs.ts | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/server/utils/llm-docs.ts b/server/utils/llm-docs.ts index 861ea6888..197567bf3 100644 --- a/server/utils/llm-docs.ts +++ b/server/utils/llm-docs.ts @@ -261,6 +261,34 @@ export async function handleLlmsTxt( return generateLlmsTxt(result) } +/** + * Fetch raw README markdown for a package. + * Returns only the README content with no metadata wrapper. + */ +export async function handlePackageMd( + packageName: string, + requestedVersion?: string, +): Promise { + const packageData = await fetchNpmPackage(packageName) + const resolvedVersion = requestedVersion ?? packageData['dist-tags']?.latest + + if (!resolvedVersion) { + throw createError({ statusCode: 404, message: 'Could not resolve package version.' }) + } + + const readmeFromPackument = getReadmeFromPackument(packageData, requestedVersion) + const readme = readmeFromPackument ?? (await fetchReadmeFromCdn(packageName, resolvedVersion)) + + if (!readme) { + throw createError({ + statusCode: 404, + message: `No README found for ${packageName}@${resolvedVersion}.`, + }) + } + + return readme +} + // Validation for org names (matches server/api/registry/org/[org]/packages.get.ts) const NPM_ORG_NAME_RE = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/i @@ -354,12 +382,24 @@ export function generateRootLlmsTxt(baseUrl: string): string { lines.push('') lines.push(`- \`${baseUrl}/package/@/llms.txt\` — organization package listing`) lines.push('') + lines.push('### Raw README Markdown (.md)') + lines.push('') + lines.push('Raw README content for a package, with no metadata wrapper.') + lines.push('') + lines.push(`- \`${baseUrl}/package/.md\` — unscoped package (latest version)`) + lines.push(`- \`${baseUrl}/package//v/.md\` — unscoped package (specific version)`) + lines.push(`- \`${baseUrl}/package/@/.md\` — scoped package (latest version)`) + lines.push( + `- \`${baseUrl}/package/@//v/.md\` — scoped package (specific version)`, + ) + lines.push('') lines.push('## Examples') lines.push('') lines.push(`- [nuxt llms.txt](${baseUrl}/package/nuxt/llms.txt)`) lines.push(`- [nuxt llms_full.txt](${baseUrl}/package/nuxt/llms_full.txt)`) lines.push(`- [@nuxt/kit llms.txt](${baseUrl}/package/@nuxt/kit/llms.txt)`) lines.push(`- [@nuxt org packages](${baseUrl}/package/@nuxt/llms.txt)`) + lines.push(`- [nuxt README](${baseUrl}/package/nuxt.md)`) lines.push('') return lines.join('\n').trimEnd() + '\n' From a481b824409207d3764d9c3da760c6176a776c4f Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Fri, 13 Feb 2026 00:03:27 +0000 Subject: [PATCH 16/19] feat(llm-docs): handle .md routes in middleware --- server/middleware/llm-docs.ts | 50 ++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/server/middleware/llm-docs.ts b/server/middleware/llm-docs.ts index eebc4ebbd..d8e635178 100644 --- a/server/middleware/llm-docs.ts +++ b/server/middleware/llm-docs.ts @@ -1,7 +1,12 @@ import * as v from 'valibot' import { PackageRouteParamsSchema } from '#shared/schemas/package' import { handleApiError } from '#server/utils/error-handler' -import { handleLlmsTxt, handleOrgLlmsTxt, generateRootLlmsTxt } from '#server/utils/llm-docs' +import { + handleLlmsTxt, + handleOrgLlmsTxt, + generateRootLlmsTxt, + handlePackageMd, +} from '#server/utils/llm-docs' const CACHE_HEADER = 's-maxage=3600, stale-while-revalidate=86400' @@ -28,6 +33,49 @@ const CACHE_HEADER = 's-maxage=3600, stale-while-revalidate=86400' export default defineEventHandler(async event => { const path = event.path.split('?')[0] ?? '/' + // Handle .md routes — raw README markdown + if (path.startsWith('/package/') && path.endsWith('.md')) { + const inner = path.slice('/package/'.length, -'.md'.length) + + let rawPackageName: string + let rawVersion: string | undefined + + if (inner.includes('/v/')) { + if (inner.startsWith('@')) { + const match = inner.match(/^(@[^/]+\/[^/]+)\/v\/(.+)$/) + if (!match?.[1] || !match[2]) return + rawPackageName = match[1] + rawVersion = match[2] + } else { + const match = inner.match(/^([^/]+)\/v\/(.+)$/) + if (!match?.[1] || !match[2]) return + rawPackageName = match[1] + rawVersion = match[2] + } + } else { + rawPackageName = inner + } + + if (!rawPackageName) return + + try { + const { packageName, version } = v.parse(PackageRouteParamsSchema, { + packageName: rawPackageName, + version: rawVersion, + }) + + const content = await handlePackageMd(packageName, version) + setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') + setHeader(event, 'Cache-Control', CACHE_HEADER) + return content + } catch (error: unknown) { + handleApiError(error, { + statusCode: 502, + message: 'Failed to generate package markdown.', + }) + } + } + if (!path.endsWith('/llms.txt') && !path.endsWith('/llms_full.txt')) return const full = path.endsWith('/llms_full.txt') From d5ef5f573ffee2fdde01acfe06b9dfefe7b10bd6 Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Fri, 13 Feb 2026 00:10:25 +0000 Subject: [PATCH 17/19] feat(llm-docs): add .md shorthand redirects --- server/middleware/canonical-redirects.global.ts | 6 +++--- server/middleware/llm-docs.ts | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/middleware/canonical-redirects.global.ts b/server/middleware/canonical-redirects.global.ts index 27772c930..555f870e9 100644 --- a/server/middleware/canonical-redirects.global.ts +++ b/server/middleware/canonical-redirects.global.ts @@ -54,7 +54,7 @@ export default defineEventHandler(async event => { // /@org/pkg or /pkg → /package/org/pkg or /package/pkg // Also handles trailing /llms.txt or /llms_full.txt suffixes let pkgMatch = path.match( - /^\/(?:(?@[^/]+)\/)?(?[^/@]+)(?\/(?:llms\.txt|llms_full\.txt))?$/, + /^\/(?:(?@[^/]+)\/)?(?[^/@]+?)(?\.md|\/(?:llms\.txt|llms_full\.txt))?$/, ) if (pkgMatch?.groups) { const args = [pkgMatch.groups.org, pkgMatch.groups.name].filter(Boolean).join('/') @@ -68,10 +68,10 @@ export default defineEventHandler(async event => { // Also handles trailing /llms.txt or /llms_full.txt suffixes const pkgVersionMatch = path.match( - /^\/(?:(?@[^/]+)\/)?(?[^/@]+)\/v\/(?[^/]+)(?\/(?:llms\.txt|llms_full\.txt))?$/, + /^\/(?:(?@[^/]+)\/)?(?[^/@]+)\/v\/(?[^/]+?)(?\.md|\/(?:llms\.txt|llms_full\.txt))?$/, ) || path.match( - /^\/(?:(?@[^/]+)\/)?(?[^/@]+)@(?[^/]+)(?\/(?:llms\.txt|llms_full\.txt))?$/, + /^\/(?:(?@[^/]+)\/)?(?[^/@]+)@(?[^/]+?)(?\.md|\/(?:llms\.txt|llms_full\.txt))?$/, ) if (pkgVersionMatch?.groups) { diff --git a/server/middleware/llm-docs.ts b/server/middleware/llm-docs.ts index d8e635178..d97c8a804 100644 --- a/server/middleware/llm-docs.ts +++ b/server/middleware/llm-docs.ts @@ -11,7 +11,7 @@ import { const CACHE_HEADER = 's-maxage=3600, stale-while-revalidate=86400' /** - * Middleware to handle ALL llms.txt / llms_full.txt routes. + * Middleware to handle ALL llms.txt / llms_full.txt / .md routes. * * All llms.txt handling lives here rather than in file-based routes because * Vercel's ISR route rules with glob patterns (e.g. `/package/ ** /llms.txt`) @@ -20,6 +20,10 @@ const CACHE_HEADER = 's-maxage=3600, stale-while-revalidate=86400' * * Handles: * - /llms.txt (root discovery page) + * - /package/:name.md (unscoped, latest, raw README) + * - /package/@:org/:name.md (scoped, latest, raw README) + * - /package/:name/v/:version.md (unscoped, versioned, raw README) + * - /package/@:org/:name/v/:version.md (scoped, versioned, raw README) * - /package/@:org/llms.txt (org package listing) * - /package/:name/llms.txt (unscoped, latest) * - /package/:name/llms_full.txt (unscoped, latest, full) From 7bc5f23b0547a8bf5c4ba47abf5b459acbe76ca6 Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Fri, 13 Feb 2026 00:39:46 +0000 Subject: [PATCH 18/19] fix: remove no-op replace Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- server/utils/llm-docs.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/utils/llm-docs.ts b/server/utils/llm-docs.ts index 197567bf3..51c573a96 100644 --- a/server/utils/llm-docs.ts +++ b/server/utils/llm-docs.ts @@ -323,8 +323,7 @@ export async function handleOrgLlmsTxt(orgName: string, baseUrl: string): Promis lines.push('') for (const pkg of packages) { - const encodedPkg = pkg.replace('/', '/') - lines.push(`- [${pkg}](${baseUrl}/package/${encodedPkg}/llms.txt)`) + lines.push(`- [${pkg}](${baseUrl}/package/${pkg}/llms.txt)`) } lines.push('') From 895314f74c994653d509b6da39266d16380fc616 Mon Sep 17 00:00:00 2001 From: Luke Oliff Date: Fri, 13 Feb 2026 00:49:36 +0000 Subject: [PATCH 19/19] fix(llm-docs): remove versioned .md routes due to Vercel ISR conflict Versioned .md paths (e.g. /package/nuxt/v/3.16.2.md) conflict with Vercel's ISR route rules which match /package/:name/v/:version and intercept the request before middleware can handle it. Keep .md for latest-only (unscoped and scoped). --- scripts/smoke-test-llm-docs.sh | 69 +++++++++++++++++++ .../middleware/canonical-redirects.global.ts | 4 +- server/middleware/llm-docs.ts | 32 ++------- server/utils/llm-docs.ts | 4 -- test/unit/server/utils/llm-docs.spec.ts | 3 +- 5 files changed, 77 insertions(+), 35 deletions(-) create mode 100755 scripts/smoke-test-llm-docs.sh diff --git a/scripts/smoke-test-llm-docs.sh b/scripts/smoke-test-llm-docs.sh new file mode 100755 index 000000000..d25eaba03 --- /dev/null +++ b/scripts/smoke-test-llm-docs.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# Smoke test all llm-docs routes (llms.txt, llms_full.txt, .md) +# Usage: ./scripts/smoke-test-llm-docs.sh http://localhost:3333 + +set -euo pipefail + +BASE="${1:?Usage: $0 }" +BASE="${BASE%/}" # strip trailing slash + +PASS=0 +FAIL=0 + +check() { + local label="$1" + local url="$2" + local expect_status="${3:-200}" + + status=$(curl -s -o /dev/null -w "%{http_code}" -L "$url") + + if [ "$status" = "$expect_status" ]; then + echo " PASS GET $url $status $label" + PASS=$((PASS + 1)) + else + echo " FAIL GET $url $status $label (expected $expect_status)" + FAIL=$((FAIL + 1)) + fi +} + +echo "=== Root ===" +check "Root llms.txt" "$BASE/llms.txt" + +echo "" +echo "=== Unscoped package (latest) ===" +check "llms.txt" "$BASE/package/nuxt/llms.txt" +check "llms_full.txt" "$BASE/package/nuxt/llms_full.txt" +check ".md" "$BASE/package/nuxt.md" + +echo "" +echo "=== Unscoped package (versioned) ===" +check "llms.txt" "$BASE/package/nuxt/v/3.16.2/llms.txt" +check "llms_full.txt" "$BASE/package/nuxt/v/3.16.2/llms_full.txt" + +echo "" +echo "=== Scoped package (latest) ===" +check "llms.txt" "$BASE/package/@nuxt/kit/llms.txt" +check "llms_full.txt" "$BASE/package/@nuxt/kit/llms_full.txt" +check ".md" "$BASE/package/@nuxt/kit.md" + +echo "" +echo "=== Scoped package (versioned) ===" +check "llms.txt" "$BASE/package/@nuxt/kit/v/4.3.1/llms.txt" +check "llms_full.txt" "$BASE/package/@nuxt/kit/v/4.3.1/llms_full.txt" + +echo "" +echo "=== Org-level ===" +check "Org llms.txt" "$BASE/package/@nuxt/llms.txt" + +echo "" +echo "=== Shorthand redirects (follow → 200) ===" +check "Unscoped .md redirect" "$BASE/nuxt.md" +check "Scoped .md redirect" "$BASE/@nuxt/kit.md" +check "Unscoped llms.txt redirect" "$BASE/nuxt/llms.txt" +check "Scoped llms.txt redirect" "$BASE/@nuxt/kit/llms.txt" + +echo "" +echo "=== Results ===" +echo " $PASS passed, $FAIL failed" +exit $FAIL diff --git a/server/middleware/canonical-redirects.global.ts b/server/middleware/canonical-redirects.global.ts index 555f870e9..19559cf73 100644 --- a/server/middleware/canonical-redirects.global.ts +++ b/server/middleware/canonical-redirects.global.ts @@ -68,10 +68,10 @@ export default defineEventHandler(async event => { // Also handles trailing /llms.txt or /llms_full.txt suffixes const pkgVersionMatch = path.match( - /^\/(?:(?@[^/]+)\/)?(?[^/@]+)\/v\/(?[^/]+?)(?\.md|\/(?:llms\.txt|llms_full\.txt))?$/, + /^\/(?:(?@[^/]+)\/)?(?[^/@]+)\/v\/(?[^/]+)(?\/(?:llms\.txt|llms_full\.txt))?$/, ) || path.match( - /^\/(?:(?@[^/]+)\/)?(?[^/@]+)@(?[^/]+?)(?\.md|\/(?:llms\.txt|llms_full\.txt))?$/, + /^\/(?:(?@[^/]+)\/)?(?[^/@]+)@(?[^/]+)(?\/(?:llms\.txt|llms_full\.txt))?$/, ) if (pkgVersionMatch?.groups) { diff --git a/server/middleware/llm-docs.ts b/server/middleware/llm-docs.ts index d97c8a804..2de126fb9 100644 --- a/server/middleware/llm-docs.ts +++ b/server/middleware/llm-docs.ts @@ -22,8 +22,6 @@ const CACHE_HEADER = 's-maxage=3600, stale-while-revalidate=86400' * - /llms.txt (root discovery page) * - /package/:name.md (unscoped, latest, raw README) * - /package/@:org/:name.md (scoped, latest, raw README) - * - /package/:name/v/:version.md (unscoped, versioned, raw README) - * - /package/@:org/:name/v/:version.md (scoped, versioned, raw README) * - /package/@:org/llms.txt (org package listing) * - /package/:name/llms.txt (unscoped, latest) * - /package/:name/llms_full.txt (unscoped, latest, full) @@ -37,38 +35,18 @@ const CACHE_HEADER = 's-maxage=3600, stale-while-revalidate=86400' export default defineEventHandler(async event => { const path = event.path.split('?')[0] ?? '/' - // Handle .md routes — raw README markdown - if (path.startsWith('/package/') && path.endsWith('.md')) { - const inner = path.slice('/package/'.length, -'.md'.length) - - let rawPackageName: string - let rawVersion: string | undefined - - if (inner.includes('/v/')) { - if (inner.startsWith('@')) { - const match = inner.match(/^(@[^/]+\/[^/]+)\/v\/(.+)$/) - if (!match?.[1] || !match[2]) return - rawPackageName = match[1] - rawVersion = match[2] - } else { - const match = inner.match(/^([^/]+)\/v\/(.+)$/) - if (!match?.[1] || !match[2]) return - rawPackageName = match[1] - rawVersion = match[2] - } - } else { - rawPackageName = inner - } + // Handle .md routes — raw README markdown (latest version only) + if (path.startsWith('/package/') && path.endsWith('.md') && !path.includes('/v/')) { + const rawPackageName = path.slice('/package/'.length, -'.md'.length) if (!rawPackageName) return try { - const { packageName, version } = v.parse(PackageRouteParamsSchema, { + const { packageName } = v.parse(PackageRouteParamsSchema, { packageName: rawPackageName, - version: rawVersion, }) - const content = await handlePackageMd(packageName, version) + const content = await handlePackageMd(packageName) setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8') setHeader(event, 'Cache-Control', CACHE_HEADER) return content diff --git a/server/utils/llm-docs.ts b/server/utils/llm-docs.ts index 51c573a96..dc23fee18 100644 --- a/server/utils/llm-docs.ts +++ b/server/utils/llm-docs.ts @@ -386,11 +386,7 @@ export function generateRootLlmsTxt(baseUrl: string): string { lines.push('Raw README content for a package, with no metadata wrapper.') lines.push('') lines.push(`- \`${baseUrl}/package/.md\` — unscoped package (latest version)`) - lines.push(`- \`${baseUrl}/package//v/.md\` — unscoped package (specific version)`) lines.push(`- \`${baseUrl}/package/@/.md\` — scoped package (latest version)`) - lines.push( - `- \`${baseUrl}/package/@//v/.md\` — scoped package (specific version)`, - ) lines.push('') lines.push('## Examples') lines.push('') diff --git a/test/unit/server/utils/llm-docs.spec.ts b/test/unit/server/utils/llm-docs.spec.ts index 9ae9e83d0..e46db73fa 100644 --- a/test/unit/server/utils/llm-docs.spec.ts +++ b/test/unit/server/utils/llm-docs.spec.ts @@ -337,9 +337,8 @@ describe('generateRootLlmsTxt', () => { const output = generateRootLlmsTxt('https://npmx.dev') expect(output).toContain('https://npmx.dev/package/.md') - expect(output).toContain('https://npmx.dev/package//v/.md') expect(output).toContain('https://npmx.dev/package/@/.md') - expect(output).toContain('https://npmx.dev/package/@//v/.md') + expect(output).not.toContain('/v/.md') }) it('includes .md example links', () => {