feat(ui): add llms.txt generation for npm packages

scripts/smoke-test-llm-docs.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+#
+# Smoke test all llm-docs routes (llms.txt, llms_full.txt, .md)
+# Usage: ./scripts/smoke-test-llm-docs.sh http://localhost:3333
+
+set -euo pipefail
+
+BASE="${1:?Usage: $0 <base-url>}"
+BASE="${BASE%/}" # strip trailing slash
+
+PASS=0
+FAIL=0
+
+check() {
+  local label="$1"
+  local url="$2"
+  local expect_status="${3:-200}"
+
+  status=$(curl -s -o /dev/null -w "%{http_code}" -L "$url")
+
+  if [ "$status" = "$expect_status" ]; then
+    echo "  PASS  GET $url $status $label"
+    PASS=$((PASS + 1))
+  else
+    echo "  FAIL  GET $url $status $label (expected $expect_status)"
+    FAIL=$((FAIL + 1))
+  fi
+}
+
+echo "=== Root ==="
+check "Root llms.txt" "$BASE/llms.txt"
+
+echo ""
+echo "=== Unscoped package (latest) ==="
+check "llms.txt"      "$BASE/package/nuxt/llms.txt"
+check "llms_full.txt" "$BASE/package/nuxt/llms_full.txt"
+check ".md"           "$BASE/package/nuxt.md"
+
+echo ""
+echo "=== Unscoped package (versioned) ==="
+check "llms.txt"      "$BASE/package/nuxt/v/3.16.2/llms.txt"
+check "llms_full.txt" "$BASE/package/nuxt/v/3.16.2/llms_full.txt"
+
+echo ""
+echo "=== Scoped package (latest) ==="
+check "llms.txt"      "$BASE/package/@nuxt/kit/llms.txt"
+check "llms_full.txt" "$BASE/package/@nuxt/kit/llms_full.txt"
+check ".md"           "$BASE/package/@nuxt/kit.md"
+
+echo ""
+echo "=== Scoped package (versioned) ==="
+check "llms.txt"      "$BASE/package/@nuxt/kit/v/4.3.1/llms.txt"
+check "llms_full.txt" "$BASE/package/@nuxt/kit/v/4.3.1/llms_full.txt"
+
+echo ""
+echo "=== Org-level ==="
+check "Org llms.txt"  "$BASE/package/@nuxt/llms.txt"
+
+echo ""
+echo "=== Shorthand redirects (follow → 200) ==="
+check "Unscoped .md redirect"           "$BASE/nuxt.md"
+check "Scoped .md redirect"             "$BASE/@nuxt/kit.md"
+check "Unscoped llms.txt redirect"      "$BASE/nuxt/llms.txt"
+check "Scoped llms.txt redirect"        "$BASE/@nuxt/kit/llms.txt"
+
+echo ""
+echo "=== Results ==="
+echo "  $PASS passed, $FAIL failed"
+exit $FAIL

server/middleware/canonical-redirects.global.ts

@@ -47,26 +47,42 @@ export default defineEventHandler(async event => {
     return
   }
 
+  // /llms.txt at root is handled by the llm-docs middleware
+  if (path === '/llms.txt') {
+    return
+  }
+
   // /@org/pkg or /pkg → /package/org/pkg or /package/pkg
-  let pkgMatch = path.match(/^\/(?:(?<org>@[^/]+)\/)?(?<name>[^/@]+)$/)
+  // Also handles trailing /llms.txt or /llms_full.txt suffixes
+  let pkgMatch = path.match(
+    /^\/(?:(?<org>@[^/]+)\/)?(?<name>[^/@]+?)(?<suffix>\.md|\/(?:llms\.txt|llms_full\.txt))?$/,
+  )
   if (pkgMatch?.groups) {
     const args = [pkgMatch.groups.org, pkgMatch.groups.name].filter(Boolean).join('/')
+    const suffix = pkgMatch.groups.suffix ?? ''
     setHeader(event, 'cache-control', cacheControl)
-    return sendRedirect(event, `/package/${args}` + (query ? '?' + query : ''), 301)
+    return sendRedirect(event, `/package/${args}${suffix}` + (query ? '?' + query : ''), 301)
   }
 
   // /@org/pkg/v/version or /@org/pkg@version → /package/org/pkg/v/version
   // /pkg/v/version or /pkg@version → /package/pkg/v/version
+  // Also handles trailing /llms.txt or /llms_full.txt suffixes
   const pkgVersionMatch =
-    path.match(/^\/(?:(?<org>@[^/]+)\/)?(?<name>[^/@]+)\/v\/(?<version>[^/]+)$/) ||
-    path.match(/^\/(?:(?<org>@[^/]+)\/)?(?<name>[^/@]+)@(?<version>[^/]+)$/)
+    path.match(
+      /^\/(?:(?<org>@[^/]+)\/)?(?<name>[^/@]+)\/v\/(?<version>[^/]+)(?<suffix>\/(?:llms\.txt|llms_full\.txt))?$/,
+    ) ||
+    path.match(
+      /^\/(?:(?<org>@[^/]+)\/)?(?<name>[^/@]+)@(?<version>[^/]+)(?<suffix>\/(?:llms\.txt|llms_full\.txt))?$/,
+    )
 
   if (pkgVersionMatch?.groups) {
     const args = [pkgVersionMatch.groups.org, pkgVersionMatch.groups.name].filter(Boolean).join('/')
+    const versionSuffix = pkgVersionMatch.groups.suffix ?? ''
     setHeader(event, 'cache-control', cacheControl)
     return sendRedirect(
       event,
-      `/package/${args}/v/${pkgVersionMatch.groups.version}` + (query ? '?' + query : ''),
+      `/package/${args}/v/${pkgVersionMatch.groups.version}${versionSuffix}` +
+        (query ? '?' + query : ''),
       301,
     )
   }

server/middleware/llm-docs.ts

@@ -0,0 +1,135 @@
+import * as v from 'valibot'
+import { PackageRouteParamsSchema } from '#shared/schemas/package'
+import { handleApiError } from '#server/utils/error-handler'
+import {
+  handleLlmsTxt,
+  handleOrgLlmsTxt,
+  generateRootLlmsTxt,
+  handlePackageMd,
+} from '#server/utils/llm-docs'
+
+const CACHE_HEADER = 's-maxage=3600, stale-while-revalidate=86400'
+
+/**
+ * Middleware to handle ALL llms.txt / llms_full.txt / .md routes.
+ *
+ * All llms.txt handling lives here rather than in file-based routes because
+ * Vercel's ISR route rules with glob patterns (e.g. `/package/ ** /llms.txt`)
+ * create catch-all serverless functions that interfere with Nitro's file-based
+ * route resolution — scoped packages and versioned paths fail to match.
+ *
+ * Handles:
+ * - /llms.txt (root discovery page)
+ * - /package/:name.md (unscoped, latest, raw README)
+ * - /package/@:org/:name.md (scoped, latest, raw README)
+ * - /package/@:org/llms.txt (org package listing)
+ * - /package/:name/llms.txt (unscoped, latest)
+ * - /package/:name/llms_full.txt (unscoped, latest, full)
+ * - /package/@:org/:name/llms.txt (scoped, latest)
+ * - /package/@:org/:name/llms_full.txt (scoped, latest, full)
+ * - /package/:name/v/:version/llms.txt (unscoped, versioned)
+ * - /package/:name/v/:version/llms_full.txt (unscoped, versioned, full)
+ * - /package/@:org/:name/v/:version/llms.txt (scoped, versioned)
+ * - /package/@:org/:name/v/:version/llms_full.txt (scoped, versioned, full)
+ */
+export default defineEventHandler(async event => {
+  const path = event.path.split('?')[0] ?? '/'
+
+  // Handle .md routes — raw README markdown (latest version only)
+  if (path.startsWith('/package/') && path.endsWith('.md') && !path.includes('/v/')) {
+    const rawPackageName = path.slice('/package/'.length, -'.md'.length)
+
+    if (!rawPackageName) return
+
+    try {
+      const { packageName } = v.parse(PackageRouteParamsSchema, {
+        packageName: rawPackageName,
+      })
+
+      const content = await handlePackageMd(packageName)
+      setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8')
+      setHeader(event, 'Cache-Control', CACHE_HEADER)
+      return content
+    } catch (error: unknown) {
+      handleApiError(error, {
+        statusCode: 502,
+        message: 'Failed to generate package markdown.',
+      })
+    }
+  }
+
+  if (!path.endsWith('/llms.txt') && !path.endsWith('/llms_full.txt')) return
+
+  const full = path.endsWith('/llms_full.txt')
+  const suffix = full ? '/llms_full.txt' : '/llms.txt'
+
+  // Root /llms.txt
+  if (path === '/llms.txt') {
+    const url = getRequestURL(event)
+    const baseUrl = `${url.protocol}//${url.host}`
+    setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8')
+    setHeader(event, 'Cache-Control', CACHE_HEADER)
+    return generateRootLlmsTxt(baseUrl)
+  }
+
+  if (!path.startsWith('/package/')) return
+
+  // Strip /package/ prefix and /llms[_full].txt suffix
+  const inner = path.slice('/package/'.length, -suffix.length)
+
+  // Org-level: /package/@org/llms.txt (inner = "@org")
+  if (!full && inner.startsWith('@') && !inner.includes('/')) {
+    const orgName = inner.slice(1)
+    try {
+      const url = getRequestURL(event)
+      const baseUrl = `${url.protocol}//${url.host}`
+      const content = await handleOrgLlmsTxt(orgName, baseUrl)
+      setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8')
+      setHeader(event, 'Cache-Control', CACHE_HEADER)
+      return content
+    } catch (error: unknown) {
+      handleApiError(error, { statusCode: 502, message: 'Failed to generate org llms.txt.' })
+    }
+  }
+
+  // Parse package name and optional version from inner path
+  let rawPackageName: string
+  let rawVersion: string | undefined
+
+  if (inner.includes('/v/')) {
+    // Versioned path
+    if (inner.startsWith('@')) {
+      const match = inner.match(/^(@[^/]+\/[^/]+)\/v\/(.+)$/)
+      if (!match?.[1] || !match[2]) return
+      rawPackageName = match[1]
+      rawVersion = match[2]
+    } else {
+      const match = inner.match(/^([^/]+)\/v\/(.+)$/)
+      if (!match?.[1] || !match[2]) return
+      rawPackageName = match[1]
+      rawVersion = match[2]
+    }
+  } else {
+    // Latest version — inner is just the package name
+    rawPackageName = inner
+  }
+
+  if (!rawPackageName) return
+
+  try {
+    const { packageName, version } = v.parse(PackageRouteParamsSchema, {
+      packageName: rawPackageName,
+      version: rawVersion,
+    })
+
+    const content = await handleLlmsTxt(packageName, version, { includeAgentFiles: full })
+    setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8')
+    setHeader(event, 'Cache-Control', CACHE_HEADER)
+    return content
+  } catch (error: unknown) {
+    handleApiError(error, {
+      statusCode: 502,
+      message: `Failed to generate ${full ? 'llms_full.txt' : 'llms.txt'}.`,
+    })
+  }
+})