From 7e787f2cdc5de0320b2b1eecf3dddf6ab9a44594 Mon Sep 17 00:00:00 2001 From: Duncan Crawbuck Date: Tue, 17 Mar 2026 11:14:14 -0700 Subject: [PATCH] Restore markdown and llms routes --- source.config.ts | 2 +- src/lib/llms.ts | 90 ++++++++++++++++++++++++ src/lib/source.ts | 12 ++-- src/routeTree.gen.ts | 45 ++++++++++++ src/routes/llms-full-{$section}[.]txt.ts | 15 ++++ src/routes/llms-full[.]txt.ts | 10 +-- src/routes/llms-{$section}[.]txt.ts | 15 ++++ src/routes/llms[.]mdx.docs.$.ts | 7 +- src/routes/llms[.]txt.ts | 14 +--- src/start.ts | 64 +++++++++++++---- 10 files changed, 231 insertions(+), 43 deletions(-) create mode 100644 src/lib/llms.ts create mode 100644 src/routes/llms-full-{$section}[.]txt.ts create mode 100644 src/routes/llms-{$section}[.]txt.ts diff --git a/source.config.ts b/source.config.ts index bd2e5822..0710dda9 100644 --- a/source.config.ts +++ b/source.config.ts @@ -16,7 +16,7 @@ export const docs = defineDocs({ docs: { async: isDevelopment, postprocess: { - includeProcessedMarkdown: !isDevelopment, + includeProcessedMarkdown: true, }, }, }); diff --git a/src/lib/llms.ts b/src/lib/llms.ts new file mode 100644 index 00000000..77183aa7 --- /dev/null +++ b/src/lib/llms.ts @@ -0,0 +1,90 @@ +export const llmsSectionConfigs = { + dashboard: { + label: "Dashboard", + urlPrefix: "/docs/dashboard", + }, + ios: { + label: "iOS", + urlPrefix: "/docs/ios", + }, + android: { + label: "Android", + urlPrefix: "/docs/android", + }, + flutter: { + label: "Flutter", + urlPrefix: "/docs/flutter", + }, + expo: { + label: "Expo", + urlPrefix: "/docs/expo", + }, + "react-native": { + label: "React Native", + urlPrefix: "/docs/react-native", + }, + integrations: { + label: "Integrations", + urlPrefix: "/docs/integrations", + }, + "web-checkout": { + label: "Web Checkout", + urlPrefix: "/docs/web-checkout", + }, +} as const; + +export type LLMSection = keyof typeof llmsSectionConfigs; +export type LLMPageLike = { + url: string; + data: { + title: string; + description?: string; + }; +}; + +export function getLLMSectionConfig(section?: string) { + if (!section) return undefined; + return llmsSectionConfigs[section as LLMSection]; +} + +export function getPagesForLLMSection(pages: LLMPageLike[], section?: string) { + const config = getLLMSectionConfig(section); + if (!config) return pages; + + return pages.filter( + (page) => page.url === config.urlPrefix || page.url.startsWith(`${config.urlPrefix}/`), + ); +} + +export function buildLLMSummaryTextFromPages(pages: LLMPageLike[], section?: string) { + const config = getLLMSectionConfig(section); + const title = config ? `${config.label} Documentation` : "Documentation"; + const lines: string[] = [`# ${title}`, ""]; + + for (const page of getPagesForLLMSection(pages, section)) { + const description = page.data.description ? `: ${page.data.description}` : ""; + lines.push(`- [${page.data.title}](${page.url})${description}`); + } + + return lines.join("\n"); +} + +export async function buildLLMSummaryText(section?: string) { + const { source } = await import("@/lib/source"); + return buildLLMSummaryTextFromPages(source.getPages(), section); +} + +export async function buildLLMFullText(section?: string) { + const { source, getLLMText } = await import("@/lib/source"); + const scanned = await Promise.all(getPagesForLLMSection(source.getPages(), section).map(getLLMText)); + return scanned.join("\n\n"); +} + +export function buildLLMResponse(body: string) { + return new Response(body, { + headers: { + "Access-Control-Allow-Origin": "*", + "Content-Type": "text/plain; charset=utf-8", + }, + }); +} diff --git a/src/lib/source.ts b/src/lib/source.ts index 1edc49c5..a6ba994c 100644 --- a/src/lib/source.ts +++ b/src/lib/source.ts @@ -134,11 +134,15 @@ export const source = loader({ }, }); +export async function getPageMarkdownText( + page: InferPageType, + _type: "raw" | "processed" = "processed", +) { + return page.data.getText("processed"); +} + export async function getLLMText(page: InferPageType) { - // Dev disables processed markdown generation to avoid paying that extra MDX - // postprocess cost on the first docs request, so fall back to raw text there. - const textType = process.env.NODE_ENV === "development" ? "raw" : "processed"; - const processed = await page.data.getText(textType); + const processed = await getPageMarkdownText(page); return `# ${page.data.title} diff --git a/src/routeTree.gen.ts b/src/routeTree.gen.ts index b7db8a12..cc79350a 100644 --- a/src/routeTree.gen.ts +++ b/src/routeTree.gen.ts @@ -12,7 +12,9 @@ import { Route as rootRouteImport } from './routes/__root' import { Route as SitemapDotxmlRouteImport } from './routes/sitemap[.]xml' import { Route as RobotsDottxtRouteImport } from './routes/robots[.]txt' import { Route as LlmsDottxtRouteImport } from './routes/llms[.]txt' +import { Route as LlmsChar123sectionChar125DottxtRouteImport } from './routes/llms-{$section}[.]txt' import { Route as LlmsFullDottxtRouteImport } from './routes/llms-full[.]txt' +import { Route as LlmsFullChar123sectionChar125DottxtRouteImport } from './routes/llms-full-{$section}[.]txt' import { Route as SplatRouteImport } from './routes/$' import { Route as IndexRouteImport } from './routes/index' import { Route as ApiSearchRouteImport } from './routes/api/search' @@ -36,11 +38,23 @@ const LlmsDottxtRoute = LlmsDottxtRouteImport.update({ path: '/llms.txt', getParentRoute: () => rootRouteImport, } as any) +const LlmsChar123sectionChar125DottxtRoute = + LlmsChar123sectionChar125DottxtRouteImport.update({ + id: '/llms-{$section}.txt', + path: '/llms-{$section}.txt', + getParentRoute: () => rootRouteImport, + } as any) const LlmsFullDottxtRoute = LlmsFullDottxtRouteImport.update({ id: '/llms-full.txt', path: '/llms-full.txt', getParentRoute: () => rootRouteImport, } as any) +const LlmsFullChar123sectionChar125DottxtRoute = + LlmsFullChar123sectionChar125DottxtRouteImport.update({ + id: '/llms-full-{$section}.txt', + path: '/llms-full-{$section}.txt', + getParentRoute: () => rootRouteImport, + } as any) const SplatRoute = SplatRouteImport.update({ id: '/$', path: '/$', @@ -80,7 +94,9 @@ const ApiRawSplatRoute = ApiRawSplatRouteImport.update({ export interface FileRoutesByFullPath { '/': typeof IndexRoute '/$': typeof SplatRoute + '/llms-full-{$section}.txt': typeof LlmsFullChar123sectionChar125DottxtRoute '/llms-full.txt': typeof LlmsFullDottxtRoute + '/llms-{$section}.txt': typeof LlmsChar123sectionChar125DottxtRoute '/llms.txt': typeof LlmsDottxtRoute '/robots.txt': typeof RobotsDottxtRoute '/sitemap.xml': typeof SitemapDotxmlRoute @@ -93,7 +109,9 @@ export interface FileRoutesByFullPath { export interface FileRoutesByTo { '/': typeof IndexRoute '/$': typeof SplatRoute + '/llms-full-{$section}.txt': typeof LlmsFullChar123sectionChar125DottxtRoute '/llms-full.txt': typeof LlmsFullDottxtRoute + '/llms-{$section}.txt': typeof LlmsChar123sectionChar125DottxtRoute '/llms.txt': typeof LlmsDottxtRoute '/robots.txt': typeof RobotsDottxtRoute '/sitemap.xml': typeof SitemapDotxmlRoute @@ -107,7 +125,9 @@ export interface FileRoutesById { __root__: typeof rootRouteImport '/': typeof IndexRoute '/$': typeof SplatRoute + '/llms-full-{$section}.txt': typeof LlmsFullChar123sectionChar125DottxtRoute '/llms-full.txt': typeof LlmsFullDottxtRoute + '/llms-{$section}.txt': typeof LlmsChar123sectionChar125DottxtRoute '/llms.txt': typeof LlmsDottxtRoute '/robots.txt': typeof RobotsDottxtRoute '/sitemap.xml': typeof SitemapDotxmlRoute @@ -122,7 +142,9 @@ export interface FileRouteTypes { fullPaths: | '/' | '/$' + | '/llms-full-{$section}.txt' | '/llms-full.txt' + | '/llms-{$section}.txt' | '/llms.txt' | '/robots.txt' | '/sitemap.xml' @@ -135,7 +157,9 @@ export interface FileRouteTypes { to: | '/' | '/$' + | '/llms-full-{$section}.txt' | '/llms-full.txt' + | '/llms-{$section}.txt' | '/llms.txt' | '/robots.txt' | '/sitemap.xml' @@ -148,7 +172,9 @@ export interface FileRouteTypes { | '__root__' | '/' | '/$' + | '/llms-full-{$section}.txt' | '/llms-full.txt' + | '/llms-{$section}.txt' | '/llms.txt' | '/robots.txt' | '/sitemap.xml' @@ -162,7 +188,9 @@ export interface FileRouteTypes { export interface RootRouteChildren { IndexRoute: typeof IndexRoute SplatRoute: typeof SplatRoute + LlmsFullChar123sectionChar125DottxtRoute: typeof LlmsFullChar123sectionChar125DottxtRoute LlmsFullDottxtRoute: typeof LlmsFullDottxtRoute + LlmsChar123sectionChar125DottxtRoute: typeof LlmsChar123sectionChar125DottxtRoute LlmsDottxtRoute: typeof LlmsDottxtRoute RobotsDottxtRoute: typeof RobotsDottxtRoute SitemapDotxmlRoute: typeof SitemapDotxmlRoute @@ -196,6 +224,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof LlmsDottxtRouteImport parentRoute: typeof rootRouteImport } + '/llms-{$section}.txt': { + id: '/llms-{$section}.txt' + path: '/llms-{$section}.txt' + fullPath: '/llms-{$section}.txt' + preLoaderRoute: typeof LlmsChar123sectionChar125DottxtRouteImport + parentRoute: typeof rootRouteImport + } '/llms-full.txt': { id: '/llms-full.txt' path: '/llms-full.txt' @@ -203,6 +238,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof LlmsFullDottxtRouteImport parentRoute: typeof rootRouteImport } + '/llms-full-{$section}.txt': { + id: '/llms-full-{$section}.txt' + path: '/llms-full-{$section}.txt' + fullPath: '/llms-full-{$section}.txt' + preLoaderRoute: typeof LlmsFullChar123sectionChar125DottxtRouteImport + parentRoute: typeof rootRouteImport + } '/$': { id: '/$' path: '/$' @@ -258,7 +300,10 @@ declare module '@tanstack/react-router' { const rootRouteChildren: RootRouteChildren = { IndexRoute: IndexRoute, SplatRoute: SplatRoute, + LlmsFullChar123sectionChar125DottxtRoute: + LlmsFullChar123sectionChar125DottxtRoute, LlmsFullDottxtRoute: LlmsFullDottxtRoute, + LlmsChar123sectionChar125DottxtRoute: LlmsChar123sectionChar125DottxtRoute, LlmsDottxtRoute: LlmsDottxtRoute, RobotsDottxtRoute: RobotsDottxtRoute, SitemapDotxmlRoute: SitemapDotxmlRoute, diff --git a/src/routes/llms-full-{$section}[.]txt.ts b/src/routes/llms-full-{$section}[.]txt.ts new file mode 100644 index 00000000..b61117e3 --- /dev/null +++ b/src/routes/llms-full-{$section}[.]txt.ts @@ -0,0 +1,15 @@ +import { buildLLMFullText, buildLLMResponse, getLLMSectionConfig } from "@/lib/llms"; +import { createFileRoute } from "@tanstack/react-router"; + +export const Route = createFileRoute("/llms-full-{$section}.txt")({ + server: { + handlers: { + GET: async ({ params }) => { + if (!getLLMSectionConfig(params.section)) { + return new Response("LLMS section not found", { status: 404 }); + } + return buildLLMResponse(await buildLLMFullText(params.section)); + }, + }, + }, +}); diff --git a/src/routes/llms-full[.]txt.ts b/src/routes/llms-full[.]txt.ts index 0957c811..222e5ab9 100644 --- a/src/routes/llms-full[.]txt.ts +++ b/src/routes/llms-full[.]txt.ts @@ -1,17 +1,11 @@ import { createFileRoute } from "@tanstack/react-router"; -import { source, getLLMText } from "@/lib/source"; +import { buildLLMFullText, buildLLMResponse } from "@/lib/llms"; export const Route = createFileRoute("/llms-full.txt")({ server: { handlers: { GET: async () => { - const scan = source.getPages().map(getLLMText); - const scanned = await Promise.all(scan); - return new Response(scanned.join("\n\n"), { - headers: { - "Access-Control-Allow-Origin": "*", - }, - }); + return buildLLMResponse(await buildLLMFullText()); }, }, }, diff --git a/src/routes/llms-{$section}[.]txt.ts b/src/routes/llms-{$section}[.]txt.ts new file mode 100644 index 00000000..8087bc44 --- /dev/null +++ b/src/routes/llms-{$section}[.]txt.ts @@ -0,0 +1,15 @@ +import { buildLLMResponse, buildLLMSummaryText, getLLMSectionConfig } from "@/lib/llms"; +import { createFileRoute } from "@tanstack/react-router"; + +export const Route = createFileRoute("/llms-{$section}.txt")({ + server: { + handlers: { + GET: async ({ params }) => { + if (!getLLMSectionConfig(params.section)) { + return new Response("LLMS section not found", { status: 404 }); + } + return buildLLMResponse(await buildLLMSummaryText(params.section)); + }, + }, + }, +}); diff --git a/src/routes/llms[.]mdx.docs.$.ts b/src/routes/llms[.]mdx.docs.$.ts index b947d145..737e36f2 100644 --- a/src/routes/llms[.]mdx.docs.$.ts +++ b/src/routes/llms[.]mdx.docs.$.ts @@ -1,5 +1,5 @@ import { createFileRoute, notFound } from "@tanstack/react-router"; -import { source } from "@/lib/source"; +import { getPageMarkdownText, source } from "@/lib/source"; export const Route = createFileRoute("/llms.mdx/docs/$")({ server: { @@ -8,11 +8,8 @@ export const Route = createFileRoute("/llms.mdx/docs/$")({ const slugs = params._splat?.split("/") ?? []; const page = source.getPage(slugs); if (!page) throw notFound(); - // Dev disables processed markdown generation to keep the first docs - // request fast, so the LLM endpoint serves raw markdown in development. - const textType = process.env.NODE_ENV === "development" ? "raw" : "processed"; - return new Response(await page.data.getText(textType), { + return new Response(await getPageMarkdownText(page), { headers: { "Content-Type": "text/markdown", "Access-Control-Allow-Origin": "*", diff --git a/src/routes/llms[.]txt.ts b/src/routes/llms[.]txt.ts index c724fa74..69fd2509 100644 --- a/src/routes/llms[.]txt.ts +++ b/src/routes/llms[.]txt.ts @@ -1,21 +1,11 @@ -import { source } from "@/lib/source"; +import { buildLLMResponse, buildLLMSummaryText } from "@/lib/llms"; import { createFileRoute } from "@tanstack/react-router"; export const Route = createFileRoute("/llms.txt")({ server: { handlers: { GET: async () => { - const lines: string[] = []; - lines.push("# Documentation"); - lines.push(""); - for (const page of source.getPages()) { - lines.push(`- [${page.data.title}](${page.url}): ${page.data.description}`); - } - return new Response(lines.join("\n"), { - headers: { - "Access-Control-Allow-Origin": "*", - }, - }); + return buildLLMResponse(await buildLLMSummaryText()); }, }, }, diff --git a/src/start.ts b/src/start.ts index 84edf128..9bc3bcce 100644 --- a/src/start.ts +++ b/src/start.ts @@ -1,13 +1,21 @@ import { createMiddleware, createStart } from "@tanstack/react-start"; -import { rewritePath } from "fumadocs-core/negotiation"; +import { isMarkdownPreferred, rewritePath } from "fumadocs-core/negotiation"; import { redirect } from "@tanstack/react-router"; import { buildRedirectRouteRules } from "./lib/redirect-route-rules"; import { DOCS_BASE } from "./lib/url-base"; -const { rewrite: rewriteLLM } = rewritePath( +const { rewrite: rewriteLLMMarkdown } = rewritePath( + `${DOCS_BASE}{/*path}.md`, + `${DOCS_BASE}/llms.mdx/docs{/*path}`, +); +const { rewrite: rewriteLLMMdx } = rewritePath( `${DOCS_BASE}{/*path}.mdx`, `${DOCS_BASE}/llms.mdx/docs{/*path}`, ); +const { rewrite: rewriteLLMPreferred } = rewritePath( + `${DOCS_BASE}{/*path}`, + `${DOCS_BASE}/llms.mdx/docs{/*path}`, +); const legacyRedirectRules = buildRedirectRouteRules(); const redirectBypassPrefixes = [ `${DOCS_BASE}/_serverFn`, @@ -27,6 +35,14 @@ function isSelfRedirect(requestUrl: URL, destination: string): boolean { return normalizePathname(requestUrl.pathname) === normalizePathname(targetUrl.pathname); } +function withRequestSearch(requestUrl: URL, destination: string): URL { + const targetUrl = new URL(destination, requestUrl); + if (!targetUrl.search && requestUrl.search) { + targetUrl.search = requestUrl.search; + } + return targetUrl; +} + function shouldBypassLegacyRedirect(pathname: string): boolean { if (!pathname.startsWith(DOCS_BASE)) return true; if ( @@ -39,6 +55,34 @@ function shouldBypassLegacyRedirect(pathname: string): boolean { return staticAssetPathPattern.test(pathname); } +function shouldBypassLLMRewrite(pathname: string): boolean { + if (!pathname.startsWith(DOCS_BASE)) return true; + if ( + pathname === `${DOCS_BASE}/llms.mdx` || + pathname.startsWith(`${DOCS_BASE}/llms.mdx/`) || + redirectBypassPrefixes.some( + (prefix) => pathname === prefix || pathname.startsWith(`${prefix}/`), + ) + ) { + return true; + } + + return /\.(?!mdx?$)[a-z0-9]{2,8}$/i.test(pathname); +} + +export function resolveLLMPath(request: Request): string | undefined { + const url = new URL(request.url); + if (shouldBypassLLMRewrite(url.pathname)) return undefined; + + const directRewrite = + rewriteLLMMarkdown(url.pathname) || rewriteLLMMdx(url.pathname); + if (directRewrite) return directRewrite; + + if (!isMarkdownPreferred(request)) return undefined; + + return rewriteLLMPreferred(url.pathname) || undefined; +} + function resolveDeprecatedSdkPath(normalizedPath: string): string | undefined { if (normalizedPath === `${DOCS_BASE}/sdk`) { return `${DOCS_BASE}/ios`; @@ -77,13 +121,8 @@ const legacyRedirectMiddleware = createMiddleware().server(({ next, request }) = } if (destination) { - const targetUrl = new URL(destination, url); - if (!targetUrl.search && url.search) { - targetUrl.search = url.search; - } - throw redirect({ - href: targetUrl.toString(), + href: withRequestSearch(url, destination).toString(), statusCode: 308, }); } @@ -93,18 +132,17 @@ const legacyRedirectMiddleware = createMiddleware().server(({ next, request }) = const llmMiddleware = createMiddleware().server(({ next, request }) => { const url = new URL(request.url); - let path: string | null; + let destination: string | undefined; try { - // @ts-expect-error - path = rewriteLLM(url.pathname); + destination = resolveLLMPath(request); } catch { return next(); } - if (path) { + if (destination) { throw redirect({ - href: new URL(path, url).toString(), + href: withRequestSearch(url, destination).toString(), statusCode: 307, }); }