feat: add robots and sitemap generation, enhance documentation page with copy markdown button

- Introduced `robots.ts` for defining robots.txt rules and sitemap URL.
- Added `sitemap.ts` to generate a sitemap based on available documentation pages.
- Enhanced the documentation page to include a "Copy as Markdown" button for easy copying of content.
- Created `copy-markdown-button.tsx` component for clipboard functionality.
- Implemented `llms.txt` route to provide a text-based overview of documentation links.

apps/docs/app/docs/[[...slug]]/page.tsx

@@ -1,4 +1,5 @@
-import { getPageImage, source } from "@/lib/source";
+import { CopyMarkdownButton } from "@/components/copy-markdown-button";
+import { getPageImage, getLLMText, source } from "@/lib/source";
 import { getMDXComponents } from "@/mdx-components";
 import {
 	DocsBody,
@@ -16,10 +17,14 @@ export default async function Page(props: PageProps<"/docs/[[...slug]]">) {
 	if (!page) notFound();
 
 	const MDX = page.data.body;
+	const markdown = await getLLMText(page);
 
 	return (
 		<DocsPage toc={page.data.toc} full={page.data.full}>
-			<DocsTitle>{page.data.title}</DocsTitle>
+			<div className="flex items-center justify-between">
+				<DocsTitle>{page.data.title}</DocsTitle>
+				<CopyMarkdownButton markdown={markdown} />
+			</div>
 			<DocsDescription>{page.data.description}</DocsDescription>
 			<DocsBody>
 				<MDX

apps/docs/app/llms.txt/route.ts

@@ -0,0 +1,28 @@
+import { source } from "@/lib/source";
+
+export const revalidate = false;
+
+const baseUrl = "https://docs.dokploy.com";
+
+export function GET() {
+	const pages = source.getPages();
+
+	const lines = [
+		"# Dokploy Documentation",
+		"",
+		"> Dokploy is an open-source, self-hostable Platform as a Service (PaaS) that simplifies the deployment and management of applications, databases, and services.",
+		"",
+		"## Docs",
+		"",
+		...pages.map(
+			(page) =>
+				`- [${page.data.title}](${baseUrl}${page.url})${page.data.description ? `: ${page.data.description}` : ""}`,
+		),
+		"",
+		"## Full Documentation",
+		"",
+		`- [llms-full.txt](${baseUrl}/llms-full.txt)`,
+	];
+
+	return new Response(lines.join("\n"));
+}

apps/docs/app/robots.ts

@@ -0,0 +1,11 @@
+import type { MetadataRoute } from "next";
+
+export default function robots(): MetadataRoute.Robots {
+	return {
+		rules: {
+			userAgent: "*",
+			allow: "/",
+		},
+		sitemap: "https://docs.dokploy.com/sitemap.xml",
+	};
+}

apps/docs/app/sitemap.ts

@@ -0,0 +1,23 @@
+import { source } from "@/lib/source";
+import type { MetadataRoute } from "next";
+
+const baseUrl = "https://docs.dokploy.com";
+
+export default function sitemap(): MetadataRoute.Sitemap {
+	const pages = source.getPages().map((page) => ({
+		url: `${baseUrl}${page.url}`,
+		lastModified: new Date(),
+		changeFrequency: "weekly" as const,
+		priority: 0.7,
+	}));
+
+	return [
+		{
+			url: baseUrl,
+			lastModified: new Date(),
+			changeFrequency: "monthly",
+			priority: 1,
+		},
+		...pages,
+	];
+}

apps/docs/components/copy-markdown-button.tsx

@@ -0,0 +1,30 @@
+"use client";
+
+import { useCopyButton } from "fumadocs-ui/utils/use-copy-button";
+import { Check, Copy } from "lucide-react";
+
+export function CopyMarkdownButton({ markdown }: { markdown: string }) {
+	const [checked, onClick] = useCopyButton(() => {
+		navigator.clipboard.writeText(markdown);
+	});
+
+	return (
+		<button
+			type="button"
+			className="inline-flex items-center gap-1.5 rounded-md border bg-fd-secondary px-3 py-1.5 text-xs font-medium text-fd-secondary-foreground transition-colors hover:bg-fd-accent hover:text-fd-accent-foreground"
+			onClick={onClick}
+		>
+			{checked ? (
+				<>
+					<Check className="size-3.5" />
+					Copied!
+				</>
+			) : (
+				<>
+					<Copy className="size-3.5" />
+					Copy as Markdown
+				</>
+			)}
+		</button>
+	);
+}