fix(nx-dev): make sure only prod is indexed (#33922)

Note: required updating inputs for tasks to make sure env vars are
correctly cache busting.

Prod
<img width="1459" height="724" alt="image"
src="https://github.com/user-attachments/assets/1289b745-b68f-4176-9812-ef783d03ba26"
/>
non prod
<img width="1585" height="589" alt="image"
src="https://github.com/user-attachments/assets/dd0603b2-6487-4690-8bed-4c3d82644304"
/>

Author: barbados-clemens

astro-docs/project.json

@@ -23,6 +23,7 @@
           "target": "build"
         }
       ],
+      "inputs": ["production", "^production", { "env": "NX_DEV_URL" }],
       "outputs": [
         "{projectRoot}/dist",
         "{projectRoot}/.astro",

astro-docs/public/robots.txt

@@ -0,0 +1,33 @@
+import type { APIRoute } from 'astro';
+
+const siteUrl = import.meta.env.SITE ?? 'https://nx.dev';
+const isProductionDomain = siteUrl === 'https://nx.dev';
+
+const productionRobotsTxt = `# https://www.robotstxt.org/robotstxt.html
+User-agent: *
+Allow: /
+
+# Host
+Host: https://nx.dev
+
+# Sitemaps
+Sitemap: https://nx.dev/sitemap-index.xml
+`;
+
+// Block all crawling on non-production domains (canary, versioned, etc.)
+const nonProductionRobotsTxt = `# This is a non-production deployment (canary, versioned, preview, etc.)
+# All crawling is disallowed to prevent duplicate content in search results.
+User-agent: *
+Disallow: /
+`;
+
+export const GET: APIRoute = () => {
+  return new Response(
+    isProductionDomain ? productionRobotsTxt : nonProductionRobotsTxt,
+    {
+      headers: {
+        'Content-Type': 'text/plain; charset=utf-8',
+      },
+    }
+  );
+};

astro-docs/src/pages/robots.txt.ts

@@ -1,6 +1,8 @@
 const path = require('path');
 
-const siteUrl = process.env.SITE_URL || 'https://nx.dev';
+const siteUrl = process.env.NX_DEV_URL || 'https://nx.dev';
+const noIndex = process.env.NEXT_PUBLIC_NO_INDEX === 'true';
+
 /**
  * @type {import('next-sitemap').IConfig}
  **/
@@ -10,7 +12,11 @@ module.exports = {
   exclude: [],
   sourceDir: path.resolve(__dirname, '../../dist/nx-dev/nx-dev/.next'),
   outDir: path.resolve(__dirname, '../../dist/nx-dev/nx-dev/public'),
-  robotsTxtOptions: {
-    additionalSitemaps: [`${siteUrl}/docs/sitemap-index.xml`],
-  },
+  robotsTxtOptions: noIndex
+    ? {
+        policies: [{ userAgent: '*', disallow: '/' }],
+      }
+    : {
+        additionalSitemaps: [`${siteUrl}/docs/sitemap-index.xml`],
+      },
 };

nx-dev/nx-dev/next-sitemap.config.js

@@ -44,6 +44,11 @@
     },
     "sitemap": {
       "executor": "nx:run-commands",
+      "inputs": [
+        "{workspaceRoot}/nx-dev/nx-dev/next-sitemap.config.js",
+        { "env": "NX_DEV_URL" },
+        { "env": "NEXT_PUBLIC_NO_INDEX" }
+      ],
       "outputs": ["{workspaceRoot}/dist/nx-dev/nx-dev/public"],
       "options": {
         "command": "pnpm next-sitemap --config ./nx-dev/nx-dev/next-sitemap.config.js"