fix(web): add robots/sitemap metadata and bypass middleware

apps/web/src/app/robots.ts

@@ -0,0 +1,34 @@
+import type { MetadataRoute } from "next";
+
+const DEFAULT_SITE_URL = "https://www.solid-connection.com";
+
+const getSiteUrl = () => (process.env.NEXT_PUBLIC_WEB_URL ?? DEFAULT_SITE_URL).replace(/\/$/, "");
+
+export default function robots(): MetadataRoute.Robots {
+  const siteUrl = getSiteUrl();
+  const vercelEnv = process.env.VERCEL_ENV;
+  const isNonIndexEnvironment =
+    vercelEnv === "preview" ||
+    vercelEnv === "development" ||
+    siteUrl.includes("stage") ||
+    siteUrl.includes("localhost") ||
+    siteUrl.includes("127.0.0.1");
+
+  if (isNonIndexEnvironment) {
+    return {
+      rules: {
+        userAgent: "*",
+        disallow: "/",
+      },
+    };
+  }
+
+  return {
+    rules: {
+      userAgent: "*",
+      allow: "/",
+    },
+    sitemap: `${siteUrl}/sitemap.xml`,
+    host: new URL(siteUrl).host,
+  };
+}

apps/web/src/app/sitemap.ts

@@ -0,0 +1,42 @@
+import type { MetadataRoute } from "next";
+
+const DEFAULT_SITE_URL = "https://www.solid-connection.com";
+
+const getSiteUrl = () => (process.env.NEXT_PUBLIC_WEB_URL ?? DEFAULT_SITE_URL).replace(/\/$/, "");
+
+export default function sitemap(): MetadataRoute.Sitemap {
+  const siteUrl = getSiteUrl();
+
+  if (siteUrl.includes("stage") || siteUrl.includes("localhost") || siteUrl.includes("127.0.0.1")) {
+    return [];
+  }
+
+  const lastModified = new Date();
+
+  return [
+    {
+      url: `${siteUrl}/`,
+      lastModified,
+      changeFrequency: "daily",
+      priority: 1,
+    },
+    {
+      url: `${siteUrl}/mentor`,
+      lastModified,
+      changeFrequency: "weekly",
+      priority: 0.8,
+    },
+    {
+      url: `${siteUrl}/university`,
+      lastModified,
+      changeFrequency: "weekly",
+      priority: 0.8,
+    },
+    {
+      url: `${siteUrl}/terms`,
+      lastModified,
+      changeFrequency: "yearly",
+      priority: 0.5,
+    },
+  ];
+}

apps/web/src/middleware.ts

@@ -105,6 +105,6 @@ export function middleware(request: NextRequest) {
 }
 export const config = {
   matcher: [
-    "/((?!api|_next/static|_next/image|static/chunks|images|assets|favicon.ico|sw.js|viewer|fonts|.*\\.splat).*)",
+    "/((?!api|_next/static|_next/image|static/chunks|images|assets|favicon.ico|robots.txt|sitemap.xml|sw.js|viewer|fonts|.*\\.splat).*)",
   ],
 };