Merge pull request #765 from javaistic/fix/seo

arghyaxcodes · web-flow · commit a5aab1456961 · 2025-10-21T05:29:13.000+05:30
Update `robots.txt` to improve crawl accessibility
diff --git a/src/app/robots.ts b/src/app/robots.ts
@@ -9,11 +9,9 @@ export default function robots(): MetadataRoute.Robots {
         allow: "/",
         disallow: [
           "/api/", // Block API endpoints from crawling
-          "/_next/", // Block Next.js internal files
         ],
       },
     ],
     sitemap: `${baseUrl}/sitemap.xml`,
-    host: baseUrl,
   };
 }
