Merge pull request #601 from dzcode-io/robot-text

fix: robots.txt

api/package.json

@@ -22,7 +22,6 @@
     "drizzle-orm": "^0.33.0",
     "express": "^4.21.0",
     "express-rate-limit": "^7.4.0",
-    "express-robots-txt": "^1.0.0",
     "fs-extra": "^11.2.0",
     "helmet": "^7.1.0",
     "lodash": "^4.17.21",

api/src/app/index.ts

@@ -20,7 +20,7 @@ import Container from "typedi";
 
 import { ErrorMiddleware } from "./middlewares/error";
 import { LoggerMiddleware } from "./middlewares/logger";
-import { RobotsMiddleware } from "./middlewares/robots";
+import { RobotsController } from "./middlewares/robots";
 import { SecurityMiddleware } from "./middlewares/security";
 
 // Use typedi container
@@ -43,8 +43,9 @@ const routingControllersOptions: RoutingControllersOptions = {
     MilestoneController,
     ProjectController,
     ContributorController,
+    RobotsController,
   ],
-  middlewares: [SecurityMiddleware, ErrorMiddleware, LoggerMiddleware, RobotsMiddleware],
+  middlewares: [SecurityMiddleware, ErrorMiddleware, LoggerMiddleware],
   defaultErrorHandler: false,
   cors: Container.get(SecurityMiddleware).cors(),
 };

api/src/app/middlewares/robots.ts

@@ -1,10 +1,22 @@
-import { RequestHandler } from "express";
-import { ExpressMiddlewareInterface, Middleware } from "routing-controllers";
+import { ConfigService } from "src/config/service";
 import { Service } from "typedi";
-const robots = require("express-robots-txt"); // eslint-disable-line @typescript-eslint/no-require-imports
+import { Response } from "express";
+import { Controller, Res, Get } from "routing-controllers";
 
 @Service()
-@Middleware({ type: "before", priority: 0 })
-export class RobotsMiddleware implements ExpressMiddlewareInterface {
-  use: RequestHandler = robots({ UserAgent: "*", Disallow: "/docs" });
+@Controller("/robots.txt")
+export class RobotsController {
+  private readonly stage = this.configService.env().NODE_ENV;
+
+  constructor(private readonly configService: ConfigService) {}
+
+  @Get("/")
+  public async robotsTxt(@Res() response: Response): Promise<Response> {
+    response.set("Content-Type", "text/plain");
+    if (this.stage === "production") {
+      return response.send("User-agent: *\nAllow: /");
+    } else {
+      return response.send("User-agent: *\nDisallow: /");
+    }
+  }
 }

web/package.json

@@ -61,11 +61,12 @@
     "bundle:alone": "rsbuild build",
     "clean": "lerna run clean:alone [email protected]/web --include-dependencies --stream",
     "clean:alone": "rimraf dist coverage bundle node_modules/.cache && rimraf ./cloudflare/public",
-    "deploy": "npm run generate:htmls && npm run generate:sitemap && rimraf ./cloudflare/public && cpy \"./bundle/**/*\" ./cloudflare/public && cd ./cloudflare && npm install && npm run deploy:prd ",
-    "deploy:stg": "npm run generate:htmls && npm run generate:sitemap && rimraf ./cloudflare/public && cpy \"./bundle/**/*\" ./cloudflare/public && cd ./cloudflare && npm install && npm run deploy:stg",
+    "deploy": "npm run generate:htmls && npm run generate:robots-txt && npm run generate:sitemap && rimraf ./cloudflare/public && cpy \"./bundle/**/*\" ./cloudflare/public && cd ./cloudflare && npm install && npm run deploy:prd ",
+    "deploy:stg": "npm run generate:htmls && npm run generate:robots-txt && npm run generate:sitemap && rimraf ./cloudflare/public && cpy \"./bundle/**/*\" ./cloudflare/public && cd ./cloudflare && npm install && npm run deploy:stg",
     "e2e:dev": "npx wait-on http://localhost:8080/ && npx cypress open",
     "generate:bundle-info": "ts-node ../packages/tooling/bundle-info.ts",
     "generate:htmls": "cross-env TS_NODE_SKIP_PROJECT=true ts-node --compilerOptions \"{\\\"baseUrl\\\": \\\".\\\"}\" src/_build/gen-multiple-htmls.ts",
+    "generate:robots-txt": "cross-env TS_NODE_SKIP_PROJECT=true ts-node --compilerOptions \"{\\\"baseUrl\\\": \\\".\\\"}\" src/_build/gen-robots-txt.ts",
     "generate:sentry-release": "ts-node ../packages/tooling/sentry-release.ts web bundle",
     "generate:sitemap": "cross-env TS_NODE_SKIP_PROJECT=true ts-node --compilerOptions \"{\\\"baseUrl\\\": \\\".\\\"}\" src/_build/sitemap.ts",
     "lint": "npm run build && npm run lint:alone",

web/src/_build/gen-robots-txt.ts

@@ -0,0 +1,27 @@
+// for dev, run:
+// npm run clean && npm run bundle && npm run generate:robots-txt
+// npm run rsbuild preview
+
+import { join } from "path";
+import { writeFileSync } from "fs";
+import { Environment, environments } from "@dzcode.io/utils/dist/config/environment";
+
+let stage = process.env.STAGE as Environment;
+if (!environments.includes(stage)) {
+  console.log(`⚠️  No STAGE provided, falling back to "development"`);
+  stage = "development";
+}
+
+const distFolder = "./bundle";
+
+const robotsTxtPath = join(distFolder, "robots.txt");
+
+console.log(`generating $robot.txt ...`);
+
+const robotsTxt = `User-agent: *
+${stage === "production" ? "Allow: /" : "Disallow: /"}
+`;
+
+writeFileSync(robotsTxtPath, robotsTxt);
+
+console.log("done");