From 86ed8e2361907c4d6d65635d7005af4e7e099ab1 Mon Sep 17 00:00:00 2001
From: sundasnoreen12 <sundasnoreen12@gmail.com>
Date: Wed, 5 Mar 2025 15:27:09 +0500
Subject: [PATCH] fix: blocked web crawlers for auth MFE

---
 package-lock.json      | 72 ++++++++++++++++++++++++++++++++++++++++++
 package.json           |  1 +
 public/index.html      |  1 +
 public/robots.txt      |  2 ++
 webpack.dev.config.js  | 12 +++++++
 webpack.prod.config.js | 12 +++++++
 6 files changed, 100 insertions(+)
 create mode 100644 public/robots.txt

diff --git a/package-lock.json b/package-lock.json
index 0f288c4b..22ebf61c 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -54,6 +54,7 @@
         "@edx/reactifex": "1.1.0",
         "@openedx/frontend-build": "^14.0.3",
         "babel-plugin-formatjs": "10.5.26",
+        "copy-webpack-plugin": "^11.0.0",
         "eslint-plugin-import": "2.31.0",
         "glob": "7.2.3",
         "history": "5.3.0",
@@ -6495,6 +6496,77 @@
       "version": "2.1.4",
       "license": "MIT"
     },
+    "node_modules/copy-webpack-plugin": {
+      "version": "11.0.0",
+      "resolved": "https://registry.npmjs.org/copy-webpack-plugin/-/copy-webpack-plugin-11.0.0.tgz",
+      "integrity": "sha512-fX2MWpamkW0hZxMEg0+mYnA40LTosOSa5TqZ9GYIBzyJa9C3QUaMPSE2xAi/buNr8u89SfD9wHSQVBzrRa/SOQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "fast-glob": "^3.2.11",
+        "glob-parent": "^6.0.1",
+        "globby": "^13.1.1",
+        "normalize-path": "^3.0.0",
+        "schema-utils": "^4.0.0",
+        "serialize-javascript": "^6.0.0"
+      },
+      "engines": {
+        "node": ">= 14.15.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
+      },
+      "peerDependencies": {
+        "webpack": "^5.1.0"
+      }
+    },
+    "node_modules/copy-webpack-plugin/node_modules/glob-parent": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
+      "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "is-glob": "^4.0.3"
+      },
+      "engines": {
+        "node": ">=10.13.0"
+      }
+    },
+    "node_modules/copy-webpack-plugin/node_modules/globby": {
+      "version": "13.2.2",
+      "resolved": "https://registry.npmjs.org/globby/-/globby-13.2.2.tgz",
+      "integrity": "sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "dir-glob": "^3.0.1",
+        "fast-glob": "^3.3.0",
+        "ignore": "^5.2.4",
+        "merge2": "^1.4.1",
+        "slash": "^4.0.0"
+      },
+      "engines": {
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/copy-webpack-plugin/node_modules/slash": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz",
+      "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/core-js": {
       "version": "3.39.0",
       "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.39.0.tgz",
diff --git a/package.json b/package.json
index 854399ec..dcfb08fe 100644
--- a/package.json
+++ b/package.json
@@ -77,6 +77,7 @@
     "@edx/reactifex": "1.1.0",
     "@openedx/frontend-build": "^14.0.3",
     "babel-plugin-formatjs": "10.5.26",
+    "copy-webpack-plugin": "^11.0.0",
     "eslint-plugin-import": "2.31.0",
     "glob": "7.2.3",
     "history": "5.3.0",
diff --git a/public/index.html b/public/index.html
index b21df089..666912f6 100644
--- a/public/index.html
+++ b/public/index.html
@@ -7,6 +7,7 @@
     </title>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta name="robots" content="noindex, nofollow" />
     <link
       rel="shortcut icon"
       href="<%=htmlWebpackPlugin.options.FAVICON_URL%>"
diff --git a/public/robots.txt b/public/robots.txt
new file mode 100644
index 00000000..1f53798b
--- /dev/null
+++ b/public/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
diff --git a/webpack.dev.config.js b/webpack.dev.config.js
index f70fa45a..cc71d86b 100644
--- a/webpack.dev.config.js
+++ b/webpack.dev.config.js
@@ -1,6 +1,7 @@
 const path = require('path');
 
 const { createConfig } = require('@openedx/frontend-build');
+const CopyPlugin = require('copy-webpack-plugin');
 
 const config = createConfig('webpack-dev');
 
@@ -11,4 +12,15 @@ config.resolve.modules = [
 
 config.module.rules[0].exclude = /node_modules\/(?!(fastest-levenshtein|@edx))/;
 
+config.plugins.push(
+  new CopyPlugin({
+    patterns: [
+      {
+        from: path.resolve(__dirname, './public/robots.txt'),
+        to: path.resolve(__dirname, './dist/robots.txt'),
+      },
+    ],
+  }),
+);
+
 module.exports = config;
diff --git a/webpack.prod.config.js b/webpack.prod.config.js
index eb59b941..d7a3403d 100644
--- a/webpack.prod.config.js
+++ b/webpack.prod.config.js
@@ -1,6 +1,7 @@
 const path = require('path');
 
 const { createConfig } = require('@openedx/frontend-build');
+const CopyPlugin = require('copy-webpack-plugin');
 
 const config = createConfig('webpack-prod');
 
@@ -11,4 +12,15 @@ config.resolve.modules = [
 
 config.module.rules[0].exclude = /node_modules\/(?!(fastest-levenshtein|@edx))/;
 
+config.plugins.push(
+  new CopyPlugin({
+    patterns: [
+      {
+        from: path.resolve(__dirname, './public/robots.txt'),
+        to: path.resolve(__dirname, './dist/robots.txt'),
+      },
+    ],
+  }),
+);
+
 module.exports = config;