fix: blocked web crawlers for account MFE

package-lock.json

@@ -69,6 +69,7 @@
         "@openedx/frontend-build": "14.3.2",
         "@testing-library/jest-dom": "6.6.3",
         "@testing-library/react": "12.1.5",
+        "copy-webpack-plugin": "^11.0.0",
         "react-test-renderer": "17.0.2",
         "reactifex": "1.1.1",
         "redux-mock-store": "1.5.5"
@@ -6677,6 +6678,77 @@
       "version": "2.1.4",
       "license": "MIT"
     },
+    "node_modules/copy-webpack-plugin": {
+      "version": "11.0.0",
+      "resolved": "https://registry.npmjs.org/copy-webpack-plugin/-/copy-webpack-plugin-11.0.0.tgz",
+      "integrity": "sha512-fX2MWpamkW0hZxMEg0+mYnA40LTosOSa5TqZ9GYIBzyJa9C3QUaMPSE2xAi/buNr8u89SfD9wHSQVBzrRa/SOQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "fast-glob": "^3.2.11",
+        "glob-parent": "^6.0.1",
+        "globby": "^13.1.1",
+        "normalize-path": "^3.0.0",
+        "schema-utils": "^4.0.0",
+        "serialize-javascript": "^6.0.0"
+      },
+      "engines": {
+        "node": ">= 14.15.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
+      },
+      "peerDependencies": {
+        "webpack": "^5.1.0"
+      }
+    },
+    "node_modules/copy-webpack-plugin/node_modules/glob-parent": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
+      "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "is-glob": "^4.0.3"
+      },
+      "engines": {
+        "node": ">=10.13.0"
+      }
+    },
+    "node_modules/copy-webpack-plugin/node_modules/globby": {
+      "version": "13.2.2",
+      "resolved": "https://registry.npmjs.org/globby/-/globby-13.2.2.tgz",
+      "integrity": "sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "dir-glob": "^3.0.1",
+        "fast-glob": "^3.3.0",
+        "ignore": "^5.2.4",
+        "merge2": "^1.4.1",
+        "slash": "^4.0.0"
+      },
+      "engines": {
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/copy-webpack-plugin/node_modules/slash": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz",
+      "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/core-js": {
       "version": "3.41.0",
       "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.41.0.tgz",

package.json

@@ -83,6 +83,7 @@
     "universal-cookie": "7.2.2"
   },
   "devDependencies": {
+    "copy-webpack-plugin": "^11.0.0",
     "@edx/browserslist-config": "1.5.0",
     "@edx/reactifex": "2.2.0",
     "@openedx/frontend-build": "14.3.2",

public/index.html

@@ -4,6 +4,7 @@
     <title>Account | <%= process.env.SITE_NAME %></title>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta name="robots" content="noindex, nofollow" />
     <link
       rel="shortcut icon"
       href="<%=htmlWebpackPlugin.options.FAVICON_URL%>"

public/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

webpack.dev.config.js

@@ -0,0 +1,19 @@
+const path = require('path');
+
+const { createConfig } = require('@openedx/frontend-build');
+const CopyPlugin = require('copy-webpack-plugin');
+
+const config = createConfig('webpack-dev');
+
+config.plugins.push(
+  new CopyPlugin({
+    patterns: [
+      {
+        from: path.resolve(__dirname, './public/robots.txt'),
+        to: path.resolve(__dirname, './dist/robots.txt'),
+      },
+    ],
+  }),
+);
+
+module.exports = config;

webpack.prod.config.js

@@ -0,0 +1,19 @@
+const path = require('path');
+
+const { createConfig } = require('@openedx/frontend-build');
+const CopyPlugin = require('copy-webpack-plugin');
+
+const config = createConfig('webpack-prod');
+
+config.plugins.push(
+  new CopyPlugin({
+    patterns: [
+      {
+        from: path.resolve(__dirname, './public/robots.txt'),
+        to: path.resolve(__dirname, './dist/robots.txt'),
+      },
+    ],
+  }),
+);
+
+module.exports = config;