Merge pull request #1445 from openedx/sundas/INF-1819

fix: blocked web crawlers for auth MFE

package-lock.json

@@ -54,6 +54,7 @@
         "@edx/reactifex": "1.1.0",
         "@openedx/frontend-build": "^14.0.3",
         "babel-plugin-formatjs": "10.5.26",
+        "copy-webpack-plugin": "^11.0.0",
         "eslint-plugin-import": "2.31.0",
         "glob": "7.2.3",
         "history": "5.3.0",
@@ -6495,6 +6496,77 @@
       "version": "2.1.4",
       "license": "MIT"
     },
+    "node_modules/copy-webpack-plugin": {
+      "version": "11.0.0",
+      "resolved": "https://registry.npmjs.org/copy-webpack-plugin/-/copy-webpack-plugin-11.0.0.tgz",
+      "integrity": "sha512-fX2MWpamkW0hZxMEg0+mYnA40LTosOSa5TqZ9GYIBzyJa9C3QUaMPSE2xAi/buNr8u89SfD9wHSQVBzrRa/SOQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "fast-glob": "^3.2.11",
+        "glob-parent": "^6.0.1",
+        "globby": "^13.1.1",
+        "normalize-path": "^3.0.0",
+        "schema-utils": "^4.0.0",
+        "serialize-javascript": "^6.0.0"
+      },
+      "engines": {
+        "node": ">= 14.15.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
+      },
+      "peerDependencies": {
+        "webpack": "^5.1.0"
+      }
+    },
+    "node_modules/copy-webpack-plugin/node_modules/glob-parent": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
+      "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "is-glob": "^4.0.3"
+      },
+      "engines": {
+        "node": ">=10.13.0"
+      }
+    },
+    "node_modules/copy-webpack-plugin/node_modules/globby": {
+      "version": "13.2.2",
+      "resolved": "https://registry.npmjs.org/globby/-/globby-13.2.2.tgz",
+      "integrity": "sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "dir-glob": "^3.0.1",
+        "fast-glob": "^3.3.0",
+        "ignore": "^5.2.4",
+        "merge2": "^1.4.1",
+        "slash": "^4.0.0"
+      },
+      "engines": {
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/copy-webpack-plugin/node_modules/slash": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz",
+      "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/core-js": {
       "version": "3.39.0",
       "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.39.0.tgz",

package.json

@@ -77,6 +77,7 @@
     "@edx/reactifex": "1.1.0",
     "@openedx/frontend-build": "^14.0.3",
     "babel-plugin-formatjs": "10.5.26",
+    "copy-webpack-plugin": "^11.0.0",
     "eslint-plugin-import": "2.31.0",
     "glob": "7.2.3",
     "history": "5.3.0",

public/index.html

@@ -7,6 +7,7 @@
     </title>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta name="robots" content="noindex, nofollow" />
     <link
       rel="shortcut icon"
       href="<%=htmlWebpackPlugin.options.FAVICON_URL%>"

public/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

webpack.dev.config.js

@@ -1,6 +1,7 @@
 const path = require('path');
 
 const { createConfig } = require('@openedx/frontend-build');
+const CopyPlugin = require('copy-webpack-plugin');
 
 const config = createConfig('webpack-dev');
 
@@ -11,4 +12,15 @@ config.resolve.modules = [
 
 config.module.rules[0].exclude = /node_modules\/(?!(fastest-levenshtein|@edx))/;
 
+config.plugins.push(
+  new CopyPlugin({
+    patterns: [
+      {
+        from: path.resolve(__dirname, './public/robots.txt'),
+        to: path.resolve(__dirname, './dist/robots.txt'),
+      },
+    ],
+  }),
+);
+
 module.exports = config;

webpack.prod.config.js

@@ -1,6 +1,7 @@
 const path = require('path');
 
 const { createConfig } = require('@openedx/frontend-build');
+const CopyPlugin = require('copy-webpack-plugin');
 
 const config = createConfig('webpack-prod');
 
@@ -11,4 +12,15 @@ config.resolve.modules = [
 
 config.module.rules[0].exclude = /node_modules\/(?!(fastest-levenshtein|@edx))/;
 
+config.plugins.push(
+  new CopyPlugin({
+    patterns: [
+      {
+        from: path.resolve(__dirname, './public/robots.txt'),
+        to: path.resolve(__dirname, './dist/robots.txt'),
+      },
+    ],
+  }),
+);
+
 module.exports = config;