feat: Remove Learner Home from web crawling (#133)

webpack.prod.config.js

@@ -1,5 +1,6 @@
 const path = require('path');
 const { createConfig } = require('@edx/frontend-build');
+const CopyPlugin = require('copy-webpack-plugin');
 
 const config = createConfig('webpack-prod');
 
@@ -10,4 +11,15 @@ config.resolve.modules = [
 
 config.module.rules[0].exclude = /node_modules\/(?!(query-string|split-on-first|strict-uri-encode|@edx))/;
 
+config.plugins.push(
+  new CopyPlugin({
+    patterns: [
+      {
+        from: path.resolve(__dirname, './public/robots.txt'),
+        to: path.resolve(__dirname, './dist/robots.txt'),
+      },
+    ],
+  }),
+);
+
 module.exports = config;