HDFS-9651. All web UIs should include a robots.txt file. Contributed by Lars Francke.

(cherry picked from commit 09e6caec3dbba257f11fe7246d4c25ee8fc95d8b)

hadoop-hdfs-project/hadoop-hdfs/pom.xml

@@ -381,6 +381,9 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
             <exclude>src/test/resources/editsStored*</exclude>
             <exclude>src/test/resources/empty-file</exclude>
             <exclude>src/main/webapps/datanode/robots.txt</exclude>
+            <exclude>src/main/webapps/hdfs/robots.txt</exclude>
+            <exclude>src/main/webapps/journal/robots.txt</exclude>
+            <exclude>src/main/webapps/secondary/robots.txt</exclude>
             <exclude>src/contrib/**</exclude>
             <exclude>src/site/resources/images/*</exclude>
             <exclude>src/main/webapps/static/bootstrap-3.0.2/**</exclude>