YARN-7856. Validate Node Attributes from NM. Contributed by Weiwei Yang.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java

@@ -46,6 +46,8 @@ import org.apache.hadoop.yarn.util.Records;
 public abstract class NodeAttribute {
 
   public static final String DEFAULT_PREFIX = "";
+  public static final String PREFIX_DISTRIBUTED = "nm.yarn.io";
+  public static final String PREFIX_CENTRALIZED = "rm.yarn.io";
 
   public static NodeAttribute newInstance(String attributeName,
       NodeAttributeType attributeType, String attributeValue) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java

@@ -17,7 +17,11 @@
  */
 package org.apache.hadoop.yarn.nodelabels;
 
+import com.google.common.base.Strings;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+
 import java.io.IOException;
+import java.util.Set;
 import java.util.regex.Pattern;
 
 /**
@@ -94,4 +98,31 @@ public final class NodeLabelUtil {
           + ", now it is= " + prefix);
     }
   }
+
+  /**
+   * Validate if a given set of attributes are valid. Attributes could be
+   * invalid if any of following conditions is met:
+   *
+   * <ul>
+   *   <li>Missing prefix: the attribute doesn't have prefix defined</li>
+   *   <li>Malformed attribute prefix: the prefix is not in valid format</li>
+   * </ul>
+   * @param attributeSet
+   * @throws IOException
+   */
+  public static void validateNodeAttributes(Set<NodeAttribute> attributeSet)
+      throws IOException {
+    if (attributeSet != null && !attributeSet.isEmpty()) {
+      for (NodeAttribute nodeAttribute : attributeSet) {
+        String prefix = nodeAttribute.getAttributePrefix();
+        if (Strings.isNullOrEmpty(prefix)) {
+          throw new IOException("Attribute prefix must be set");
+        }
+        // Verify attribute prefix format.
+        checkAndThrowAttributePrefix(prefix);
+        // Verify attribute name format.
+        checkAndThrowLabelName(nodeAttribute.getAttributeName());
+      }
+    }
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.NodeAttribute;
 import org.apache.hadoop.yarn.api.records.NodeAttributeType;
+import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil;
 
 import java.io.IOException;
 import java.util.HashSet;
@@ -116,13 +117,33 @@ public class ScriptBasedNodeAttributesProvider extends NodeAttributesProvider{
                 + NODE_ATTRIBUTE_DELIMITER + "ATTRIBUTE_VALUE; but get "
                 + nodeAttribute);
           }
+          // Automatically setup prefix for collected attributes
           NodeAttribute na = NodeAttribute
-              .newInstance(attributeStrs[0],
+              .newInstance(NodeAttribute.PREFIX_DISTRIBUTED,
+                  attributeStrs[0],
                   NodeAttributeType.valueOf(attributeStrs[1]),
                   attributeStrs[2]);
-          attributeSet.add(na);
+
+          // Since a NodeAttribute is identical with another one as long as
+          // their prefix and name are same, to avoid attributes getting
+          // overwritten by ambiguous attribute, make sure it fails in such
+          // case.
+          if (!attributeSet.add(na)) {
+            throw new IOException("Ambiguous node attribute is found: "
+                + na.toString() + ", a same attribute already exists");
+          }
         }
       }
+
+      // Before updating the attributes to the provider,
+      // verify if they are valid
+      try {
+        NodeLabelUtil.validateNodeAttributes(attributeSet);
+      } catch (IOException e) {
+        throw new IOException("Node attributes collected by the script "
+            + "contains some invalidate entries. Detail message: "
+            + e.getMessage());
+      }
       return attributeSet;
     }
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java

@@ -220,4 +220,31 @@ public class TestScriptBasedNodeAttributesProvider {
       }
     }, 500, 3000);
   }
+
+  @Test
+  public void testNodeAttributesValidation() throws Exception{
+    // Script output contains ambiguous node attributes
+    String scriptContent = "echo NODE_ATTRIBUTE:host,STRING,host1234\n "
+        + "echo NODE_ATTRIBUTE:host,STRING,host2345\n "
+        + "echo NODE_ATTRIBUTE:ip,STRING,10.0.0.1";
+
+    writeNodeAttributeScriptFile(scriptContent, true);
+
+    nodeAttributesProvider.init(getConfForNodeAttributeScript());
+    nodeAttributesProvider.start();
+
+    // There should be no attributes found, and we should
+    // see Malformed output warnings in the log
+    try {
+      GenericTestUtils
+          .waitFor(() -> nodeAttributesProvider
+                  .getDescriptors().size() == 3,
+              500, 3000);
+      Assert.fail("This test should timeout because the provide is unable"
+          + " to parse any attributes from the script output.");
+    } catch (TimeoutException e) {
+      Assert.assertEquals(0, nodeAttributesProvider
+          .getDescriptors().size());
+    }
+  }
 }