YARN-9087. Improve logging for initialization of Resource plugins. (Contributed by Szilard Nemeth)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java

@@ -315,11 +315,12 @@ public class LinuxContainerExecutor extends ContainerExecutor {
       resourceHandlerChain = ResourceHandlerModule
           .getConfiguredResourceHandlerChain(conf, nmContext);
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Resource handler chain enabled = " + (resourceHandlerChain
-            != null));
+        final boolean enabled = resourceHandlerChain != null;
+        LOG.debug("Resource handler chain enabled = " + enabled);
       }
       if (resourceHandlerChain != null) {
-        LOG.debug("Bootstrapping resource handler chain");
+        LOG.debug("Bootstrapping resource handler chain: " +
+            resourceHandlerChain);
         resourceHandlerChain.bootstrap(conf);
       }
     } catch (ResourceHandlerException e) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java

@@ -173,4 +173,9 @@ public class CGroupsBlkioResourceHandlerImpl implements DiskResourceHandler {
   public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
     return null;
   }
+
+  @Override
+  public String toString() {
+    return CGroupsBlkioResourceHandlerImpl.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java

@@ -256,4 +256,9 @@ public class CGroupsCpuResourceHandlerImpl implements CpuResourceHandler {
       throws ResourceHandlerException {
     return null;
   }
+
+  @Override
+  public String toString() {
+    return CGroupsCpuResourceHandlerImpl.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java

@@ -64,7 +64,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
   private static final String MTAB_FILE = "/proc/mounts";
   private static final String CGROUPS_FSTYPE = "cgroup";
 
-  private String mtabFile;
+  private final String mtabFile;
   private final String cGroupPrefix;
   private final boolean enableCGroupMount;
   private final String cGroupMountPath;
@@ -622,4 +622,16 @@ class CGroupsHandlerImpl implements CGroupsHandler {
   public String getCGroupMountPath() {
     return cGroupMountPath;
   }
+
+  @Override
+  public String toString() {
+    return CGroupsHandlerImpl.class.getName() + "{" +
+        "mtabFile='" + mtabFile + '\'' +
+        ", cGroupPrefix='" + cGroupPrefix + '\'' +
+        ", enableCGroupMount=" + enableCGroupMount +
+        ", cGroupMountPath='" + cGroupMountPath + '\'' +
+        ", deleteCGroupTimeout=" + deleteCGroupTimeout +
+        ", deleteCGroupDelay=" + deleteCGroupDelay +
+        '}';
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java

@@ -171,4 +171,9 @@ public class CGroupsMemoryResourceHandlerImpl implements MemoryResourceHandler {
   public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
     return null;
   }
+
+  @Override
+  public String toString() {
+    return CGroupsMemoryResourceHandlerImpl.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkPacketTaggingHandlerImpl.java

@@ -166,4 +166,9 @@ public class NetworkPacketTaggingHandlerImpl
       Configuration conf) {
     return NetworkTagMappingManagerFactory.getManager(conf);
   }
+
+  @Override
+  public String toString() {
+    return NetworkPacketTaggingHandlerImpl.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerChain.java

@@ -159,4 +159,10 @@ public class ResourceHandlerChain implements ResourceHandler {
     return Collections.unmodifiableList(resourceHandlers);
   }
 
+  @Override
+  public String toString() {
+    return ResourceHandlerChain.class.getName() + "{" +
+        "resourceHandlers=" + resourceHandlers +
+        '}';
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java

@@ -84,6 +84,9 @@ public class ResourceHandlerModule {
         if (cGroupsHandler == null) {
           cGroupsHandler = new CGroupsHandlerImpl(conf,
               PrivilegedOperationExecutor.getInstance(conf));
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Value of CGroupsHandler is: " + cGroupsHandler);
+          }
         }
       }
     }
@@ -306,16 +309,32 @@ public class ResourceHandlerModule {
       List<ResourceHandler> handlerList, Configuration conf,
       Context nmContext) throws ResourceHandlerException {
     ResourcePluginManager pluginManager = nmContext.getResourcePluginManager();
-    if (pluginManager != null) {
-       Map<String, ResourcePlugin> pluginMap = pluginManager.getNameToPlugins();
-       if (pluginMap != null) {
-        for (ResourcePlugin plugin : pluginMap.values()) {
-          addHandlerIfNotNull(handlerList, plugin
-              .createResourceHandler(nmContext,
-                  getInitializedCGroupsHandler(conf),
-                  PrivilegedOperationExecutor.getInstance(conf)));
-        }
+
+    if (pluginManager == null) {
+      LOG.warn("Plugin manager was null while trying to add " +
+          "ResourceHandlers from configuration!");
+      return;
+    }
+
+    Map<String, ResourcePlugin> pluginMap = pluginManager.getNameToPlugins();
+    if (pluginMap == null) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("List of plugins of ResourcePluginManager was empty " +
+            "while trying to add ResourceHandlers from configuration!");
       }
+      return;
+    } else {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("List of plugins of ResourcePluginManager: " +
+            pluginManager.getNameToPlugins());
+      }
+    }
+
+    for (ResourcePlugin plugin : pluginMap.values()) {
+      addHandlerIfNotNull(handlerList,
+          plugin.createResourceHandler(nmContext,
+              getInitializedCGroupsHandler(conf),
+              PrivilegedOperationExecutor.getInstance(conf)));
     }
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java

@@ -283,4 +283,9 @@ public class TrafficControlBandwidthHandlerImpl
 
     return null;
   }
+
+  @Override
+  public String toString() {
+    return TrafficControlBandwidthHandlerImpl.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java

@@ -55,11 +55,11 @@ public class FpgaResourceHandlerImpl implements ResourceHandler {
 
   private final String REQUEST_FPGA_IP_ID_KEY = "REQUESTED_FPGA_IP_ID";
 
-  private AbstractFpgaVendorPlugin vendorPlugin;
+  private final AbstractFpgaVendorPlugin vendorPlugin;
 
-  private FpgaResourceAllocator allocator;
+  private final FpgaResourceAllocator allocator;
 
-  private CGroupsHandler cGroupsHandler;
+  private final CGroupsHandler cGroupsHandler;
 
   public static final String EXCLUDED_FPGAS_CLI_OPTION = "--excluded_fpgas";
   public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
@@ -223,4 +223,12 @@ public class FpgaResourceHandlerImpl implements ResourceHandler {
   public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
     return null;
   }
+
+  @Override
+  public String toString() {
+    return FpgaResourceHandlerImpl.class.getName() + "{" +
+        "vendorPlugin=" + vendorPlugin +
+        ", allocator=" + allocator +
+        '}';
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java

@@ -304,4 +304,9 @@ public class GpuResourceAllocator {
     }
     return assigns;
   }
+
+  @Override
+  public String toString() {
+    return GpuResourceAllocator.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java

@@ -48,10 +48,10 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
   public static final String EXCLUDED_GPUS_CLI_OPTION = "--excluded_gpus";
   public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
 
-  private Context nmContext;
-  private GpuResourceAllocator gpuAllocator;
-  private CGroupsHandler cGroupsHandler;
-  private PrivilegedOperationExecutor privilegedOperationExecutor;
+  private final Context nmContext;
+  private final GpuResourceAllocator gpuAllocator;
+  private final CGroupsHandler cGroupsHandler;
+  private final PrivilegedOperationExecutor privilegedOperationExecutor;
 
   public GpuResourceHandlerImpl(Context nmContext,
       CGroupsHandler cGroupsHandler,
@@ -185,4 +185,11 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
   public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
     return null;
   }
+
+  @Override
+  public String toString() {
+    return GpuResourceHandlerImpl.class.getName() + "{" +
+        "gpuAllocator=" + gpuAllocator +
+        '}';
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceHandlerImpl.java

@@ -41,8 +41,8 @@ public class NumaResourceHandlerImpl implements ResourceHandler {
 
   private static final Log LOG = LogFactory
       .getLog(NumaResourceHandlerImpl.class);
-  private NumaResourceAllocator numaResourceAllocator;
-  private String numaCtlCmd;
+  private final NumaResourceAllocator numaResourceAllocator;
+  private final String numaCtlCmd;
 
   public NumaResourceHandlerImpl(Configuration conf, Context nmContext) {
     LOG.info("NUMA resources allocation is enabled, initializing NUMA resources"
@@ -111,4 +111,12 @@ public class NumaResourceHandlerImpl implements ResourceHandler {
   public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
     return null;
   }
+
+  @Override
+  public String toString() {
+    return NumaResourceHandlerImpl.class.getName() + "{" +
+        "numaResourceAllocator=" + numaResourceAllocator +
+        ", numaCtlCmd='" + numaCtlCmd + '\'' +
+        '}';
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java

@@ -39,6 +39,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.lang.reflect.Method;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -67,6 +68,12 @@ public class ResourcePluginManager {
     Map<String, ResourcePlugin> pluginMap = new HashMap<>();
 
     String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS);
+    if (plugins == null || plugins.length == 0) {
+      LOG.info("No Resource plugins found from configuration!");
+    }
+    LOG.info("Found Resource plugins from configuration: "
+        + Arrays.toString(plugins));
+
     if (plugins != null) {
       // Initialize each plugins
       for (String resourceName : plugins) {
@@ -75,23 +82,21 @@ public class ResourcePluginManager {
           String msg =
               "Trying to initialize resource plugin with name=" + resourceName
                   + ", it is not supported, list of supported plugins:"
-                  + StringUtils.join(",",
-                  SUPPORTED_RESOURCE_PLUGINS);
+                  + StringUtils.join(",", SUPPORTED_RESOURCE_PLUGINS);
           LOG.error(msg);
           throw new YarnException(msg);
         }
 
         if (pluginMap.containsKey(resourceName)) {
-          // Duplicated items, ignore ...
+          LOG.warn("Ignoring duplicate Resource plugin definition: " +
+              resourceName);
           continue;
         }
 
         ResourcePlugin plugin = null;
         if (resourceName.equals(GPU_URI)) {
           plugin = new GpuResourcePlugin();
-        }
-
-        if (resourceName.equals(FPGA_URI)) {
+        } else if (resourceName.equals(FPGA_URI)) {
           plugin = new FpgaResourcePlugin();
         }
 
@@ -101,6 +106,7 @@ public class ResourcePluginManager {
                   + " should be loaded and initialized");
         }
         plugin.initialize(context);
+        LOG.info("Initialized plugin {}", plugin);
         pluginMap.put(resourceName, plugin);
       }
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java

@@ -99,4 +99,9 @@ public class DevicePluginAdapter implements ResourcePlugin {
   public DeviceResourceHandlerImpl getDeviceResourceHandler() {
     return deviceResourceHandler;
   }
+
+  @Override
+  public String toString() {
+    return DevicePluginAdapter.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceHandlerImpl.java

@@ -46,12 +46,12 @@ public class DeviceResourceHandlerImpl implements ResourceHandler {
 
   static final Log LOG = LogFactory.getLog(DeviceResourceHandlerImpl.class);
 
-  private String resourceName;
-  private DevicePlugin devicePlugin;
-  private DeviceMappingManager deviceMappingManager;
-  private CGroupsHandler cGroupsHandler;
-  private PrivilegedOperationExecutor privilegedOperationExecutor;
-  private DevicePluginAdapter devicePluginAdapter;
+  private final String resourceName;
+  private final DevicePlugin devicePlugin;
+  private final DeviceMappingManager deviceMappingManager;
+  private final CGroupsHandler cGroupsHandler;
+  private final PrivilegedOperationExecutor privilegedOperationExecutor;
+  private final DevicePluginAdapter devicePluginAdapter;
 
   public DeviceResourceHandlerImpl(String reseName,
       DevicePlugin devPlugin,
@@ -142,4 +142,13 @@ public class DeviceResourceHandlerImpl implements ResourceHandler {
       throws ResourceHandlerException {
     return null;
   }
+
+  @Override
+  public String toString() {
+    return DeviceResourceHandlerImpl.class.getName() + "{" +
+        "resourceName='" + resourceName + '\'' +
+        ", devicePlugin=" + devicePlugin +
+        ", devicePluginAdapter=" + devicePluginAdapter +
+        '}';
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaResourcePlugin.java

@@ -102,4 +102,9 @@ public class FpgaResourcePlugin implements ResourcePlugin {
   public NMResourceInfo getNMResourceInfo() throws YarnException {
     return null;
   }
+
+  @Override
+  public String toString() {
+    return FpgaResourcePlugin.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java

@@ -89,4 +89,9 @@ public class GpuResourcePlugin implements ResourcePlugin {
     return new NMGpuResourceInfo(gpuDeviceInformation, totalGpus,
         assignedGpuDevices);
   }
+
+  @Override
+  public String toString() {
+    return GpuResourcePlugin.class.getName();
+  }
 }