YARN-1228. Clean up Fair Scheduler configuration loading. (Sandy Ryza)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1528203 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -70,6 +70,8 @@ Release 2.1.2 - UNRELEASED
     YARN-899. Added back queue level administrator-acls so that there is no
     regression w.r.t 1.x. (Xuan Gong via vinodkv)
 
+    YARN-1228. Clean up Fair Scheduler configuration loading. (Sandy Ryza)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java

@@ -18,9 +18,12 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
 import java.io.File;
+import java.net.URL;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.conf.Configuration;
@@ -33,6 +36,9 @@ import org.apache.hadoop.yarn.util.resource.Resources;
 @Evolving
 public class FairSchedulerConfiguration extends Configuration {
 
+  public static final Log LOG = LogFactory.getLog(
+      FairSchedulerConfiguration.class.getName());
+  
   /** Increment request grant-able by the RM scheduler. 
    * These properties are looked up in the yarn-site.xml  */
   public static final String RM_SCHEDULER_INCREMENT_ALLOCATION_MB =
@@ -42,11 +48,10 @@ public class FairSchedulerConfiguration extends Configuration {
     YarnConfiguration.YARN_PREFIX + "scheduler.increment-allocation-vcores";
   public static final int DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES = 1;
   
-  public static final String FS_CONFIGURATION_FILE = "fair-scheduler.xml";
-
   private static final String CONF_PREFIX =  "yarn.scheduler.fair.";
 
   protected static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file";
+  protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml";
   protected static final String EVENT_LOG_DIR = "eventlog.dir";
 
   /** Whether to use the user name as the queue name (instead of "default") if
@@ -105,7 +110,6 @@ public class FairSchedulerConfiguration extends Configuration {
 
   public FairSchedulerConfiguration(Configuration conf) {
     super(conf);
-    addResource(FS_CONFIGURATION_FILE);
   }
 
   public Resource getMinimumAllocation() {
@@ -182,8 +186,28 @@ public class FairSchedulerConfiguration extends Configuration {
     return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT);
   }
 
-  public String getAllocationFile() {
-    return get(ALLOCATION_FILE);
+  /**
+   * Path to XML file containing allocations. If the
+   * path is relative, it is searched for in the
+   * classpath, but loaded like a regular File.
+   */
+  public File getAllocationFile() {
+    String allocFilePath = get(ALLOCATION_FILE, DEFAULT_ALLOCATION_FILE);
+    File allocFile = new File(allocFilePath);
+    if (!allocFile.isAbsolute()) {
+      URL url = Thread.currentThread().getContextClassLoader()
+          .getResource(allocFilePath);
+      if (url == null) {
+        LOG.warn(allocFilePath + " not found on the classpath.");
+        allocFile = null;
+      } else if (!url.getProtocol().equalsIgnoreCase("file")) {
+        throw new RuntimeException("Allocation file " + url
+            + " found on the classpath is not on the local filesystem.");
+      } else {
+        allocFile = new File(url.getPath());
+      }
+    }
+    return allocFile;
   }
 
   public String getEventlogDir() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java

@@ -75,11 +75,8 @@ public class QueueManager {
 
   private final FairScheduler scheduler;
 
-  private Object allocFile; // Path to XML file containing allocations. This
-                            // is either a URL to specify a classpath resource
-                            // (if the fair-scheduler.xml on the classpath is
-                            // used) or a String to specify an absolute path (if
-                            // mapred.fairscheduler.allocation.file is used).
+  // Path to XML file containing allocations. 
+  private File allocFile; 
 
   private final Collection<FSLeafQueue> leafQueues = 
       new CopyOnWriteArrayList<FSLeafQueue>();
@@ -107,16 +104,7 @@ public class QueueManager {
     queues.put(rootQueue.getName(), rootQueue);
     
     this.allocFile = conf.getAllocationFile();
-    if (allocFile == null) {
-      // No allocation file specified in jobconf. Use the default allocation
-      // file, fair-scheduler.xml, looking for it on the classpath.
-      allocFile = new Configuration().getResource("fair-scheduler.xml");
-      if (allocFile == null) {
-        LOG.error("The fair scheduler allocation file fair-scheduler.xml was "
-            + "not found on the classpath, and no other config file is given "
-            + "through mapred.fairscheduler.allocation.file.");
-      }
-    }
+    
     reloadAllocs();
     lastSuccessfulReload = scheduler.getClock().getTime();
     lastReloadAttempt = scheduler.getClock().getTime();
@@ -255,14 +243,7 @@ public class QueueManager {
       try {
         // Get last modified time of alloc file depending whether it's a String
         // (for a path name) or an URL (for a classloader resource)
-        long lastModified;
-        if (allocFile instanceof String) {
-          File file = new File((String) allocFile);
-          lastModified = file.lastModified();
-        } else { // allocFile is an URL
-          URLConnection conn = ((URL) allocFile).openConnection();
-          lastModified = conn.getLastModified();
-        }
+        long lastModified = allocFile.lastModified();
         if (lastModified > lastSuccessfulReload &&
             time > lastModified + ALLOC_RELOAD_WAIT) {
           reloadAllocs();
@@ -321,12 +302,7 @@ public class QueueManager {
       DocumentBuilderFactory.newInstance();
     docBuilderFactory.setIgnoringComments(true);
     DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
-    Document doc;
-    if (allocFile instanceof String) {
-      doc = builder.parse(new File((String) allocFile));
-    } else {
-      doc = builder.parse(allocFile.toString());
-    }
+    Document doc = builder.parse(allocFile);
     Element root = doc.getDocumentElement();
     if (!"allocations".equals(root.getTagName()))
       throw new AllocationConfigurationException("Bad fair scheduler config " +

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java

@@ -20,6 +20,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.parseResourceConfigValue;
 import static org.junit.Assert.assertEquals;
 
+import java.io.File;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.junit.Test;
 
@@ -55,4 +60,15 @@ public class TestFairSchedulerConfiguration {
   public void testGibberish() throws Exception {
     parseResourceConfigValue("1o24vc0res");
   }
+  
+  @Test
+  public void testGetAllocationFileFromClasspath() {
+    FairSchedulerConfiguration conf = new FairSchedulerConfiguration(
+        new Configuration());
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+        "test-fair-scheduler.xml");
+    File allocationFile = conf.getAllocationFile();
+    Assert.assertEquals("test-fair-scheduler.xml", allocationFile.getName());
+    Assert.assertTrue(allocationFile.exists());
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/test-fair-scheduler.xml

@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<?xml version="1.0"?>
+<allocations>
+  <queue name="queue1"></queue>
+</allocations>
+

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm

@@ -81,7 +81,7 @@ Hadoop MapReduce Next Generation - Fair Scheduler
   distribute the resources assigned to them to their children in the same
   fashion.  Applications may only be scheduled on leaf queues. Queues can be
   specified as children of other queues by placing them as sub-elements of 
-  their parents in the fair scheduler configuration file.
+  their parents in the fair scheduler allocation file.
   
   A queue's name starts with the names of its parents, with periods as
   separators. So a queue named "queue1" under the root queue, would be referred
@@ -118,16 +118,20 @@ Hadoop MapReduce Next Generation - Fair Scheduler
   Customizing the Fair Scheduler typically involves altering two files. First, 
   scheduler-wide options can be set by adding configuration properties in the 
   yarn-site.xml file in your existing configuration directory. Second, in 
-  most cases users will want to create a manifest file listing which queues 
-  exist and their respective weights and capacities. The location of this file 
-  is flexible - but it must be declared in yarn-site.xml.
+  most cases users will want to create an allocation file listing which queues 
+  exist and their respective weights and capacities. The allocation file
+  is reloaded every 10 seconds, allowing changes to be made on the fly.
+
+Properties that can be placed in yarn-site.xml
 
  * <<<yarn.scheduler.fair.allocation.file>>>
 
    * Path to allocation file. An allocation file is an XML manifest describing
      queues and their properties, in addition to certain policy defaults. This file
-     must be in XML format as described in the next section.
-     Defaults to fair-scheduler.xml in configuration directory.
+     must be in the XML format described in the next section. If a relative path is
+     given, the file is searched for on the classpath (which typically includes
+     the Hadoop conf directory).
+     Defaults to fair-scheduler.xml.
 
  * <<<yarn.scheduler.fair.user-as-default-queue>>>