Merge -r 1304596:1304597 from trunk to branch-0.23. Fixes: HADOOP-7030

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1304599 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -22,6 +22,9 @@ Release 0.23.3 - UNRELEASED
     HADOOP-8121. Active Directory Group Mapping Service. (Jonathan Natkins via
     atm)
 
+    HADOOP-7030. Add TableMapping topology implementation to read host to rack
+    mapping from a file. (Patrick Angeles and tomwhite via tomwhite)
+
   IMPROVEMENTS
 
     HADOOP-7524. Change RPC to allow multiple protocols including multuple

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java

@@ -63,6 +63,10 @@ public class CommonConfigurationKeysPublic {
   /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
   public static final String  NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY =
     "net.topology.node.switch.mapping.impl";
+  
+  /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+  public static final String  NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY =
+    "net.topology.table.file.name";
 
   /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
   public static final String  FS_TRASH_CHECKPOINT_INTERVAL_KEY =

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java

@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.net;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+
+/**
+ * <p>
+ * Simple {@link DNSToSwitchMapping} implementation that reads a 2 column text
+ * file. The columns are separated by whitespace. The first column is a DNS or
+ * IP address and the second column specifies the rack where the address maps.
+ * </p>
+ * <p>
+ * This class uses the configuration parameter {@code
+ * net.topology.table.file.name} to locate the mapping file.
+ * </p>
+ * <p>
+ * Calls to {@link #resolve(List)} will look up the address as defined in the
+ * mapping file. If no entry corresponding to the address is found, the value
+ * {@code /default-rack} is returned.
+ * </p>
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class TableMapping extends CachedDNSToSwitchMapping {
+
+  private static final Log LOG = LogFactory.getLog(TableMapping.class);
+  
+  public TableMapping() {
+    super(new RawTableMapping());
+  }
+  
+  private RawTableMapping getRawMapping() {
+    return (RawTableMapping) rawMapping;
+  }
+
+  @Override
+  public Configuration getConf() {
+    return getRawMapping().getConf();
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    super.setConf(conf);
+    getRawMapping().setConf(conf);
+  }
+  
+  private static final class RawTableMapping extends Configured
+      implements DNSToSwitchMapping {
+    
+    private final Map<String, String> map = new HashMap<String, String>();
+    private boolean initialized = false;
+  
+    private synchronized void load() {
+      map.clear();
+  
+      String filename = getConf().get(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, null);
+      if (StringUtils.isBlank(filename)) {
+        LOG.warn(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY + " not configured. "
+            + NetworkTopology.DEFAULT_RACK + " will be returned.");
+        return;
+      }
+  
+      BufferedReader reader = null;
+      try {
+        reader = new BufferedReader(new FileReader(filename));
+        String line = reader.readLine();
+        while (line != null) {
+          line = line.trim();
+          if (line.length() != 0 && line.charAt(0) != '#') {
+            String[] columns = line.split("\\s+");
+            if (columns.length == 2) {
+              map.put(columns[0], columns[1]);
+            } else {
+              LOG.warn("Line does not have two columns. Ignoring. " + line);
+            }
+          }
+          line = reader.readLine();
+        }
+      } catch (Exception e) {
+        LOG.warn(filename + " cannot be read. " + NetworkTopology.DEFAULT_RACK
+            + " will be returned.", e);
+        map.clear();
+      } finally {
+        if (reader != null) {
+          try {
+            reader.close();
+          } catch (IOException e) {
+            LOG.warn(filename + " cannot be read. "
+                + NetworkTopology.DEFAULT_RACK + " will be returned.", e);
+            map.clear();
+          }
+        }
+      }
+    }
+  
+    public synchronized List<String> resolve(List<String> names) {
+      if (!initialized) {
+        initialized = true;
+        load();
+      }
+  
+      List<String> results = new ArrayList<String>(names.size());
+      for (String name : names) {
+        String result = map.get(name);
+        if (result != null) {
+          results.add(result);
+        } else {
+          results.add(NetworkTopology.DEFAULT_RACK);
+        }
+      }
+      return results;
+    }
+    
+  }
+}

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -688,6 +688,19 @@
   </description>
 </property>
 
+<property>
+  <name>net.topology.table.file.name</name>
+  <value></value>
+  <description> The file name for a topology file, which is used when the
+    net.topology.script.file.name property is set to
+    org.apache.hadoop.net.TableMapping. The file format is a two column text
+    file, with columns separated by whitespace. The first column is a DNS or
+    IP address and the second column specifies the rack where the address maps.
+    If no entry corresponding to a host in the cluster is found, then 
+    /default-rack is assumed.
+  </description>
+</property>
+
 <!-- Local file system -->
 <property>
   <name>file.stream-buffer-size</name>

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java

@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.net;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestTableMapping {
+  
+  private File mappingFile;
+  
+  @Before
+  public void setUp() throws IOException {
+    mappingFile = File.createTempFile(getClass().getSimpleName(), ".txt");
+    Files.write("a.b.c /rack1\n" +
+                "1.2.3\t/rack2\n", mappingFile, Charsets.UTF_8);
+    mappingFile.deleteOnExit();
+  }
+
+  @Test
+  public void testResolve() throws IOException {
+    TableMapping mapping = new TableMapping();
+
+    Configuration conf = new Configuration();
+    conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, mappingFile.getCanonicalPath());
+    mapping.setConf(conf);
+
+    List<String> names = new ArrayList<String>();
+    names.add("a.b.c");
+    names.add("1.2.3");
+
+    List<String> result = mapping.resolve(names);
+    assertEquals(names.size(), result.size());
+    assertEquals("/rack1", result.get(0));
+    assertEquals("/rack2", result.get(1));
+  }
+
+  @Test
+  public void testTableCaching() throws IOException {
+    TableMapping mapping = new TableMapping();
+
+    Configuration conf = new Configuration();
+    conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, mappingFile.getCanonicalPath());
+    mapping.setConf(conf);
+
+    List<String> names = new ArrayList<String>();
+    names.add("a.b.c");
+    names.add("1.2.3");
+
+    List<String> result1 = mapping.resolve(names);
+    assertEquals(names.size(), result1.size());
+    assertEquals("/rack1", result1.get(0));
+    assertEquals("/rack2", result1.get(1));
+
+    // unset the file, see if it gets read again
+    conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, "some bad value for a file");
+
+    List<String> result2 = mapping.resolve(names);
+    assertEquals(result1, result2);
+  }
+
+  @Test
+  public void testNoFile() {
+    TableMapping mapping = new TableMapping();
+
+    Configuration conf = new Configuration();
+    mapping.setConf(conf);
+
+    List<String> names = new ArrayList<String>();
+    names.add("a.b.c");
+    names.add("1.2.3");
+
+    List<String> result = mapping.resolve(names);
+    assertEquals(names.size(), result.size());
+    assertEquals(NetworkTopology.DEFAULT_RACK, result.get(0));
+    assertEquals(NetworkTopology.DEFAULT_RACK, result.get(1));
+  }
+
+  @Test
+  public void testFileDoesNotExist() {
+    TableMapping mapping = new TableMapping();
+
+    Configuration conf = new Configuration();
+    conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, "/this/file/does/not/exist");
+    mapping.setConf(conf);
+
+    List<String> names = new ArrayList<String>();
+    names.add("a.b.c");
+    names.add("1.2.3");
+
+    List<String> result = mapping.resolve(names);
+    assertEquals(names.size(), result.size());
+    assertEquals(result.get(0), NetworkTopology.DEFAULT_RACK);
+    assertEquals(result.get(1), NetworkTopology.DEFAULT_RACK);
+  }
+
+  @Test
+  public void testBadFile() throws IOException {
+    Files.write("bad contents", mappingFile, Charsets.UTF_8);
+    
+    TableMapping mapping = new TableMapping();
+
+    Configuration conf = new Configuration();
+    conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, mappingFile.getCanonicalPath());
+    mapping.setConf(conf);
+
+    List<String> names = new ArrayList<String>();
+    names.add("a.b.c");
+    names.add("1.2.3");
+
+    List<String> result = mapping.resolve(names);
+    assertEquals(names.size(), result.size());
+    assertEquals(result.get(0), NetworkTopology.DEFAULT_RACK);
+    assertEquals(result.get(1), NetworkTopology.DEFAULT_RACK);
+  }
+
+}