HADOOP-1589 Exception handling in HBase is broken over client server

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@555813 13f79535-47bb-0310-9956-ffa450edef68

src/contrib/hbase/CHANGES.txt

@@ -58,3 +58,4 @@ Trunk (unreleased changes)
  34. HADOOP-1589 Exception handling in HBase is broken over client server connections
  35. HADOOP-1375 a simple parser for hbase (Edward Yoon via Stack)
  36. HADOOP-1600 Update license in HBase code
+ 37. HADOOP-1589 Exception handling in HBase is broken over client server

src/contrib/hbase/src/java/org/apache/hadoop/hbase/HClient.java

@@ -19,19 +19,15 @@
  */
 package org.apache.hadoop.hbase;
 
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Random;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.TreeSet;
-import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -40,9 +36,6 @@ import org.apache.hadoop.hbase.filter.RowFilterInterface;
 import org.apache.hadoop.hbase.io.KeyedData;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.io.retry.RetryProxy;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RemoteException;
 
@@ -265,8 +258,8 @@ public class HClient implements HConstants {
     checkMaster();
     try {
       this.master.createTable(desc);
-    } catch (Exception e) {
-      RemoteExceptionHandler.handleRemoteException(e);
+    } catch (RemoteException e) {
+      throw RemoteExceptionHandler.decodeRemoteException(e);
     }
   }
 
@@ -283,8 +276,8 @@ public class HClient implements HConstants {
 
     try {
       this.master.deleteTable(tableName);
-    } catch(Exception e) {
-      RemoteExceptionHandler.handleRemoteException(e);
+    } catch(RemoteException e) {
+      throw RemoteExceptionHandler.decodeRemoteException(e);
     }
 
     // Wait until first region is deleted
@@ -315,9 +308,12 @@ public class HClient implements HConstants {
           break;
         }
 
-      } catch (Exception ex) {
+      } catch (IOException ex) {
         if(tries == numRetries - 1) {           // no more tries left
-          RemoteExceptionHandler.handleRemoteException(ex);
+          if(ex instanceof RemoteException) {
+            ex = RemoteExceptionHandler.decodeRemoteException((RemoteException) ex);
+          }
+          throw ex;
         }
 
       } finally {
@@ -353,8 +349,8 @@ public class HClient implements HConstants {
     try {
       this.master.addColumn(tableName, column);
       
-    } catch (Exception e) {
-      RemoteExceptionHandler.handleRemoteException(e);
+    } catch (RemoteException e) {
+      throw RemoteExceptionHandler.decodeRemoteException(e);
     }
   }
 
@@ -372,8 +368,8 @@ public class HClient implements HConstants {
     try {
       this.master.deleteColumn(tableName, columnName);
       
-    } catch(Exception e) {
-      RemoteExceptionHandler.handleRemoteException(e);
+    } catch(RemoteException e) {
+      throw RemoteExceptionHandler.decodeRemoteException(e);
     }
   }
   
@@ -391,8 +387,8 @@ public class HClient implements HConstants {
     try {
       this.master.enableTable(tableName);
       
-    } catch(Exception e) {
-      RemoteExceptionHandler.handleRemoteException(e);
+    } catch(RemoteException e) {
+      throw RemoteExceptionHandler.decodeRemoteException(e);
     }
 
     // Wait until first region is enabled
@@ -433,9 +429,12 @@ public class HClient implements HConstants {
           break;
         }
         
-      } catch (Exception e) {
+      } catch (IOException e) {
         if(tries == numRetries - 1) {                   // no more retries
-          RemoteExceptionHandler.handleRemoteException(e);
+          if(e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
+          throw e;
         }
         
       } finally {
@@ -479,8 +478,8 @@ public class HClient implements HConstants {
     try {
       this.master.disableTable(tableName);
       
-    } catch(Exception e) {
-      RemoteExceptionHandler.handleRemoteException(e);
+    } catch(RemoteException e) {
+      throw RemoteExceptionHandler.decodeRemoteException(e);
     }
 
     // Wait until first region is disabled
@@ -521,9 +520,12 @@ public class HClient implements HConstants {
           break;
         }
         
-      } catch(Exception e) {
+      } catch(IOException e) {
         if(tries == numRetries - 1) {                   // no more retries
-          RemoteExceptionHandler.handleRemoteException(e);
+          if(e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
+          throw e;
         }
         
       } finally {
@@ -559,8 +561,8 @@ public class HClient implements HConstants {
     checkMaster();
     try {
       this.master.shutdown();
-    } catch(Exception e) {
-      RemoteExceptionHandler.handleRemoteException(e);
+    } catch(RemoteException e) {
+      throw RemoteExceptionHandler.decodeRemoteException(e);
     }
   }
 
@@ -741,10 +743,13 @@ public class HClient implements HConstants {
       try {
         rootRegion.getRegionInfo(HGlobals.rootRegionInfo.regionName);
         break;
-      } catch(Exception e) {
+      } catch(IOException e) {
         if(tries == numRetries - 1) {
           // Don't bother sleeping. We've run out of retries.
-          RemoteExceptionHandler.handleRemoteException(e);
+          if(e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
+          throw e;
         }
         
         // Sleep and retry finding root region.
@@ -868,9 +873,12 @@ public class HClient implements HConstants {
           servers.put(regionInfo.startKey, 
               new RegionLocation(regionInfo, new HServerAddress(serverAddress)));
         }
-      } catch (Exception e) {
+      } catch (IOException e) {
         if(tries == numRetries - 1) {                   // no retries left
-          RemoteExceptionHandler.handleRemoteException(e);
+          if(e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
+          throw e;
         }
         
       } finally {
@@ -943,8 +951,8 @@ public class HClient implements HConstants {
         server = (HRegionInterface) RPC.waitForProxy(serverInterfaceClass,
             versionId, regionServer.getInetSocketAddress(), this.conf);
         
-      } catch (Exception e) {
-        RemoteExceptionHandler.handleRemoteException(e);
+      } catch (RemoteException e) {
+        throw RemoteExceptionHandler.decodeRemoteException(e);
       }
 
       this.servers.put(regionServer.toString(), server);
@@ -1000,8 +1008,8 @@ public class HClient implements HConstants {
             }
           }
         }
-      } catch (Exception ex) {
-        RemoteExceptionHandler.handleRemoteException(ex);
+      } catch (RemoteException ex) {
+        throw RemoteExceptionHandler.decodeRemoteException(ex);
         
       } finally {
         if(scannerId != -1L) {
@@ -1069,9 +1077,12 @@ public class HClient implements HConstants {
         value = server.get(info.regionInfo.regionName, row, column);
         break;
         
-      } catch (Exception e) {
+      } catch (IOException e) {
         if (tries == numRetries - 1) {
-          RemoteExceptionHandler.handleRemoteException(e);
+          if(e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
+          throw e;
         }
         findRegion(info);
       }
@@ -1104,10 +1115,13 @@ public class HClient implements HConstants {
         values = server.get(info.regionInfo.regionName, row, column, numVersions);
         break;
         
-      } catch(Exception e) {
+      } catch(IOException e) {
         if(tries == numRetries - 1) {
           // No more tries
-          RemoteExceptionHandler.handleRemoteException(e);
+          if(e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
+          throw e;
         }
         findRegion(info);
       }
@@ -1150,10 +1164,13 @@ public class HClient implements HConstants {
         values = server.get(info.regionInfo.regionName, row, column, timestamp, numVersions);
         break;
     
-      } catch(Exception e) {
+      } catch(IOException e) {
         if(tries == numRetries - 1) {
           // No more tries
-          RemoteExceptionHandler.handleRemoteException(e);
+          if(e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
+          throw e;
         }
         findRegion(info);
       }
@@ -1192,10 +1209,13 @@ public class HClient implements HConstants {
         value = server.getRow(info.regionInfo.regionName, row);
         break;
         
-      } catch(NotServingRegionException e) {
+      } catch(IOException e) {
         if(tries == numRetries - 1) {
           // No more tries
-          RemoteExceptionHandler.handleRemoteException(e);
+          if(e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
+          throw e;
         }
         findRegion(info);
       }
@@ -1279,37 +1299,6 @@ public class HClient implements HConstants {
     return new ClientScanner(columns, startRow, timestamp, filter);
   }
 
-  /*
-   * @return General HClient RetryPolicy instance.
-   */
-  RetryPolicy getRetryPolicy() {
-    Map<Class <? extends Exception>, RetryPolicy> exceptionToPolicyMap =
-      new HashMap<Class <? extends Exception>, RetryPolicy>();
-    // Pass numRetries - 1 because it does less-than-equal internally rather
-    // than the less-than we do elsewhere where we use numRetries.
-    RetryPolicy rp =
-      RetryPolicies.retryUpToMaximumCountWithProportionalSleep(numRetries,
-        this.pause, TimeUnit.MILLISECONDS);
-    exceptionToPolicyMap.put(NotServingRegionException.class, rp);
-    exceptionToPolicyMap.put(WrongRegionException.class, rp);
-    exceptionToPolicyMap.put(RegionNotFoundException.class, rp);
-    return RetryPolicies.retryByRemoteException(RetryPolicies.TRY_ONCE_THEN_FAIL,
-      exceptionToPolicyMap);
-    
-  }
-  
-  /*
-   * Interface for {@link #startUpate()} used by the
-   * {@link org.apache.hadoop.io.retry} mechanism. 
-   */
-  private interface StartUpdateInterface {
-    /**
-     * @return row lockid for the update
-     * @throws IOException
-     */
-    long startUpdate() throws IOException;
-  }
-
   /** 
    * Start an atomic row insertion/update.  No changes are committed until the 
    * call to commit() returns. A call to abort() will abandon any updates in progress.
@@ -1326,44 +1315,40 @@ public class HClient implements HConstants {
    * @throws IOException
    */
   public long startUpdate(final Text row) throws IOException {
-    // Implemention of the StartUpdate interface.
-    StartUpdateInterface implementation = new StartUpdateInterface() {
-      private RegionLocation info = null;
-      private int attempts = 0;
-      
-      /*
-       * Wrapped method.  Proxy wrapper is configured to judge whether
-       * exception merits retry.
-       * @return lockid
-       * @throws IOException
-       */
-      public long startUpdate() throws IOException {
-        this.attempts++;
-        if (this.info != null) {
-          LOG.info("Retry of startUpdate.  Attempt " + this.attempts +
-            " for row " + row);
-          // If a retry. Something wrong w/ region we have. Refind.
-          try {
-            findRegion(info);
-          } catch (RegionNotFoundException e) {
-            // continue.  If no longer exists, perhaps we just came through
-            // a split and region is now gone. Below getRegionLocation should
-            // recalibrate client.
-          }
-        }
-        this.info = getRegionLocation(row);
+    long lockid = -1;
+    for(int tries = 0; tries < numRetries; tries++) {
+      IOException e = null;
+      RegionLocation info = getRegionLocation(row);
+      try {
         currentServer = getHRegionConnection(info.serverAddress);
         currentRegion = info.regionInfo.regionName;
         clientid = rand.nextLong();
-        return currentServer.startUpdate(currentRegion, clientid, row);
+        lockid = currentServer.startUpdate(currentRegion, clientid, row);
+        break;
+        
+      } catch (IOException ex) {
+        e = ex;
       }
-    };
-    
-    // Get retry proxy wrapper around 'implementation'.
-    StartUpdateInterface retryProxy = (StartUpdateInterface)RetryProxy.
-      create(StartUpdateInterface.class, implementation, getRetryPolicy());
-    // Run retry.
-    return retryProxy.startUpdate();
+      if(tries < numRetries - 1) {
+        try {
+          Thread.sleep(this.pause);
+          
+        } catch (InterruptedException ex) {
+        }
+        try {
+          findRegion(info);
+          
+        } catch (IOException ex) {
+          e = ex;
+        }
+      } else {
+        if(e instanceof RemoteException) {
+          e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+        }
+        throw e;
+      }
+    }
+    return lockid;
   }
   
   /** 
@@ -1378,7 +1363,7 @@ public class HClient implements HConstants {
     try {
       this.currentServer.put(this.currentRegion, this.clientid, lockid, column,
         val);
-    } catch(Exception e) {
+    } catch(IOException e) {
       try {
         this.currentServer.abort(this.currentRegion, this.clientid, lockid);
       } catch(IOException e2) {
@@ -1386,7 +1371,10 @@ public class HClient implements HConstants {
       }
       this.currentServer = null;
       this.currentRegion = null;
-      RemoteExceptionHandler.handleRemoteException(e);
+      if(e instanceof RemoteException) {
+        e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+      }
+      throw e;
     }
   }
   
@@ -1401,7 +1389,7 @@ public class HClient implements HConstants {
     try {
       this.currentServer.delete(this.currentRegion, this.clientid, lockid,
         column);
-    } catch(Exception e) {
+    } catch(IOException e) {
       try {
         this.currentServer.abort(this.currentRegion, this.clientid, lockid);
       } catch(IOException e2) {
@@ -1409,7 +1397,10 @@ public class HClient implements HConstants {
       }
       this.currentServer = null;
       this.currentRegion = null;
-      RemoteExceptionHandler.handleRemoteException(e);
+      if(e instanceof RemoteException) {
+        e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+      }
+      throw e;
     }
   }
   
@@ -1422,10 +1413,13 @@ public class HClient implements HConstants {
   public void abort(long lockid) throws IOException {
     try {
       this.currentServer.abort(this.currentRegion, this.clientid, lockid);
-    } catch(Exception e) {
+    } catch(IOException e) {
       this.currentServer = null;
       this.currentRegion = null;
-      RemoteExceptionHandler.handleRemoteException(e);
+      if(e instanceof RemoteException) {
+        e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+      }
+      throw e;
     }
   }
   
@@ -1451,10 +1445,13 @@ public class HClient implements HConstants {
       this.currentServer.commit(this.currentRegion, this.clientid, lockid,
           timestamp);
       
-    } catch (Exception e) {
+    } catch (IOException e) {
       this.currentServer = null;
       this.currentRegion = null;
-      RemoteExceptionHandler.handleRemoteException(e);
+      if(e instanceof RemoteException) {
+        e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+      }
+      throw e;
     }
   }
   
@@ -1467,7 +1464,7 @@ public class HClient implements HConstants {
   public void renewLease(long lockid) throws IOException {
     try {
       this.currentServer.renewLease(lockid, this.clientid);
-    } catch(Exception e) {
+    } catch(IOException e) {
       try {
         this.currentServer.abort(this.currentRegion, this.clientid, lockid);
       } catch(IOException e2) {
@@ -1475,7 +1472,10 @@ public class HClient implements HConstants {
       }
       this.currentServer = null;
       this.currentRegion = null;
-      RemoteExceptionHandler.handleRemoteException(e);
+      if(e instanceof RemoteException) {
+        e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+      }
+      throw e;
     }
   }
 
@@ -1563,19 +1563,25 @@ public class HClient implements HConstants {
 
             break;
         
-          } catch(Exception e) {
+          } catch(IOException e) {
             if(tries == numRetries - 1) {
               // No more tries
-              RemoteExceptionHandler.handleRemoteException(e);
+              if(e instanceof RemoteException) {
+                e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+              }
+              throw e;
             }
             findRegion(info);
             loadRegions();
           }
         }
 
-      } catch(Exception e) {
+      } catch(IOException e) {
         close();
-        RemoteExceptionHandler.handleRemoteException(e);
+        if(e instanceof RemoteException) {
+          e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+        }
+        throw e;
       }
       return true;
     }

src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java

@@ -44,6 +44,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.io.KeyedData;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.util.StringUtils;
@@ -215,6 +216,9 @@ public class HMaster implements HConstants, HMasterInterface,
             }
           }
         } catch (IOException e) {
+          if (e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
           LOG.error(e);
         }
       }
@@ -313,7 +317,15 @@ public class HMaster implements HConstants, HMasterInterface,
           }
           tries = 0;
 
-        } catch(IOException e) {
+        } catch (IOException e) {
+          if (e instanceof RemoteException) {
+            try {
+              e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+              
+            } catch (IOException ex) {
+              LOG.warn(ex);
+            }
+          }
           tries++;
           if(tries < numRetries) {
             LOG.warn("ROOT scanner", e);
@@ -465,6 +477,9 @@ public class HMaster implements HConstants, HMasterInterface,
               tries = 0;
               
             } catch (IOException e) {
+              if (e instanceof RemoteException) {
+                e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+              }
               tries++;
               if(tries < numRetries) {
                 LOG.warn("META scanner", e);
@@ -475,7 +490,15 @@ public class HMaster implements HConstants, HMasterInterface,
             }
           } while(true);
 
-        } catch(IOException e) {
+        } catch (IOException e) {
+          if (e instanceof RemoteException) {
+            try {
+              e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+              
+            } catch (IOException ex) {
+              LOG.warn(ex);
+            }
+          }
           LOG.error("META scanner", e);
           closed = true;
         }
@@ -601,7 +624,10 @@ public class HMaster implements HConstants, HMasterInterface,
         root.getLog().closeAndDelete();
         meta.close();
         meta.getLog().closeAndDelete();
-      } catch(IOException e) {
+      } catch (IOException e) {
+        if (e instanceof RemoteException) {
+          e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+        }
         LOG.error(e);
       }
     }
@@ -695,7 +721,15 @@ public class HMaster implements HConstants, HMasterInterface,
       // Start the server last so everything else is running before we start
       // receiving requests
       this.server.start();
-    } catch(IOException e) {
+    } catch (IOException e) {
+      if (e instanceof RemoteException) {
+        try {
+          e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          
+        } catch (IOException ex) {
+          LOG.warn(ex);
+        }
+      }
       // Something happened during startup. Shut things down.
       this.closed = true;
       LOG.error(e);
@@ -721,7 +755,16 @@ public class HMaster implements HConstants, HMasterInterface,
           LOG.debug("Processing " + op.toString());
         }
         op.process();
-      } catch(Exception ex) {
+        
+      } catch (Exception ex) {
+        if (ex instanceof RemoteException) {
+          try {
+            ex = RemoteExceptionHandler.decodeRemoteException((RemoteException) ex);
+            
+          } catch (IOException e) {
+            LOG.warn(e);
+          }
+        }
         LOG.warn(ex);
         synchronized(msgQueue) {
           msgQueue.addLast(op);
@@ -750,7 +793,7 @@ public class HMaster implements HConstants, HMasterInterface,
     try {
       // Wait for the root scanner to finish.
       rootScannerThread.join();
-    } catch(Exception iex) {
+    } catch (Exception iex) {
       // Print if ever there is an interrupt (Just for kicks. Remove if it
       // ever happens).
       LOG.warn(iex);
@@ -1209,13 +1252,12 @@ public class HMaster implements HConstants, HMasterInterface,
           try {
             values = server.next(scannerId);
             
-          } catch(Exception e) {
-            try {
-              RemoteExceptionHandler.handleRemoteException(e);
+          } catch (IOException e) {
+            if (e instanceof RemoteException) {
+              e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
               
-            } catch(Exception ex) {
-              LOG.error(ex);
             }
+            LOG.error(e);
             break;
           }
           
@@ -1287,7 +1329,10 @@ public class HMaster implements HConstants, HMasterInterface,
           try {
             info.readFields(inbuf);
             
-          } catch(IOException e) {
+          } catch (IOException e) {
+            if (e instanceof RemoteException) {
+              e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+            }
             LOG.error(e);
             break;
           }
@@ -1335,9 +1380,11 @@ public class HMaster implements HConstants, HMasterInterface,
           try {
             server.close(scannerId);
             
-          } catch(IOException e) {
+          } catch (IOException e) {
+            if (e instanceof RemoteException) {
+              e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+            }
             LOG.error(e);
-            
           }
         }
       }
@@ -1411,9 +1458,12 @@ public class HMaster implements HConstants, HMasterInterface,
           scanMetaRegion(server, scannerId, HGlobals.rootRegionInfo.regionName);
           break;
           
-        } catch(Exception e) {
-          if(tries == numRetries - 1) {
-            RemoteExceptionHandler.handleRemoteException(e);
+        } catch (IOException e) {
+          if (tries == numRetries - 1) {
+            if (e instanceof RemoteException) {
+              e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+            }
+            throw e;
           }
         }
       }
@@ -1443,9 +1493,12 @@ public class HMaster implements HConstants, HMasterInterface,
           }
           break;
             
-        } catch(Exception e) {
-          if(tries == numRetries - 1) {
-            RemoteExceptionHandler.handleRemoteException(e);
+        } catch (IOException e) {
+          if (tries == numRetries - 1) {
+            if (e instanceof RemoteException) {
+              e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+            }
+            throw e;
           }
         }
       }
@@ -1542,9 +1595,12 @@ public class HMaster implements HConstants, HMasterInterface,
           
           break;
 
-        } catch(Exception e) {
-          if(tries == numRetries - 1) {
-            RemoteExceptionHandler.handleRemoteException(e);
+        } catch (IOException e) {
+          if (tries == numRetries - 1) {
+            if (e instanceof RemoteException) {
+              e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+            }
+            throw e;
           }
           continue;
         }
@@ -1562,7 +1618,10 @@ public class HMaster implements HConstants, HMasterInterface,
         try {
           HRegion.deleteRegion(fs, dir, regionInfo.regionName);
 
-        } catch(IOException e) {
+        } catch (IOException e) {
+          if (e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
           LOG.error("failed to delete region " + regionInfo.regionName);
           LOG.error(e);
           throw e;
@@ -1653,9 +1712,12 @@ public class HMaster implements HConstants, HMasterInterface,
           
           break;
           
-        } catch(Exception e) {
+        } catch (IOException e) {
           if(tries == numRetries - 1) {
-            RemoteExceptionHandler.handleRemoteException(e);
+            if (e instanceof RemoteException) {
+              e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+            }
+            throw e;
           }
         }
         pendingRegions.remove(regionName);
@@ -1758,9 +1820,12 @@ public class HMaster implements HConstants, HMasterInterface,
         assignAttempts.put(regionName, Long.valueOf(0L));
         break;
 
-      } catch(Exception e) {
+      } catch (IOException e) {
         if(tries == numRetries - 1) {
-          RemoteExceptionHandler.handleRemoteException(e);
+          if (e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
+          throw e;
         }
       }
     }
@@ -1930,7 +1995,10 @@ public class HMaster implements HConstants, HMasterInterface,
                   try {
                     server.close(scannerId);
 
-                  } catch(IOException e) {
+                  } catch (IOException e) {
+                    if (e instanceof RemoteException) {
+                      e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+                    }
                     LOG.error(e);
                   }
                 }
@@ -1947,9 +2015,12 @@ public class HMaster implements HConstants, HMasterInterface,
             } // for(MetaRegion m:)
           } // synchronized(metaScannerLock)
           
-        } catch(Exception e) {
+        } catch (IOException e) {
           if(tries == numRetries - 1) {
-            RemoteExceptionHandler.handleRemoteException(e);
+            if (e instanceof RemoteException) {
+              e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+            }
+            throw e;
           }
           continue;
         }
@@ -2034,10 +2105,12 @@ public class HMaster implements HConstants, HMasterInterface,
             LOG.debug("updated columns in row: " + i.regionName);
           }
 
-        } catch(Exception e) {
+        } catch (IOException e) {
+          if (e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
           LOG.error("column update failed in row: " + i.regionName);
           LOG.error(e);
-          RemoteExceptionHandler.handleRemoteException(e);
 
         } finally {
           try {
@@ -2045,7 +2118,10 @@ public class HMaster implements HConstants, HMasterInterface,
               server.abort(m.regionName, clientId, lockid);
             }
 
-          } catch(IOException iex) {
+          } catch (IOException iex) {
+            if (iex instanceof RemoteException) {
+              iex = RemoteExceptionHandler.decodeRemoteException((RemoteException) iex);
+            }
             LOG.error(iex);
           }
         }
@@ -2136,7 +2212,10 @@ public class HMaster implements HConstants, HMasterInterface,
         // Delete the region
         try {
           HRegion.deleteRegion(fs, dir, i.regionName);
-        } catch(IOException e) {
+        } catch (IOException e) {
+          if (e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
           LOG.error("failed to delete region " + i.regionName);
           LOG.error(e);
         }
@@ -2186,17 +2265,22 @@ public class HMaster implements HConstants, HMasterInterface,
         if(LOG.isDebugEnabled()) {
           LOG.debug("updated columns in row: " + i.regionName);
         }
-      } catch(Exception e) {
+      } catch (Exception e) {
+        if (e instanceof RemoteException) {
+          e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+        }
         LOG.error("column update failed in row: " + i.regionName);
         LOG.error(e);
-        RemoteExceptionHandler.handleRemoteException(e);
 
       } finally {
         if(lockid != -1L) {
           try {
             server.abort(regionName, clientId, lockid);
             
-          } catch(IOException iex) {
+          } catch (IOException iex) {
+            if (iex instanceof RemoteException) {
+              iex = RemoteExceptionHandler.decodeRemoteException((RemoteException) iex);
+            }
             LOG.error(iex);
           }
         }
@@ -2226,14 +2310,20 @@ public class HMaster implements HConstants, HMasterInterface,
         try {
           fs.delete(HStoreFile.getMapDir(dir, i.regionName, columnName));
           
-        } catch(IOException e) {
+        } catch (IOException e) {
+          if (e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
           LOG.error(e);
         }
         
         try {
           fs.delete(HStoreFile.getInfoDir(dir, i.regionName, columnName));
           
-        } catch(IOException e) {
+        } catch (IOException e) {
+          if (e instanceof RemoteException) {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          }
           LOG.error(e);
         }
         

src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java

@@ -40,7 +40,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.filter.RowFilterInterface;
 import org.apache.hadoop.hbase.io.KeyedData;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.retry.RetryProxy;
+import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.net.DNS;
@@ -96,17 +96,6 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
   private final Thread splitOrCompactCheckerThread;
   protected final Integer splitOrCompactLock = new Integer(0);
   
-  /**
-   * Interface used by the {@link org.apache.hadoop.io.retry} mechanism.
-   */
-  public interface UpdateMetaInterface {
-    /**
-     * @return True if succeeded.
-     * @throws IOException
-     */
-   public boolean update() throws IOException;
-  }
-
   /** Runs periodically to determine if regions need to be compacted or split */
   class SplitOrCompactChecker implements Runnable, RegionUnavailableListener {
     HClient client = new HClient(conf);
@@ -207,55 +196,71 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
       // When a region is split, the META table needs to updated if we're
       // splitting a 'normal' region, and the ROOT table needs to be
       // updated if we are splitting a META region.
+      
       final Text tableToUpdate =
         region.getRegionInfo().tableDesc.getName().equals(META_TABLE_NAME) ?
             ROOT_TABLE_NAME : META_TABLE_NAME;
       if(LOG.isDebugEnabled()) {
         LOG.debug("Updating " + tableToUpdate + " with region split info");
       }
+
+      // Remove old region from META
       
-      // Wrap the update of META region with an org.apache.hadoop.io.retry.
-      UpdateMetaInterface implementation = new UpdateMetaInterface() {
-        /* (non-Javadoc)
-         * @see org.apache.hadoop.hbase.HRegionServer.UpdateMetaInterface#update()
-         */
-        public boolean update() throws IOException {
+      for (int tries = 0; tries < numRetries; tries++) {
+        try {
           HRegion.removeRegionFromMETA(client, tableToUpdate,
-            region.getRegionName());
-          for (int i = 0; i < newRegions.length; i++) {
-            HRegion.addRegionToMETA(client, tableToUpdate, newRegions[i],
-              serverInfo.getServerAddress(), serverInfo.getStartCode());
-          }
+              region.getRegionName());
           
-          // Now tell the master about the new regions
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Reporting region split to master");
+        } catch (IOException e) {
+          if(tries == numRetries - 1) {
+            if(e instanceof RemoteException) {
+              e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+            }
+            throw e;
           }
-          reportSplit(newRegions[0].getRegionInfo(), newRegions[1].
-            getRegionInfo());
-          LOG.info("region split, META update, and report to master all" +
-            " successful. Old region=" + oldRegion + ", new regions: " +
-            newRegions[0].getRegionName() + ", " +
-            newRegions[1].getRegionName());
-
-          // Finally, start serving the new regions
-          lock.writeLock().lock();
+        }
+      }
+      
+      // Add new regions to META
+      
+      for (int i = 0; i < newRegions.length; i++) {
+        for (int tries = 0; tries < numRetries; tries ++) {
           try {
-            onlineRegions.put(newRegions[0].getRegionName(), newRegions[0]);
-            onlineRegions.put(newRegions[1].getRegionName(), newRegions[1]);
-          } finally {
-            lock.writeLock().unlock();
+            HRegion.addRegionToMETA(client, tableToUpdate, newRegions[i],
+                serverInfo.getServerAddress(), serverInfo.getStartCode());
+
+          } catch(IOException e) {
+            if(tries == numRetries - 1) {
+              if(e instanceof RemoteException) {
+                e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+              }
+              throw e;
+            }
           }
-          return true;
         }
-      };
+      }
+          
+      // Now tell the master about the new regions
       
-      // Get retry proxy wrapper around 'implementation'.
-      UpdateMetaInterface retryProxy = (UpdateMetaInterface)RetryProxy.
-        create(UpdateMetaInterface.class, implementation,
-        client.getRetryPolicy());
-      // Run retry.
-      retryProxy.update();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Reporting region split to master");
+      }
+      reportSplit(newRegions[0].getRegionInfo(), newRegions[1].
+          getRegionInfo());
+      LOG.info("region split, META update, and report to master all" +
+          " successful. Old region=" + oldRegion + ", new regions: " +
+          newRegions[0].getRegionName() + ", " +
+          newRegions[1].getRegionName());
+
+      // Finally, start serving the new regions
+      
+      lock.writeLock().lock();
+      try {
+        onlineRegions.put(newRegions[0].getRegionName(), newRegions[0]);
+        onlineRegions.put(newRegions[1].getRegionName(), newRegions[1]);
+      } finally {
+        lock.writeLock().unlock();
+      }
     }
   }
 
@@ -293,7 +298,15 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
 
             try {
               cur.optionallyFlush();
-            } catch(IOException iex) {
+            } catch (IOException iex) {
+              if (iex instanceof RemoteException) {
+                try {
+                  iex = RemoteExceptionHandler.decodeRemoteException((RemoteException) iex);
+                  
+                } catch (IOException x) {
+                  iex = x;
+                }
+              }
               LOG.error(iex);
             }
           }
@@ -346,8 +359,16 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
                 LOG.debug("Rolling log. Number of entries is: " + nEntries);
               }
               log.rollWriter();
-            } catch(IOException iex) {
-              // continue
+            } catch (IOException iex) {
+              if (iex instanceof RemoteException) {
+                try {
+                  iex = RemoteExceptionHandler.decodeRemoteException((RemoteException) iex);
+                  
+                } catch (IOException x) {
+                  iex = x;
+                }
+              }
+              LOG.warn(iex);
             }
           }
         }
@@ -470,8 +491,11 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
           HMasterRegionInterface.class, HMasterRegionInterface.versionID,
           new HServerAddress(conf.get(MASTER_ADDRESS)).getInetSocketAddress(),
           conf);
-    } catch(IOException e) {
+    } catch (IOException e) {
       this.stopRequested = true;
+      if (e instanceof RemoteException) {
+        e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+      }
       throw e;
     }
   }
@@ -558,8 +582,16 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
       this.server.start();
       LOG.info("HRegionServer started at: " + serverInfo.getServerAddress().toString());
     } catch(IOException e) {
-      LOG.error(e);
       stopRequested = true;
+      if (e instanceof RemoteException) {
+        try {
+          e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          
+        } catch (IOException ex) {
+          e = ex;
+        }
+      }
+      LOG.error(e);
     }
 
     while(! stopRequested) {
@@ -644,7 +676,15 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
               }
             }
 
-          } catch(IOException e) {
+          } catch (IOException e) {
+            if (e instanceof RemoteException) {
+              try {
+                e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+                
+              } catch (IOException ex) {
+                e = ex;
+              }
+            }
             LOG.error(e);
           }
         }
@@ -683,7 +723,15 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
     if (abortRequested) {
       try {
         log.rollWriter();
-      } catch(IOException e) {
+      } catch (IOException e) {
+        if (e instanceof RemoteException) {
+          try {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+            
+          } catch (IOException ex) {
+            e = ex;
+          }
+        }
         LOG.warn(e);
       }
       LOG.info("aborting server at: " +
@@ -692,7 +740,15 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
       Vector<HRegion> closedRegions = closeAllRegions();
       try {
         log.closeAndDelete();
-      } catch(IOException e) {
+      } catch (IOException e) {
+        if (e instanceof RemoteException) {
+          try {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+            
+          } catch (IOException ex) {
+            e = ex;
+          }
+        }
         LOG.error(e);
       }
       try {
@@ -708,7 +764,15 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
         LOG.info("telling master that region server is shutting down at: "
             + serverInfo.getServerAddress().toString());
         hbaseMaster.regionServerReport(serverInfo, exitMsg);
-      } catch(IOException e) {
+      } catch (IOException e) {
+        if (e instanceof RemoteException) {
+          try {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+            
+          } catch (IOException ex) {
+            e = ex;
+          }
+        }
         LOG.warn(e);
       }
       LOG.info("stopping server at: " +
@@ -822,7 +886,15 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
                 "Impossible state during msg processing.  Instruction: "
                 + e.msg.toString());
           }
-        } catch(IOException ie) {
+        } catch (IOException ie) {
+          if (ie instanceof RemoteException) {
+            try {
+              ie = RemoteExceptionHandler.decodeRemoteException((RemoteException) ie);
+              
+            } catch (IOException x) {
+              ie = x;
+            }
+          }
           if(e.tries < numRetries) {
             LOG.warn(ie);
             e.tries++;
@@ -888,7 +960,15 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
       try {
         region.close();
         LOG.debug("region closed " + region.getRegionName());
-      } catch(IOException e) {
+      } catch (IOException e) {
+        if (e instanceof RemoteException) {
+          try {
+            e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+            
+          } catch (IOException x) {
+            e = x;
+          }
+        }
         LOG.error("error closing region " + region.getRegionName(), e);
       }
     }
@@ -1025,7 +1105,15 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
     public void leaseExpired() {
       try {
         localRegion.abort(localLockId);
-      } catch(IOException iex) {
+      } catch (IOException iex) {
+        if (iex instanceof RemoteException) {
+          try {
+            iex = RemoteExceptionHandler.decodeRemoteException((RemoteException) iex);
+            
+          } catch (IOException x) {
+            iex = x;
+          }
+        }
         LOG.error(iex);
       }
     }
@@ -1176,7 +1264,15 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
       }
       leases.createLease(scannerId, scannerId,
         new ScannerListener(scannerName));
-    } catch(IOException e) {
+    } catch (IOException e) {
+      if (e instanceof RemoteException) {
+        try {
+          e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+          
+        } catch (IOException x) {
+          e = x;
+        }
+      }
       LOG.error(e);
       throw e;
     }

src/contrib/hbase/src/java/org/apache/hadoop/hbase/RemoteExceptionHandler.java

@@ -34,62 +34,46 @@ public class RemoteExceptionHandler {
   
   /**
    * Converts org.apache.hadoop.ipc.RemoteException into original exception,
-   * if possible.
+   * if possible. If the original exception is an Error or a RuntimeException,
+   * throws the original exception.
    * 
-   * @param e original exception
-   * @throws IOException
+   * @param re original exception
+   * @return decoded RemoteException if it is an instance of or a subclass of
+   *         IOException, or the original RemoteException if it cannot be decoded.
+   * 
+   * @throws IOException indicating a server error ocurred if the decoded 
+   *         exception is not an IOException. The decoded exception is set as
+   *         the cause.
    */
   @SuppressWarnings("unchecked")
-  public static void handleRemoteException(final Exception e) throws IOException {
-    Exception ex = e;
-    if (e instanceof RemoteException) {
-      RemoteException r = (RemoteException) e;
+  public static IOException decodeRemoteException(final RemoteException re)
+  throws IOException {
+    IOException i = re;
 
-      Class c = null;
-      try {
-        c = Class.forName(r.getClassName());
-      
-      } catch (ClassNotFoundException x) {
-        throw r;
-      }
+    try {
+      Class c = Class.forName(re.getClassName());
+
+      Class[] parameterTypes = { String.class };
+      Constructor ctor = c.getConstructor(parameterTypes);
       
-      Constructor ctor = null;
-      try {
-        Class[] parameterTypes = { String.class };
-        ctor = c.getConstructor(parameterTypes);
-        
-      } catch (NoSuchMethodException x) {
-        throw r;
-      }
+      Object[] arguments = { re.getMessage() };
+      Throwable t = (Throwable) ctor.newInstance(arguments);
       
-      try {
-        Object[] arguments = { r.getMessage() };
+      if (t instanceof IOException) {
+        i = (IOException) t;
 
-        ex = (Exception) ctor.newInstance(arguments);
-        
-      } catch (IllegalAccessException x) {
-        throw r;
-      
-      } catch (InvocationTargetException x) {
-        throw r;
-        
-      } catch (InstantiationException x) {
-        throw r;
+      } else {
+        i = new IOException("server error");
+        i.initCause(t);
+        throw i;
       }
-    } 
 
-    if (ex instanceof IOException) {
-      IOException io = (IOException) ex;
-      throw io;
-        
-    } else if (ex instanceof RuntimeException) {
-      RuntimeException re = (RuntimeException) ex;
-      throw re;
-        
-    } else {
-      AssertionError a = new AssertionError("unexpected exception");
-      a.initCause(ex);
-      throw a;
+    } catch (ClassNotFoundException x) {
+    } catch (NoSuchMethodException x) {
+    } catch (IllegalAccessException x) {
+    } catch (InvocationTargetException x) {
+    } catch (InstantiationException x) {
     }
+    return i;
   }
 }