|
|
@@ -918,40 +918,45 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt
|
|
|
// If the file is under construction , then it must be in our
|
|
|
// leases. Find the appropriate lease record.
|
|
|
//
|
|
|
- Lease lease = leaseManager.getLease(holder);
|
|
|
- //
|
|
|
- // We found the lease for this file. And surprisingly the original
|
|
|
- // holder is trying to recreate this file. This should never occur.
|
|
|
- //
|
|
|
- if (lease != null) {
|
|
|
+ Lease lease = leaseManager.getLeaseByPath(src);
|
|
|
+ if (lease == null) {
|
|
|
throw new AlreadyBeingCreatedException(
|
|
|
- "failed to create file " + src + " for " + holder +
|
|
|
- " on client " + clientMachine +
|
|
|
- " because current leaseholder is trying to recreate file.");
|
|
|
+ "failed to create file " + src + " for " + holder +
|
|
|
+ " on client " + clientMachine +
|
|
|
+ " because pendingCreates is non-null but no leases found.");
|
|
|
}
|
|
|
//
|
|
|
- // Find the original holder.
|
|
|
+ // We found the lease for this file. And surprisingly the original
|
|
|
+ // holder is trying to recreate this file. This should never occur.
|
|
|
//
|
|
|
- lease = leaseManager.getLease(pendingFile.clientName);
|
|
|
- if (lease == null) {
|
|
|
+ if (lease.getHolder().equals(holder)) {
|
|
|
throw new AlreadyBeingCreatedException(
|
|
|
- "failed to create file " + src + " for " + holder +
|
|
|
- " on client " + clientMachine +
|
|
|
- " because pendingCreates is non-null but no leases found.");
|
|
|
+ "failed to create file " + src + " for " + holder +
|
|
|
+ " on client " + clientMachine +
|
|
|
+ " because current leaseholder is trying to recreate file.");
|
|
|
}
|
|
|
+ assert lease.getHolder().equals(pendingFile.getClientName()) :
|
|
|
+ "Current lease holder " + lease.getHolder() +
|
|
|
+ " does not match file creator " + pendingFile.getClientName();
|
|
|
//
|
|
|
+ // Current lease holder is different from the requester.
|
|
|
// If the original holder has not renewed in the last SOFTLIMIT
|
|
|
- // period, then start lease recovery.
|
|
|
+ // period, then start lease recovery, otherwise fail.
|
|
|
//
|
|
|
if (lease.expiredSoftLimit()) {
|
|
|
LOG.info("startFile: recover lease " + lease + ", src=" + src);
|
|
|
- internalReleaseLease(lease, src);
|
|
|
- }
|
|
|
- throw new AlreadyBeingCreatedException("failed to create file " + src + " for " + holder +
|
|
|
- " on client " + clientMachine +
|
|
|
- ", because this file is already being created by " +
|
|
|
- pendingFile.getClientName() +
|
|
|
- " on " + pendingFile.getClientMachine());
|
|
|
+ boolean isClosed = internalReleaseLease(lease, src, null);
|
|
|
+ if(!isClosed)
|
|
|
+ throw new RecoveryInProgressException(
|
|
|
+ "Failed to close file " + src +
|
|
|
+ ". Lease recovery is in progress. Try again later.");
|
|
|
+
|
|
|
+ } else
|
|
|
+ throw new AlreadyBeingCreatedException("failed to create file " +
|
|
|
+ src + " for " + holder + " on client " + clientMachine +
|
|
|
+ ", because this file is already being created by " +
|
|
|
+ pendingFile.getClientName() +
|
|
|
+ " on " + pendingFile.getClientMachine());
|
|
|
}
|
|
|
|
|
|
try {
|
|
|
@@ -1004,7 +1009,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt
|
|
|
clientMachine,
|
|
|
clientNode);
|
|
|
dir.replaceNode(src, node, cons);
|
|
|
- leaseManager.addLease(cons.clientName, src);
|
|
|
+ leaseManager.addLease(cons.getClientName(), src);
|
|
|
|
|
|
} else {
|
|
|
// Now we can add the name to the filesystem. This file has no
|
|
|
@@ -1020,7 +1025,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt
|
|
|
throw new IOException("DIR* NameSystem.startFile: " +
|
|
|
"Unable to add file to namespace.");
|
|
|
}
|
|
|
- leaseManager.addLease(newNode.clientName, src);
|
|
|
+ leaseManager.addLease(newNode.getClientName(), src);
|
|
|
if (NameNode.stateChangeLog.isDebugEnabled()) {
|
|
|
NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: "
|
|
|
+"add "+src+" to namespace for "+holder);
|
|
|
@@ -1632,20 +1637,31 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt
|
|
|
* Move a file that is being written to be immutable.
|
|
|
* @param src The filename
|
|
|
* @param lease The lease for the client creating the file
|
|
|
+ * @param recoveryLeaseHolder reassign lease to this holder if the last block
|
|
|
+ * needs recovery; keep current holder if null.
|
|
|
+ * @throws AlreadyBeingCreatedException if file is waiting to achieve minimal
|
|
|
+ * replication;<br>
|
|
|
+ * RecoveryInProgressException if lease recovery is in progress.<br>
|
|
|
+ * IOException in case of an error.
|
|
|
+ * @return true if file has been successfully finalized and closed or
|
|
|
+ * false if block recovery has been initiated
|
|
|
*/
|
|
|
- void internalReleaseLease(Lease lease, String src) throws IOException {
|
|
|
+ boolean internalReleaseLease(
|
|
|
+ Lease lease, String src, String recoveryLeaseHolder)
|
|
|
+ throws AlreadyBeingCreatedException,
|
|
|
+ IOException {
|
|
|
LOG.info("Recovering lease=" + lease + ", src=" + src);
|
|
|
|
|
|
INodeFile iFile = dir.getFileINode(src);
|
|
|
if (iFile == null) {
|
|
|
- final String message = "DIR* NameSystem.internalReleaseCreate: "
|
|
|
+ final String message = "DIR* NameSystem.internalReleaseLease: "
|
|
|
+ "attempt to release a create lock on "
|
|
|
+ src + " file does not exist.";
|
|
|
NameNode.stateChangeLog.warn(message);
|
|
|
throw new IOException(message);
|
|
|
}
|
|
|
if (!iFile.isUnderConstruction()) {
|
|
|
- final String message = "DIR* NameSystem.internalReleaseCreate: "
|
|
|
+ final String message = "DIR* NameSystem.internalReleaseLease: "
|
|
|
+ "attempt to release a create lock on "
|
|
|
+ src + " but file is already closed.";
|
|
|
NameNode.stateChangeLog.warn(message);
|
|
|
@@ -1653,35 +1669,112 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt
|
|
|
}
|
|
|
|
|
|
INodeFileUnderConstruction pendingFile = (INodeFileUnderConstruction) iFile;
|
|
|
- BlockInfoUnderConstruction lastBlock = pendingFile.getLastBlock();
|
|
|
+ int nrBlocks = pendingFile.numBlocks();
|
|
|
+ BlockInfo[] blocks = pendingFile.getBlocks();
|
|
|
+
|
|
|
+ int nrCompleteBlocks;
|
|
|
+ BlockInfo curBlock = null;
|
|
|
+ for(nrCompleteBlocks = 0; nrCompleteBlocks < nrBlocks; nrCompleteBlocks++) {
|
|
|
+ curBlock = blocks[nrCompleteBlocks];
|
|
|
+ if(!curBlock.isComplete())
|
|
|
+ break;
|
|
|
+ assert blockManager.checkMinReplication(curBlock) :
|
|
|
+ "A COMPLETE block is not minimally replicated in " + src;
|
|
|
+ }
|
|
|
|
|
|
- // Initialize lease recovery for pendingFile. If there are no blocks
|
|
|
- // associated with this file, then reap lease immediately. Otherwise
|
|
|
- // renew the lease and trigger lease recovery.
|
|
|
- if (lastBlock == null) {
|
|
|
- assert pendingFile.getBlocks().length == 0 :
|
|
|
- "file is not empty but the last block does not exist";
|
|
|
+ // If there are no incomplete blocks associated with this file,
|
|
|
+ // then reap lease immediately and close the file.
|
|
|
+ if(nrCompleteBlocks == nrBlocks) {
|
|
|
finalizeINodeFileUnderConstruction(src, pendingFile);
|
|
|
NameNode.stateChangeLog.warn("BLOCK*"
|
|
|
- + " internalReleaseLease: No blocks found, lease removed.");
|
|
|
- return;
|
|
|
+ + " internalReleaseLease: All existing blocks are COMPLETE,"
|
|
|
+ + " lease removed, file closed.");
|
|
|
+ return true; // closed!
|
|
|
}
|
|
|
|
|
|
- // setup the last block locations from the blockManager if not known
|
|
|
- if(lastBlock.getNumLocations() == 0) {
|
|
|
- DatanodeDescriptor targets[] = blockManager.getNodes(lastBlock);
|
|
|
- lastBlock.setLocations(targets);
|
|
|
+ // Only the last and the penultimate blocks may be in non COMPLETE state.
|
|
|
+ // If the penultimate block is not COMPLETE, then it must be COMMITTED.
|
|
|
+ if(nrCompleteBlocks < nrBlocks - 2 ||
|
|
|
+ nrCompleteBlocks == nrBlocks - 2 &&
|
|
|
+ curBlock.getBlockUCState() != BlockUCState.COMMITTED) {
|
|
|
+ final String message = "DIR* NameSystem.internalReleaseLease: "
|
|
|
+ + "attempt to release a create lock on "
|
|
|
+ + src + " but file is already closed.";
|
|
|
+ NameNode.stateChangeLog.warn(message);
|
|
|
+ throw new IOException(message);
|
|
|
+ }
|
|
|
+
|
|
|
+ // no we know that the last block is not COMPLETE, and
|
|
|
+ // that the penultimate block if exists is either COMPLETE or COMMITTED
|
|
|
+ BlockInfoUnderConstruction lastBlock = pendingFile.getLastBlock();
|
|
|
+ BlockUCState lastBlockState = lastBlock.getBlockUCState();
|
|
|
+ BlockInfo penultimateBlock = pendingFile.getPenultimateBlock();
|
|
|
+ BlockUCState penultimateBlockState = (penultimateBlock == null ?
|
|
|
+ BlockUCState.COMPLETE : penultimateBlock.getBlockUCState());
|
|
|
+ assert penultimateBlockState == BlockUCState.COMPLETE ||
|
|
|
+ penultimateBlockState == BlockUCState.COMMITTED :
|
|
|
+ "Unexpected state of penultimate block in " + src;
|
|
|
+
|
|
|
+ switch(lastBlockState) {
|
|
|
+ case COMPLETE:
|
|
|
+ assert false : "Already checked that the last block is incomplete";
|
|
|
+ break;
|
|
|
+ case COMMITTED:
|
|
|
+ // Close file if committed blocks are minimally replicated
|
|
|
+ if(blockManager.checkMinReplication(penultimateBlock) &&
|
|
|
+ blockManager.checkMinReplication(lastBlock)) {
|
|
|
+ finalizeINodeFileUnderConstruction(src, pendingFile);
|
|
|
+ NameNode.stateChangeLog.warn("BLOCK*"
|
|
|
+ + " internalReleaseLease: Committed blocks are minimally replicated,"
|
|
|
+ + " lease removed, file closed.");
|
|
|
+ return true; // closed!
|
|
|
+ }
|
|
|
+ // Cannot close file right now, since some blocks
|
|
|
+ // are not yet minimally replicated.
|
|
|
+ // This may potentially cause infinite loop in lease recovery
|
|
|
+ // if there are no valid replicas on data-nodes.
|
|
|
+ String message = "DIR* NameSystem.internalReleaseLease: " +
|
|
|
+ "Failed to release lease for file " + src +
|
|
|
+ ". Committed blocks are waiting to be minimally replicated." +
|
|
|
+ " Try again later.";
|
|
|
+ NameNode.stateChangeLog.warn(message);
|
|
|
+ throw new AlreadyBeingCreatedException(message);
|
|
|
+ case UNDER_CONSTRUCTION:
|
|
|
+ case UNDER_RECOVERY:
|
|
|
+ // setup the last block locations from the blockManager if not known
|
|
|
+ if(lastBlock.getNumExpectedLocations() == 0)
|
|
|
+ lastBlock.setExpectedLocations(blockManager.getNodes(lastBlock));
|
|
|
+ // start recovery of the last block for this file
|
|
|
+ long blockRecoveryId = nextGenerationStamp();
|
|
|
+ lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
|
|
|
+ lastBlock.initializeBlockRecovery(blockRecoveryId);
|
|
|
+ leaseManager.renewLease(lease);
|
|
|
+ // Cannot close file right now, since the last block requires recovery.
|
|
|
+ // This may potentially cause infinite loop in lease recovery
|
|
|
+ // if there are no valid replicas on data-nodes.
|
|
|
+ NameNode.stateChangeLog.warn(
|
|
|
+ "DIR* NameSystem.internalReleaseLease: " +
|
|
|
+ "File " + src + " has not been closed." +
|
|
|
+ " Lease recovery is in progress. " +
|
|
|
+ "RecoveryId = " + blockRecoveryId + " for block " + lastBlock);
|
|
|
+ break;
|
|
|
}
|
|
|
+ return false;
|
|
|
+ }
|
|
|
|
|
|
- // start lease recovery of the last block for this file.
|
|
|
- lastBlock.assignPrimaryDatanode();
|
|
|
- leaseManager.renewLease(lease);
|
|
|
+ Lease reassignLease(Lease lease, String src, String newHolder,
|
|
|
+ INodeFileUnderConstruction pendingFile) {
|
|
|
+ if(newHolder == null)
|
|
|
+ return lease;
|
|
|
+ pendingFile.setClientName(newHolder);
|
|
|
+ return leaseManager.reassignLease(lease, src, newHolder);
|
|
|
}
|
|
|
|
|
|
+
|
|
|
private void finalizeINodeFileUnderConstruction(
|
|
|
String src,
|
|
|
INodeFileUnderConstruction pendingFile) throws IOException {
|
|
|
- leaseManager.removeLease(pendingFile.clientName, src);
|
|
|
+ leaseManager.removeLease(pendingFile.getClientName(), src);
|
|
|
|
|
|
// complete the penultimate block
|
|
|
blockManager.completeBlock(pendingFile, pendingFile.numBlocks()-2);
|
|
|
@@ -1715,11 +1808,20 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt
|
|
|
throw new IOException("Block (=" + lastblock + ") not found");
|
|
|
}
|
|
|
INodeFile iFile = oldblockinfo.getINode();
|
|
|
- if (!iFile.isUnderConstruction()) {
|
|
|
+ if (!iFile.isUnderConstruction() || oldblockinfo.isComplete()) {
|
|
|
throw new IOException("Unexpected block (=" + lastblock
|
|
|
+ ") since the file (=" + iFile.getLocalName()
|
|
|
+ ") is not under construction");
|
|
|
}
|
|
|
+
|
|
|
+ long recoveryId =
|
|
|
+ ((BlockInfoUnderConstruction)oldblockinfo).getBlockRecoveryId();
|
|
|
+ if(recoveryId != newgenerationstamp) {
|
|
|
+ throw new IOException("The recovery id " + newgenerationstamp
|
|
|
+ + " does not match current recovery id "
|
|
|
+ + recoveryId + " for block " + lastblock);
|
|
|
+ }
|
|
|
+
|
|
|
INodeFileUnderConstruction pendingFile = (INodeFileUnderConstruction)iFile;
|
|
|
|
|
|
|
Konstantin Shvachko