HADOOP-15346 S3ARetryPolicy for 400/BadArgument to be "fail". Contributed by Steve Loughran.

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java

@@ -175,9 +175,9 @@ public class S3ARetryPolicy implements RetryPolicy {
     // which isn't going to be recovered from
     policyMap.put(EOFException.class, retryIdempotentCalls);
 
-    // policy on a 400/bad request still ambiguous. Given it
-    // comes and goes on test runs: try again
-    policyMap.put(AWSBadRequestException.class, connectivityFailure);
+    // policy on a 400/bad request still ambiguous.
+    // Treated as an immediate failure
+    policyMap.put(AWSBadRequestException.class, fail);
 
     // Status 500 error code is also treated as a connectivity problem
     policyMap.put(AWSStatus500Exception.class, connectivityFailure);

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java

@@ -283,18 +283,12 @@ public class TestInvoker extends Assert {
   /**
    * Repeatedly retry until eventually a bad request succeeds.
    */
-  @Test
-  public void testRetryBadRequestIdempotent() throws Throwable {
-    final AtomicInteger counter = new AtomicInteger(0);
-    final int attemptsBeforeSuccess = ACTIVE_RETRY_LIMIT;
-    invoker.retry("test", null, true,
+  @Test(expected = AWSBadRequestException.class)
+  public void testRetryBadRequestNotIdempotent() throws Throwable {
+    invoker.retry("test", null, false,
         () -> {
-          if (counter.incrementAndGet() < attemptsBeforeSuccess) {
-            throw BAD_REQUEST;
-          }
+          throw BAD_REQUEST;
         });
-    assertEquals(attemptsBeforeSuccess, counter.get());
-    assertEquals("retry count ", attemptsBeforeSuccess - 1, retryCount);
   }
 
   @Test