HADOOP-11804. Shaded Hadoop client artifacts and minicluster. Contributed by Sean Busbey.

.gitignore

@@ -15,6 +15,7 @@
 .settings
 target
 build
+dependency-reduced-pom.xml
 
 # External tool builders
 */.externalToolBuilders

dev-support/bin/dist-layout-stitching

@@ -137,6 +137,12 @@ run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERS
 run cp -pr "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${VERSION}"/* .
 run cp -pr "${ROOT}/hadoop-common-project/hadoop-kms/target/hadoop-kms-${VERSION}"/* .
 
+# copy client jars as-is
+run mkdir -p "share/hadoop/client"
+run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-${VERSION}.jar" share/hadoop/client/
+run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-${VERSION}.jar" share/hadoop/client/
+run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/
+
 echo
 echo "Hadoop dist layout available at: ${BASEDIR}/hadoop-${VERSION}"
-echo
+echo

hadoop-client-modules/hadoop-client-api/pom.xml

@@ -0,0 +1,216 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+<parent>
+   <groupId>org.apache.hadoop</groupId>
+   <artifactId>hadoop-project</artifactId>
+   <version>3.0.0-alpha2-SNAPSHOT</version>
+   <relativePath>../../hadoop-project</relativePath>
+</parent>
+  <artifactId>hadoop-client-api</artifactId>
+  <version>3.0.0-alpha2-SNAPSHOT</version>
+  <packaging>jar</packaging>
+
+  <description>Apache Hadoop Client</description>
+  <name>Apache Hadoop Client API</name>
+
+  <properties>
+    <shaded.dependency.prefix>org.apache.hadoop.shaded</shaded.dependency.prefix>
+    <!-- We contain no source -->
+    <maven.javadoc.skip>true</maven.javadoc.skip>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+      <!-- We list this as optional because as a type-pom it won't get included in the shading.
+           Marking it optional means it doesn't count as a transitive dependency of this artifact.
+        -->
+      <optional>true</optional>
+      <exclusions>
+        <!-- these APIs are a part of the SE JDK -->
+        <exclusion>
+          <groupId>javax.xml.bind</groupId>
+          <artifactId>jaxb-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>xml-apis</groupId>
+          <artifactId>xml-apis</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <!-- This comes from our parent pom. If we don't expressly change it here to get included,
+         downstream will get warnings at compile time. -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-annotations</artifactId>
+      <scope>compile</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>jdk.tools</groupId>
+          <artifactId>jdk.tools</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <!-- We contain no source -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-source-plugin</artifactId>
+          <configuration>
+            <skipSource>true</skipSource>
+          </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-maven-plugins</artifactId>
+            <version>${project.version}</version>
+          </dependency>
+        </dependencies>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <artifactSet>
+                <includes>
+                  <include>org.apache.hadoop:*</include>
+                </includes>
+              </artifactSet>
+              <filters>
+                <!-- We get these package level classes from various yarn api jars -->
+                <filter>
+                  <artifact>org.apache.hadoop:hadoop-yarn-common</artifact>
+                  <excludes>
+                    <exclude>org/apache/hadoop/yarn/factories/package-info.class</exclude>
+                    <exclude>org/apache/hadoop/yarn/util/package-info.class</exclude>
+                    <exclude>org/apache/hadoop/yarn/factory/providers/package-info.class</exclude>
+                    <exclude>org/apache/hadoop/yarn/client/api/impl/package-info.class</exclude>
+                    <exclude>org/apache/hadoop/yarn/client/api/package-info.class</exclude>
+                  </excludes>
+                </filter>
+              </filters>
+              <relocations>
+                <relocation>
+                  <pattern>org/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.org.</shadedPattern>
+                  <excludes>
+                    <exclude>org/apache/hadoop/*</exclude>
+                    <exclude>org/apache/hadoop/**/*</exclude>
+                    <!-- Our non-shaded htrace and logging libraries -->
+                    <exclude>org/apache/htrace/*</exclude>
+                    <exclude>org/apache/htrace/**/*</exclude>
+                    <exclude>org/slf4j/*</exclude>
+                    <exclude>org/slf4j/**/*</exclude>
+                    <exclude>org/apache/commons/logging/*</exclude>
+                    <exclude>org/apache/commons/logging/**/*</exclude>
+                    <exclude>org/apache/log4j/*</exclude>
+                    <exclude>org/apache/log4j/**/*</exclude>
+                    <exclude>**/pom.xml</exclude>
+                    <!-- Not the org/ packages that are a part of the jdk -->
+                    <exclude>org/ietf/jgss/*</exclude>
+                    <exclude>org/omg/**/*</exclude>
+                    <exclude>org/w3c/dom/*</exclude>
+                    <exclude>org/w3c/dom/**/*</exclude>
+                    <exclude>org/xml/sax/*</exclude>
+                    <exclude>org/xml/sax/**/*</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>com/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.com.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                    <!-- Not the com/ packages that are a part of particular jdk implementations -->
+                    <exclude>com/sun/tools/*</exclude>
+                    <exclude>com/sun/javadoc/*</exclude>
+                    <exclude>com/sun/security/*</exclude>
+                    <exclude>com/sun/jndi/*</exclude>
+                    <exclude>com/sun/management/*</exclude>
+                    <exclude>com/sun/tools/**/*</exclude>
+                    <exclude>com/sun/javadoc/**/*</exclude>
+                    <exclude>com/sun/security/**/*</exclude>
+                    <exclude>com/sun/jndi/**/*</exclude>
+                    <exclude>com/sun/management/**/*</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>io/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.io.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                    <!-- Exclude config keys for Hadoop that look like package names -->
+                    <exclude>io/compression/*</exclude>
+                    <exclude>io/compression/**/*</exclude>
+                    <exclude>io/mapfile/*</exclude>
+                    <exclude>io/mapfile/**/*</exclude>
+                    <exclude>io/map/index/*</exclude>
+                    <exclude>io/seqfile/*</exclude>
+                    <exclude>io/seqfile/**/*</exclude>
+                    <exclude>io/file/buffer/size</exclude>
+                    <exclude>io/skip/checksum/errors</exclude>
+                    <exclude>io/sort/*</exclude>
+                    <exclude>io/serializations</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>javax/servlet/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.javax.servlet.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>net/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.net.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                    <!-- Exclude config keys for Hadoop that look like package names -->
+                    <exclude>net/topology/*</exclude>
+                    <exclude>net/topology/**/*</exclude>
+                  </excludes>
+                </relocation>
+              </relocations>
+              <transformers>
+                <!-- Needed until MSHADE-182 -->
+                <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
+                    <addHeader>false</addHeader>
+                </transformer>
+              </transformers>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>license-maven-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+</project>
+

hadoop-client-modules/hadoop-client-check-invariants/pom.xml

@@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project</artifactId>
+    <version>3.0.0-alpha2-SNAPSHOT</version>
+    <relativePath>../../hadoop-project</relativePath>
+  </parent>
+  <artifactId>hadoop-client-check-invariants</artifactId>
+  <version>3.0.0-alpha2-SNAPSHOT</version>
+  <packaging>pom</packaging>
+
+  <description>Enforces our invariants for the api and runtime client modules.</description>
+  <name>Apache Hadoop Client Packaging Invariants</name>
+
+  <properties>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-runtime</artifactId>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <version>1.4</version>
+        <dependencies>
+          <dependency>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>extra-enforcer-rules</artifactId>
+            <version>1.0-beta-3</version>
+          </dependency>
+        </dependencies>
+        <executions>
+          <execution>
+            <id>enforce-banned-dependencies</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration>
+              <rules>
+                <banTransitiveDependencies>
+<!--
+                  <message>
+    Our client-facing artifacts are not supposed to have additional dependencies
+    and one or more of them do. The output from the enforcer plugin should give
+    specifics.
+                  </message>
+-->
+                  <excludes>
+                    <!-- annotations is provided, and both artifacts exclude the tools transitive,
+                         but enforcer still sees it.
+                    -->
+                    <exclude>org.apache.hadoop:hadoop-annotations</exclude>
+                    <!-- We leave HTrace as an unshaded dependnecy on purpose so that tracing within a JVM will work -->
+                    <exclude>org.apache.htrace:htrace-core4</exclude>
+                    <!-- Leave slf4j unshaded so downstream users can configure logging. -->
+                    <exclude>org.slf4j:slf4j-api</exclude>
+                    <!-- Leave commons-logging unshaded so downstream users can configure logging. -->
+                    <exclude>commons-logging:commons-logging</exclude>
+                    <!-- Leave log4j unshaded so downstream users can configure logging. -->
+                    <exclude>log4j:log4j</exclude>
+                  </excludes>
+                </banTransitiveDependencies>
+                <banDuplicateClasses>
+                  <findAllDuplicates>true</findAllDuplicates>
+                  <dependencies>
+                    <dependency>
+                      <groupId>org.apache.hadoop</groupId>
+                      <artifactId>hadoop-annotations</artifactId>
+                      <ignoreClasses>
+                        <ignoreClass>*</ignoreClass>
+                      </ignoreClasses>
+                    </dependency>
+                  </dependencies>
+                </banDuplicateClasses>
+              </rules>
+<!-- TODO we need a rule for "we don't have classes that are outside of the org.apache.hadoop package" -->
+<!-- TODO we need a rule for "the constants in this set of classes haven't been shaded / don't have this prefix"
+     Manually checking the set of Keys that look like packages we relocate:
+
+      cat `find . \( -name '*Keys.java' -o -name '*KeysPublic.java' \) -a -path '*/src/main/*'`  | grep -E "\"(io\.|org\.|com\.|net\.)" | grep -v "^package" | grep -v "^import" | grep -v "\"org.apache.hadoop"
+
+     Manually check the set of shaded artifacts to see if the Keys constants have been relocated:
+
+     for clazz in `find . \( -name '*Keys.java' -o -name '*KeysPublic.java' \) -a -path '*/src/main/*'`; do
+       clazz=${clazz#*src/main/java/}
+       clazz="${clazz%.java}"
+       javap -cp hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-3.0.0-alpha2-SNAPSHOT.jar:hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-3.0.0-alpha2-SNAPSHOT.jar:hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-3.0.0-alpha2-SNAPSHOT.jar \
+           -constants "${clazz//\//.}" | grep "org.apache.hadoop.shaded"
+     done
+-->
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>
+

hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml

@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project</artifactId>
+    <version>3.0.0-alpha2-SNAPSHOT</version>
+    <relativePath>../../hadoop-project</relativePath>
+  </parent>
+  <artifactId>hadoop-client-check-test-invariants</artifactId>
+  <version>3.0.0-alpha2-SNAPSHOT</version>
+  <packaging>pom</packaging>
+
+  <description>Enforces our invariants for the testing client modules.</description>
+  <name>Apache Hadoop Client Packaging Invariants for Test</name>
+
+  <properties>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-runtime</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-minicluster</artifactId>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <version>1.4</version>
+        <dependencies>
+          <dependency>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>extra-enforcer-rules</artifactId>
+            <version>1.0-beta-3</version>
+          </dependency>
+        </dependencies>
+        <executions>
+          <execution>
+            <id>enforce-banned-dependencies</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration>
+              <rules>
+                <banTransitiveDependencies>
+<!--
+                  <message>
+    Our client-facing artifacts are not supposed to have additional dependencies
+    and one or more of them do. The output from the enforcer plugin should give
+    specifics.
+                  </message>
+-->
+                  <excludes>
+                    <!-- annotations is provided, and both artifacts exclude the tools transitive,
+                         but enforcer still sees it.
+                    -->
+                    <exclude>org.apache.hadoop:hadoop-annotations</exclude>
+                    <!-- We leave HTrace as an unshaded dependnecy on purpose so that tracing within a JVM will work -->
+                    <exclude>org.apache.htrace:htrace-core4</exclude>
+                    <!-- Leave slf4j unshaded so downstream users can configure logging. -->
+                    <exclude>org.slf4j:slf4j-api</exclude>
+                    <!-- Leave commons-logging unshaded so downstream users can configure logging. -->
+                    <exclude>commons-logging:commons-logging</exclude>
+                    <!-- Leave log4j unshaded so downstream users can configure logging. -->
+                    <exclude>log4j:log4j</exclude>
+                    <!-- Leave JUnit unshaded so downstream can use our test helper classes -->
+                    <exclude>junit:junit</exclude>
+                    <!-- JUnit brings in hamcrest -->
+                    <exclude> org.hamcrest:hamcrest-core</exclude>
+                  </excludes>
+                </banTransitiveDependencies>
+                <banDuplicateClasses>
+                  <findAllDuplicates>true</findAllDuplicates>
+                  <dependencies>
+                    <dependency>
+                      <groupId>org.apache.hadoop</groupId>
+                      <artifactId>hadoop-annotations</artifactId>
+                      <ignoreClasses>
+                        <ignoreClass>*</ignoreClass>
+                      </ignoreClasses>
+                    </dependency>
+                  </dependencies>
+                </banDuplicateClasses>
+              </rules>
+<!-- TODO we need a rule for "we don't have classes that are outside of the org.apache.hadoop package" -->
+<!-- TODO we need a rule for "the constants in this set of classes haven't been shaded / don't have this prefix"
+     Manually checking the set of Keys that look like packages we relocate:
+
+      cat `find . \( -name '*Keys.java' -o -name '*KeysPublic.java' \) -a -path '*/src/main/*'`  | grep -E "\"(io\.|org\.|com\.|net\.)" | grep -v "^package" | grep -v "^import" | grep -v "\"org.apache.hadoop"
+
+     Manually check the set of shaded artifacts to see if the Keys constants have been relocated:
+
+     for clazz in `find . \( -name '*Keys.java' -o -name '*KeysPublic.java' \) -a -path '*/src/main/*'`; do
+       clazz=${clazz#*src/main/java/}
+       clazz="${clazz%.java}"
+       javap -cp hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-3.0.0-alpha2-SNAPSHOT.jar:hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-3.0.0-alpha2-SNAPSHOT.jar:hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-3.0.0-alpha2-SNAPSHOT.jar \
+           -constants "${clazz//\//.}" | grep "org.apache.hadoop.shaded"
+     done
+-->
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>
+

hadoop-client-modules/hadoop-client-integration-tests/pom.xml

@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project</artifactId>
+    <version>3.0.0-alpha2-SNAPSHOT</version>
+    <relativePath>../../hadoop-project</relativePath>
+  </parent>
+  <artifactId>hadoop-client-integration-tests</artifactId>
+  <version>3.0.0-alpha2-SNAPSHOT</version>
+
+  <description>Checks that we can use the generated artifacts</description>
+  <name>Apache Hadoop Client Packaging Integration Tests</name>
+
+  <properties>
+    <failsafe.timeout>400</failsafe.timeout>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-runtime</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-minicluster</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>log4j</groupId>
+      <artifactId>log4j</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <!-- Because our tests rely on our shaded artifacts, we can't compile
+           them until after the package phase has run.
+        -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <executions>
+        <!--
+           First, let's make sure the normal test-compile doesn't try to
+           compile our integration tests.
+        -->
+          <execution>
+            <id>default-testCompile</id>
+            <phase>test-compile</phase>
+            <configuration>
+              <testExcludes>
+                <testExclude>**/IT*</testExclude>
+                <testExclude>**/*IT</testExclude>
+              </testExcludes>
+            </configuration>
+          </execution>
+        <!--
+           Finally, let's make a 'just for integration tests'-compile that
+           fires off prior to our integration tests but after the package
+           phase has created our shaded artifacts.
+        -->
+          <execution>
+            <id>compile-integration-tests</id>
+            <phase>pre-integration-test</phase>
+            <goals>
+              <goal>testCompile</goal>
+            </goals>
+            <configuration>
+              <testIncludes>
+                <testInclude>**/IT*</testInclude>
+                <testInclude>**/*IT</testInclude>
+              </testIncludes>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-failsafe-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>integration-test</goal>
+              <goal>verify</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>
+

hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java

@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.hadoop.example;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+
+import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
+import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+
+/**
+ * Ensure that we can perform operations against the shaded minicluster
+ * given the API and runtime jars by performing some simple smoke tests.
+ */
+public class ITUseMiniCluster {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ITUseMiniCluster.class);
+
+  private MiniDFSCluster cluster;
+
+  private static final String TEST_PATH = "/foo/bar/cats/dee";
+  private static final String FILENAME = "test.file";
+
+  private static final String TEXT = "Lorem ipsum dolor sit amet, consectetur "
+      + "adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore "
+      + "magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation "
+      + "ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute "
+      + "irure dolor in reprehenderit in voluptate velit esse cillum dolore eu "
+      + "fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident,"
+      + " sunt in culpa qui officia deserunt mollit anim id est laborum.";
+
+  @Before
+  public void clusterUp() throws IOException {
+    final Configuration conf = new HdfsConfiguration();
+    cluster = new MiniDFSCluster.Builder(conf)
+        .numDataNodes(3)
+        .build();
+    cluster.waitActive();
+  }
+
+  @After
+  public void clusterDown() {
+    cluster.close();
+  }
+
+  @Test
+  public void useHdfsFileSystem() throws IOException {
+    try (final FileSystem fs = cluster.getFileSystem()) {
+      simpleReadAfterWrite(fs);
+    }
+  }
+
+  public void simpleReadAfterWrite(final FileSystem fs) throws IOException {
+    LOG.info("Testing read-after-write with FS implementation: {}", fs);
+    final Path path = new Path(TEST_PATH, FILENAME);
+    if (!fs.mkdirs(path.getParent())) {
+      throw new IOException("Mkdirs failed to create " +
+          TEST_PATH);
+    }
+    try (final FSDataOutputStream out = fs.create(path)) {
+      out.writeUTF(TEXT);
+    }
+    try (final FSDataInputStream in = fs.open(path)) {
+      final String result = in.readUTF();
+      Assert.assertEquals("Didn't read back text we wrote.", TEXT, result);
+    }
+  }
+
+  @Test
+  public void useWebHDFS() throws IOException, URISyntaxException {
+    try (final FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(
+        cluster.getConfiguration(0), WebHdfsConstants.WEBHDFS_SCHEME)) {
+      simpleReadAfterWrite(fs);
+    }
+  }
+}

hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/hdfs-site.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+  <!-- Turn security off for tests by default -->
+  <property>
+    <name>hadoop.security.authentication</name>
+    <value>simple</value>
+  </property>
+  <!-- Disable min block size since most tests use tiny blocks -->
+  <property>
+    <name>dfs.namenode.fs-limits.min-block-size</name>
+    <value>0</value>
+  </property>
+
+</configuration>

hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/log4j.properties

@@ -0,0 +1,24 @@
+#
+#   Licensed to the Apache Software Foundation (ASF) under one or more
+#   contributor license agreements.  See the NOTICE file distributed with
+#   this work for additional information regarding copyright ownership.
+#   The ASF licenses this file to You under the Apache License, Version 2.0
+#   (the "License"); you may not use this file except in compliance with
+#   the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+

hadoop-client-modules/hadoop-client-minicluster/pom.xml

@@ -0,0 +1,745 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project</artifactId>
+    <version>3.0.0-alpha2-SNAPSHOT</version>
+    <relativePath>../../hadoop-project</relativePath>
+  </parent>
+  <artifactId>hadoop-client-minicluster</artifactId>
+  <version>3.0.0-alpha2-SNAPSHOT</version>
+  <packaging>jar</packaging>
+
+  <description>Apache Hadoop Minicluster for Clients</description>
+  <name>Apache Hadoop Client Test Minicluster</name>
+
+  <properties>
+    <shaded.dependency.prefix>org.apache.hadoop.shaded</shaded.dependency.prefix>
+    <!-- We contain no source -->
+    <maven.javadoc.skip>true</maven.javadoc.skip>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-api</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-runtime</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <!-- Leave JUnit as a direct dependency -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <!-- Adding hadoop-annotations so we can make it optional to remove from our transitive tree -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-annotations</artifactId>
+      <scope>compile</scope>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>jdk.tools</groupId>
+          <artifactId>jdk.tools</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <!-- uncomment this dependency if you need to use
+         `mvn dependency:tree -Dverbose` to determine if a dependency shows up
+         in both the hadoop-client-* artifacts and something under minicluster.
+    -->
+<!--
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+      <scope>provided</scope>
+    </dependency>
+ -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-minicluster</artifactId>
+      <optional>true</optional>
+      <exclusions>
+        <!-- Exclude the in-development timeline service and
+             add it as an optional runtime dependency
+          -->
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-server-timelineservice</artifactId>
+        </exclusion>
+        <!-- exclude everything that comes in via the shaded runtime and api  TODO remove once we have a filter for "is in these artifacts" -->
+        <!-- Skip jersey, since we need it again here. -->
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-hdfs-client</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-mapreduce-client-app</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-mapreduce-client-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+        </exclusion>
+        <!-- exclude things that came in via transitive in shaded runtime and api -->
+        <exclusion>
+          <groupId>io.netty</groupId>
+          <artifactId>netty</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>javax.servlet</groupId>
+          <artifactId>javax.servlet-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>javax.xml.bind</groupId>
+          <artifactId>jaxb-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.avro</groupId>
+          <artifactId>avro</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.curator</groupId>
+          <artifactId>curator-client</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-mapreduce-client-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-server-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.zookeeper</groupId>
+          <artifactId>zookeeper</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.fusesource.leveldbjni</groupId>
+          <artifactId>leveldbjni-all</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-util</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.protobuf</groupId>
+          <artifactId>protobuf-java</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-collections</groupId>
+          <artifactId>commons-collections</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-io</groupId>
+          <artifactId>commons-io</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-lang</groupId>
+          <artifactId>commons-lang</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>log4j</groupId>
+          <artifactId>log4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-annotations</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-databind</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-cli</groupId>
+          <artifactId>commons-cli</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-codec</groupId>
+          <artifactId>commons-codec</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>javax.servlet</groupId>
+          <artifactId>javax.servlet-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>xmlenc</groupId>
+          <artifactId>xmlenc</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <!-- Add optional runtime dependency on the in-development timeline server module
+         to indicate that downstream folks interested in turning it on need that dep.
+      -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-server-timelineservice</artifactId>
+      <scope>runtime</scope>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <!-- Add back in transitive dependencies of hadoop-minicluster that are test-jar artifacts excluded as a side effect of excluding the jar
+         Note that all of these must be marked "optional" because they won't be removed from the reduced-dependencies pom after they're included.
+      -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>compile</scope>
+      <type>test-jar</type>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <scope>compile</scope>
+      <type>test-jar</type>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+      <scope>compile</scope>
+      <type>test-jar</type>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <!-- Add back in the transitive dependencies excluded from hadoop-common in client TODO remove once we have a filter for "is in these artifacts" -->
+    <!-- skip javax.servlet:servlet-api because it's in client -->
+    <!-- Skip commons-logging:commons-logging-api because it looks like nothing actually included it -->
+    <!-- Skip jetty-util because it's in client -->
+    <dependency>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-core</artifactId>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>javax.ws.rs</groupId>
+          <artifactId>jsr311-api</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-client</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+      <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-json</artifactId>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>javax.xml.bind</groupId>
+          <artifactId>jaxb-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>jackson-core-asl</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>jackson-mapper-asl</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>jackson-jaxrs</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>jackson-xc</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-server</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-servlet</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jdt</groupId>
+      <artifactId>core</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <!-- skip org.apache.avro:avro-ipc because it doesn't look like hadoop-common actually uses it -->
+    <dependency>
+      <groupId>net.sf.kosmosfs</groupId>
+      <artifactId>kfs</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+      <groupId>net.java.dev.jets3t</groupId>
+      <artifactId>jets3t</artifactId>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>commons-codec</groupId>
+          <artifactId>commons-codec</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.httpcomponents</groupId>
+          <artifactId>httpclient</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.httpcomponents</groupId>
+          <artifactId>httpcore</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>com.jcraft</groupId>
+      <artifactId>jsch</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <!-- add back in transitive dependencies of hadoop-mapreduce-client-app removed in client -->
+    <!-- Skipping javax.servlet:servlet-api because it's in client -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-server-nodemanager</artifactId>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>javax.servlet</groupId>
+          <artifactId>javax.servlet-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-server-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.fusesource.leveldbjni</groupId>
+          <artifactId>leveldbjni-all</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-util</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.protobuf</groupId>
+          <artifactId>protobuf-java</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-codec</groupId>
+          <artifactId>commons-codec</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-lang</groupId>
+          <artifactId>commons-lang</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>javax.xml.bind</groupId>
+          <artifactId>jaxb-api</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>javax.servlet</groupId>
+          <artifactId>javax.servlet-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-server-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-httpclient</groupId>
+          <artifactId>commons-httpclient</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-util</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <!-- skipping hadoop-annotations -->
+    <dependency>
+      <groupId>com.google.inject.extensions</groupId>
+      <artifactId>guice-servlet</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <!-- skipping junit:junit because it is test scope -->
+    <!-- skipping avro because it is in client via hadoop-common -->
+    <!-- skipping jline:jline because it is only present at test scope in the original -->
+    <!-- skipping io.netty:netty because it's in client -->
+    <!-- add back in transitive dependencies of hadoop-yarn-api removed in client -->
+    <!-- skipping hadoop-annotations -->
+    <dependency>
+      <groupId>com.google.inject</groupId>
+      <artifactId>guice</artifactId>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>com.sun.jersey.jersey-test-framework</groupId>
+      <artifactId>jersey-test-framework-grizzly2</artifactId>
+      <optional>true</optional>
+      <exclusions>
+        <!-- excluding because client already has the tomcat version -->
+        <exclusion>
+          <groupId>org.glassfish</groupId>
+          <artifactId>javax.servlet</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <!-- skipping jersey-server because it's above -->
+    <dependency>
+      <groupId>com.sun.jersey.contribs</groupId>
+      <artifactId>jersey-guice</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <!-- skipping guice-servlet because it's above -->
+    <!-- skipping avro because it is in client via hadoop-common -->
+    <!-- skipping jersey-core because it's above -->
+    <!-- skipping jersey-json because it's above. -->
+    <!-- skipping io.netty:netty because it's in client -->
+    <!-- Add back in transitive dependencies from hadoop-mapreduce-client-core that were excluded by client -->
+    <!-- skipping junit:junit because it is test scope -->
+    <!-- skipping guice because it's above -->
+    <!-- skipping jersey-test-framework-grizzly2 because it's above -->
+    <!-- skipping jersey-server because it's above -->
+    <!-- skipping jersey-guice because it's above -->
+    <!-- skipping avro because it is in client via hadoop-common -->
+    <!-- skipping hadoop-annotations -->
+    <!-- skipping guice-servlet because it's above -->
+    <!-- skipping jersey-json because it's above. -->
+    <!-- skipping io.netty:netty because it's in client -->
+    <!-- add back in transitive dependencies of hadoop-mapreduce-client-jobclient that were excluded from client -->
+    <!-- skipping junit:junit because it is test scope -->
+    <!-- skipping avro because it is in client via hadoop-common -->
+    <!-- skipping hadoop-annotations -->
+    <!-- skipping guice-servlet because it's above -->
+    <!-- skipping io.netty:netty because it's in client -->
+  </dependencies>
+  <build>
+    <plugins>
+      <!-- We contain no source -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-source-plugin</artifactId>
+          <configuration>
+            <skipSource>true</skipSource>
+          </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-maven-plugins</artifactId>
+            <version>${project.version}</version>
+          </dependency>
+        </dependencies>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <createSourceJar>true</createSourceJar>
+              <artifactSet>
+                <excludes>
+                  <!-- Fine to expose our already-shaded deps as dependencies -->
+                  <exclude>org.apache.hadoop:hadoop-annotations</exclude>
+                  <exclude>org.apache.hadoop:hadoop-client-api</exclude>
+                  <exclude>org.apache.hadoop:hadoop-client-runtime</exclude>
+                  <!-- Fine to expose our purposefully not-shaded deps as dependencies -->
+                  <exclude>org.apache.htrace:htrace-core4</exclude>
+                  <exclude>org.slf4j:slf4j-api</exclude>
+                  <exclude>commons-logging:commons-logging</exclude>
+                  <exclude>junit:junit</exclude>
+                  <!-- Keep optional runtime deps out of the shading -->
+                  <exclude>org.apache.hadoop:hadoop-yarn-server-timelineservice</exclude>
+                  <exclude>log4j:log4j</exclude>
+                  <!-- We need a filter that matches just those things that are included in the above artiacts -->
+                </excludes>
+              </artifactSet>
+              <filters>
+                <!-- Some of our dependencies include source, so remove it. -->
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>**/*.java</exclude>
+                  </excludes>
+                </filter>
+                <!-- We pull in several test jars; keep out the actual test classes -->
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>**/Test*.class</exclude>
+                  </excludes>
+                </filter>
+                <!-- Since runtime has classes for these jars, we exclude them.
+                     We still want the java services api files, since those were excluded in runtime
+                  -->
+                <filter>
+                  <artifact>com.sun.jersey:jersey-client</artifact>
+                  <excludes>
+                    <exclude>**/*.class</exclude>
+                  </excludes>
+                </filter>
+                <filter>
+                  <artifact>com.sun.jersey:jersey-core</artifact>
+                  <excludes>
+                    <exclude>**/*.class</exclude>
+                  </excludes>
+                </filter>
+                <filter>
+                  <artifact>com.sun.jersey:jersey-servlet</artifact>
+                  <excludes>
+                    <exclude>**/*.class</exclude>
+                  </excludes>
+                </filter>
+              </filters>
+              <relocations>
+                <relocation>
+                  <pattern>org/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.org.</shadedPattern>
+                  <excludes>
+                    <exclude>org/apache/hadoop/*</exclude>
+                    <exclude>org/apache/hadoop/**/*</exclude>
+                    <!-- Our non-shaded htrace and logging libraries -->
+                    <exclude>org/apache/htrace/*</exclude>
+                    <exclude>org/apache/htrace/**/*</exclude>
+                    <exclude>org/slf4j/*</exclude>
+                    <exclude>org/slf4j/**/*</exclude>
+                    <exclude>org/apache/commons/logging/*</exclude>
+                    <exclude>org/apache/commons/logging/**/*</exclude>
+                    <exclude>org/apache/log4j/*</exclude>
+                    <exclude>org/apache/log4j/**/*</exclude>
+                    <exclude>**/pom.xml</exclude>
+                    <!-- Our non-shaded JUnit library -->
+                    <exclude>org/junit/*</exclude>
+                    <exclude>org/junit/**/*</exclude>
+                    <!-- Not the org/ packages that are a part of the jdk -->
+                    <exclude>org/ietf/jgss/*</exclude>
+                    <exclude>org/omg/**/*</exclude>
+                    <exclude>org/w3c/dom/*</exclude>
+                    <exclude>org/w3c/dom/**/*</exclude>
+                    <exclude>org/xml/sax/*</exclude>
+                    <exclude>org/xml/sax/**/*</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>com/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.com.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                    <!-- Not the com/ packages that are a part of particular jdk implementations -->
+                    <exclude>com/sun/tools/*</exclude>
+                    <exclude>com/sun/javadoc/*</exclude>
+                    <exclude>com/sun/security/*</exclude>
+                    <exclude>com/sun/jndi/*</exclude>
+                    <exclude>com/sun/management/*</exclude>
+                    <exclude>com/sun/tools/**/*</exclude>
+                    <exclude>com/sun/javadoc/**/*</exclude>
+                    <exclude>com/sun/security/**/*</exclude>
+                    <exclude>com/sun/jndi/**/*</exclude>
+                    <exclude>com/sun/management/**/*</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>io/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.io.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                    <!-- Exclude config keys for Hadoop that look like package names -->
+                    <exclude>io/compression/*</exclude>
+                    <exclude>io/compression/**/*</exclude>
+                    <exclude>io/mapfile/*</exclude>
+                    <exclude>io/mapfile/**/*</exclude>
+                    <exclude>io/map/index/*</exclude>
+                    <exclude>io/seqfile/*</exclude>
+                    <exclude>io/seqfile/**/*</exclude>
+                    <exclude>io/file/buffer/size</exclude>
+                    <exclude>io/skip/checksum/errors</exclude>
+                    <exclude>io/sort/*</exclude>
+                    <exclude>io/serializations</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>javax/el/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.javax.el.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>javax/inject/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.javax.inject.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>javax/servlet/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.javax.servlet.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>net/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.net.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                    <!-- Exclude config keys for Hadoop that look like package names -->
+                    <exclude>net/topology/*</exclude>
+                    <exclude>net/topology/**/*</exclude>
+                  </excludes>
+                </relocation>
+              </relocations>
+              <transformers>
+                <!-- Needed until MSHADE-182 -->
+                <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
+                  <addHeader>false</addHeader>
+                </transformer>
+              </transformers>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>license-maven-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>
+

hadoop-client-modules/hadoop-client-runtime/pom.xml

@@ -0,0 +1,318 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+<parent>
+   <groupId>org.apache.hadoop</groupId>
+   <artifactId>hadoop-project</artifactId>
+   <version>3.0.0-alpha2-SNAPSHOT</version>
+   <relativePath>../../hadoop-project</relativePath>
+</parent>
+  <artifactId>hadoop-client-runtime</artifactId>
+  <version>3.0.0-alpha2-SNAPSHOT</version>
+  <packaging>jar</packaging>
+
+  <description>Apache Hadoop Client</description>
+  <name>Apache Hadoop Client Runtime</name>
+
+  <properties>
+    <shaded.dependency.prefix>org.apache.hadoop.shaded</shaded.dependency.prefix>
+    <!-- We contain no source -->
+    <maven.javadoc.skip>true</maven.javadoc.skip>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+      <!-- We list this as optional because as a type-pom it won't get included in the shading.
+           Marking it optional means it doesn't count as a transitive dependency of this artifact.
+        -->
+      <optional>true</optional>
+      <exclusions>
+        <!-- these APIs are a part of the SE JDK -->
+        <exclusion>
+          <groupId>javax.xml.bind</groupId>
+          <artifactId>jaxb-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>xml-apis</groupId>
+          <artifactId>xml-apis</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <!-- At runtime anyone using us must have the api present -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-api</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <!-- This comes from our parent pom. If we don't expressly change it here to get included,
+         downstream will get warnings at compile time. -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-annotations</artifactId>
+      <scope>compile</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>jdk.tools</groupId>
+          <artifactId>jdk.tools</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <!-- Since hadoop-client is listed as optional, we have to list transitive
+         dependencies that we still want to show up.
+         * HTrace
+         * Slf4j API
+         * commons-logging
+      -->
+    <dependency>
+      <groupId>org.apache.htrace</groupId>
+      <artifactId>htrace-core4</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>commons-logging</groupId>
+      <artifactId>commons-logging</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <!-- Move log4j to optional, since it is needed for some pieces folks might not use:
+         * one of the three custom log4j appenders we have
+         * JobConf (?!) (so essentially any user of MapReduce)
+      -->
+    <dependency>
+      <groupId>log4j</groupId>
+      <artifactId>log4j</artifactId>
+      <scope>runtime</scope>
+      <optional>true</optional>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <!-- We contain no source -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-source-plugin</artifactId>
+          <configuration>
+            <skipSource>true</skipSource>
+          </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-maven-plugins</artifactId>
+            <version>${project.version}</version>
+          </dependency>
+        </dependencies>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <createSourceJar>true</createSourceJar>
+              <artifactSet>
+                <excludes>
+                  <!-- We need a filter that matches just those things that aer included in the api jar -->
+                  <exclude>org.apache.hadoop:hadoop-client-api</exclude>
+                  <!-- Leave HTrace as an unshaded dependency on purpose, since a static class member is used to trace within a given JVM instance -->
+                  <exclude>org.apache.htrace:htrace-core4</exclude>
+                  <!-- Leave slf4j unshaded so downstream users can configure logging. -->
+                  <exclude>org.slf4j:slf4j-api</exclude>
+                  <!-- Leave commons-logging unshaded so downstream users can configure logging. -->
+                  <exclude>commons-logging:commons-logging</exclude>
+                  <!-- Leave log4j unshaded so downstream users can configure logging. -->
+                  <exclude>log4j:log4j</exclude>
+                </excludes>
+              </artifactSet>
+              <filters>
+                <!-- We need a filter that matches just those things that are included in the api jar -->
+                <filter>
+                  <artifact>org.apache.hadoop:*</artifact>
+                  <excludes>
+                    <exclude>**/*</exclude>
+                    <exclude>*</exclude>
+                  </excludes>
+                </filter>
+                <!-- Some of our dependencies include source, so remove it. -->
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>**/*.java</exclude>
+                  </excludes>
+                </filter>
+                <!-- We only want one copy of the Localizer class. So long as we keep jasper compiler and runtime on the same version, which one doesn't matter -->
+                <filter>
+                  <artifact>tomcat:jasper-compiler</artifact>
+                  <excludes>
+                    <exclude>org/apache/jasper/compiler/Localizer.class</exclude>
+                  </excludes>
+                </filter>
+                <!-- We only have xerces as a dependency for XML output for the fsimage edits, we don't need anything specific to it for javax xml support -->
+                <filter>
+                  <artifact>xerces:xercesImpl</artifact>
+                  <excludes>
+                    <exclude>META-INF/services/*</exclude>
+                  </excludes>
+                </filter>
+                <!-- We rely on jersey for our web interfaces. We want to use its java services stuff only internal to jersey -->
+                <filter>
+                  <artifact>com.sun.jersey:*</artifact>
+                  <excludes>
+                    <exclude>META-INF/services/javax.*</exclude>
+                  </excludes>
+                </filter>
+              </filters>
+              <relocations>
+                <relocation>
+                  <pattern>org/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.org.</shadedPattern>
+                  <excludes>
+                    <exclude>org/apache/hadoop/*</exclude>
+                    <exclude>org/apache/hadoop/**/*</exclude>
+                    <!-- Our non-shaded htrace and logging libraries -->
+                    <exclude>org/apache/htrace/*</exclude>
+                    <exclude>org/apache/htrace/**/*</exclude>
+                    <exclude>org/slf4j/*</exclude>
+                    <exclude>org/slf4j/**/*</exclude>
+                    <exclude>org/apache/commons/logging/*</exclude>
+                    <exclude>org/apache/commons/logging/**/*</exclude>
+                    <exclude>org/apache/log4j/*</exclude>
+                    <exclude>org/apache/log4j/**/*</exclude>
+                    <exclude>**/pom.xml</exclude>
+                    <!-- Not the org/ packages that are a part of the jdk -->
+                    <exclude>org/ietf/jgss/*</exclude>
+                    <exclude>org/omg/**/*</exclude>
+                    <exclude>org/w3c/dom/*</exclude>
+                    <exclude>org/w3c/dom/**/*</exclude>
+                    <exclude>org/xml/sax/*</exclude>
+                    <exclude>org/xml/sax/**/*</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>com/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.com.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                    <!-- Not the com/ packages that are a part of particular jdk implementations -->
+                    <exclude>com/sun/tools/*</exclude>
+                    <exclude>com/sun/javadoc/*</exclude>
+                    <exclude>com/sun/security/*</exclude>
+                    <exclude>com/sun/jndi/*</exclude>
+                    <exclude>com/sun/management/*</exclude>
+                    <exclude>com/sun/tools/**/*</exclude>
+                    <exclude>com/sun/javadoc/**/*</exclude>
+                    <exclude>com/sun/security/**/*</exclude>
+                    <exclude>com/sun/jndi/**/*</exclude>
+                    <exclude>com/sun/management/**/*</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>io/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.io.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                    <!-- Exclude config keys for Hadoop that look like package names -->
+                    <exclude>io/compression/*</exclude>
+                    <exclude>io/compression/**/*</exclude>
+                    <exclude>io/mapfile/*</exclude>
+                    <exclude>io/mapfile/**/*</exclude>
+                    <exclude>io/map/index/*</exclude>
+                    <exclude>io/seqfile/*</exclude>
+                    <exclude>io/seqfile/**/*</exclude>
+                    <exclude>io/file/buffer/size</exclude>
+                    <exclude>io/skip/checksum/errors</exclude>
+                    <exclude>io/sort/*</exclude>
+                    <exclude>io/serializations</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>javax/el/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.javax.el.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>javax/servlet/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.javax.servlet.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                  </excludes>
+                </relocation>
+                <relocation>
+                  <pattern>net/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.net.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                    <!-- Exclude config keys for Hadoop that look like package names -->
+                    <exclude>net/topology/*</exclude>
+                    <exclude>net/topology/**/*</exclude>
+                  </excludes>
+                </relocation>
+                <!-- probably not. -->
+<!--
+                <relocation>
+                  <pattern>javax/</pattern>
+                  <shadedPattern>${shaded.dependency.prefix}.javax.</shadedPattern>
+                  <excludes>
+                    <exclude>**/pom.xml</exclude>
+                  </excludes>
+                </relocation>
+-->
+              </relocations>
+              <transformers>
+                <!-- Needed until MSHADE-182 -->
+                <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
+                  <addHeader>false</addHeader>
+                </transformer>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.XmlAppendingTransformer">
+                  <resource>META-INF/jboss-beans.xml</resource>
+                  <!-- Add this to enable loading of DTDs
+                  <ignoreDtd>false</ignoreDtd>
+                  -->
+                </transformer>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                  <resource>META-INF/mailcap.default</resource>
+                </transformer>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                  <resource>META-INF/mimetypes.default</resource>
+                </transformer>
+              </transformers>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>license-maven-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>
+

hadoop-client/pom.xml → hadoop-client-modules/hadoop-client/pom.xml

@@ -19,14 +19,14 @@
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-project-dist</artifactId>
    <version>3.0.0-alpha2-SNAPSHOT</version>
-   <relativePath>../hadoop-project-dist</relativePath>
- </parent>
+   <relativePath>../../hadoop-project-dist</relativePath>
+</parent>
   <artifactId>hadoop-client</artifactId>
   <version>3.0.0-alpha2-SNAPSHOT</version>
-  <packaging>jar</packaging>
+  <packaging>pom</packaging>
 
-  <description>Apache Hadoop Client</description>
-  <name>Apache Hadoop Client</name>
+  <description>Apache Hadoop Client aggregation pom with dependencies exposed</description>
+  <name>Apache Hadoop Client Aggregator</name>
 
 <properties>
    <hadoop.component>client</hadoop.component>
@@ -98,6 +98,11 @@
           <groupId>org.apache.zookeeper</groupId>
           <artifactId>zookeeper</artifactId>
         </exclusion>
+        <!-- No slf4j backends for downstream clients -->
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -170,6 +175,11 @@
           <groupId>io.netty</groupId>
           <artifactId>netty</artifactId>
         </exclusion>
+        <!-- No slf4j backends for downstream clients -->
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>org.apache.zookeeper</groupId>
           <artifactId>zookeeper</artifactId>
@@ -270,6 +280,11 @@
           <groupId>io.netty</groupId>
           <artifactId>netty</artifactId>
         </exclusion>
+        <!-- No slf4j backends for downstream clients -->
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -298,6 +313,11 @@
           <groupId>io.netty</groupId>
           <artifactId>netty</artifactId>
         </exclusion>
+        <!-- No slf4j backends for downstream clients -->
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -312,7 +332,6 @@
         </exclusion>
       </exclusions>
     </dependency>
-    
   </dependencies>
 
 </project>

hadoop-client-modules/pom.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project</artifactId>
+    <version>3.0.0-alpha2-SNAPSHOT</version>
+    <relativePath>../hadoop-project</relativePath>
+  </parent>
+  <artifactId>hadoop-client-modules</artifactId>
+  <packaging>pom</packaging>
+  <description>multi-module for Apache Hadoop client artifacts</description>
+  <name>Apache Hadoop Client Modules</name>
+
+  <modules>
+    <!-- Left as an empty artifact w/dep for compat -->
+    <module>hadoop-client</module>
+    <!-- Should be used at compile scope for access to IA.Public classes -->
+    <module>hadoop-client-api</module>
+    <!-- Should be used at runtime scope for remaining classes necessary for hadoop-client-api to function -->
+    <module>hadoop-client-runtime</module>
+    <!-- Should be used at test scope for those that need access to mini cluster that works with above api and runtime -->
+    <module>hadoop-client-minicluster</module>
+    <!-- Checks invariants above -->
+    <module>hadoop-client-check-invariants</module>
+    <module>hadoop-client-check-test-invariants</module>
+    <!-- Attempt to use the created libraries -->
+    <module>hadoop-client-integration-tests</module>
+  </modules>
+
+</project>

hadoop-dist/pom.xml

@@ -51,6 +51,23 @@
       <artifactId>hadoop-yarn-api</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-check-invariants</artifactId>
+      <type>pom</type>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-check-test-invariants</artifactId>
+      <type>pom</type>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client-integration-tests</artifactId>
+      <scope>provided</scope>
+    </dependency>
   </dependencies>
 
   <build>

hadoop-maven-plugins/pom.xml

@@ -54,6 +54,28 @@
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.maven.plugins</groupId>
+      <artifactId>maven-shade-plugin</artifactId>
+      <version>${maven-shade-plugin.version}</version>
+      <scope>provided</scope>
+      <exclusions>
+        <!-- shade conflicts with itself and our other maven internals
+             by including a combination of 3.x and 2.x
+          -->
+        <exclusion>
+          <groupId>org.apache.maven.shared</groupId>
+          <artifactId>maven-dependency-tree</artifactId>
+        </exclusion>
+        <!-- again, shade conflicts with itself on the transitives of this
+             dependency
+          -->
+        <exclusion>
+          <groupId>org.vafer</groupId>
+          <artifactId>jdependency</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
   </dependencies>
   <build>
     <plugins>

hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java

@@ -0,0 +1,164 @@
+package org.apache.hadoop.maven.plugin.shade.resource;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+import java.io.BufferedReader;
+import org.apache.maven.plugins.shade.relocation.Relocator;
+import org.apache.maven.plugins.shade.resource.ResourceTransformer;
+import org.codehaus.plexus.util.IOUtil;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+
+/**
+ * Resources transformer that appends entries in META-INF/services resources
+ * into a single resource. For example, if there are several
+ * META-INF/services/org.apache.maven.project.ProjectBuilder resources spread
+ * across many JARs the individual entries will all be concatenated into a
+ * single META-INF/services/org.apache.maven.project.ProjectBuilder resource
+ * packaged into the resultant JAR produced by the shading process.
+ *
+ * From following sources, only needed until MSHADE-182 gets released
+ * * https://s.apache.org/vwjl (source in maven-shade-plugin repo)
+ * * https://issues.apache.org/jira/secure/attachment/12718938/MSHADE-182.patch
+ *
+ * Has been reformatted according to Hadoop checkstyle rules and modified
+ * to meet Hadoop's threshold for Findbugs problems.
+ */
+public class ServicesResourceTransformer
+    implements ResourceTransformer {
+
+  private static final String SERVICES_PATH = "META-INF/services";
+
+  private Map<String, ServiceStream> serviceEntries = new HashMap<>();
+
+  private List<Relocator> relocators;
+
+  public boolean canTransformResource(String resource) {
+    if (resource.startsWith(SERVICES_PATH)) {
+      return true;
+    }
+
+    return false;
+  }
+
+  public void processResource(String resource, InputStream is,
+      List<Relocator> relocatorz) throws IOException {
+    ServiceStream out = serviceEntries.get(resource);
+    if (out == null) {
+      out = new ServiceStream();
+      serviceEntries.put(resource, out);
+    }
+
+    out.append(is);
+    is.close();
+
+    if (this.relocators == null) {
+      this.relocators = relocatorz;
+    }
+  }
+
+  public boolean hasTransformedResource() {
+    return serviceEntries.size() > 0;
+  }
+
+  public void modifyOutputStream(JarOutputStream jos)
+      throws IOException {
+    for (Map.Entry<String, ServiceStream> entry : serviceEntries.entrySet()) {
+      String key = entry.getKey();
+      ServiceStream data = entry.getValue();
+
+      if (relocators != null) {
+        key = key.substring(SERVICES_PATH.length() + 1);
+        for (Relocator relocator : relocators) {
+          if (relocator.canRelocateClass(key)) {
+            key = relocator.relocateClass(key);
+            break;
+          }
+        }
+
+        key = SERVICES_PATH + '/' + key;
+      }
+
+      jos.putNextEntry(new JarEntry(key));
+
+      //read the content of service file for candidate classes for relocation
+      //presume everything is UTF8, because Findbugs barfs on default
+      //charset and this seems no worse a choice ¯\_(ツ)_/¯
+      PrintWriter writer = new PrintWriter(new OutputStreamWriter(jos,
+          StandardCharsets.UTF_8));
+      InputStreamReader streamReader =
+          new InputStreamReader(data.toInputStream(), StandardCharsets.UTF_8);
+      BufferedReader reader = new BufferedReader(streamReader);
+      String className;
+
+      while ((className = reader.readLine()) != null) {
+
+        if (relocators != null) {
+          for (Relocator relocator : relocators) {
+            //if the class can be relocated then relocate it
+            if (relocator.canRelocateClass(className)) {
+              className = relocator.applyToSourceContent(className);
+              break;
+            }
+          }
+        }
+
+        writer.println(className);
+        writer.flush();
+      }
+
+      reader.close();
+      data.reset();
+    }
+  }
+
+  static class ServiceStream extends ByteArrayOutputStream {
+
+    public ServiceStream() {
+      super(1024);
+    }
+
+    public void append(InputStream is)
+        throws IOException {
+      if (count > 0 && buf[count - 1] != '\n' && buf[count - 1] != '\r') {
+        write('\n');
+      }
+
+      IOUtil.copy(is, this);
+    }
+
+    public InputStream toInputStream() {
+      return new ByteArrayInputStream(buf, 0, count);
+    }
+
+  }
+
+}

hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Resource handling plugins used internal to the Hadoop build.
+ * IA.Private (build structure encourages not using the actual annotations)
+ */
+package org.apache.hadoop.maven.plugin.shade.resource;

hadoop-project/pom.xml

@@ -27,6 +27,7 @@
   <description>Apache Hadoop Project POM</description>
   <name>Apache Hadoop Project POM</name>
   <packaging>pom</packaging>
+  <inceptionYear>2008</inceptionYear>
 
   <properties>
     <!-- Set the Release year during release -->
@@ -114,6 +115,7 @@
     <maven-compiler-plugin.version>3.1</maven-compiler-plugin.version>
     <maven-install-plugin.version>2.5.1</maven-install-plugin.version>
     <maven-resources-plugin.version>2.6</maven-resources-plugin.version>
+    <maven-shade-plugin.version>2.4.3</maven-shade-plugin.version>
     <maven-jar-plugin.version>2.5</maven-jar-plugin.version>
     <maven-war-plugin.version>2.4</maven-war-plugin.version>
     <maven-source-plugin.version>2.3</maven-source-plugin.version>
@@ -148,6 +150,44 @@
         <artifactId>hadoop-annotations</artifactId>
         <version>${project.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-client-modules</artifactId>
+        <version>${project.version}</version>
+        <type>pom</type>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-client-api</artifactId>
+        <version>${project.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-client-check-invariants</artifactId>
+        <version>${project.version}</version>
+        <type>pom</type>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-client-check-test-invariants</artifactId>
+        <version>${project.version}</version>
+        <type>pom</type>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-client-integration-tests</artifactId>
+        <version>${project.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-client-runtime</artifactId>
+        <version>${project.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-client-minicluster</artifactId>
+        <version>${project.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-common</artifactId>
@@ -1244,6 +1284,40 @@
   <build>
     <pluginManagement>
       <plugins>
+        <plugin>
+          <groupId>org.codehaus.mojo</groupId>
+          <artifactId>build-helper-maven-plugin</artifactId>
+          <version>${build-helper-maven-plugin.version}</version>
+        </plugin>
+        <!-- We include the configuration for license-maven-plugin to correct
+             maven-shade-plugin generated poms because it's always the same. We
+             can't simply configure the plugin because we must ensure execution
+             happens in the package phase after the shade plugin runs.
+          -->
+        <plugin>
+          <groupId>org.codehaus.mojo</groupId>
+          <artifactId>license-maven-plugin</artifactId>
+          <version>1.10</version>
+          <configuration>
+            <canUpdateCopyright>false</canUpdateCopyright>
+            <roots><root>${project.basedir}</root></roots>
+          </configuration>
+          <executions>
+            <execution>
+              <id>update-pom-license</id>
+              <goals>
+                <goal>update-file-header</goal>
+              </goals>
+              <phase>package</phase>
+              <configuration>
+                <licenseName>apache_v2</licenseName>
+                <includes>
+                  <include>dependency-reduced-pom.xml</include>
+                </includes>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
         <plugin>
           <artifactId>maven-clean-plugin</artifactId>
           <version>${maven-clean-plugin.version}</version>
@@ -1260,13 +1334,8 @@
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
-          <artifactId>maven-dependency-plugin</artifactId>
-          <version>2.2</version>
-        </plugin>
-        <plugin>
-          <groupId>org.codehaus.mojo</groupId>
-          <artifactId>build-helper-maven-plugin</artifactId>
-          <version>${build-helper-maven-plugin.version}</version>
+          <artifactId>maven-shade-plugin</artifactId>
+          <version>${maven-shade-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
@@ -1355,6 +1424,19 @@
     </pluginManagement>
 
     <plugins>
+      <plugin>
+        <artifactId>maven-clean-plugin</artifactId>
+        <configuration>
+          <filesets>
+            <fileset>
+              <directory>${project.basedir}</directory>
+              <includes>
+                <include>dependency-reduced-pom.xml</include>
+              </includes>
+            </fileset>
+          </filesets>
+        </configuration>
+      </plugin>
       <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>findbugs-maven-plugin</artifactId>

pom.xml

@@ -96,7 +96,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
     <maven-stylus-skin.version>1.5</maven-stylus-skin.version>
     <maven-antrun-plugin.version>1.7</maven-antrun-plugin.version>
     <maven-assembly-plugin.version>2.4</maven-assembly-plugin.version>
-    <maven-dependency-plugin.version>2.8</maven-dependency-plugin.version>
+    <maven-dependency-plugin.version>2.10</maven-dependency-plugin.version>
     <maven-enforcer-plugin.version>1.4.1</maven-enforcer-plugin.version>
     <maven-javadoc-plugin.version>2.9.1</maven-javadoc-plugin.version>
     <maven-gpg-plugin.version>1.5</maven-gpg-plugin.version>
@@ -125,8 +125,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
     <module>hadoop-mapreduce-project</module>
     <module>hadoop-tools</module>
     <module>hadoop-dist</module>
-    <module>hadoop-client</module>
     <module>hadoop-minicluster</module>
+    <module>hadoop-client-modules</module>
     <module>hadoop-build-tools</module>
   </modules>