hadoop/hadoop-tools/hadoop-azure/pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <parent>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-project</artifactId>
    <version>3.1.5-SNAPSHOT</version>
    <relativePath>../../hadoop-project</relativePath>
  </parent>
  <artifactId>hadoop-azure</artifactId>
  <name>Apache Hadoop Azure support</name>
  <description>
    This module contains code to support integration with Azure.
    Currently this consists of a filesystem client to read data from
    and write data to Azure Storage.
  </description>
  <packaging>jar</packaging>

  <properties>
    <file.encoding>UTF-8</file.encoding>
    <downloadSources>true</downloadSources>
    <hadoop.tmp.dir>${project.build.directory}/test</hadoop.tmp.dir>
    <!-- are scale tests enabled ? -->
    <fs.azure.scale.test.enabled>unset</fs.azure.scale.test.enabled>
    <!-- Size in MB of huge files. -->
    <fs.azure.scale.test.huge.filesize>unset</fs.azure.scale.test.huge.filesize>
    <!-- Size in MB of the partion size in huge file uploads. -->
    <fs.azure.scale.test.huge.partitionsize>unset</fs.azure.scale.test.huge.partitionsize>
    <!-- Timeout in seconds for scale tests.-->
    <fs.azure.scale.test.timeout>7200</fs.azure.scale.test.timeout>
    <fs.azure.scale.test.list.performance.threads>10</fs.azure.scale.test.list.performance.threads>
    <fs.azure.scale.test.list.performance.files>1000</fs.azure.scale.test.list.performance.files>
  </properties>

  <build>
    <plugins>
      <plugin>
        <groupId>org.codehaus.mojo</groupId>
        <artifactId>findbugs-maven-plugin</artifactId>
        <configuration>
          <findbugsXmlOutput>true</findbugsXmlOutput>
          <xmlOutput>true</xmlOutput>
          <excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml
          </excludeFilterFile>
          <effort>Max</effort>
        </configuration>
      </plugin>
      <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-checkstyle-plugin</artifactId>
          <configuration>
            <!-- To run with the default Sun ruleset, 
                 comment out the configLocation line -->
            <configLocation>src/config/checkstyle.xml</configLocation>
          </configuration>
          
      </plugin>

      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-jar-plugin</artifactId>
        <executions>
          <execution>
            <goals>
              <goal>test-jar</goal>
            </goals>
          </execution>
        </executions>
      </plugin>

       <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-dependency-plugin</artifactId>
        <executions>
          <execution>
            <id>deplist</id>
            <phase>compile</phase>
            <goals>
              <goal>list</goal>
            </goals>
            <configuration>
              <!-- build a shellprofile -->
              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt</outputFile>
            </configuration>
          </execution>
        </executions>
      </plugin>

    </plugins>

    
    <!--
       The following is to suppress a m2e warning in eclipse
       (m2e doesn't know how to handle maven-enforcer:enforce, so we have to tell m2e to ignore it)
       see: https://stackoverflow.com/questions/13040788/how-to-elimate-the-maven-enforcer-plugin-goal-enforce-is-ignored-by-m2e-wa
    -->
    <pluginManagement>
      <plugins>
        <plugin>
          <groupId>org.eclipse.m2e</groupId>
          <artifactId>lifecycle-mapping</artifactId>
          <version>1.0.0</version>
          <configuration>
            <lifecycleMappingMetadata>
              <pluginExecutions>
                <pluginExecution>
                  <pluginExecutionFilter>
                    <groupId>org.apache.maven.plugins</groupId>
                    <artifactId>maven-enforcer-plugin</artifactId>
                    <versionRange>[1.0.0,)</versionRange>
                    <goals>
                      <goal>enforce</goal>
                    </goals>
                  </pluginExecutionFilter>
                  <action>
                    <ignore />
                  </action>
                </pluginExecution>
              </pluginExecutions>
            </lifecycleMappingMetadata>
          </configuration>
        </plugin>
      </plugins>
    </pluginManagement>

  </build>

  <!-- see hadoop-project/pom.xml for version number declarations -->
  <dependencies>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <scope>compile</scope>
    </dependency>

    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-core</artifactId>
      <scope>compile</scope>
    </dependency>
    
    <dependency>
      <groupId>org.apache.httpcomponents</groupId>
      <artifactId>httpclient</artifactId>
      <scope>compile</scope>
    </dependency>
    
    <dependency>
      <groupId>com.microsoft.azure</groupId>
      <artifactId>azure-storage</artifactId>
      <scope>compile</scope>
      <exclusions>
        <exclusion>
          <groupId>org.apache.commons</groupId>
          <artifactId>commons-lang3</artifactId>
        </exclusion>
      </exclusions>
    </dependency>

    <dependency>
      <groupId>com.google.guava</groupId>
      <artifactId>guava</artifactId>
      <scope>compile</scope>
    </dependency>

    <dependency>
      <groupId>org.eclipse.jetty</groupId>
      <artifactId>jetty-util-ajax</artifactId>
      <scope>compile</scope>
    </dependency>
    
    
    <!-- dependencies use for test only -->
    <dependency>
      <groupId>commons-io</groupId>
      <artifactId>commons-io</artifactId>
      <scope>test</scope>
    </dependency>

    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <scope>test</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <scope>test</scope>
      <type>test-jar</type>
    </dependency>
    
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
      <scope>test</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-distcp</artifactId>
      <scope>test</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-distcp</artifactId>
      <scope>test</scope>
      <type>test-jar</type>
    </dependency>

    <dependency>
      <groupId>org.mockito</groupId>
      <artifactId>mockito-all</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-databind</artifactId>
    </dependency>

  </dependencies>

  <profiles>
    <profile>
      <id>parallel-tests</id>
      <activation>
        <property>
          <name>parallel-tests</name>
        </property>
      </activation>
      <build>
        <plugins>
          <plugin>
            <artifactId>maven-antrun-plugin</artifactId>
            <executions>
              <execution>
                <id>create-parallel-tests-dirs</id>
                <phase>test-compile</phase>
                <configuration>
                  <target>
                    <script language="javascript"><![CDATA[
                      var baseDirs = [
                        project.getProperty("test.build.data"),
                        project.getProperty("test.build.dir"),
                        project.getProperty("hadoop.tmp.dir")
                      ];
                      for (var i in baseDirs) {
                        for (var j = 1; j <= ${testsThreadCount}; ++j) {
                          var mkdir = project.createTask("mkdir");
                          mkdir.setDir(new java.io.File(baseDirs[i], j));
                          mkdir.perform();
                        }
                      }
                    ]]></script>
                  </target>
                </configuration>
                <goals>
                  <goal>run</goal>
                </goals>
              </execution>
            </executions>
          </plugin>
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-surefire-plugin</artifactId>
            <executions>
              <execution>
                <id>default-test</id>
                <goals>
                  <goal>test</goal>
                </goals>
                <configuration>
                  <forkCount>1</forkCount>
                  <forkCount>${testsThreadCount}</forkCount>
                  <reuseForks>false</reuseForks>
                  <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
                  <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
                  <systemPropertyVariables>
                    <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
                    <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
                    <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
                    <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
                    <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
                    <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
                    <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
                    <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
                    <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
                  </systemPropertyVariables>
                  <includes>
                    <include>**/Test*.java</include>
                  </includes>
                  <excludes>
                    <exclude>**/TestRollingWindowAverage*.java</exclude>
                  </excludes>
                </configuration>
              </execution>
              <execution>
                <id>serialized-test</id>
                <goals>
                  <goal>test</goal>
                </goals>
                <configuration>
                  <forkCount>1</forkCount>
                  <reuseForks>false</reuseForks>
                  <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
                  <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
                  <systemPropertyVariables>
                    <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
                    <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
                    <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
                    <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
                    <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
                    <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
                    <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
                    <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
                    <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
                  </systemPropertyVariables>
                  <includes>
                    <include>**/TestRollingWindowAverage*.java</include>
                  </includes>
                </configuration>
              </execution>
            </executions>
          </plugin>
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-failsafe-plugin</artifactId>
            <executions>
              <execution>
                <id>default-integration-test</id>
                <goals>
                  <goal>integration-test</goal>
                  <goal>verify</goal>
                </goals>
                <configuration>
                  <forkCount>${testsThreadCount}</forkCount>
                  <reuseForks>false</reuseForks>
                  <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
                  <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
                  <trimStackTrace>false</trimStackTrace>
                  <systemPropertyVariables>
                    <!-- Tell tests that they are being executed in parallel -->
                    <test.parallel.execution>true</test.parallel.execution>
                    <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
                    <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
                    <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>

                    <!-- Due to a Maven quirk, setting this to just -->
                    <!-- surefire.forkNumber won't do the parameter -->
                    <!-- substitution.  Putting a prefix in front of it like -->
                    <!-- "fork-" makes it work. -->
                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
                    <!-- Propagate scale parameters -->
                    <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
                    <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
                    <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
                    <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
                    <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
                    <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
                  </systemPropertyVariables>
                  <!-- Some tests cannot run in parallel.  Tests that cover -->
                  <!-- access to the root directory must run in isolation -->
                  <!-- from anything else that could modify the bucket. -->
                  <!-- azure tests that cover multi-part upload must run in -->
                  <!-- isolation, because the file system is configured to -->
                  <!-- purge existing multi-part upload data on -->
                  <!-- initialization.  MiniYARNCluster has not yet been -->
                  <!-- changed to handle parallel test execution gracefully. -->
                  <!-- Exclude all of these tests from parallel execution, -->
                  <!-- and instead run them sequentially in a separate -->
                  <!-- Surefire execution step later. -->
                  <includes>
                    <include>**/ITest*.java</include>
                  </includes>
                  <excludes>
                    <exclude>**/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java</exclude>
                    <exclude>**/ITestFileSystemOperationsWithThreads.java</exclude>
                    <exclude>**/ITestOutOfBandAzureBlobOperationsLive.java</exclude>
                    <exclude>**/ITestNativeAzureFileSystemAuthorizationWithOwner.java</exclude>
                    <exclude>**/ITestNativeAzureFileSystemConcurrencyLive.java</exclude>
                    <exclude>**/ITestNativeAzureFileSystemLive.java</exclude>
                    <exclude>**/ITestNativeAzureFSPageBlobLive.java</exclude>
                    <exclude>**/ITestWasbRemoteCallHelper.java</exclude>
                    <exclude>**/ITestBlockBlobInputStream.java</exclude>
                  </excludes>
                </configuration>
              </execution>
              <!-- Do a sequential run for tests that cannot handle -->
              <!-- parallel execution. -->
              <execution>
                <id>sequential-integration-tests</id>
                <goals>
                  <goal>integration-test</goal>
                  <goal>verify</goal>
                </goals>
                <configuration>
                  <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
                  <trimStackTrace>false</trimStackTrace>
                  <systemPropertyVariables>
                    <test.parallel.execution>false</test.parallel.execution>
                    <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
                    <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
                    <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
                    <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
                    <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
                    <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
                  </systemPropertyVariables>
                  <includes>
                    <include>**/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java</include>
                    <include>**/ITestFileSystemOperationsWithThreads.java</include>
                    <include>**/ITestOutOfBandAzureBlobOperationsLive.java</include>
                    <include>**/ITestNativeAzureFileSystemAuthorizationWithOwner.java</include>
                    <include>**/ITestNativeAzureFileSystemConcurrencyLive.java</include>
                    <include>**/ITestNativeAzureFileSystemLive.java</include>
                    <include>**/ITestWasbRemoteCallHelper.java</include>
                    <include>**/ITestBlockBlobInputStream.java</include>
                  </includes>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>
    <profile>
      <id>sequential-tests</id>
      <activation>
        <property>
          <name>!parallel-tests</name>
        </property>
      </activation>
      <build>
        <plugins>
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-failsafe-plugin</artifactId>
            <executions>
              <execution>
                <goals>
                  <goal>integration-test</goal>
                  <goal>verify</goal>
                </goals>
                <configuration>
                  <systemPropertyVariables>
                    <!-- Propagate scale parameters -->
                    <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
                    <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
                    <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
                    <fs.azure.scale.test.list.performance.threads>${fs.azure.scale.test.list.performance.threads}</fs.azure.scale.test.list.performance.threads>
                    <fs.azure.scale.test.list.performance.files>${fs.azure.scale.test.list.performance.files}</fs.azure.scale.test.list.performance.files>
                  </systemPropertyVariables>
                  <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
                  <trimStackTrace>false</trimStackTrace>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>

    <!-- Turn on scale tests-->
    <profile>
      <id>scale</id>
      <activation>
        <property>
          <name>scale</name>
        </property>
      </activation>
      <properties>
        <fs.azure.scale.test.enabled>true</fs.azure.scale.test.enabled>
      </properties>
    </profile>
  </profiles>
</project>