apache
/
hadoop
mirror da https://github.com/apache/hadoop.git


			
				
					
						
						
							123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605
							<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<!--
 | Generated by Apache Maven Doxia at 2025-05-26
 | Rendered using Apache Maven Stylus Skin 1.5
-->
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>
    <title>Apache Hadoop Compatibility Benchmark &#x2013; Compatibility Benchmark over HCFS Implementations</title>
    <style type="text/css" media="all">
      @import url("./css/maven-base.css");
      @import url("./css/maven-theme.css");
      @import url("./css/site.css");
    </style>
    <link rel="stylesheet" href="./css/print.css" type="text/css" media="print" />
        <meta name="Date-Revision-yyyymmdd" content="20250526" />
    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
                </head>
  <body class="composite">
    <div id="banner">
                        <a href="http://hadoop.apache.org/" id="bannerLeft">
                                        <img src="http://hadoop.apache.org/images/hadoop-logo.jpg" alt="" />
                </a>
                              <a href="http://www.apache.org/" id="bannerRight">
                                        <img src="http://www.apache.org/images/asf_logo_wide.png" alt="" />
                </a>
            <div class="clear">
        <hr/>
      </div>
    </div>
    <div id="breadcrumbs">
            
                                     <div class="xright">            <a href="http://wiki.apache.org/hadoop" class="externalLink">Wiki</a>
            |
                <a href="https://gitbox.apache.org/repos/asf/hadoop.git" class="externalLink">git</a>
              
                                   &nbsp;| Last Published: 2025-05-26
              &nbsp;| Version: 3.5.0-SNAPSHOT
            </div>
      <div class="clear">
        <hr/>
      </div>
    </div>
    <div id="leftColumn">
      <div id="navcolumn">
             
                                                   <h5>General</h5>
                  <ul>
                  <li class="none">
                  <a href="../index.html">Overview</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/SingleCluster.html">Single Node Setup</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/ClusterSetup.html">Cluster Setup</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/CommandsManual.html">Commands Reference</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/FileSystemShell.html">FileSystem Shell</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/Compatibility.html">Compatibility Specification</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/DownstreamDev.html">Downstream Developer's Guide</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/AdminCompatibilityGuide.html">Admin Compatibility Guide</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/InterfaceClassification.html">Interface Classification</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/filesystem/index.html">FileSystem Specification</a>
            </li>
          </ul>
                       <h5>Common</h5>
                  <ul>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/CLIMiniCluster.html">CLI Mini Cluster</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/FairCallQueue.html">Fair Call Queue</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/NativeLibraries.html">Native Libraries</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/Superusers.html">Proxy User</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/RackAwareness.html">Rack Awareness</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/SecureMode.html">Secure Mode</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/ServiceLevelAuth.html">Service Level Authorization</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/HttpAuthentication.html">HTTP Authentication</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html">Credential Provider API</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-kms/index.html">Hadoop KMS</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/Tracing.html">Tracing</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/UnixShellGuide.html">Unix Shell Guide</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/registry/index.html">Registry</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/AsyncProfilerServlet.html">Async Profiler</a>
            </li>
          </ul>
                       <h5>HDFS</h5>
                  <ul>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HdfsDesign.html">Architecture</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HdfsUserGuide.html">User Guide</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HDFSCommands.html">Commands Reference</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithQJM.html">NameNode HA With QJM</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithNFS.html">NameNode HA With NFS</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/ObserverNameNode.html">Observer NameNode</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/Federation.html">Federation</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/ViewFs.html">ViewFs</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/ViewFsOverloadScheme.html">ViewFsOverloadScheme</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HdfsSnapshots.html">Snapshots</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HdfsEditsViewer.html">Edits Viewer</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HdfsImageViewer.html">Image Viewer</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html">Permissions and HDFS</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HdfsQuotaAdminGuide.html">Quotas and HDFS</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/LibHdfs.html">libhdfs (C API)</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/WebHDFS.html">WebHDFS (REST API)</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-hdfs-httpfs/index.html">HttpFS</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/ShortCircuitLocalReads.html">Short Circuit Local Reads</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/CentralizedCacheManagement.html">Centralized Cache Management</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HdfsNfsGateway.html">NFS Gateway</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HdfsRollingUpgrade.html">Rolling Upgrade</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/ExtendedAttributes.html">Extended Attributes</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/TransparentEncryption.html">Transparent Encryption</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HdfsMultihoming.html">Multihoming</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html">Storage Policies</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/MemoryStorage.html">Memory Storage Support</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/SLGUserGuide.html">Synthetic Load Generator</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HDFSErasureCoding.html">Erasure Coding</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HDFSDiskbalancer.html">Disk Balancer</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HdfsUpgradeDomain.html">Upgrade Domain</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HdfsDataNodeAdminGuide.html">DataNode Admin</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs-rbf/HDFSRouterFederation.html">Router Federation</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/HdfsProvidedStorage.html">Provided Storage</a>
            </li>
          </ul>
                       <h5>MapReduce</h5>
                  <ul>
                  <li class="none">
                  <a href="../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduceTutorial.html">Tutorial</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredCommands.html">Commands Reference</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduce_Compatibility_Hadoop1_Hadoop2.html">Compatibility with 1.x</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-mapreduce-client/hadoop-mapreduce-client-core/EncryptedShuffle.html">Encrypted Shuffle</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-mapreduce-client/hadoop-mapreduce-client-core/PluggableShuffleAndPluggableSort.html">Pluggable Shuffle/Sort</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-mapreduce-client/hadoop-mapreduce-client-core/DistributedCacheDeploy.html">Distributed Cache Deploy</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-mapreduce-client/hadoop-mapreduce-client-core/SharedCacheSupport.html">Support for YARN Shared Cache</a>
            </li>
          </ul>
                       <h5>MapReduce REST APIs</h5>
                  <ul>
                  <li class="none">
                  <a href="../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredAppMasterRest.html">MR Application Master</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-mapreduce-client/hadoop-mapreduce-client-hs/HistoryServerRest.html">MR History Server</a>
            </li>
          </ul>
                       <h5>YARN</h5>
                  <ul>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/YARN.html">Architecture</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/YarnCommands.html">Commands Reference</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html">Capacity Scheduler</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/FairScheduler.html">Fair Scheduler</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/ResourceManagerRestart.html">ResourceManager Restart</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/ResourceManagerHA.html">ResourceManager HA</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/ResourceModel.html">Resource Model</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/NodeLabel.html">Node Labels</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/NodeAttributes.html">Node Attributes</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/WebApplicationProxy.html">Web Application Proxy</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/TimelineServer.html">Timeline Server</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/TimelineServiceV2.html">Timeline Service V.2</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/WritingYarnApplications.html">Writing YARN Applications</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/YarnApplicationSecurity.html">YARN Application Security</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/NodeManager.html">NodeManager</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/DockerContainers.html">Running Applications in Docker Containers</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/RuncContainers.html">Running Applications in runC Containers</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html">Using CGroups</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/SecureContainer.html">Secure Containers</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/ReservationSystem.html">Reservation System</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/GracefulDecommission.html">Graceful Decommission</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/OpportunisticContainers.html">Opportunistic Containers</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/Federation.html">YARN Federation</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/SharedCache.html">Shared Cache</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/UsingGpus.html">Using GPU</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/UsingFPGA.html">Using FPGA</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/PlacementConstraints.html">Placement Constraints</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/YarnUI2.html">YARN UI2</a>
            </li>
          </ul>
                       <h5>YARN REST APIs</h5>
                  <ul>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/WebServicesIntro.html">Introduction</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html">Resource Manager</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/NodeManagerRest.html">Node Manager</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/TimelineServer.html#Timeline_Server_REST_API_v1">Timeline Server</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/TimelineServiceV2.html#Timeline_Service_v.2_REST_API">Timeline Service V.2</a>
            </li>
          </ul>
                       <h5>YARN Service</h5>
                  <ul>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/yarn-service/Overview.html">Overview</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/yarn-service/QuickStart.html">QuickStart</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/yarn-service/Concepts.html">Concepts</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/yarn-service/YarnServiceAPI.html">Yarn Service API</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/yarn-service/ServiceDiscovery.html">Service Discovery</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-site/yarn-service/SystemServices.html">System Services</a>
            </li>
          </ul>
                       <h5>Hadoop Compatible File Systems</h5>
                  <ul>
                  <li class="none">
                  <a href="../hadoop-aliyun/tools/hadoop-aliyun/index.html">Aliyun OSS</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-aws/tools/hadoop-aws/index.html">Amazon S3</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-azure/index.html">Azure Blob Storage</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-azure-datalake/index.html">Azure Data Lake Storage</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-cos/cloud-storage/index.html">Tencent COS</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-huaweicloud/index.html">Huaweicloud OBS</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-tos/cloud-storage/index.html">VolcanoEngine TOS</a>
            </li>
          </ul>
                       <h5>Auth</h5>
                  <ul>
                  <li class="none">
                  <a href="../hadoop-auth/index.html">Overview</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-auth/Examples.html">Examples</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-auth/Configuration.html">Configuration</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-auth/BuildingIt.html">Building</a>
            </li>
          </ul>
                       <h5>Tools</h5>
                  <ul>
                  <li class="none">
                  <a href="../hadoop-streaming/HadoopStreaming.html">Hadoop Streaming</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-archives/HadoopArchives.html">Hadoop Archives</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-archive-logs/HadoopArchiveLogs.html">Hadoop Archive Logs</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-distcp/DistCp.html">DistCp</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-federation-balance/HDFSFederationBalance.html">HDFS Federation Balance</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-gridmix/GridMix.html">GridMix</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-rumen/Rumen.html">Rumen</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-resourceestimator/ResourceEstimator.html">Resource Estimator Service</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-sls/SchedulerLoadSimulator.html">Scheduler Load Simulator</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/Benchmarking.html">Hadoop Benchmarking</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-dynamometer/Dynamometer.html">Dynamometer</a>
            </li>
          </ul>
                       <h5>Reference</h5>
                  <ul>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/release/">Changelog and Release Notes</a>
            </li>
                  <li class="none">
                  <a href="../api/index.html">Java API docs</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/UnixShellAPI.html">Unix Shell API</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/Metrics.html">Metrics</a>
            </li>
          </ul>
                       <h5>Configuration</h5>
                  <ul>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/core-default.xml">core-default.xml</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs/hdfs-default.xml">hdfs-default.xml</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-hdfs-rbf/hdfs-rbf-default.xml">hdfs-rbf-default.xml</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml">mapred-default.xml</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-yarn/hadoop-yarn-common/yarn-default.xml">yarn-default.xml</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-kms/kms-default.html">kms-default.xml</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-hdfs-httpfs/httpfs-default.html">httpfs-default.xml</a>
            </li>
                  <li class="none">
                  <a href="../hadoop-project-dist/hadoop-common/DeprecatedProperties.html">Deprecated Properties</a>
            </li>
          </ul>
                                 <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
          <img alt="Built by Maven" src="./images/logos/maven-feather.png"/>
        </a>
                       
                               </div>
    </div>
    <div id="bodyColumn">
      <div id="contentBox">
        <!---
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<h1>Compatibility Benchmark over HCFS Implementations</h1>
<ul>
<li><a href="#Overview"> Overview</a></li>
<li><a href="#Prepare"> Prepare</a>
<ul>
<li><a href="#a"></a>
<ul>
<li><a href="#FileSystem_implementation">FileSystem implementation</a></li>
<li><a href="#Optional_configuration">Optional configuration</a></li></ul></li></ul></li>
<li><a href="#Usage">Usage</a>
<ul>
<li><a href="#a"></a>
<ul>
<li><a href="#uri">uri</a></li>
<li><a href="#suite">suite</a></li>
<li><a href="#output">output</a></li></ul></li></ul></li></ul>
<section>
<h2><a name="Overview"></a> Overview</h2>
<p>Hadoop-Compatible File System (HCFS) is a core conception in big data storage ecosystem, providing unified interfaces and generally clear semantics, and has become the de-factor standard for industry storage systems to follow and conform with. There have been a series of HCFS implementations in Hadoop, such as S3AFileSystem for Amazon&#x2019;s S3 Object Store, WASB for Microsoft&#x2019;s Azure Blob Storage, OSS connector for Alibaba Cloud Object Storage, and more from storage service&#x2019;s providers on their own.</p>
<p>Meanwhile, Hadoop is also developing and new features are continuously contributing to HCFS interfaces for existing implementations to follow and update. However, we need a tool to check whether the features are supported by a specific implementation.</p>
<p>This module defines an HCFS compatibility benchmark and provides a corresponding tool to do the compatibility assessment for an HCFS storage system. The tool is a jar file which is executable by <code>hadoop jar</code>, after which an HCFS compatibility report is generated showing an overall score, and a detailed list of passed and failed cases (optional).</p></section><section>
<h2><a name="Prepare"></a> Prepare</h2>
<p>First of all, there must be a properly installed Hadoop environment to run <code>hadoop jar</code> command. See <a href="./HdfsUserGuide.html">HdfsUserGuide</a> for more information about how to set up a Hadoop environment. Then, two things should be done before a quick benchmark assessment.</p><section><section>
<h4><a name="FileSystem_implementation"></a>FileSystem implementation</h4>
<p>There must be a Java FileSystem implementation. The FS is known to Hadoop by config key <code>fs.&lt;scheme&gt;impl</code>. <code>org.apache.hadoop.fs.s3a.S3AFileSystem</code> is an example, while for implementations that has not been directly supported by Hadoop community, an extra jar file is needed.</p>
<p>The jar file should be placed at the last part of hadoop classpath. A common practice is to modify <code>hadoop-env.sh</code> to append an extra classpath like:</p>

<div class="source">
<div class="source">
<pre>export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/hadoop/extra/classpath/*
</pre></div></div>

<p>Then we are able to place extra jar files to <code>/hadoop/extra/classpath/</code></p></section><section>
<h4><a name="Optional_configuration"></a>Optional configuration</h4>
<p>Some FS APIs may need additional information to run. Additional information can be passed via optional configurations. There is an example:</p>

<div class="source">
<div class="source">
<pre>&lt;property&gt;
  &lt;name&gt;fs.{scheme}.compatibility.storage.policies&lt;/name&gt;
  &lt;value&gt;Hot,WARM,COLD&lt;/value&gt;
  &lt;description&gt;
    Storage policy names used by HCFS compatibility benchmark tool.
    The config key is fs.{scheme}.compatibility.storage.policies.
    The config value is Comma-separated storage policy names for the FS.
  &lt;/description&gt;
&lt;/property&gt;
</pre></div></div>

<p>Optional configurations are defined in <code>org.apache.hadoop.fs.compat.common.HdfsCompatEnvironment</code>.</p></section></section></section><section>
<h2><a name="Usage"></a>Usage</h2>

<div class="source">
<div class="source">
<pre>hadoop jar hadoop-compat-bench.jar -uri &lt;uri&gt; [-suite &lt;suite-name&gt;] [-output &lt;output-file&gt;]
</pre></div></div>

<p>This command generates a report with text format, showing an overall score and optionally passed and failed case lists.</p><section><section>
<h4><a name="uri"></a>uri</h4>
<p><code>uri</code> points to the target storage service path that you&#x2019;d like to evaluate. Some new files or directories would be created under the path.</p></section><section>
<h4><a name="suite"></a>suite</h4>
<p><code>suite-name</code> corresponds to a subset of all test cases. For example, a suite with name &#x2018;shell&#x2019; contains only shell command cases. There are three default suites of the tool: * ALL: run all test cases of the benchmark. This is the default suite if <code>-suite</code> is absent. * Shell: run only shell command cases. * TPCDS: run cases for APIs that a TPC-DS program may require.</p></section><section>
<h4><a name="output"></a>output</h4>
<p><code>output-file</code> points to a local file to save a detailed compatibility report document. The detailed report contains not only an overall score but also passed and failed case lists.</p></section></section></section>
      </div>
    </div>
    <div class="clear">
      <hr/>
    </div>
    <div id="footer">
      <div class="xright">
        &#169;            2008-2025
              Apache Software Foundation
            
                          - <a href="http://maven.apache.org/privacy-policy.html">Privacy Policy</a>.
        Apache Maven, Maven, Apache, the Apache feather logo, and the Apache Maven project logos are trademarks of The Apache Software Foundation.
      </div>
      <div class="clear">
        <hr/>
      </div>
    </div>
  </body>
</html>