Revert MAPREDUCE-3868. Reenable Raid.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1363572 13f79535-47bb-0310-9956-ffa450edef68

hadoop-assemblies/src/main/resources/assemblies/hadoop-raid-dist.xml

@@ -1,60 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-<assembly>
-  <id>hadoop-raid-dist</id>
-  <formats>
-    <format>dir</format>
-  </formats>
-  <includeBaseDirectory>false</includeBaseDirectory>
-  <fileSets>
-    <!-- Configuration files -->
-    <fileSet>
-      <directory>${basedir}/src/main/conf</directory>
-      <outputDirectory>/etc/hadoop</outputDirectory>
-      <includes>
-        <include>*</include>
-      </includes>
-    </fileSet>
-    <fileSet>
-      <directory>${basedir}/src/main/sbin</directory>
-      <outputDirectory>/sbin</outputDirectory>
-      <includes>
-        <include>*</include>
-      </includes>
-      <fileMode>0755</fileMode>
-    </fileSet>
-    <fileSet>
-      <directory>${basedir}/src/main/libexec</directory>
-      <outputDirectory>/libexec</outputDirectory>
-      <includes>
-        <include>*</include>
-      </includes>
-      <fileMode>0755</fileMode>
-    </fileSet>
-    <!-- Documentation -->
-    <fileSet>
-      <directory>${project.build.directory}/site</directory>
-      <outputDirectory>/share/doc/hadoop/raid</outputDirectory>
-    </fileSet>
-  </fileSets>
-  <dependencySets>
-    <dependencySet>
-      <outputDirectory>/share/hadoop/${hadoop.component}/lib</outputDirectory>
-      <unpack>false</unpack>
-      <scope>runtime</scope>
-      <useProjectArtifact>true</useProjectArtifact>
-    </dependencySet>
-  </dependencySets>
-</assembly>

hadoop-dist/pom.xml

@@ -52,11 +52,6 @@
       <artifactId>hadoop-yarn-api</artifactId>
       <scope>provided</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs-raid</artifactId>
-      <scope>provided</scope>
-    </dependency>
   </dependencies>
 
   <build>
@@ -125,7 +120,6 @@
                       run cp -r $ROOT/hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}/* .
                       run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}/* .
                       run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}/* .
-                      run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-raid/target/hadoop-hdfs-raid-${project.version}/* .
                       run cp -r $ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* .
                       run cp -r $ROOT/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}/* .
                       echo

hadoop-hdfs-project/hadoop-hdfs-raid/pom.xml

@@ -1,170 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-
--->
-<project>
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>org.apache.hadoop</groupId>
-    <artifactId>hadoop-project-dist</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
-    <relativePath>../../hadoop-project-dist</relativePath>
-  </parent>
-  <groupId>org.apache.hadoop</groupId>
-  <artifactId>hadoop-hdfs-raid</artifactId>
-  <version>3.0.0-SNAPSHOT</version>
-  <packaging>jar</packaging>
-
-  <name>Apache Hadoop HDFS Raid</name>
-  <description>Apache Hadoop HDFS Raid</description>
-
-
-  <properties>
-    <hadoop.component>raid</hadoop.component>
-    <is.hadoop.component>false</is.hadoop.component>
-  </properties>
-
-  <dependencies>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-annotations</artifactId>
-      <scope>provided</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-minicluster</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
-      <scope>provided</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-archives</artifactId>
-      <scope>provided</scope>
-    </dependency>
-  </dependencies>
-
-  <build>
-
-    <plugins>
-      <plugin>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>create-mrapp-generated-classpath</id>
-            <phase>generate-test-resources</phase>
-            <goals>
-              <goal>build-classpath</goal>
-            </goals>
-            <configuration>
-              <!--
-              This is needed to run the unit tests. It generates the required classpath
-              that is required in the env of the launch container in the mini mr/yarn cluster.
-              -->
-              <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-      <groupId>org.apache.rat</groupId>
-        <artifactId>apache-rat-plugin</artifactId>
-        <configuration>
-          <excludes>
-          </excludes>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>findbugs-maven-plugin</artifactId>
-        <configuration>
-          <excludeFilterFile combine.self="override"></excludeFilterFile>
-        </configuration>
-      </plugin>
-    </plugins>
-  </build>
-
-  <profiles>
-    <profile>
-      <id>docs</id>
-      <activation>
-        <activeByDefault>false</activeByDefault>
-      </activation>
-      <build>
-        <plugins>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-site-plugin</artifactId>
-            <executions>
-              <execution>
-                <id>docs</id>
-                <phase>prepare-package</phase>
-                <goals>
-                  <goal>site</goal>
-                </goals>
-              </execution>
-            </executions>
-          </plugin>
-        </plugins>
-      </build>
-    </profile>
-
-    <profile>
-      <id>dist</id>
-      <activation>
-        <activeByDefault>false</activeByDefault>
-      </activation>
-      <build>
-        <plugins>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-assembly-plugin</artifactId>
-            <dependencies>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-assemblies</artifactId>
-                <version>${project.version}</version>
-              </dependency>
-            </dependencies>
-            <executions>
-              <execution>
-                <id>dist</id>
-                <phase>prepare-package</phase>
-                <goals>
-                  <goal>single</goal>
-                </goals>
-                <configuration>
-                  <finalName>${project.artifactId}-${project.version}</finalName>
-                  <appendAssemblyId>false</appendAssemblyId>
-                  <attach>false</attach>
-                  <descriptorRefs>
-                    <descriptorRef>hadoop-raid-dist</descriptorRef>
-                  </descriptorRefs>
-                </configuration>
-              </execution>
-            </executions>
-          </plugin>
-        </plugins>
-      </build>
-    </profile>
-  </profiles>
-</project>

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/conf/raid.xml

@@ -1,58 +0,0 @@
-   <configuration>
-    <srcPath prefix="hdfs://dfs1.xxx.com:8000/user/dhruba/">
-      <policy name = "dhruba">
-        <property>
-          <name>srcReplication</name>
-          <value>3</value>
-          <description> pick files for RAID only if their replication factor is
-                        greater than or equal to this value.
-          </description>
-        </property>
-        <property>
-          <name>targetReplication</name>
-          <value>2</value>
-          <description> after RAIDing, decrease the replication factor of a file to 
-                        this value.
-          </description>
-        </property>
-        <property>
-          <name>metaReplication</name>
-          <value>2</value>
-          <description> the replication factor of the RAID meta file
-          </description>
-        </property>
-        <property>
-          <name>modTimePeriod</name>
-          <value>3600000</value>
-          <description> time (milliseconds) after a file is modified to make it a
-                        candidate for RAIDing
-          </description>
-        </property>
-      </policy>
-    </srcPath>
-    <srcPath prefix="hdfs://dfs1.xxx.com:9000/warehouse/table1">
-      <policy name = "table1">
-        <property>
-          <name>targetReplication</name>
-          <value>1</value>
-          <description> after RAIDing, decrease the replication factor of a file to 
-                        this value.
-          </description>
-        </property>
-        <property>
-          <name>metaReplication</name>
-          <value>2</value>
-          <description> the replication factor of the RAID meta file
-          </description>
-        </property>
-        <property>
-          <name>modTimePeriod</name>
-          <value>3600000</value>
-          <description> time (milliseconds) after a file is modified to make it a
-                        candidate for RAIDing
-          </description>
-        </property>
-      </policy>
-    </srcPath>
-   </configuration>
-

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java

@@ -1,509 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs;
-
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.URI;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.ChecksumException;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FilterFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.raid.Decoder;
-import org.apache.hadoop.raid.RaidNode;
-import org.apache.hadoop.raid.ReedSolomonDecoder;
-import org.apache.hadoop.raid.XORDecoder;
-import org.apache.hadoop.raid.protocol.PolicyInfo.ErasureCodeType;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * This is an implementation of the Hadoop  RAID Filesystem. This FileSystem 
- * wraps an instance of the DistributedFileSystem.
- * If a file is corrupted, this FileSystem uses the parity blocks to 
- * regenerate the bad block.
- */
-
-public class DistributedRaidFileSystem extends FilterFileSystem {
-
-  // these are alternate locations that can be used for read-only access
-  DecodeInfo[] alternates;
-  Configuration conf;
-  int stripeLength;
-
-  DistributedRaidFileSystem() throws IOException {
-  }
-
-  DistributedRaidFileSystem(FileSystem fs) throws IOException {
-    super(fs);
-    alternates = null;
-    stripeLength = 0;
-  }
-
-  // Information required for decoding a source file
-  static private class DecodeInfo {
-    final Path destPath;
-    final ErasureCodeType type;
-    final Configuration conf;
-    final int stripeLength;
-    private DecodeInfo(Configuration conf, ErasureCodeType type, Path destPath) {
-      this.conf = conf;
-      this.type = type;
-      this.destPath = destPath;
-      this.stripeLength = RaidNode.getStripeLength(conf);
-    }
-
-    Decoder createDecoder() {
-      if (this.type == ErasureCodeType.XOR) {
-        return new XORDecoder(conf, stripeLength);
-      } else if (this.type == ErasureCodeType.RS) {
-        return new ReedSolomonDecoder(conf, stripeLength,
-                              RaidNode.rsParityLength(conf));
-      }
-      return null;
-    }
-  }
-
-  /* Initialize a Raid FileSystem
-   */
-  public void initialize(URI name, Configuration conf) throws IOException {
-    this.conf = conf;
-
-    Class<?> clazz = conf.getClass("fs.raid.underlyingfs.impl",
-        DistributedFileSystem.class);
-    if (clazz == null) {
-      throw new IOException("No FileSystem for fs.raid.underlyingfs.impl.");
-    }
-    
-    this.fs = (FileSystem)ReflectionUtils.newInstance(clazz, null); 
-    super.initialize(name, conf);
-    
-    // find stripe length configured
-    stripeLength = RaidNode.getStripeLength(conf);
-    if (stripeLength == 0) {
-      LOG.info("dfs.raid.stripeLength is incorrectly defined to be " + 
-               stripeLength + " Ignoring...");
-      return;
-    }
-
-    // Put XOR and RS in alternates
-    alternates= new DecodeInfo[2];
-    Path xorPath = RaidNode.xorDestinationPath(conf, fs);
-    alternates[0] = new DecodeInfo(conf, ErasureCodeType.XOR, xorPath);
-    Path rsPath = RaidNode.rsDestinationPath(conf, fs);
-    alternates[1] = new DecodeInfo(conf, ErasureCodeType.RS, rsPath);
-  }
-
-  /*
-   * Returns the underlying filesystem
-   */
-  public FileSystem getFileSystem() throws IOException {
-    return fs;
-  }
-
-  @Override
-  public FSDataInputStream open(Path f, int bufferSize) throws IOException {
-    ExtFSDataInputStream fd = new ExtFSDataInputStream(conf, this, alternates, f,
-                                                       stripeLength, bufferSize);
-    return fd;
-  }
-
-  public void close() throws IOException {
-    if (fs != null) {
-      try {
-        fs.close();
-      } catch(IOException ie) {
-        //this might already be closed, ignore
-      }
-    }
-    super.close();
-  }
-
-  /**
-   * Layered filesystem input stream. This input stream tries reading
-   * from alternate locations if it encoumters read errors in the primary location.
-   */
-  private static class ExtFSDataInputStream extends FSDataInputStream {
-
-    private static class UnderlyingBlock {
-      // File that holds this block. Need not be the same as outer file.
-      public Path path;
-      // Offset within path where this block starts.
-      public long actualFileOffset;
-      // Offset within the outer file where this block starts.
-      public long originalFileOffset;
-      // Length of the block (length <= blk sz of outer file).
-      public long length;
-      public UnderlyingBlock(Path path, long actualFileOffset,
-          long originalFileOffset, long length) {
-        this.path = path;
-        this.actualFileOffset = actualFileOffset;
-        this.originalFileOffset = originalFileOffset;
-        this.length = length;
-      }
-    }
-
-    /**
-     * Create an input stream that wraps all the reads/positions/seeking.
-     */
-    private static class ExtFsInputStream extends FSInputStream {
-
-      // Extents of "good" underlying data that can be read.
-      private UnderlyingBlock[] underlyingBlocks;
-      private long currentOffset;
-      private FSDataInputStream currentStream;
-      private UnderlyingBlock currentBlock;
-      private byte[] oneBytebuff = new byte[1];
-      private int nextLocation;
-      private DistributedRaidFileSystem lfs;
-      private Path path;
-      private FileStatus stat;
-      private final DecodeInfo[] alternates;
-      private final int buffersize;
-      private final Configuration conf;
-      private final int stripeLength;
-
-      ExtFsInputStream(Configuration conf, DistributedRaidFileSystem lfs,
-          DecodeInfo[] alternates, Path path, int stripeLength, int buffersize)
-          throws IOException {
-        this.path = path;
-        this.nextLocation = 0;
-        // Construct array of blocks in file.
-        this.stat = lfs.getFileStatus(path);
-        long numBlocks = (this.stat.getLen() % this.stat.getBlockSize() == 0) ?
-                        this.stat.getLen() / this.stat.getBlockSize() :
-                        1 + this.stat.getLen() / this.stat.getBlockSize();
-        this.underlyingBlocks = new UnderlyingBlock[(int)numBlocks];
-        for (int i = 0; i < numBlocks; i++) {
-          long actualFileOffset = i * stat.getBlockSize();
-          long originalFileOffset = i * stat.getBlockSize();
-          long length = Math.min(
-            stat.getBlockSize(), stat.getLen() - originalFileOffset);
-          this.underlyingBlocks[i] = new UnderlyingBlock(
-            path, actualFileOffset, originalFileOffset, length);
-        }
-        this.currentOffset = 0;
-        this.currentBlock = null;
-        this.alternates = alternates;
-        this.buffersize = buffersize;
-        this.conf = conf;
-        this.lfs = lfs;
-        this.stripeLength = stripeLength;
-        // Open a stream to the first block.
-        openCurrentStream();
-      }
-
-      private void closeCurrentStream() throws IOException {
-        if (currentStream != null) {
-          currentStream.close();
-          currentStream = null;
-        }
-      }
-
-      /**
-       * Open a stream to the file containing the current block
-       * and seek to the appropriate offset
-       */
-      private void openCurrentStream() throws IOException {
-        int blockIdx = (int)(currentOffset/stat.getBlockSize());
-        UnderlyingBlock block = underlyingBlocks[blockIdx];
-        // If the current path is the same as we want.
-        if (currentBlock == block ||
-           currentBlock != null && currentBlock.path == block.path) {
-          // If we have a valid stream, nothing to do.
-          if (currentStream != null) {
-            currentBlock = block;
-            return;
-          }
-        } else {
-          closeCurrentStream();
-        }
-        currentBlock = block;
-        currentStream = lfs.fs.open(currentBlock.path, buffersize);
-        long offset = block.actualFileOffset +
-          (currentOffset - block.originalFileOffset);
-        currentStream.seek(offset);
-      }
-
-      /**
-       * Returns the number of bytes available in the current block.
-       */
-      private int blockAvailable() {
-        return (int) (currentBlock.length -
-                (currentOffset - currentBlock.originalFileOffset));
-      }
-
-      @Override
-      public synchronized int available() throws IOException {
-        // Application should not assume that any bytes are buffered here.
-        nextLocation = 0;
-        return Math.min(blockAvailable(), currentStream.available());
-      }
-
-      @Override
-      public synchronized  void close() throws IOException {
-        closeCurrentStream();
-        super.close();
-      }
-
-      @Override
-      public boolean markSupported() { return false; }
-
-      @Override
-      public void mark(int readLimit) {
-        // Mark and reset are not supported.
-        nextLocation = 0;
-      }
-
-      @Override
-      public void reset() throws IOException {
-        // Mark and reset are not supported.
-        nextLocation = 0;
-      }
-
-      @Override
-      public synchronized int read() throws IOException {
-        int value = read(oneBytebuff);
-        if (value < 0) {
-          return value;
-        } else {
-          return oneBytebuff[0];
-        }
-      }
-
-      @Override
-      public synchronized int read(byte[] b) throws IOException {
-        int value = read(b, 0, b.length);
-        nextLocation = 0;
-        return value;
-      }
-
-      @Override
-      public synchronized int read(byte[] b, int offset, int len) 
-        throws IOException {
-        while (true) {
-          openCurrentStream();
-          try{
-            int limit = Math.min(blockAvailable(), len);
-            int value = currentStream.read(b, offset, limit);
-            currentOffset += value;
-            nextLocation = 0;
-            return value;
-          } catch (BlockMissingException e) {
-            setAlternateLocations(e, currentOffset);
-          } catch (ChecksumException e) {
-            setAlternateLocations(e, currentOffset);
-          }
-        }
-      }
-      
-      @Override
-      public synchronized int read(long position, byte[] b, int offset, int len) 
-        throws IOException {
-        long oldPos = currentOffset;
-        seek(position);
-        try {
-          return read(b, offset, len);
-        } finally {
-          seek(oldPos);
-        }
-      }
-      
-      @Override
-      public synchronized long skip(long n) throws IOException {
-        long skipped = 0;
-        while (skipped < n) {
-          int val = read();
-          if (val < 0) {
-            break;
-          }
-          skipped++;
-        }
-        nextLocation = 0;
-        return skipped;
-      }
-      
-      @Override
-      public synchronized long getPos() throws IOException {
-        nextLocation = 0;
-        return currentOffset;
-      }
-      
-      @Override
-      public synchronized void seek(long pos) throws IOException {
-        if (pos != currentOffset) {
-          closeCurrentStream();
-          currentOffset = pos;
-          openCurrentStream();
-        }
-        nextLocation = 0;
-      }
-
-      @Override
-      public boolean seekToNewSource(long targetPos) throws IOException {
-        seek(targetPos);
-        boolean value = currentStream.seekToNewSource(currentStream.getPos());
-        nextLocation = 0;
-        return value;
-      }
-      
-      /**
-       * position readable again.
-       */
-      @Override
-      public void readFully(long pos, byte[] b, int offset, int length) 
-        throws IOException {
-        long oldPos = currentOffset;
-        seek(pos);
-        try {
-          while (true) {
-            // This loop retries reading until successful. Unrecoverable errors
-            // cause exceptions.
-            // currentOffset is changed by read().
-            try {
-              while (length > 0) {
-                int n = read(b, offset, length);
-                if (n < 0) {
-                  throw new IOException("Premature EOF");
-                }
-                offset += n;
-                length -= n;
-              }
-              nextLocation = 0;
-              return;
-            } catch (BlockMissingException e) {
-              setAlternateLocations(e, currentOffset);
-            } catch (ChecksumException e) {
-              setAlternateLocations(e, currentOffset);
-            }
-          }
-        } finally {
-          seek(oldPos);
-        }
-      }
-
-      @Override
-      public void readFully(long pos, byte[] b) throws IOException {
-        readFully(pos, b, 0, b.length);
-        nextLocation = 0;
-      }
-
-      /**
-       * Extract good block from RAID
-       * @throws IOException if all alternate locations are exhausted
-       */
-      private void setAlternateLocations(IOException curexp, long offset) 
-        throws IOException {
-        while (alternates != null && nextLocation < alternates.length) {
-          try {
-            int idx = nextLocation++;
-            // Start offset of block.
-            long corruptOffset =
-              (offset / stat.getBlockSize()) * stat.getBlockSize();
-            // Make sure we use DFS and not DistributedRaidFileSystem for unRaid.
-            Configuration clientConf = new Configuration(conf);
-            Class<?> clazz = conf.getClass("fs.raid.underlyingfs.impl",
-                                                DistributedFileSystem.class);
-            clientConf.set("fs.hdfs.impl", clazz.getName());
-            // Disable caching so that a previously cached RaidDfs is not used.
-            clientConf.setBoolean("fs.hdfs.impl.disable.cache", true);
-            Path npath = RaidNode.unRaidCorruptBlock(clientConf, path,
-                         alternates[idx].destPath,
-                         alternates[idx].createDecoder(),
-                         stripeLength, corruptOffset);
-            if (npath == null)
-              continue;
-            try {
-              String outdir = conf.get("fs.raid.recoverylogdir");
-              if (outdir != null) {
-                DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
-                java.util.Date date = new java.util.Date();
-                String fname = path.getName() + dateFormat.format(date) +
-                (new Random()).nextInt() + ".txt";
-                Path outputunraid = new Path(outdir, fname);
-                FileSystem fs = outputunraid.getFileSystem(conf);
-                FSDataOutputStream dout = fs.create(outputunraid);
-                PrintStream ps = new PrintStream(dout);
-                ps.println("Recovery attempt log");
-                ps.println("Source path : " + path );
-                ps.println("Alternate path : " + alternates[idx].destPath);
-                ps.println("Stripe lentgh : " + stripeLength);
-                ps.println("Corrupt offset : " + corruptOffset);
-                String output = (npath==null) ? "UNSUCCESSFUL" : npath.toString();
-                ps.println("Output from unRaid : " + output);
-                ps.close();
-              }
-            } catch (Exception exc) {
-              LOG.info("Error while creating recovery log: " + exc);
-            }
-
-            closeCurrentStream();
-            LOG.info("Using block at offset " + corruptOffset + " from " +
-              npath);
-            currentBlock.path = npath;
-            currentBlock.actualFileOffset = 0;  // Single block in file.
-            // Dont change currentOffset, in case the user had done a seek?
-            openCurrentStream();
-
-            return;
-          } catch (Exception e) {
-            LOG.info("Error in using alternate path " + path + ". " + e +
-                     " Ignoring...");
-          }
-        }
-        throw curexp;
-      }
-
-      /**
-       * The name of the file system that is immediately below the
-       * DistributedRaidFileSystem. This is specified by the
-       * configuration parameter called fs.raid.underlyingfs.impl.
-       * If this parameter is not specified in the configuration, then
-       * the default class DistributedFileSystem is returned.
-       * @param conf the configuration object
-       * @return the filesystem object immediately below DistributedRaidFileSystem
-       * @throws IOException if all alternate locations are exhausted
-       */
-      private FileSystem getUnderlyingFileSystem(Configuration conf) {
-        Class<?> clazz = conf.getClass("fs.raid.underlyingfs.impl", DistributedFileSystem.class);
-        FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
-        return fs;
-      }
-    }
-  
-    /**
-     * constructor for ext input stream.
-     * @param fs the underlying filesystem
-     * @param p the path in the underlying file system
-     * @param buffersize the size of IO
-     * @throws IOException
-     */
-    public ExtFSDataInputStream(Configuration conf, DistributedRaidFileSystem lfs,
-      DecodeInfo[] alternates, Path  p, int stripeLength, int buffersize) throws IOException {
-        super(new ExtFsInputStream(conf, lfs, alternates, p, stripeLength, buffersize));
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/RaidDFSUtil.java

@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs;
-
-import java.io.IOException;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.io.PrintStream;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.HashSet;
-import java.util.Set;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.tools.DFSck;
-import org.apache.hadoop.util.ToolRunner;
-
-public abstract class RaidDFSUtil {
-  /**
-   * Returns the corrupt blocks in a file.
-   */
-  public static List<LocatedBlock> corruptBlocksInFile(
-    DistributedFileSystem dfs, String path, long offset, long length)
-  throws IOException {
-    List<LocatedBlock> corrupt = new LinkedList<LocatedBlock>();
-    LocatedBlocks locatedBlocks =
-      getBlockLocations(dfs, path, offset, length);
-    for (LocatedBlock b: locatedBlocks.getLocatedBlocks()) {
-      if (b.isCorrupt() ||
-         (b.getLocations().length == 0 && b.getBlockSize() > 0)) {
-        corrupt.add(b);
-      }
-    }
-    return corrupt;
-  }
-
-  public static LocatedBlocks getBlockLocations(
-    DistributedFileSystem dfs, String path, long offset, long length)
-    throws IOException {
-    return dfs.getClient().namenode.getBlockLocations(path, offset, length);
-  }
-
-  /**
-   * Make successive calls to listCorruptFiles to obtain all 
-   * corrupt files.
-   */ 
-  public static String[] getCorruptFiles(DistributedFileSystem dfs)
-    throws IOException {
-    Set<String> corruptFiles = new HashSet<String>();
-    RemoteIterator<Path> cfb = dfs.listCorruptFileBlocks(new Path("/"));
-    while (cfb.hasNext()) {
-      corruptFiles.add(cfb.next().toUri().getPath());
-    }
-
-    return corruptFiles.toArray(new String[corruptFiles.size()]);
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRaid.java

@@ -1,632 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.blockmanagement;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.LinkedHashMap;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.Comparator;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.server.namenode.*;
-import org.apache.hadoop.net.NetworkTopology;
-import org.apache.hadoop.net.Node;
-import org.apache.hadoop.raid.RaidNode;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Time;
-
-/**
- * This BlockPlacementPolicy spreads out the group of blocks which used by RAID
- * for recovering each other. This is important for the availability
- * of the blocks. This class can be used by multiple threads. It has to be
- * thread safe.
- */
-public class BlockPlacementPolicyRaid extends BlockPlacementPolicy {
-  public static final Log LOG =
-    LogFactory.getLog(BlockPlacementPolicyRaid.class);
-  Configuration conf;
-  private int stripeLength;
-  private int xorParityLength;
-  private int rsParityLength;
-  private String xorPrefix = null;
-  private String rsPrefix = null;
-  private String raidTempPrefix = null;
-  private String raidrsTempPrefix = null;
-  private String raidHarTempPrefix = null;
-  private String raidrsHarTempPrefix = null;
-  private FSNamesystem namesystem = null;
-  private BlockPlacementPolicyDefault defaultPolicy;
-
-  CachedLocatedBlocks cachedLocatedBlocks;
-  CachedFullPathNames cachedFullPathNames;
-
-  /** {@inheritDoc} */
-  @Override
-  public void initialize(Configuration conf,  FSClusterStats stats,
-                         NetworkTopology clusterMap) {
-    this.conf = conf;
-    this.stripeLength = RaidNode.getStripeLength(conf);
-    this.rsParityLength = RaidNode.rsParityLength(conf);
-    this.xorParityLength = 1;
-    try {
-      this.xorPrefix = RaidNode.xorDestinationPath(conf).toUri().getPath();
-      this.rsPrefix = RaidNode.rsDestinationPath(conf).toUri().getPath();
-    } catch (IOException e) {
-    }
-    if (this.xorPrefix == null) {
-      this.xorPrefix = RaidNode.DEFAULT_RAID_LOCATION;
-    }
-    if (this.rsPrefix == null) {
-      this.rsPrefix = RaidNode.DEFAULT_RAIDRS_LOCATION;
-    }
-    // Throws ClassCastException if we cannot cast here.
-    this.namesystem = (FSNamesystem) stats;
-    this.cachedLocatedBlocks = new CachedLocatedBlocks(namesystem);
-    this.cachedFullPathNames = new CachedFullPathNames(namesystem);
-    this.raidTempPrefix = RaidNode.xorTempPrefix(conf);
-    this.raidrsTempPrefix = RaidNode.rsTempPrefix(conf);
-    this.raidHarTempPrefix = RaidNode.xorHarTempPrefix(conf);
-    this.raidrsHarTempPrefix = RaidNode.rsHarTempPrefix(conf);
-    defaultPolicy = new BlockPlacementPolicyDefault(conf, stats, clusterMap);
-  }
-
-  @Override
-  DatanodeDescriptor[] chooseTarget(String srcPath, int numOfReplicas,
-      DatanodeDescriptor writer, List<DatanodeDescriptor> chosenNodes,
-      long blocksize) {
-    return chooseTarget(srcPath, numOfReplicas, writer, chosenNodes,
-        null, blocksize);
-  }
-
-  @Override
-  public DatanodeDescriptor[] chooseTarget(String srcPath, int numOfReplicas,
-      DatanodeDescriptor writer, List<DatanodeDescriptor> chosenNodes,
-      boolean returnChosenNodes,
-      HashMap<Node, Node> excludedNodes, long blocksize) {
-    try {
-      FileType type = getFileType(srcPath);
-      if (type == FileType.NOT_RAID) {
-        return defaultPolicy.chooseTarget(
-            srcPath, numOfReplicas, writer, chosenNodes, blocksize);
-      }
-      if (excludedNodes == null) {
-        excludedNodes = new HashMap<Node, Node>();
-      }
-      addExcludedNodes(srcPath, type, excludedNodes);
-      DatanodeDescriptor[] result =
-        defaultPolicy.chooseTarget(numOfReplicas, writer,
-          chosenNodes, returnChosenNodes, excludedNodes, blocksize);
-      // Add the added block locations in the block locations cache.
-      // So the rest of the blocks know about these locations.
-      cachedLocatedBlocks.get(srcPath).
-          add(new LocatedBlock(new ExtendedBlock(), result));
-      return result;
-    } catch (Exception e) {
-      LOG.debug("Error happend when choosing datanode to write:" +
-        StringUtils.stringifyException(e));
-      return defaultPolicy.chooseTarget(srcPath, numOfReplicas, writer,
-                                chosenNodes, blocksize);
-    }
-  }
-
-  @Override
-  public int verifyBlockPlacement(String srcPath, LocatedBlock lBlk,
-      int minRacks) {
-    return defaultPolicy.verifyBlockPlacement(srcPath, lBlk, minRacks);
-  }
-
-  /** {@inheritDoc} */
-  @Override
-  public DatanodeDescriptor chooseReplicaToDelete(BlockCollection bc,
-      Block block, short replicationFactor,
-      Collection<DatanodeDescriptor> first,
-      Collection<DatanodeDescriptor> second) {
-
-    DatanodeDescriptor chosenNode = null;
-    try {
-      String path = cachedFullPathNames.get(bc);
-      FileType type = getFileType(path);
-      if (type == FileType.NOT_RAID) {
-        return defaultPolicy.chooseReplicaToDelete(
-            bc, block, replicationFactor, first, second);
-      }
-      List<LocatedBlock> companionBlocks =
-          getCompanionBlocks(path, type, block);
-      if (companionBlocks == null || companionBlocks.size() == 0) {
-        // Use the default method if it is not a valid raided or parity file
-        return defaultPolicy.chooseReplicaToDelete(
-            bc, block, replicationFactor, first, second);
-      }
-      // Delete from the first collection first
-      // This ensures the number of unique rack of this block is not reduced
-      Collection<DatanodeDescriptor> all = new HashSet<DatanodeDescriptor>();
-      all.addAll(first);
-      all.addAll(second);
-      chosenNode = chooseReplicaToDelete(companionBlocks, all);
-      if (chosenNode != null) {
-        return chosenNode;
-      }
-      return defaultPolicy.chooseReplicaToDelete(
-          bc, block, replicationFactor, first, second);
-    } catch (Exception e) {
-      LOG.debug("Error happend when choosing replica to delete" +
-        StringUtils.stringifyException(e));
-      return defaultPolicy.chooseReplicaToDelete(
-          bc, block, replicationFactor, first, second);
-    }
-  }
-
-  /**
-   * Obtain the excluded nodes for the current block that is being written
-   */
-  void addExcludedNodes(String file, FileType type, HashMap<Node, Node> excluded)
-      throws IOException {
-    Collection<LocatedBlock> blocks = getCompanionBlocks(file, type, null);
-    if (blocks == null) {
-      return;
-    }
-    for (LocatedBlock b : blocks) {
-      for (Node n : b.getLocations()) {
-        excluded.put(n, n);
-      }
-    }
-  }
-
-  private DatanodeDescriptor chooseReplicaToDelete(
-      Collection<LocatedBlock> companionBlocks,
-      Collection<DatanodeDescriptor> dataNodes) throws IOException {
-
-    if (dataNodes.isEmpty()) {
-      return null;
-    }
-    // Count the number of replicas on each node and rack
-    final Map<String, Integer> nodeCompanionBlockCount =
-      countCompanionBlocks(companionBlocks, false);
-    final Map<String, Integer> rackCompanionBlockCount =
-      countCompanionBlocks(companionBlocks, true);
-
-    NodeComparator comparator =
-      new NodeComparator(nodeCompanionBlockCount, rackCompanionBlockCount);
-    return Collections.max(dataNodes, comparator);
-  }
-
-  /**
-   * Count how many companion blocks are on each datanode or the each rack
-   * @param companionBlocks a collection of all the companion blocks
-   * @param doRackCount count the companion blocks on the racks of datanodes
-   * @param result the map from node name to the number of companion blocks
-   */
-  static Map<String, Integer> countCompanionBlocks(
-      Collection<LocatedBlock> companionBlocks, boolean doRackCount) {
-    Map<String, Integer> result = new HashMap<String, Integer>();
-    for (LocatedBlock block : companionBlocks) {
-      for (DatanodeInfo d : block.getLocations()) {
-        String name = doRackCount ? d.getParent().getName() : d.getName();
-        if (result.containsKey(name)) {
-          int count = result.get(name) + 1;
-          result.put(name, count);
-        } else {
-          result.put(name, 1);
-        }
-      }
-    }
-    return result;
-  }
-
-  /**
-   * Compares the datanodes based on the number of companion blocks on the same
-   * node and rack. If even, compare the remaining space on the datanodes.
-   */
-  class NodeComparator implements Comparator<DatanodeDescriptor> {
-    private Map<String, Integer> nodeBlockCount;
-    private Map<String, Integer> rackBlockCount;
-    private NodeComparator(Map<String, Integer> nodeBlockCount,
-                           Map<String, Integer> rackBlockCount) {
-      this.nodeBlockCount = nodeBlockCount;
-      this.rackBlockCount = rackBlockCount;
-    }
-    @Override
-    public int compare(DatanodeDescriptor d1, DatanodeDescriptor d2) {
-      int res = compareBlockCount(d1, d2, nodeBlockCount);
-      if (res != 0) {
-        return res;
-      }
-      res = compareBlockCount(d1.getParent(), d2.getParent(), rackBlockCount);
-      if (res != 0) {
-        return res;
-      }
-      if (d1.getRemaining() > d2.getRemaining()) {
-        return -1;
-      }
-      if (d1.getRemaining() < d2.getRemaining()) {
-        return 1;
-      }
-      return 0;
-    }
-    private int compareBlockCount(Node node1, Node node2,
-                                  Map<String, Integer> blockCount) {
-      Integer count1 = blockCount.get(node1.getName());
-      Integer count2 = blockCount.get(node2.getName());
-      count1 = count1 == null ? 0 : count1;
-      count2 = count2 == null ? 0 : count2;
-      if (count1 > count2) {
-        return 1;
-      }
-      if (count1 < count2) {
-        return -1;
-      }
-      return 0;
-    }
-  }
-
-  /**
-   * Obtain the companion blocks of the give block
-   * Companion blocks are defined as the blocks that can help recover each
-   * others by using raid decoder.
-   * @param path The path of the file contains the block
-   * @param type The type of this file
-   * @param block The given block
-   *              null if it is the block which is currently being written to
-   * @return the block locations of companion blocks
-   */
-  List<LocatedBlock> getCompanionBlocks(String path, FileType type,
-      Block block) throws IOException {
-    switch (type) {
-      case NOT_RAID:
-        return new ArrayList<LocatedBlock>();
-      case XOR_HAR_TEMP_PARITY:
-        return getCompanionBlocksForHarParityBlock(
-            path, xorParityLength, block);
-      case RS_HAR_TEMP_PARITY:
-        return getCompanionBlocksForHarParityBlock(
-            path, rsParityLength, block);
-      case XOR_TEMP_PARITY:
-        return getCompanionBlocksForParityBlock(
-            getSourceFile(path, raidTempPrefix), path, xorParityLength, block);
-      case RS_TEMP_PARITY:
-        return getCompanionBlocksForParityBlock(
-            getSourceFile(path, raidrsTempPrefix), path, rsParityLength, block);
-      case XOR_PARITY:
-        return getCompanionBlocksForParityBlock(getSourceFile(path, xorPrefix),
-            path, xorParityLength, block);
-      case RS_PARITY:
-        return getCompanionBlocksForParityBlock(getSourceFile(path, rsPrefix),
-            path, rsParityLength, block);
-      case XOR_SOURCE:
-        return getCompanionBlocksForSourceBlock(
-            path, getParityFile(path), xorParityLength, block);
-      case RS_SOURCE:
-        return getCompanionBlocksForSourceBlock(
-            path, getParityFile(path), xorParityLength, block);
-    }
-    return new ArrayList<LocatedBlock>();
-  }
-
-  private List<LocatedBlock> getCompanionBlocksForHarParityBlock(
-      String parity, int parityLength, Block block)
-      throws IOException {
-    int blockIndex = getBlockIndex(parity, block);
-    // consider only parity file in this case because source file block
-    // location is not easy to obtain
-    List<LocatedBlock> parityBlocks = cachedLocatedBlocks.get(parity);
-    List<LocatedBlock> result = new ArrayList<LocatedBlock>();
-    synchronized (parityBlocks) {
-      int start = Math.max(0, blockIndex - parityLength + 1);
-      int end = Math.min(parityBlocks.size(), blockIndex + parityLength);
-      result.addAll(parityBlocks.subList(start, end));
-    }
-    return result;
-  }
-
-  private List<LocatedBlock> getCompanionBlocksForParityBlock(
-      String src, String parity, int parityLength, Block block)
-      throws IOException {
-    int blockIndex = getBlockIndex(parity, block);
-    List<LocatedBlock> result = new ArrayList<LocatedBlock>();
-    List<LocatedBlock> parityBlocks = cachedLocatedBlocks.get(parity);
-    int stripeIndex = blockIndex / parityLength;
-    synchronized (parityBlocks) {
-      int parityStart = stripeIndex * parityLength;
-      int parityEnd = Math.min(parityStart + parityLength,
-                               parityBlocks.size());
-      // for parity, always consider the neighbor blocks as companion blocks
-      if (parityStart < parityBlocks.size()) {
-        result.addAll(parityBlocks.subList(parityStart, parityEnd));
-      }
-    }
-
-    if (src == null) {
-      return result;
-    }
-    List<LocatedBlock> sourceBlocks = cachedLocatedBlocks.get(src);
-    synchronized (sourceBlocks) {
-      int sourceStart = stripeIndex * stripeLength;
-      int sourceEnd = Math.min(sourceStart + stripeLength,
-                               sourceBlocks.size());
-      if (sourceStart < sourceBlocks.size()) {
-        result.addAll(sourceBlocks.subList(sourceStart, sourceEnd));
-      }
-    }
-    return result;
-  }
-
-  private List<LocatedBlock> getCompanionBlocksForSourceBlock(
-      String src, String parity, int parityLength, Block block)
-      throws IOException {
-    int blockIndex = getBlockIndex(src, block);
-    List<LocatedBlock> result = new ArrayList<LocatedBlock>();
-    List<LocatedBlock> sourceBlocks = cachedLocatedBlocks.get(src);
-    int stripeIndex = blockIndex / stripeLength;
-    synchronized (sourceBlocks) {
-      int sourceStart = stripeIndex * stripeLength;
-      int sourceEnd = Math.min(sourceStart + stripeLength,
-                               sourceBlocks.size());
-      if (sourceStart < sourceBlocks.size()) {
-        result.addAll(sourceBlocks.subList(sourceStart, sourceEnd));
-      }
-    }
-    if (parity == null) {
-      return result;
-    }
-    List<LocatedBlock> parityBlocks = cachedLocatedBlocks.get(parity);
-    synchronized (parityBlocks) {
-      int parityStart = stripeIndex * parityLength;
-      int parityEnd = Math.min(parityStart + parityLength,
-                               parityBlocks.size());
-      if (parityStart < parityBlocks.size()) {
-        result.addAll(parityBlocks.subList(parityStart, parityEnd));
-      }
-    }
-    return result;
-  }
-
-  private int getBlockIndex(String file, Block block) throws IOException {
-    List<LocatedBlock> blocks = cachedLocatedBlocks.get(file);
-    synchronized (blocks) {
-      // null indicates that this block is currently added. Return size()
-      // as the index in this case
-	    if (block == null) {
-	      return blocks.size();
-	    }
-      for (int i = 0; i < blocks.size(); i++) {
-        if (blocks.get(i).getBlock().getLocalBlock().equals(block)) {
-          return i;
-        }
-      }
-    }
-    throw new IOException("Cannot locate " + block + " in file " + file);
-  }
-
-  /**
-   * Cache results for getFullPathName()
-   */
-  static class CachedFullPathNames {
-    FSNamesystem namesystem;
-    CachedFullPathNames(FSNamesystem namesystem) {
-      this.namesystem = namesystem;
-    }
-    private Cache<INodeWithHashCode, String> cacheInternal =
-      new Cache<INodeWithHashCode, String>() {
-        @Override
-        public String getDirectly(INodeWithHashCode inode) throws IOException {
-          namesystem.readLock();
-          try {
-            return inode.getFullPathName();
-          } finally {
-            namesystem.readUnlock();
-          }
-        }
-      };
-
-    static private class INodeWithHashCode {
-      BlockCollection bc;
-      INodeWithHashCode(BlockCollection bc) {
-        this.bc= bc;
-      }
-      @Override
-      public boolean equals(Object obj) {
-        return bc== obj;
-      }
-      @Override
-      public int hashCode() {
-        return System.identityHashCode(bc);
-      }
-      String getFullPathName() {
-        return bc.getName();
-      }
-    }
-
-    public String get(BlockCollection bc) throws IOException {
-      return cacheInternal.get(new INodeWithHashCode(bc));
-    }
-  }
-
-  /**
-   * Cache results for FSNamesystem.getBlockLocations()
-   */
-  static class CachedLocatedBlocks extends Cache<String, List<LocatedBlock>> {
-    FSNamesystem namesystem;
-    CachedLocatedBlocks(FSNamesystem namesystem) {
-      this.namesystem = namesystem;
-    }
-    @Override
-    public List<LocatedBlock> getDirectly(String file) throws IOException {
-      long len = NameNodeRaidUtil.getFileInfo(namesystem, file, true).getLen();
-      List<LocatedBlock> result = NameNodeRaidUtil.getBlockLocations(namesystem,
-          file, 0L, len, false, false).getLocatedBlocks();
-      if (result == null || result.isEmpty()) {
-        result = new ArrayList<LocatedBlock>();
-      }
-      return Collections.synchronizedList(result);
-    }
-  }
-
-  static abstract class Cache<K, V> {
-    private Map<K, ValueWithTime> cache;
-    private static final long CACHE_TIMEOUT = 300000L; // 5 minutes
-    // The timeout is long but the consequence of stale value is not serious
-    Cache() {
-      Map<K, ValueWithTime> map = new LinkedHashMap<K, ValueWithTime>() {
-        private static final long serialVersionUID = 1L;
-          final private int MAX_ENTRIES = 50000;
-          @Override
-          protected boolean removeEldestEntry(
-            Map.Entry<K, ValueWithTime> eldest) {
-            return size() > MAX_ENTRIES;
-          }
-        };
-      this.cache = Collections.synchronizedMap(map);
-    }
-
-    // Note that this method may hold FSNamesystem.readLock() and it may
-    // be called inside FSNamesystem.writeLock(). If we make this method
-    // synchronized, it will deadlock.
-    abstract protected V getDirectly(K key) throws IOException;
-
-    public V get(K key) throws IOException {
-      // The method is not synchronized so we may get some stale value here but
-      // it's OK.
-      ValueWithTime result = cache.get(key);
-      long now = Time.now();
-      if (result != null &&
-          now - result.cachedTime < CACHE_TIMEOUT) {
-        return result.value;
-      }
-      result = new ValueWithTime();
-      result.value = getDirectly(key);
-      result.cachedTime = now;
-      cache.put(key, result);
-      return result.value;
-    }
-    private class ValueWithTime {
-      V value = null;
-      long cachedTime = 0L;
-    }
-  }
-
-  /**
-   * Get path for the corresponding source file for a valid parity
-   * file. Returns null if it does not exists
-   * @param parity the toUri path of the parity file
-   * @return the toUri path of the source file
-   */
-  String getSourceFile(String parity, String prefix) throws IOException {
-    if (isHarFile(parity)) {
-      return null;
-    }
-    // remove the prefix
-    String src = parity.substring(prefix.length());
-    if (NameNodeRaidUtil.getFileInfo(namesystem, src, true) == null) {
-      return null;
-    }
-    return src;
-  }
-
-  /**
-   * Get path for the corresponding parity file for a source file.
-   * Returns null if it does not exists
-   * @param src the toUri path of the source file
-   * @return the toUri path of the parity file
-   */
-  String getParityFile(String src) throws IOException {
-    String xorParity = getParityFile(xorPrefix, src);
-    if (xorParity != null) {
-      return xorParity;
-    }
-    String rsParity = getParityFile(rsPrefix, src);
-    if (rsParity != null) {
-      return rsParity;
-    }
-    return null;
-  }
-
-  /**
-   * Get path for the parity file. Returns null if it does not exists
-   * @param parityPrefix usuall "/raid/" or "/raidrs/"
-   * @return the toUri path of the parity file
-   */
-  private String getParityFile(String parityPrefix, String src)
-      throws IOException {
-    String parity = parityPrefix + src;
-    if (NameNodeRaidUtil.getFileInfo(namesystem, parity, true) == null) {
-      return null;
-    }
-    return parity;
-  }
-
-  private boolean isHarFile(String path) {
-    return path.lastIndexOf(RaidNode.HAR_SUFFIX) != -1;
-  }
-
-  enum FileType {
-    NOT_RAID,
-    XOR_HAR_TEMP_PARITY,
-    XOR_TEMP_PARITY,
-    XOR_PARITY,
-    XOR_SOURCE,
-    RS_HAR_TEMP_PARITY,
-    RS_TEMP_PARITY,
-    RS_PARITY,
-    RS_SOURCE,
-  }
-
-  FileType getFileType(String path) throws IOException {
-    if (path.startsWith(raidHarTempPrefix + Path.SEPARATOR)) {
-      return FileType.XOR_HAR_TEMP_PARITY;
-    }
-    if (path.startsWith(raidrsHarTempPrefix + Path.SEPARATOR)) {
-      return FileType.RS_HAR_TEMP_PARITY;
-    }
-    if (path.startsWith(raidTempPrefix + Path.SEPARATOR)) {
-      return FileType.XOR_TEMP_PARITY;
-    }
-    if (path.startsWith(raidrsTempPrefix + Path.SEPARATOR)) {
-      return FileType.RS_TEMP_PARITY;
-    }
-    if (path.startsWith(xorPrefix + Path.SEPARATOR)) {
-      return FileType.XOR_PARITY;
-    }
-    if (path.startsWith(rsPrefix + Path.SEPARATOR)) {
-      return FileType.RS_PARITY;
-    }
-    String parity = getParityFile(path);
-    if (parity == null) {
-      return FileType.NOT_RAID;
-    }
-    if (parity.startsWith(xorPrefix + Path.SEPARATOR)) {
-      return FileType.XOR_SOURCE;
-    }
-    if (parity.startsWith(rsPrefix + Path.SEPARATOR)) {
-      return FileType.RS_SOURCE;
-    }
-    return FileType.NOT_RAID;
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/datanode/RaidBlockSender.java

@@ -1,505 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.SocketException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.Arrays;
-
-import org.apache.commons.logging.Log;
-import org.apache.hadoop.fs.ChecksumException;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
-import org.apache.hadoop.hdfs.util.DataTransferThrottler;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.net.SocketOutputStream;
-import org.apache.hadoop.util.DataChecksum;
-import org.apache.hadoop.util.StringUtils;
-
-/**
- * Reads a block from the disk and sends it to a recipient.
- */
-public class RaidBlockSender implements java.io.Closeable {
-  public static final Log LOG = DataNode.LOG;
-  static final Log ClientTraceLog = DataNode.ClientTraceLog;
-  
-  private ExtendedBlock block; // the block to read from
-
-  /** The visible length of a replica. */
-  private final long replicaVisibleLength;
-
-  private InputStream blockIn; // data stream
-  private long blockInPosition = -1; // updated while using transferTo().
-  private DataInputStream checksumIn; // checksum datastream
-  private DataChecksum checksum; // checksum stream
-  private long offset; // starting position to read
-  /** Initial position to read */
-  private long initialOffset;
-  private long endOffset; // ending position
-  private int chunkSize; // chunk size
-  private int checksumSize; // checksum size
-  private boolean corruptChecksumOk; // if need to verify checksum
-  private boolean chunkOffsetOK; // if need to send chunk offset
-  private long seqno; // sequence number of packet
-
-  private boolean transferToAllowed = true;
-  private boolean blockReadFully; //set when the whole block is read
-  private boolean verifyChecksum; //if true, check is verified while reading
-  private final String clientTraceFmt; // format of client trace log message
-
-  /**
-   * Minimum buffer used while sending data to clients. Used only if
-   * transferTo() is enabled. 64KB is not that large. It could be larger, but
-   * not sure if there will be much more improvement.
-   */
-  private static final int MIN_BUFFER_WITH_TRANSFERTO = 64*1024;
-  private static final int TRANSFERTO_BUFFER_SIZE = Math.max(
-      HdfsConstants.IO_FILE_BUFFER_SIZE, MIN_BUFFER_WITH_TRANSFERTO);
-  private volatile ChunkChecksum lastChunkChecksum = null;
-
-  
-  public RaidBlockSender(ExtendedBlock block, long blockLength, long startOffset, long length,
-              boolean corruptChecksumOk, boolean chunkOffsetOK,
-              boolean verifyChecksum, boolean transferToAllowed,
-              DataInputStream metadataIn, InputStreamFactory streamFactory
-              ) throws IOException {
-    this(block, blockLength, startOffset, length,
-        corruptChecksumOk, chunkOffsetOK,
-        verifyChecksum, transferToAllowed,
-        metadataIn, streamFactory, null);
-  }
-
-  public RaidBlockSender(ExtendedBlock block, long blockLength, long startOffset, long length,
-              boolean corruptChecksumOk, boolean chunkOffsetOK,
-              boolean verifyChecksum, boolean transferToAllowed,
-              DataInputStream metadataIn, InputStreamFactory streamFactory,
-              String clientTraceFmt) throws IOException {
-    try {
-      this.block = block;
-      this.chunkOffsetOK = chunkOffsetOK;
-      this.corruptChecksumOk = corruptChecksumOk;
-      this.verifyChecksum = verifyChecksum;
-      this.replicaVisibleLength = blockLength;
-      this.transferToAllowed = transferToAllowed;
-      this.clientTraceFmt = clientTraceFmt;
-
-      if ( !corruptChecksumOk || metadataIn != null) {
-        this.checksumIn = metadataIn;
-
-        // read and handle the common header here. For now just a version
-       BlockMetadataHeader header = BlockMetadataHeader.readHeader(checksumIn);
-       short version = header.getVersion();
-
-        if (version != BlockMetadataHeader.VERSION) {
-          LOG.warn("Wrong version (" + version + ") for metadata file for "
-              + block + " ignoring ...");
-        }
-        checksum = header.getChecksum();
-      } else {
-        LOG.warn("Could not find metadata file for " + block);
-        // This only decides the buffer size. Use BUFFER_SIZE?
-        checksum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_NULL,
-            16 * 1024);
-      }
-
-      /* If bytesPerChecksum is very large, then the metadata file
-       * is mostly corrupted. For now just truncate bytesPerchecksum to
-       * blockLength.
-       */        
-      int size = checksum.getBytesPerChecksum();
-      if (size > 10*1024*1024 && size > replicaVisibleLength) {
-        checksum = DataChecksum.newDataChecksum(checksum.getChecksumType(),
-            Math.max((int)replicaVisibleLength, 10*1024*1024));
-        size = checksum.getBytesPerChecksum();        
-      }
-      chunkSize = size;
-      checksumSize = checksum.getChecksumSize();
-
-      if (length < 0) {
-        length = replicaVisibleLength;
-      }
-
-      endOffset = blockLength;
-      
-      if (startOffset < 0 || startOffset > endOffset
-          || (length + startOffset) > endOffset) {
-        String msg = " Offset " + startOffset + " and length " + length
-        + " don't match block " + block + " ( blockLen " + endOffset + " )";
-        LOG.warn("sendBlock() : " + msg);
-        throw new IOException(msg);
-      }
-      
-      offset = (startOffset - (startOffset % chunkSize));
-      if (length >= 0) {
-        // Make sure endOffset points to end of a checksumed chunk.
-        long tmpLen = startOffset + length;
-        if (tmpLen % chunkSize != 0) {
-          tmpLen += (chunkSize - tmpLen % chunkSize);
-        }
-        if (tmpLen < endOffset) {
-          // will use on-disk checksum here since the end is a stable chunk
-          endOffset = tmpLen;
-        }
-      }
-
-      // seek to the right offsets
-      if (offset > 0) {
-        long checksumSkip = (offset / chunkSize) * checksumSize;
-        // note blockInStream is seeked when created below
-        if (checksumSkip > 0) {
-          // Should we use seek() for checksum file as well?
-          IOUtils.skipFully(checksumIn, checksumSkip);
-        }
-      }
-      seqno = 0;
-      
-      blockIn = streamFactory.createStream(offset);
-    } catch (IOException ioe) {
-      IOUtils.closeStream(this);
-      IOUtils.closeStream(blockIn);
-      throw ioe;
-    }
-  }
-  
-  /**
-   * close opened files.
-   */
-  public void close() throws IOException {
-    IOException ioe = null;
-    // close checksum file
-    if(checksumIn!=null) {
-      try {
-        checksumIn.close();
-      } catch (IOException e) {
-        ioe = e;
-      }
-      checksumIn = null;
-    }
-    // close data file
-    if(blockIn!=null) {
-      try {
-        blockIn.close();
-      } catch (IOException e) {
-        ioe = e;
-      }
-      blockIn = null;
-    }
-    // throw IOException if there is any
-    if(ioe!= null) {
-      throw ioe;
-    }
-  }
-
-  /**
-   * Converts an IOExcpetion (not subclasses) to SocketException.
-   * This is typically done to indicate to upper layers that the error 
-   * was a socket error rather than often more serious exceptions like 
-   * disk errors.
-   */
-  private static IOException ioeToSocketException(IOException ioe) {
-    if (ioe.getClass().equals(IOException.class)) {
-      // "se" could be a new class in stead of SocketException.
-      IOException se = new SocketException("Original Exception : " + ioe);
-      se.initCause(ioe);
-      /* Change the stacktrace so that original trace is not truncated
-       * when printed.*/ 
-      se.setStackTrace(ioe.getStackTrace());
-      return se;
-    }
-    // otherwise just return the same exception.
-    return ioe;
-  }
-  
-  /**
-   * @param datalen Length of data 
-   * @return number of chunks for data of given size
-   */
-  private int numberOfChunks(long datalen) {
-    return (int) ((datalen + chunkSize - 1)/chunkSize);
-  }
-  
-  /**
-   * Write packet header into {@code pkt}
-   */
-  private void writePacketHeader(ByteBuffer pkt, int dataLen, int packetLen) {
-    pkt.clear();
-    PacketHeader header = new PacketHeader(packetLen, offset, seqno,
-        (dataLen == 0), dataLen, false);
-    header.putInBuffer(pkt);
-  }
-  
-  /**
-   * Read checksum into given buffer
-   * @param buf buffer to read the checksum into
-   * @param checksumOffset offset at which to write the checksum into buf
-   * @param checksumLen length of checksum to write
-   * @throws IOException on error
-   */
-  private void readChecksum(byte[] buf, final int checksumOffset,
-      final int checksumLen) throws IOException {
-    if (checksumSize <= 0 && checksumIn == null) {
-      return;
-    }
-    try {
-      checksumIn.readFully(buf, checksumOffset, checksumLen);
-    } catch (IOException e) {
-      LOG.warn(" Could not read or failed to veirfy checksum for data"
-          + " at offset " + offset + " for block " + block, e);
-      IOUtils.closeStream(checksumIn);
-      checksumIn = null;
-      if (corruptChecksumOk) {
-        if (checksumOffset < checksumLen) {
-          // Just fill the array with zeros.
-          Arrays.fill(buf, checksumOffset, checksumLen, (byte) 0);
-        }
-      } else {
-        throw e;
-      }
-    }
-  }
-  
-  /**
-   * Sends a packet with up to maxChunks chunks of data.
-   * 
-   * @param pkt buffer used for writing packet data
-   * @param maxChunks maximum number of chunks to send
-   * @param out stream to send data to
-   * @param transferTo use transferTo to send data
-   * @param throttler used for throttling data transfer bandwidth
-   */
-  private int sendPacket(ByteBuffer pkt, int maxChunks, OutputStream out,
-      boolean transferTo, DataTransferThrottler throttler) throws IOException {
-    int dataLen = (int) Math.min(endOffset - offset,
-                             (chunkSize * (long) maxChunks));
-    
-    int numChunks = numberOfChunks(dataLen); // Number of chunks be sent in the packet
-    int checksumDataLen = numChunks * checksumSize;
-    int packetLen = dataLen + checksumDataLen + 4;
-    boolean lastDataPacket = offset + dataLen == endOffset && dataLen > 0;
-
-    writePacketHeader(pkt, dataLen, packetLen);
-
-    int checksumOff = pkt.position();
-    byte[] buf = pkt.array();
-    
-    if (checksumSize > 0 && checksumIn != null) {
-      readChecksum(buf, checksumOff, checksumDataLen);
-
-      // write in progress that we need to use to get last checksum
-      if (lastDataPacket && lastChunkChecksum != null) {
-        int start = checksumOff + checksumDataLen - checksumSize;
-        byte[] updatedChecksum = lastChunkChecksum.getChecksum();
-        
-        if (updatedChecksum != null) {
-          System.arraycopy(updatedChecksum, 0, buf, start, checksumSize);
-        }
-      }
-    }
-    
-    int dataOff = checksumOff + checksumDataLen;
-    if (!transferTo) { // normal transfer
-      IOUtils.readFully(blockIn, buf, dataOff, dataLen);
-
-      if (verifyChecksum) {
-        verifyChecksum(buf, dataOff, dataLen, numChunks, checksumOff);
-      }
-    }
-    
-    try {
-      if (transferTo) {
-        SocketOutputStream sockOut = (SocketOutputStream)out;
-        sockOut.write(buf, 0, dataOff); // First write checksum
-        
-        // no need to flush. since we know out is not a buffered stream. 
-        sockOut.transferToFully(((FileInputStream)blockIn).getChannel(), 
-                                blockInPosition, dataLen);
-        blockInPosition += dataLen;
-      } else { 
-        // normal transfer
-        out.write(buf, 0, dataOff + dataLen);
-      }
-    } catch (IOException e) {
-      /* Exception while writing to the client. Connection closure from
-       * the other end is mostly the case and we do not care much about
-       * it. But other things can go wrong, especially in transferTo(),
-       * which we do not want to ignore.
-       *
-       * The message parsing below should not be considered as a good
-       * coding example. NEVER do it to drive a program logic. NEVER.
-       * It was done here because the NIO throws an IOException for EPIPE.
-       */
-      String ioem = e.getMessage();
-      if (!ioem.startsWith("Broken pipe") && !ioem.startsWith("Connection reset")) {
-        LOG.error("BlockSender.sendChunks() exception: ", e);
-      }
-      throw ioeToSocketException(e);
-    }
-
-    if (throttler != null) { // rebalancing so throttle
-      throttler.throttle(packetLen);
-    }
-
-    return dataLen;
-  }
-  
-  /**
-   * Compute checksum for chunks and verify the checksum that is read from
-   * the metadata file is correct.
-   * 
-   * @param buf buffer that has checksum and data
-   * @param dataOffset position where data is written in the buf
-   * @param datalen length of data
-   * @param numChunks number of chunks corresponding to data
-   * @param checksumOffset offset where checksum is written in the buf
-   * @throws ChecksumException on failed checksum verification
-   */
-  public void verifyChecksum(final byte[] buf, final int dataOffset,
-      final int datalen, final int numChunks, final int checksumOffset)
-      throws ChecksumException {
-    int dOff = dataOffset;
-    int cOff = checksumOffset;
-    int dLeft = datalen;
-
-    for (int i = 0; i < numChunks; i++) {
-      checksum.reset();
-      int dLen = Math.min(dLeft, chunkSize);
-      checksum.update(buf, dOff, dLen);
-      if (!checksum.compare(buf, cOff)) {
-        long failedPos = offset + datalen - dLeft;
-        throw new ChecksumException("Checksum failed at " + failedPos,
-            failedPos);
-      }
-      dLeft -= dLen;
-      dOff += dLen;
-      cOff += checksumSize;
-    }
-  }
-
-
-  /**
-   * sendBlock() is used to read block and its metadata and stream the data to
-   * either a client or to another datanode. 
-   * 
-   * @param out  stream to which the block is written to
-   * @param baseStream optional. if non-null, <code>out</code> is assumed to 
-   *        be a wrapper over this stream. This enables optimizations for
-   *        sending the data, e.g. 
-   *        {@link SocketOutputStream#transferToFully(FileChannel, 
-   *        long, int)}.
-   * @return total bytes reads, including crc.
-   */
-  public long sendBlock(DataOutputStream out, OutputStream baseStream)
-      throws IOException {
-    if (out == null) {
-      throw new IOException( "out stream is null" );
-    }
-    initialOffset = offset;
-    long totalRead = 0;
-    OutputStream streamForSendChunks = out;
-    
-    final long startTime = ClientTraceLog.isInfoEnabled() ? System.nanoTime() : 0;
-    try {
-      int maxChunksPerPacket;
-      int pktSize = PacketHeader.PKT_HEADER_LEN;
-      boolean transferTo = transferToAllowed && !verifyChecksum
-          && baseStream instanceof SocketOutputStream
-          && blockIn instanceof FileInputStream;
-      if (transferTo) {
-        FileChannel fileChannel = ((FileInputStream)blockIn).getChannel();
-        blockInPosition = fileChannel.position();
-        streamForSendChunks = baseStream;
-        maxChunksPerPacket = numberOfChunks(TRANSFERTO_BUFFER_SIZE);
-        
-        // Smaller packet size to only hold checksum when doing transferTo
-        pktSize += checksumSize * maxChunksPerPacket;
-      } else {
-        maxChunksPerPacket = Math.max(1,
-            numberOfChunks(HdfsConstants.IO_FILE_BUFFER_SIZE));
-        // Packet size includes both checksum and data
-        pktSize += (chunkSize + checksumSize) * maxChunksPerPacket;
-      }
-
-      ByteBuffer pktBuf = ByteBuffer.allocate(pktSize);
-
-      while (endOffset > offset) {
-        long len = sendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks,
-            transferTo, null);
-        offset += len;
-        totalRead += len + (numberOfChunks(len) * checksumSize);
-        seqno++;
-      }
-      try {
-        // send an empty packet to mark the end of the block
-        sendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, transferTo,
-            null);
-        out.flush();
-      } catch (IOException e) { //socket error
-        throw ioeToSocketException(e);
-      }
-      blockReadFully = true;
-    } finally {
-      if (clientTraceFmt != null) {
-        final long endTime = System.nanoTime();
-        ClientTraceLog.info(String.format(clientTraceFmt, totalRead,
-            initialOffset, endTime - startTime));
-      }
-      close();
-    }
-    return totalRead;
-  }
-  
-  boolean isBlockReadFully() {
-    return blockReadFully;
-  }
-  
-  public static interface InputStreamFactory {
-    public InputStream createStream(long offset) throws IOException; 
-  }
-  
-  /**
-   * @return the checksum type that will be used with this block transfer.
-   */
-  public DataChecksum getChecksum() {
-    return checksum;
-  }
-  
-  private static class BlockInputStreamFactory implements InputStreamFactory {
-    private final ExtendedBlock block;
-    private final FsDatasetSpi<?> data;
-
-    private BlockInputStreamFactory(ExtendedBlock block, FsDatasetSpi<?> data) {
-      this.block = block;
-      this.data = data;
-    }
-
-    @Override
-    public InputStream createStream(long offset) throws IOException {
-      return data.getBlockInputStream(block, offset);
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidUtil.java

@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.namenode;
-
-import java.io.*;
-
-import org.apache.hadoop.classification.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.hdfs.protocol.*;
-import org.apache.hadoop.ipc.StandbyException;
-import org.apache.hadoop.security.AccessControlException;
-
-/** Utilities used by RAID for accessing NameNode. */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class NameNodeRaidUtil {
-  /** Accessing FSDirectory.getFileInfo(..) */
-  public static HdfsFileStatus getFileInfo(final FSDirectory dir,
-      final String src, final boolean resolveLink
-      ) throws UnresolvedLinkException {
-    return dir.getFileInfo(src, resolveLink);
-  }
-
-  /** Accessing FSNamesystem.getFileInfo(..) 
- * @throws StandbyException */
-  public static HdfsFileStatus getFileInfo(final FSNamesystem namesystem,
-      final String src, final boolean resolveLink
-      ) throws AccessControlException, UnresolvedLinkException, StandbyException {
-    return namesystem.getFileInfo(src, resolveLink);
-  }
-
-  /** Accessing FSNamesystem.getBlockLocations(..) */
-  public static LocatedBlocks getBlockLocations(final FSNamesystem namesystem,
-      final String src, final long offset, final long length,
-      final boolean doAccessTime, final boolean needBlockToken
-      ) throws FileNotFoundException, UnresolvedLinkException, IOException {
-    return namesystem.getBlockLocations(src, offset, length,
-        doAccessTime, needBlockToken, true);
-  }
-}
-

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/BlockFixer.java

@@ -1,840 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY;
-
-import java.io.BufferedOutputStream;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Random;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.nio.channels.SocketChannel;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-
-import org.apache.hadoop.util.DataChecksum;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.protocol.datatransfer.*;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.FSConstants;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
-import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader;
-import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.RaidBlockSender;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hdfs.RaidDFSUtil;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.net.NetUtils;
-
-import org.apache.hadoop.raid.RaidNode;
-import org.apache.hadoop.raid.RaidUtils;
-
-
-/**
- * contains the core functionality of the block fixer
- *
- * configuration options:
- * raid.blockfix.classname         - the class name of the block fixer 
- *                                   implementation to use
- *
- * raid.blockfix.interval          - interval between checks for corrupt files
- *
- * raid.blockfix.history.interval  - interval before fixing same file again
- *
- * raid.blockfix.read.timeout      - read time out
- *
- * raid.blockfix.write.timeout     - write time out
- */
-public abstract class BlockFixer extends Configured implements Runnable {
-
-  public static final String BLOCKFIX_CLASSNAME = "raid.blockfix.classname";
-  public static final String BLOCKFIX_INTERVAL = "raid.blockfix.interval";
-  public static final String BLOCKFIX_HISTORY_INTERVAL =
-    "raid.blockfix.history.interval";
-  public static final String BLOCKFIX_READ_TIMEOUT =
-    "raid.blockfix.read.timeout";
-  public static final String BLOCKFIX_WRITE_TIMEOUT =
-    "raid.blockfix.write.timeout";
-
-  public static final long DEFAULT_BLOCKFIX_INTERVAL = 60 * 1000; // 1 min
-  public static final long DEFAULT_BLOCKFIX_HISTORY_INTERVAL =
-    60 * 60 * 1000; // 60 mins
-
-  public static BlockFixer createBlockFixer(Configuration conf)
-    throws ClassNotFoundException {
-    try {
-      // default to distributed block fixer
-      Class<?> blockFixerClass =
-        conf.getClass(BLOCKFIX_CLASSNAME, DistBlockFixer.class);
-      if (!BlockFixer.class.isAssignableFrom(blockFixerClass)) {
-        throw new ClassNotFoundException("not an implementation of blockfixer");
-      }
-      Constructor<?> constructor =
-        blockFixerClass.getConstructor(new Class[] {Configuration.class} );
-      return (BlockFixer) constructor.newInstance(conf);
-    } catch (NoSuchMethodException e) {
-      throw new ClassNotFoundException("cannot construct blockfixer", e);
-    } catch (InstantiationException e) {
-      throw new ClassNotFoundException("cannot construct blockfixer", e);
-    } catch (IllegalAccessException e) {
-      throw new ClassNotFoundException("cannot construct blockfixer", e);
-    } catch (InvocationTargetException e) {
-      throw new ClassNotFoundException("cannot construct blockfixer", e);
-    }
-  }
-
-  private long numFilesFixed = 0;
-
-  public volatile boolean running = true;
-
-  // interval between checks for corrupt files
-  protected long blockFixInterval;
-
-  // interval before fixing same file again
-  protected long historyInterval;
-
-  public BlockFixer(Configuration conf) {
-    super(conf);
-    blockFixInterval =
-      getConf().getLong(BLOCKFIX_INTERVAL, DEFAULT_BLOCKFIX_INTERVAL);
-    historyInterval =
-      getConf().getLong(BLOCKFIX_HISTORY_INTERVAL,
-                        DEFAULT_BLOCKFIX_HISTORY_INTERVAL);
-
-  }
-
-  @Override
-  public abstract void run();
-
-  /**
-   * returns the number of files that have been fixed by this block fixer
-   */
-  public synchronized long filesFixed() {
-    return numFilesFixed;
-  }
-
-  /**
-   * increments the number of files that have been fixed by this block fixer
-   */
-  protected synchronized void incrFilesFixed() {
-    numFilesFixed++;
-  }
-
-  /**
-   * increments the number of files that have been fixed by this block fixer
-   */
-  protected synchronized void incrFilesFixed(long incr) {
-    if (incr < 0) {
-      throw new IllegalArgumentException("cannot increment by negative value " +
-                                         incr);
-    }
-    
-    numFilesFixed += incr;
-  }
-
-  static boolean isSourceFile(Path p, String[] destPrefixes) {
-    String pathStr = p.toUri().getPath();
-    for (String destPrefix: destPrefixes) {
-      if (pathStr.startsWith(destPrefix)) {
-        return false;
-      }
-    }
-    return true;
-  }
-
-  void filterUnfixableSourceFiles(Iterator<Path> it) throws IOException {
-    String xorPrefix = RaidNode.xorDestinationPath(getConf()).toUri().getPath();
-    if (!xorPrefix.endsWith(Path.SEPARATOR)) {
-      xorPrefix += Path.SEPARATOR;
-    }
-    String rsPrefix = RaidNode.rsDestinationPath(getConf()).toUri().getPath();
-    if (!rsPrefix.endsWith(Path.SEPARATOR)) {
-      rsPrefix += Path.SEPARATOR;
-    }
-    String[] destPrefixes = new String[]{xorPrefix, rsPrefix};
-    while (it.hasNext()) {
-      Path p = it.next();
-      if (isSourceFile(p, destPrefixes) &&
-          RaidNode.xorParityForSource(p, getConf()) == null &&
-          RaidNode.rsParityForSource(p, getConf()) == null) {
-        it.remove();
-      }
-    }
-  }
-
-  /**
-   * this class implements the actual fixing functionality
-   * we keep this in a separate class so that 
-   * the distributed block fixer can use it
-   */ 
-  static class BlockFixerHelper extends Configured {
-
-    public static final Log LOG = LogFactory.getLog(BlockFixer.
-                                                    BlockFixerHelper.class);
-
-    private String xorPrefix;
-    private String rsPrefix;
-    private XOREncoder xorEncoder;
-    private XORDecoder xorDecoder;
-    private ReedSolomonEncoder rsEncoder;
-    private ReedSolomonDecoder rsDecoder;
-
-    public BlockFixerHelper(Configuration conf) throws IOException {
-      super(conf);
-
-      xorPrefix = RaidNode.xorDestinationPath(getConf()).toUri().getPath();
-      if (!xorPrefix.endsWith(Path.SEPARATOR)) {
-        xorPrefix += Path.SEPARATOR;
-      }
-      rsPrefix = RaidNode.rsDestinationPath(getConf()).toUri().getPath();
-      if (!rsPrefix.endsWith(Path.SEPARATOR)) {
-        rsPrefix += Path.SEPARATOR;
-      }
-      int stripeLength = RaidNode.getStripeLength(getConf());
-      xorEncoder = new XOREncoder(getConf(), stripeLength);
-      xorDecoder = new XORDecoder(getConf(), stripeLength);
-      int parityLength = RaidNode.rsParityLength(getConf());
-      rsEncoder = new ReedSolomonEncoder(getConf(), stripeLength, parityLength);
-      rsDecoder = new ReedSolomonDecoder(getConf(), stripeLength, parityLength);
-
-    }
-
-    /**
-     * checks whether file is xor parity file
-     */
-    boolean isXorParityFile(Path p) {
-      String pathStr = p.toUri().getPath();
-      if (pathStr.contains(RaidNode.HAR_SUFFIX)) {
-        return false;
-      }
-      return pathStr.startsWith(xorPrefix);
-    }
-
-    /**
-     * checks whether file is rs parity file
-     */
-    boolean isRsParityFile(Path p) {
-      String pathStr = p.toUri().getPath();
-      if (pathStr.contains(RaidNode.HAR_SUFFIX)) {
-        return false;
-      }
-      return pathStr.startsWith(rsPrefix);
-    }
-
-    /**
-     * Fix a file, do not report progess.
-     *
-     * @return true if file has been fixed, false if no fixing 
-     * was necessary or possible.
-     */
-    boolean fixFile(Path srcPath) throws IOException {
-      return fixFile(srcPath, new RaidUtils.DummyProgressable());
-    }
-
-    /**
-     * Fix a file, report progess.
-     *
-     * @return true if file has been fixed, false if no fixing 
-     * was necessary or possible.
-     */
-    boolean fixFile(Path srcPath, Progressable progress) throws IOException {
-
-      if (RaidNode.isParityHarPartFile(srcPath)) {
-        return processCorruptParityHarPartFile(srcPath, progress);
-      }
-
-      // The corrupted file is a XOR parity file
-      if (isXorParityFile(srcPath)) {
-        return processCorruptParityFile(srcPath, xorEncoder, progress);
-      }
-
-      // The corrupted file is a ReedSolomon parity file
-      if (isRsParityFile(srcPath)) {
-        return processCorruptParityFile(srcPath, rsEncoder, progress);
-      }
-
-      // The corrupted file is a source file
-      RaidNode.ParityFilePair ppair =
-        RaidNode.xorParityForSource(srcPath, getConf());
-      Decoder decoder = null;
-      if (ppair != null) {
-        decoder = xorDecoder;
-      } else  {
-        ppair = RaidNode.rsParityForSource(srcPath, getConf());
-        if (ppair != null) {
-          decoder = rsDecoder;
-        }
-      }
-
-      // If we have a parity file, process the file and fix it.
-      if (ppair != null) {
-        return processCorruptFile(srcPath, ppair, decoder, progress);
-      }
-      
-      // there was nothing to do
-      return false;
-    }
-
-    /**
-     * Sorts source files ahead of parity files.
-     */
-    void sortCorruptFiles(List<Path> files) {
-      // TODO: We should first fix the files that lose more blocks
-      Comparator<Path> comp = new Comparator<Path>() {
-        public int compare(Path p1, Path p2) {
-          if (isXorParityFile(p2) || isRsParityFile(p2)) {
-            // If p2 is a parity file, p1 is smaller.
-            return -1;
-          }
-          if (isXorParityFile(p1) || isRsParityFile(p1)) {
-            // If p1 is a parity file, p2 is smaller.
-            return 1;
-          }
-          // If both are source files, they are equal.
-          return 0;
-        }
-      };
-      Collections.sort(files, comp);
-    }
-
-    /**
-     * Returns a DistributedFileSystem hosting the path supplied.
-     */
-    protected DistributedFileSystem getDFS(Path p) throws IOException {
-      return (DistributedFileSystem) p.getFileSystem(getConf());
-    }
-
-    /**
-     * Reads through a corrupt source file fixing corrupt blocks on the way.
-     * @param srcPath Path identifying the corrupt file.
-     * @throws IOException
-     * @return true if file has been fixed, false if no fixing 
-     * was necessary or possible.
-     */
-    boolean processCorruptFile(Path srcPath, RaidNode.ParityFilePair parityPair,
-                               Decoder decoder, Progressable progress)
-      throws IOException {
-      LOG.info("Processing corrupt file " + srcPath);
-      
-      DistributedFileSystem srcFs = getDFS(srcPath);
-      FileStatus srcStat = srcFs.getFileStatus(srcPath);
-      long blockSize = srcStat.getBlockSize();
-      long srcFileSize = srcStat.getLen();
-      String uriPath = srcPath.toUri().getPath();
-      
-      int numBlocksFixed = 0;
-      List<LocatedBlock> corrupt =
-        RaidDFSUtil.corruptBlocksInFile(srcFs, uriPath, 0, srcFileSize);
-      if (corrupt.size() == 0) {
-        return false;
-      }
-      for (LocatedBlock lb: corrupt) {
-        ExtendedBlock corruptBlock = lb.getBlock();
-        long corruptOffset = lb.getStartOffset();
-        
-        LOG.info("Found corrupt block " + corruptBlock +
-                 ", offset " + corruptOffset);
-        
-        final long blockContentsSize =
-          Math.min(blockSize, srcFileSize - corruptOffset);
-        File localBlockFile =
-          File.createTempFile(corruptBlock.getBlockName(), ".tmp");
-        localBlockFile.deleteOnExit();
-        
-        try {
-          decoder.recoverBlockToFile(srcFs, srcPath, parityPair.getFileSystem(),
-                                     parityPair.getPath(), blockSize,
-                                     corruptOffset, localBlockFile,
-                                     blockContentsSize);
-          
-          // We have a the contents of the block, send them.
-          DatanodeInfo datanode = chooseDatanode(lb.getLocations());
-          computeMetadataAndSendFixedBlock(datanode, localBlockFile,
-                                          lb, blockContentsSize);
-          numBlocksFixed++;
-        } finally {
-          localBlockFile.delete();
-        }
-        progress.progress();
-      }
-      LOG.info("Fixed " + numBlocksFixed + " blocks in " + srcPath);
-      return true;
-    }
-
-    /**
-     * Fixes corrupt blocks in a parity file.
-     * This function uses the corresponding source file to regenerate parity
-     * file blocks.
-     * @return true if file has been fixed, false if no fixing 
-     * was necessary or possible.
-     */
-    boolean processCorruptParityFile(Path parityPath, Encoder encoder,
-                                     Progressable progress)
-      throws IOException {
-      LOG.info("Processing corrupt file " + parityPath);
-      Path srcPath = sourcePathFromParityPath(parityPath);
-      if (srcPath == null) {
-        LOG.warn("Unusable parity file " + parityPath);
-        return false;
-      }
-
-      DistributedFileSystem parityFs = getDFS(parityPath);
-      FileStatus parityStat = parityFs.getFileStatus(parityPath);
-      long blockSize = parityStat.getBlockSize();
-      long parityFileSize = parityStat.getLen();
-      FileStatus srcStat = getDFS(srcPath).getFileStatus(srcPath);
-      long srcFileSize = srcStat.getLen();
-
-      // Check timestamp.
-      if (srcStat.getModificationTime() != parityStat.getModificationTime()) {
-        LOG.info("Mismatching timestamp for " + srcPath + " and " + parityPath +
-                 ", moving on...");
-        return false;
-      }
-
-      String uriPath = parityPath.toUri().getPath();
-      int numBlocksFixed = 0;
-      List<LocatedBlock> corrupt =
-        RaidDFSUtil.corruptBlocksInFile(parityFs, uriPath, 0, parityFileSize);
-      if (corrupt.size() == 0) {
-        return false;
-      }
-      for (LocatedBlock lb: corrupt) {
-        ExtendedBlock corruptBlock = lb.getBlock();
-        long corruptOffset = lb.getStartOffset();
-        
-        LOG.info("Found corrupt block " + corruptBlock +
-                 ", offset " + corruptOffset);
-        
-        File localBlockFile =
-          File.createTempFile(corruptBlock.getBlockName(), ".tmp");
-        localBlockFile.deleteOnExit();
-        
-        try {
-          encoder.recoverParityBlockToFile(parityFs, srcPath, srcFileSize,
-                                           blockSize, parityPath,
-                                           corruptOffset, localBlockFile);
-          // We have a the contents of the block, send them.
-          DatanodeInfo datanode = chooseDatanode(lb.getLocations());
-          computeMetadataAndSendFixedBlock(datanode, localBlockFile, lb,
-                                          blockSize);
-          
-          numBlocksFixed++;
-        } finally {
-          localBlockFile.delete();
-        }
-        progress.progress();
-      }
-      LOG.info("Fixed " + numBlocksFixed + " blocks in " + parityPath);
-      return true;
-    }
-
-    /**
-     * Reads through a parity HAR part file, fixing corrupt blocks on the way.
-     * A HAR block can contain many file blocks, as long as the HAR part file
-     * block size is a multiple of the file block size.
-     * @return true if file has been fixed, false if no fixing 
-     * was necessary or possible.
-     */
-    boolean processCorruptParityHarPartFile(Path partFile,
-                                            Progressable progress)
-      throws IOException {
-      LOG.info("Processing corrupt file " + partFile);
-      // Get some basic information.
-      DistributedFileSystem dfs = getDFS(partFile);
-      FileStatus partFileStat = dfs.getFileStatus(partFile);
-      long partFileSize = partFileStat.getLen();
-      long partFileBlockSize = partFileStat.getBlockSize();
-      LOG.info(partFile + " has block size " + partFileBlockSize);
-
-      // Find the path to the index file.
-      // Parity file HARs are only one level deep, so the index files is at the
-      // same level as the part file.
-      String harDirectory = partFile.toUri().getPath(); // Temporarily.
-      harDirectory =
-        harDirectory.substring(0, harDirectory.lastIndexOf(Path.SEPARATOR));
-      Path indexFile = new Path(harDirectory + "/" + HarIndex.indexFileName);
-      FileStatus indexStat = dfs.getFileStatus(indexFile);
-      // Parses through the HAR index file.
-      HarIndex harIndex = new HarIndex(dfs.open(indexFile), indexStat.getLen());
-
-      String uriPath = partFile.toUri().getPath();
-      int numBlocksFixed = 0;
-      List<LocatedBlock> corrupt =
-        RaidDFSUtil.corruptBlocksInFile(dfs, uriPath, 0, partFileSize);
-      if (corrupt.size() == 0) {
-        return false;
-      }
-      for (LocatedBlock lb: corrupt) {
-        ExtendedBlock corruptBlock = lb.getBlock();
-        long corruptOffset = lb.getStartOffset();
-
-        File localBlockFile =
-          File.createTempFile(corruptBlock.getBlockName(), ".tmp");
-        localBlockFile.deleteOnExit();
-        processCorruptParityHarPartBlock(dfs, partFile, corruptBlock,
-                                         corruptOffset, partFileStat, harIndex,
-                                         localBlockFile, progress);
-        // Now we have recovered the part file block locally, send it.
-        try {
-          DatanodeInfo datanode = chooseDatanode(lb.getLocations());
-          computeMetadataAndSendFixedBlock(datanode, localBlockFile,
-                                          lb, localBlockFile.length());
-          numBlocksFixed++;
-        } finally {
-          localBlockFile.delete();
-        }
-        progress.progress();
-      }
-      LOG.info("Fixed " + numBlocksFixed + " blocks in " + partFile);
-      return true;
-    }
-
-    /**
-     * This fixes a single part file block by recovering in sequence each
-     * parity block in the part file block.
-     */
-    private void processCorruptParityHarPartBlock(FileSystem dfs, Path partFile,
-                                                  ExtendedBlock corruptBlock,
-                                                  long corruptOffset,
-                                                  FileStatus partFileStat,
-                                                  HarIndex harIndex,
-                                                  File localBlockFile,
-                                                  Progressable progress)
-      throws IOException {
-      String partName = partFile.toUri().getPath(); // Temporarily.
-      partName = partName.substring(1 + partName.lastIndexOf(Path.SEPARATOR));
-
-      OutputStream out = new FileOutputStream(localBlockFile);
-
-      try {
-        // A HAR part file block could map to several parity files. We need to
-        // use all of them to recover this block.
-        final long corruptEnd = Math.min(corruptOffset +
-                                         partFileStat.getBlockSize(),
-                                         partFileStat.getLen());
-        for (long offset = corruptOffset; offset < corruptEnd; ) {
-          HarIndex.IndexEntry entry = harIndex.findEntry(partName, offset);
-          if (entry == null) {
-            String msg = "Corrupt index file has no matching index entry for " +
-              partName + ":" + offset;
-            LOG.warn(msg);
-            throw new IOException(msg);
-          }
-          Path parityFile = new Path(entry.fileName);
-          Encoder encoder;
-          if (isXorParityFile(parityFile)) {
-            encoder = xorEncoder;
-          } else if (isRsParityFile(parityFile)) {
-            encoder = rsEncoder;
-          } else {
-            String msg = "Could not figure out parity file correctly";
-            LOG.warn(msg);
-            throw new IOException(msg);
-          }
-          Path srcFile = sourcePathFromParityPath(parityFile);
-          FileStatus srcStat = dfs.getFileStatus(srcFile);
-          if (srcStat.getModificationTime() != entry.mtime) {
-            String msg = "Modification times of " + parityFile + " and " +
-              srcFile + " do not match.";
-            LOG.warn(msg);
-            throw new IOException(msg);
-          }
-          long corruptOffsetInParity = offset - entry.startOffset;
-          LOG.info(partFile + ":" + offset + " maps to " +
-                   parityFile + ":" + corruptOffsetInParity +
-                   " and will be recovered from " + srcFile);
-          encoder.recoverParityBlockToStream(dfs, srcFile, srcStat.getLen(),
-                                             srcStat.getBlockSize(), parityFile,
-                                             corruptOffsetInParity, out);
-          // Finished recovery of one parity block. Since a parity block has the
-          // same size as a source block, we can move offset by source block size.
-          offset += srcStat.getBlockSize();
-          LOG.info("Recovered " + srcStat.getBlockSize() + " part file bytes ");
-          if (offset > corruptEnd) {
-            String msg =
-              "Recovered block spills across part file blocks. Cannot continue.";
-            throw new IOException(msg);
-          }
-          progress.progress();
-        }
-      } finally {
-        out.close();
-      }
-    }
-
-    /**
-     * Choose a datanode (hostname:portnumber). The datanode is chosen at
-     * random from the live datanodes.
-     * @param locationsToAvoid locations to avoid.
-     * @return A datanode
-     * @throws IOException
-     */
-    private DatanodeInfo chooseDatanode(DatanodeInfo[] locationsToAvoid)
-      throws IOException {
-      DistributedFileSystem dfs = getDFS(new Path("/"));
-      DatanodeInfo[] live =
-        dfs.getClient().datanodeReport(DatanodeReportType.LIVE);
-      LOG.info("Choosing a datanode from " + live.length +
-               " live nodes while avoiding " + locationsToAvoid.length);
-      Random rand = new Random();
-      DatanodeInfo chosen = null;
-      int maxAttempts = 1000;
-      for (int i = 0; i < maxAttempts && chosen == null; i++) {
-        int idx = rand.nextInt(live.length);
-        chosen = live[idx];
-        for (DatanodeInfo avoid: locationsToAvoid) {
-          if (chosen.getName().equals(avoid.getName())) {
-            LOG.info("Avoiding " + avoid.getName());
-            chosen = null;
-            break;
-          }
-        }
-      }
-      if (chosen == null) {
-        throw new IOException("Could not choose datanode");
-      }
-      LOG.info("Choosing datanode " + chosen.getName());
-      return chosen;
-    }
-
-    /**
-     * Reads data from the data stream provided and computes metadata.
-     */
-    static DataInputStream computeMetadata(Configuration conf,
-                                           InputStream dataStream)
-      throws IOException {
-      ByteArrayOutputStream mdOutBase = new ByteArrayOutputStream(1024*1024);
-      DataOutputStream mdOut = new DataOutputStream(mdOutBase);
-      
-      // First, write out the version.
-      mdOut.writeShort(BlockMetadataHeader.VERSION);
-      
-      // Create a summer and write out its header.
-      int bytesPerChecksum = conf.getInt("dfs.bytes-per-checksum", 512);
-      DataChecksum sum =
-        DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
-                                     bytesPerChecksum);
-      sum.writeHeader(mdOut);
-      
-      // Buffer to read in a chunk of data.
-      byte[] buf = new byte[bytesPerChecksum];
-      // Buffer to store the checksum bytes.
-      byte[] chk = new byte[sum.getChecksumSize()];
-      
-      // Read data till we reach the end of the input stream.
-      int bytesSinceFlush = 0;
-      while (true) {
-        // Read some bytes.
-        int bytesRead = dataStream.read(buf, bytesSinceFlush,
-                                        bytesPerChecksum-bytesSinceFlush);
-        if (bytesRead == -1) {
-          if (bytesSinceFlush > 0) {
-            boolean reset = true;
-            sum.writeValue(chk, 0, reset); // This also resets the sum.
-            // Write the checksum to the stream.
-            mdOut.write(chk, 0, chk.length);
-            bytesSinceFlush = 0;
-          }
-          break;
-        }
-        // Update the checksum.
-        sum.update(buf, bytesSinceFlush, bytesRead);
-        bytesSinceFlush += bytesRead;
-        
-        // Flush the checksum if necessary.
-        if (bytesSinceFlush == bytesPerChecksum) {
-          boolean reset = true;
-          sum.writeValue(chk, 0, reset); // This also resets the sum.
-          // Write the checksum to the stream.
-          mdOut.write(chk, 0, chk.length);
-          bytesSinceFlush = 0;
-        }
-      }
-      
-      byte[] mdBytes = mdOutBase.toByteArray();
-      return new DataInputStream(new ByteArrayInputStream(mdBytes));
-    }
-
-    private void computeMetadataAndSendFixedBlock(DatanodeInfo datanode,
-                                                  File localBlockFile,
-                                                  LocatedBlock block,
-                                                  long blockSize)
-      throws IOException {
-
-      LOG.info("Computing metdata");
-      InputStream blockContents = null;
-      DataInputStream blockMetadata = null;
-      try {
-        blockContents = new FileInputStream(localBlockFile);
-        blockMetadata = computeMetadata(getConf(), blockContents);
-        blockContents.close();
-        // Reopen
-        blockContents = new FileInputStream(localBlockFile);
-        sendFixedBlock(datanode, blockContents, blockMetadata, block, 
-            blockSize);
-      } finally {
-        if (blockContents != null) {
-          blockContents.close();
-          blockContents = null;
-        }
-        if (blockMetadata != null) {
-          blockMetadata.close();
-          blockMetadata = null;
-        }
-      }
-    }
-
-    /**
-     * Send a generated block to a datanode.
-     * @param datanode Chosen datanode name in host:port form.
-     * @param blockContents Stream with the block contents.
-     * @param corruptBlock Block identifying the block to be sent.
-     * @param blockSize size of the block.
-     * @throws IOException
-     */
-    private void sendFixedBlock(DatanodeInfo datanode,
-                                final InputStream blockContents,
-                                DataInputStream metadataIn,
-                                LocatedBlock block, long blockSize)
-      throws IOException {
-      InetSocketAddress target = NetUtils.createSocketAddr(datanode.getName());
-      Socket sock = SocketChannel.open().socket();
-      
-      int readTimeout =
-        getConf().getInt(BLOCKFIX_READ_TIMEOUT,
-                         HdfsServerConstants.READ_TIMEOUT);
-      NetUtils.connect(sock, target, readTimeout);
-      sock.setSoTimeout(readTimeout);
-      
-      int writeTimeout = getConf().getInt(BLOCKFIX_WRITE_TIMEOUT,
-                                          HdfsServerConstants.WRITE_TIMEOUT);
-      
-      OutputStream baseStream = NetUtils.getOutputStream(sock, writeTimeout);
-      DataOutputStream out = 
-        new DataOutputStream(new BufferedOutputStream(baseStream,
-                                                      HdfsConstants.
-                                                      SMALL_BUFFER_SIZE));
-      
-      boolean corruptChecksumOk = false;
-      boolean chunkOffsetOK = false;
-      boolean verifyChecksum = true;
-      boolean transferToAllowed = false;
-      
-      try {
-        LOG.info("Sending block " + block.getBlock() +
-                 " from " + sock.getLocalSocketAddress().toString() +
-                 " to " + sock.getRemoteSocketAddress().toString() +
-                 " " + blockSize + " bytes");
-        RaidBlockSender blockSender =
-          new RaidBlockSender(block.getBlock(), blockSize, 0, blockSize,
-                              corruptChecksumOk, chunkOffsetOK, verifyChecksum,
-                              transferToAllowed, metadataIn,
-                              new RaidBlockSender.InputStreamFactory() {
-                                @Override
-                                public InputStream
-                                  createStream(long offset) throws IOException {
-                                  // we are passing 0 as the offset above,
-                                  // so we can safely ignore
-                                  // the offset passed
-                                  return blockContents;
-                                }
-                              });
-        
-        DatanodeInfo[] nodes = new DatanodeInfo[]{datanode};
-        DataChecksum checksum = blockSender.getChecksum();
-        new Sender(out).writeBlock(block.getBlock(), block.getBlockToken(), "",
-            nodes, null, BlockConstructionStage.PIPELINE_SETUP_CREATE,
-            1, 0L, blockSize, 0L, DataChecksum.newDataChecksum(
-                checksum.getChecksumType(), checksum.getBytesPerChecksum()));
-        blockSender.sendBlock(out, baseStream);
-        
-        LOG.info("Sent block " + block.getBlock() + " to " + datanode.getName());
-      } finally {
-        out.close();
-      }
-    }
-
-    /**
-     * returns the source file corresponding to a parity file
-     */
-    Path sourcePathFromParityPath(Path parityPath) {
-      String parityPathStr = parityPath.toUri().getPath();
-      if (parityPathStr.startsWith(xorPrefix)) {
-        // Remove the prefix to get the source file.
-        String src = parityPathStr.replaceFirst(xorPrefix, "/");
-        return new Path(src);
-      } else if (parityPathStr.startsWith(rsPrefix)) {
-        // Remove the prefix to get the source file.
-        String src = parityPathStr.replaceFirst(rsPrefix, "/");
-        return new Path(src);
-      }
-      return null;
-    }
-
-    /**
-     * Returns the corrupt blocks in a file.
-     */
-    List<LocatedBlock> corruptBlocksInFile(DistributedFileSystem fs,
-                                           String uriPath, FileStatus stat)
-      throws IOException {
-      List<LocatedBlock> corrupt = new LinkedList<LocatedBlock>();
-      LocatedBlocks locatedBlocks =
-        RaidDFSUtil.getBlockLocations(fs, uriPath, 0, stat.getLen());
-      for (LocatedBlock b: locatedBlocks.getLocatedBlocks()) {
-        if (b.isCorrupt() ||
-            (b.getLocations().length == 0 && b.getBlockSize() > 0)) {
-          corrupt.add(b);
-        }
-      }
-      return corrupt;
-    }
-  }
-
-}
-

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ConfigManager.java

@@ -1,408 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.w3c.dom.Text;
-import org.xml.sax.SAXException;
-
-import org.apache.hadoop.raid.protocol.PolicyInfo;
-import org.apache.hadoop.raid.protocol.PolicyList;
-
-/**
- * Maintains the configuration xml file that is read into memory.
- */
-class ConfigManager {
-  public static final Log LOG = LogFactory.getLog(
-    "org.apache.hadoop.raid.ConfigManager");
-
-  /** Time to wait between checks of the config file */
-  public static final long RELOAD_INTERVAL = 10 * 1000;
-
-  /** Time to wait between successive runs of all policies */
-  public static final long RESCAN_INTERVAL = 3600 * 1000;
-
-  public static final long HAR_PARTFILE_SIZE = 10 * 1024 * 1024 * 1024l;
-  
-  public static final int DISTRAID_MAX_JOBS = 10;
-
-  public static final int DISTRAID_MAX_FILES = 10000;
-
-  /**
-   * Time to wait after the config file has been modified before reloading it
-   * (this is done to prevent loading a file that hasn't been fully written).
-   */
-  public static final long RELOAD_WAIT = 5 * 1000; 
-  
-  private Configuration conf;    // Hadoop configuration
-  private String configFileName; // Path to config XML file
-  
-  private long lastReloadAttempt; // Last time we tried to reload the config file
-  private long lastSuccessfulReload; // Last time we successfully reloaded config
-  private boolean lastReloadAttemptFailed = false;
-  private long reloadInterval = RELOAD_INTERVAL;
-  private long periodicity; // time between runs of all policies
-  private long harPartfileSize;
-  private int maxJobsPerPolicy; // Max no. of jobs running simultaneously for
-                                // a job.
-  private int maxFilesPerJob; // Max no. of files raided by a job.
-
-  // Reload the configuration
-  private boolean doReload;
-  private Thread reloadThread;
-  private volatile boolean running = false;
-
-  // Collection of all configured policies.
-  Collection<PolicyList> allPolicies = new ArrayList<PolicyList>();
-
-  public ConfigManager(Configuration conf) throws IOException, SAXException,
-      RaidConfigurationException, ClassNotFoundException, ParserConfigurationException {
-    this.conf = conf;
-    this.configFileName = conf.get("raid.config.file");
-    this.doReload = conf.getBoolean("raid.config.reload", true);
-    this.reloadInterval = conf.getLong("raid.config.reload.interval", RELOAD_INTERVAL);
-    this.periodicity = conf.getLong("raid.policy.rescan.interval",  RESCAN_INTERVAL);
-    this.harPartfileSize = conf.getLong("raid.har.partfile.size", HAR_PARTFILE_SIZE);
-    this.maxJobsPerPolicy = conf.getInt("raid.distraid.max.jobs",
-                                        DISTRAID_MAX_JOBS);
-    this.maxFilesPerJob = conf.getInt("raid.distraid.max.files",
-                                      DISTRAID_MAX_FILES);
-    if (configFileName == null) {
-      String msg = "No raid.config.file given in conf - " +
-                   "the Hadoop Raid utility cannot run. Aborting....";
-      LOG.warn(msg);
-      throw new IOException(msg);
-    }
-    reloadConfigs();
-    lastSuccessfulReload = RaidNode.now();
-    lastReloadAttempt = RaidNode.now();
-    running = true;
-  }
-  
-  /**
-   * Reload config file if it hasn't been loaded in a while
-   * Returns true if the file was reloaded.
-   */
-  public synchronized boolean reloadConfigsIfNecessary() {
-    long time = RaidNode.now();
-    if (time > lastReloadAttempt + reloadInterval) {
-      lastReloadAttempt = time;
-      try {
-        File file = new File(configFileName);
-        long lastModified = file.lastModified();
-        if (lastModified > lastSuccessfulReload &&
-            time > lastModified + RELOAD_WAIT) {
-          reloadConfigs();
-          lastSuccessfulReload = time;
-          lastReloadAttemptFailed = false;
-          return true;
-        }
-      } catch (Exception e) {
-        if (!lastReloadAttemptFailed) {
-          LOG.error("Failed to reload config file - " +
-              "will use existing configuration.", e);
-        }
-        lastReloadAttemptFailed = true;
-      }
-    }
-    return false;
-  }
-  
-  /**
-   * Updates the in-memory data structures from the config file. This file is
-   * expected to be in the following whitespace-separated format:
-   * 
-   <configuration>
-    <srcPath prefix="hdfs://hadoop.myhost.com:9000/user/warehouse/u_full/*">
-      <policy name = RaidScanWeekly>
-        <destPath> hdfs://dfsname.myhost.com:9000/archive/</destPath>
-        <parentPolicy> RaidScanMonthly</parentPolicy>
-        <property>
-          <name>targetReplication</name>
-          <value>2</value>
-          <description> after RAIDing, decrease the replication factor of the file to 
-                        this value.
-          </description>
-        </property>
-        <property>
-          <name>metaReplication</name>
-          <value>2</value>
-          <description> the replication factor of the RAID meta file
-          </description>
-        </property>
-        <property>
-          <name>stripeLength</name>
-          <value>10</value>
-          <description> the number of blocks to RAID together
-          </description>
-        </property>
-      </policy>
-    </srcPath>
-   </configuration>
-   *
-   * Blank lines and lines starting with # are ignored.
-   *  
-   * @throws IOException if the config file cannot be read.
-   * @throws RaidConfigurationException if configuration entries are invalid.
-   * @throws ClassNotFoundException if user-defined policy classes cannot be loaded
-   * @throws ParserConfigurationException if XML parser is misconfigured.
-   * @throws SAXException if config file is malformed.
-   * @returns A new set of policy categories.
-   */
-  void reloadConfigs() throws IOException, ParserConfigurationException, 
-      SAXException, ClassNotFoundException, RaidConfigurationException {
-
-    if (configFileName == null) {
-       return;
-    }
-    
-    File file = new File(configFileName);
-    if (!file.exists()) {
-      throw new RaidConfigurationException("Configuration file " + configFileName +
-                                           " does not exist.");
-    }
-
-    // Create some temporary hashmaps to hold the new allocs, and we only save
-    // them in our fields if we have parsed the entire allocs file successfully.
-    List<PolicyList> all = new ArrayList<PolicyList>();
-    long periodicityValue = periodicity;
-    
-    
-    // Read and parse the configuration file.
-    // allow include files in configuration file
-    DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
-    docBuilderFactory.setIgnoringComments(true);
-    docBuilderFactory.setNamespaceAware(true);
-    try {
-      docBuilderFactory.setXIncludeAware(true);
-    } catch (UnsupportedOperationException e) {
-        LOG.error("Failed to set setXIncludeAware(true) for raid parser "
-                + docBuilderFactory + ":" + e, e);
-    }
-    LOG.error("Reloading config file " + file);
-
-    DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
-    Document doc = builder.parse(file);
-    Element root = doc.getDocumentElement();
-    if (!"configuration".equalsIgnoreCase(root.getTagName()))
-      throw new RaidConfigurationException("Bad configuration file: " + 
-          "top-level element not <configuration>");
-    NodeList elements = root.getChildNodes();
-
-    Map<String, PolicyInfo> existingPolicies =
-      new HashMap<String, PolicyInfo>();
-    // loop through all the configured source paths.
-    for (int i = 0; i < elements.getLength(); i++) {
-      Node node = elements.item(i);
-      if (!(node instanceof Element)) {
-        continue;
-      }
-      Element element = (Element)node;
-      String elementTagName = element.getTagName();
-      if ("srcPath".equalsIgnoreCase(elementTagName)) {
-        String srcPathPrefix = element.getAttribute("prefix");
-
-        PolicyList policyList = null;
-        if (srcPathPrefix != null && srcPathPrefix.length() != 0) {
-          // Empty srcPath will have no effect but policies will be processed
-          // This allow us to define some "abstract" policies
-          policyList = new PolicyList();
-          all.add(policyList);
-          policyList.setSrcPath(conf, srcPathPrefix);
-        }
-
-        // loop through all the policies for this source path
-        NodeList policies = element.getChildNodes();
-        for (int j = 0; j < policies.getLength(); j++) {
-          Node node1 = policies.item(j);
-          if (!(node1 instanceof Element)) {
-            continue;
-          }
-          Element policy = (Element)node1;
-          if (!"policy".equalsIgnoreCase(policy.getTagName())) {
-            throw new RaidConfigurationException("Bad configuration file: " + 
-              "Expecting <policy> for srcPath " + srcPathPrefix);
-          }
-          String policyName = policy.getAttribute("name");
-          PolicyInfo curr = new PolicyInfo(policyName, conf);
-          if (srcPathPrefix != null && srcPathPrefix.length() > 0) {
-            curr.setSrcPath(srcPathPrefix);
-          }
-          // loop through all the properties of this policy
-          NodeList properties = policy.getChildNodes();
-          PolicyInfo parent = null;
-          for (int k = 0; k < properties.getLength(); k++) {
-            Node node2 = properties.item(k);
-            if (!(node2 instanceof Element)) {
-              continue;
-            }
-            Element property = (Element)node2;
-            String propertyName = property.getTagName();
-            if ("erasureCode".equalsIgnoreCase(propertyName)) {
-              String text = ((Text)property.getFirstChild()).getData().trim();
-              LOG.info(policyName + ".erasureCode = " + text);
-              curr.setErasureCode(text);
-            } else if ("description".equalsIgnoreCase(propertyName)) {
-              String text = ((Text)property.getFirstChild()).getData().trim();
-              curr.setDescription(text);
-            } else if ("parentPolicy".equalsIgnoreCase(propertyName)) {
-              String text = ((Text)property.getFirstChild()).getData().trim();
-              parent = existingPolicies.get(text);
-            } else if ("property".equalsIgnoreCase(propertyName)) {
-              NodeList nl = property.getChildNodes();
-              String pname=null,pvalue=null;
-              for (int l = 0; l < nl.getLength(); l++){
-                Node node3 = nl.item(l);
-                if (!(node3 instanceof Element)) {
-                  continue;
-                }
-                Element item = (Element) node3;
-                String itemName = item.getTagName();
-                if ("name".equalsIgnoreCase(itemName)){
-                  pname = ((Text)item.getFirstChild()).getData().trim();
-                } else if ("value".equalsIgnoreCase(itemName)){
-                  pvalue = ((Text)item.getFirstChild()).getData().trim();
-                }
-              }
-              if (pname != null && pvalue != null) {
-                LOG.info(policyName + "." + pname + " = " + pvalue);
-                curr.setProperty(pname,pvalue);
-              }
-            } else {
-              LOG.warn("Found bad property " + propertyName +
-                       " for srcPath" + srcPathPrefix +
-                       " policy name " + policyName +
-                       ". Ignoring."); 
-            }
-          }  // done with all properties of this policy
-
-          PolicyInfo pinfo;
-          if (parent != null) {
-            pinfo = new PolicyInfo(policyName, conf);
-            pinfo.copyFrom(parent);
-            pinfo.copyFrom(curr);
-          } else {
-            pinfo = curr;
-          }
-          if (policyList != null) {
-            policyList.add(pinfo);
-          }
-          existingPolicies.put(policyName, pinfo);
-
-        }    // done with all policies for this srcpath
-      } 
-    }        // done with all srcPaths
-    setAllPolicies(all);
-    periodicity = periodicityValue;
-    return;
-  }
-
-
-  public synchronized long getPeriodicity() {
-    return periodicity;
-  }
-
-  public synchronized long getHarPartfileSize() {
-    return harPartfileSize;
-  }
-  
-  public synchronized int getMaxJobsPerPolicy() {
-    return maxJobsPerPolicy;
-  }
-
-  public synchronized int getMaxFilesPerJob() {
-    return maxFilesPerJob;
-  }
-
-  /**
-   * Get a collection of all policies
-   */
-  public synchronized Collection<PolicyList> getAllPolicies() {
-    return new ArrayList(allPolicies);
-  }
-  
-  /**
-   * Set a collection of all policies
-   */
-  protected synchronized void setAllPolicies(Collection<PolicyList> value) {
-    this.allPolicies = value;
-  }
-
-  /**
-   * Start a background thread to reload the config file
-   */
-  void startReload() {
-    if (doReload) {
-      reloadThread = new UpdateThread();
-      reloadThread.start();
-    }
-  }
-
-  /**
-   * Stop the background thread that reload the config file
-   */
-  void stopReload() throws InterruptedException {
-    if (reloadThread != null) {
-      running = false;
-      reloadThread.interrupt();
-      reloadThread.join();
-      reloadThread = null;
-    }
-  }
-
-  /**
-   * A thread which reloads the config file.
-   */
-  private class UpdateThread extends Thread {
-    private UpdateThread() {
-      super("Raid update thread");
-    }
-
-    public void run() {
-      while (running) {
-        try {
-          Thread.sleep(reloadInterval);
-          reloadConfigsIfNecessary();
-        } catch (InterruptedException e) {
-          // do nothing 
-        } catch (Exception e) {
-          LOG.error("Failed to reload config file ", e);
-        }
-      }
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Decoder.java

@@ -1,213 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Random;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.BlockMissingException;
-import org.apache.hadoop.fs.ChecksumException;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-
-/**
- * Represents a generic decoder that can be used to read a file with
- * corrupt blocks by using the parity file.
- * This is an abstract class, concrete subclasses need to implement
- * fixErasedBlock.
- */
-public abstract class Decoder {
-  public static final Log LOG = LogFactory.getLog(
-                                  "org.apache.hadoop.raid.Decoder");
-  protected Configuration conf;
-  protected int stripeSize;
-  protected int paritySize;
-  protected Random rand;
-  protected int bufSize;
-  protected byte[][] readBufs;
-  protected byte[][] writeBufs;
-
-  Decoder(Configuration conf, int stripeSize, int paritySize) {
-    this.conf = conf;
-    this.stripeSize = stripeSize;
-    this.paritySize = paritySize;
-    this.rand = new Random();
-    this.bufSize = conf.getInt("raid.decoder.bufsize", 1024 * 1024);
-    this.readBufs = new byte[stripeSize + paritySize][];
-    this.writeBufs = new byte[paritySize][];
-    allocateBuffers();
-  }
-
-  private void allocateBuffers() {
-    for (int i = 0; i < stripeSize + paritySize; i++) {
-      readBufs[i] = new byte[bufSize];
-    }
-    for (int i = 0; i < paritySize; i++) {
-      writeBufs[i] = new byte[bufSize];
-    }
-  }
-
-  private void configureBuffers(long blockSize) {
-    if ((long)bufSize > blockSize) {
-      bufSize = (int)blockSize;
-      allocateBuffers();
-    } else if (blockSize % bufSize != 0) {
-      bufSize = (int)(blockSize / 256L); // heuristic.
-      if (bufSize == 0) {
-        bufSize = 1024;
-      }
-      bufSize = Math.min(bufSize, 1024 * 1024);
-      allocateBuffers();
-    }
-  }
-
-  /**
-   * The interface to generate a decoded file using the good portion of the
-   * source file and the parity file.
-   * @param fs The filesystem containing the source file.
-   * @param srcFile The damaged source file.
-   * @param parityFs The filesystem containing the parity file. This could be
-   *        different from fs in case the parity file is part of a HAR archive.
-   * @param parityFile The parity file.
-   * @param errorOffset Known location of error in the source file. There could
-   *        be additional errors in the source file that are discovered during
-   *        the decode process.
-   * @param decodedFile The decoded file. This will have the exact same contents
-   *        as the source file on success.
-   */
-  public void decodeFile(
-    FileSystem fs, Path srcFile, FileSystem parityFs, Path parityFile,
-    long errorOffset, Path decodedFile) throws IOException {
-
-    LOG.info("Create " + decodedFile + " for error at " +
-            srcFile + ":" + errorOffset);
-    FileStatus srcStat = fs.getFileStatus(srcFile);
-    long blockSize = srcStat.getBlockSize();
-    configureBuffers(blockSize);
-    // Move the offset to the start of the block.
-    errorOffset = (errorOffset / blockSize) * blockSize;
-
-    // Create the decoded file.
-    FSDataOutputStream out = fs.create(
-      decodedFile, false, conf.getInt("io.file.buffer.size", 64 * 1024),
-      srcStat.getReplication(), srcStat.getBlockSize());
-
-    // Open the source file.
-    FSDataInputStream in = fs.open(
-      srcFile, conf.getInt("io.file.buffer.size", 64 * 1024));
-
-    // Start copying data block-by-block.
-    for (long offset = 0; offset < srcStat.getLen(); offset += blockSize) {
-      long limit = Math.min(blockSize, srcStat.getLen() - offset);
-      long bytesAlreadyCopied = 0;
-      if (offset != errorOffset) {
-        try {
-          in = fs.open(
-            srcFile, conf.getInt("io.file.buffer.size", 64 * 1024));
-          in.seek(offset);
-          RaidUtils.copyBytes(in, out, readBufs[0], limit);
-          assert(out.getPos() == offset +limit);
-          LOG.info("Copied till " + out.getPos() + " from " + srcFile);
-          continue;
-        } catch (BlockMissingException e) {
-          LOG.warn("Encountered BME at " + srcFile + ":" + offset);
-          bytesAlreadyCopied = out.getPos() - offset;
-        } catch (ChecksumException e) {
-          LOG.warn("Encountered CE at " + srcFile + ":" + offset);
-          bytesAlreadyCopied = out.getPos() - offset;
-        }
-      }
-      // If we are here offset == errorOffset or we got an exception.
-      // Recover the block starting at offset.
-      fixErasedBlock(fs, srcFile, parityFs, parityFile, blockSize, offset,
-        bytesAlreadyCopied, limit, out);
-    }
-    out.close();
-
-    try {
-      fs.setOwner(decodedFile, srcStat.getOwner(), srcStat.getGroup());
-      fs.setPermission(decodedFile, srcStat.getPermission());
-      fs.setTimes(decodedFile, srcStat.getModificationTime(),
-                  srcStat.getAccessTime());
-    } catch (Exception exc) {
-      LOG.warn("Didn't manage to copy meta information because of " + exc +
-               " Ignoring...");
-    }
-
-  }
-
-  /**
-   * Recovers a corrupt block to local file.
-   *
-   * @param srcFs The filesystem containing the source file.
-   * @param srcPath The damaged source file.
-   * @param parityFs The filesystem containing the parity file. This could be
-   *        different from fs in case the parity file is part of a HAR archive.
-   * @param parityPath The parity file.
-   * @param blockSize The block size of the file.
-   * @param blockOffset Known location of error in the source file. There could
-   *        be additional errors in the source file that are discovered during
-   *        the decode process.
-   * @param localBlockFile The file to write the block to.
-   * @param limit The maximum number of bytes to be written out.
-   *              This is to prevent writing beyond the end of the file.
-   */
-  public void recoverBlockToFile(
-    FileSystem srcFs, Path srcPath, FileSystem parityFs, Path parityPath,
-    long blockSize, long blockOffset, File localBlockFile, long limit)
-    throws IOException {
-    OutputStream out = new FileOutputStream(localBlockFile);
-    fixErasedBlock(srcFs, srcPath, parityFs, parityPath,
-                  blockSize, blockOffset, 0, limit, out);
-    out.close();
-  }
-
-  /**
-   * Implementation-specific mechanism of writing a fixed block.
-   * @param fs The filesystem containing the source file.
-   * @param srcFile The damaged source file.
-   * @param parityFs The filesystem containing the parity file. This could be
-   *        different from fs in case the parity file is part of a HAR archive.
-   * @param parityFile The parity file.
-   * @param blockSize The maximum size of a block.
-   * @param errorOffset Known location of error in the source file. There could
-   *        be additional errors in the source file that are discovered during
-   *        the decode process.
-   * @param bytesToSkip After the block is generated, these many bytes should be
-   *       skipped before writing to the output. This is needed because the
-   *       output may have a portion of the block written from the source file
-   *       before a new corruption is discovered in the block.
-   * @param limit The maximum number of bytes to be written out, including
-   *       bytesToSkip. This is to prevent writing beyond the end of the file.
-   * @param out The output.
-   */
-  protected abstract void fixErasedBlock(
-      FileSystem fs, Path srcFile, FileSystem parityFs, Path parityFile,
-      long blockSize, long errorOffset, long bytesToSkip, long limit,
-      OutputStream out) throws IOException;
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DirectoryTraversal.java

@@ -1,323 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Stack;
-import java.util.concurrent.Executor;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.Semaphore;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.StringUtils;
-
-/**
- * Implements depth-first traversal using a Stack object. The traversal
- * can be stopped at any time and the state of traversal is saved.
- */
-public class DirectoryTraversal {
-  public static final Log LOG =
-    LogFactory.getLog("org.apache.hadoop.raid.DirectoryTraversal");
-
-  private FileSystem fs;
-  private List<FileStatus> paths;
-  private int pathIdx = 0;  // Next path to process.
-  private Stack<Node> stack = new Stack<Node>();
-  private ExecutorService executor;
-
-  private int numThreads;
-
-  /**
-   * A FileFilter object can be used to choose files during directory traversal.
-   */
-  public interface FileFilter {
-    /**
-     * @return a boolean value indicating if the file passes the filter.
-     */
-    boolean check(FileStatus f) throws IOException;
-  }
-
-  /**
-   * Represents a directory node in directory traversal.
-   */
-  static class Node {
-    private FileStatus path;  // Path that this node represents.
-    private FileStatus[] elements;  // Elements in the node.
-    private int idx = 0;
-
-    public Node(FileStatus path, FileStatus[] elements) {
-      this.path = path;
-      this.elements = elements;
-    }
-
-    public boolean hasNext() {
-      return idx < elements.length;
-    }
-
-    public FileStatus next() {
-      return elements[idx++];
-    }
-
-    public FileStatus path() {
-      return this.path;
-    }
-  }
-
-  /**
-   * Constructor.
-   * @param fs The filesystem to use.
-   * @param startPaths A list of paths that need to be traversed
-   */
-  public DirectoryTraversal(FileSystem fs, List<FileStatus> startPaths) {
-    this(fs, startPaths, 1);
-  }
-
-  public DirectoryTraversal(
-    FileSystem fs, List<FileStatus> startPaths, int numThreads) {
-    this.fs = fs;
-    paths = startPaths;
-    pathIdx = 0;
-    this.numThreads = numThreads;
-    executor = Executors.newFixedThreadPool(numThreads);
-  }
-
-  public List<FileStatus> getFilteredFiles(FileFilter filter, int limit) {
-    List<FileStatus> filtered = new ArrayList<FileStatus>();
-    if (limit == 0) 
-      return filtered;
-
-    // We need this semaphore to block when the number of running workitems
-    // is equal to the number of threads. FixedThreadPool limits the number
-    // of threads, but not the queue size. This way we will limit the memory
-    // usage.
-    Semaphore slots = new Semaphore(numThreads);
-
-    while (true) {
-      FilterFileWorkItem work = null;
-      try {
-        slots.acquire();
-        synchronized(filtered) {
-          if (filtered.size() >= limit) {
-            slots.release();
-            break;
-          }
-        }
-        Node next = getNextDirectoryNode();
-        if (next == null) {
-          slots.release();
-          break;
-        }
-        work = new FilterFileWorkItem(filter, next, filtered, slots);
-      } catch (InterruptedException ie) {
-        slots.release();
-        break;
-      } catch (IOException e) {
-        slots.release();
-        break;
-      }
-      executor.execute(work);
-    }
-
-    try {
-      // Wait for all submitted items to finish.
-      slots.acquire(numThreads);
-      // If this traversal is finished, shutdown the executor.
-      if (doneTraversal()) {
-        executor.shutdown();
-        executor.awaitTermination(1, TimeUnit.HOURS);
-      }
-    } catch (InterruptedException ie) {
-    }
-
-    return filtered;
-  }
-
-  class FilterFileWorkItem implements Runnable {
-    FileFilter filter;
-    Node dir;
-    List<FileStatus> filtered;
-    Semaphore slots;
-
-    FilterFileWorkItem(FileFilter filter, Node dir, List<FileStatus> filtered,
-      Semaphore slots) {
-      this.slots = slots;
-      this.filter = filter;
-      this.dir = dir;
-      this.filtered = filtered;
-    }
-
-    @SuppressWarnings("deprecation")
-    public void run() {
-      try {
-        LOG.info("Initiating file filtering for " + dir.path.getPath());
-        for (FileStatus f: dir.elements) {
-          if (!f.isFile()) {
-            continue;
-          }
-          if (filter.check(f)) {
-            synchronized(filtered) {
-              filtered.add(f);
-            }
-          }
-        }
-      } catch (Exception e) {
-        LOG.error("Error in directory traversal: " 
-          + StringUtils.stringifyException(e));
-      } finally {
-        slots.release();
-      }
-    }
-  }
-
-  /**
-   * Return the next file.
-   * @throws IOException
-   */
-  public FileStatus getNextFile() throws IOException {
-    // Check if traversal is done.
-    while (!doneTraversal()) {
-      // If traversal is not done, check if the stack is not empty.
-      while (!stack.isEmpty()) {
-        // If the stack is not empty, look at the top node.
-        Node node = stack.peek();
-        // Check if the top node has an element.
-        if (node.hasNext()) {
-          FileStatus element = node.next();
-          // Is the next element a directory.
-          if (!element.isDir()) {
-            // It is a file, return it.
-            return element;
-          }
-          // Next element is a directory, push it on to the stack and
-          // continue
-          try {
-            pushNewNode(element);
-          } catch (FileNotFoundException e) {
-            // Ignore and move to the next element.
-          }
-          continue;
-        } else {
-          // Top node has no next element, pop it and continue.
-          stack.pop();
-          continue;
-        }
-      }
-      // If the stack is empty, do we have more paths?
-      while (!paths.isEmpty()) {
-        FileStatus next = paths.remove(0);
-        pathIdx++;
-        if (!next.isDir()) {
-          return next;
-        }
-        try {
-          pushNewNode(next);
-        } catch (FileNotFoundException e) {
-          continue;
-        }
-        break;
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Gets the next directory in the tree. The algorithm returns deeper directories
-   * first.
-   * @return A FileStatus representing the directory.
-   * @throws IOException
-   */
-  public FileStatus getNextDirectory() throws IOException {
-    Node dirNode = getNextDirectoryNode();
-    if (dirNode != null) {
-      return dirNode.path;
-    }
-    return null;
-  }
-
-  private Node getNextDirectoryNode() throws IOException {
-
-    // Check if traversal is done.
-    while (!doneTraversal()) {
-      // If traversal is not done, check if the stack is not empty.
-      while (!stack.isEmpty()) {
-        // If the stack is not empty, look at the top node.
-        Node node = stack.peek();
-        // Check if the top node has an element.
-        if (node.hasNext()) {
-          FileStatus element = node.next();
-          // Is the next element a directory.
-          if (element.isDir()) {
-            // Next element is a directory, push it on to the stack and
-            // continue
-            try {
-              pushNewNode(element);
-            } catch (FileNotFoundException e) {
-              // Ignore and move to the next element.
-            }
-            continue;
-          }
-        } else {
-          stack.pop();
-          return node;
-        }
-      }
-      // If the stack is empty, do we have more paths?
-      while (!paths.isEmpty()) {
-        FileStatus next = paths.remove(0);
-        pathIdx++;
-        if (next.isDir()) {
-          try {
-            pushNewNode(next);
-          } catch (FileNotFoundException e) {
-            continue;
-          }
-          break;
-        }
-      }
-    }
-    return null;
-  }
-
-  private void pushNewNode(FileStatus stat) throws IOException {
-    if (!stat.isDir()) {
-      return;
-    }
-    Path p = stat.getPath();
-    FileStatus[] elements = fs.listStatus(p);
-    Node newNode = new Node(stat, (elements == null? new FileStatus[0]: elements));
-    stack.push(newNode);
-  }
-
-  public boolean doneTraversal() {
-    return paths.isEmpty() && stack.isEmpty();
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistBlockFixer.java

@@ -1,660 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.IOException;
-import java.io.PrintStream;
-import java.io.InputStreamReader;
-import java.io.BufferedReader;
-import java.util.List;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Date;
-import java.text.SimpleDateFormat;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.RaidDFSUtil;
-
-import org.apache.hadoop.conf.Configuration;
-
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
-
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Time;
-
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.InputSplit;
-
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-
-/**
- * distributed block fixer, uses map reduce jobs to fix corrupt files
- *
- * configuration options
- * raid.blockfix.filespertask       - number of corrupt files to fix in a single
- *                                    map reduce task (i.e., at one mapper node)
- *
- * raid.blockfix.maxpendingfiles    - maximum number of files to fix 
- *                                    simultaneously
- */
-public class DistBlockFixer extends BlockFixer {
-  // volatile should be sufficient since only the block fixer thread
-  // updates numJobsRunning (other threads may read)
-  private volatile int numJobsRunning = 0;
-
-  private static final String WORK_DIR_PREFIX = "blockfixer";
-  private static final String IN_FILE_SUFFIX = ".in";
-  private static final String PART_PREFIX = "part-";
-  
-  private static final String BLOCKFIX_FILES_PER_TASK = 
-    "raid.blockfix.filespertask";
-  private static final String BLOCKFIX_MAX_PENDING_FILES =
-    "raid.blockfix.maxpendingfiles";
-
-  // default number of files to fix in a task
-  private static final long DEFAULT_BLOCKFIX_FILES_PER_TASK = 10L;
-
-  // default number of files to fix simultaneously
-  private static final long DEFAULT_BLOCKFIX_MAX_PENDING_FILES = 1000L;
- 
-  protected static final Log LOG = LogFactory.getLog(DistBlockFixer.class);
-
-  // number of files to fix in a task
-  private long filesPerTask;
-
-  // number of files to fix simultaneously
-  final private long maxPendingFiles;
-
-  // number of files being fixed right now
-  private long pendingFiles;
-
-  private long lastCheckTime;
-
-  private final SimpleDateFormat dateFormat = 
-    new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-
-  private Map<String, CorruptFileInfo> fileIndex = 
-    new HashMap<String, CorruptFileInfo>();
-  private Map<Job, List<CorruptFileInfo>> jobIndex =
-    new HashMap<Job, List<CorruptFileInfo>>();
-
-  static enum Counter {
-    FILES_SUCCEEDED, FILES_FAILED, FILES_NOACTION
-  }
-
-  public DistBlockFixer(Configuration conf) {
-    super(conf);
-    filesPerTask = DistBlockFixer.filesPerTask(getConf());
-    maxPendingFiles = DistBlockFixer.maxPendingFiles(getConf());
-    pendingFiles = 0L;
-
-    // start off due for the first iteration
-    lastCheckTime = Time.now() - blockFixInterval;
-  }
-
-  /**
-   * determines how many files to fix in a single task
-   */ 
-  protected static long filesPerTask(Configuration conf) {
-    return conf.getLong(BLOCKFIX_FILES_PER_TASK, 
-                        DEFAULT_BLOCKFIX_FILES_PER_TASK);
-
-  }
-  /**
-   * determines how many files to fix simultaneously
-   */ 
-  protected static long maxPendingFiles(Configuration conf) {
-    return conf.getLong(BLOCKFIX_MAX_PENDING_FILES, 
-                        DEFAULT_BLOCKFIX_MAX_PENDING_FILES);
-  }
-
-  /**
-   * runs the block fixer periodically
-   */
-  public void run() {
-    while (running) {
-      // check if it is time to run the block fixer
-      long now = Time.now();
-      if (now >= lastCheckTime + blockFixInterval) {
-        lastCheckTime = now;
-        try {
-          checkAndFixBlocks(now);
-        } catch (InterruptedException ignore) {
-          LOG.info("interrupted");
-        } catch (Exception e) {
-          // log exceptions and keep running
-          LOG.error(StringUtils.stringifyException(e));
-        } catch (Error e) {
-          LOG.error(StringUtils.stringifyException(e));
-          throw e;
-        }
-      }
-      
-      // try to sleep for the remainder of the interval
-      long sleepPeriod = (lastCheckTime - Time.now()) + 
-        blockFixInterval;
-      
-      if ((sleepPeriod > 0L) && running) {
-        try {
-          Thread.sleep(sleepPeriod);
-        } catch (InterruptedException ignore) {
-          LOG.info("interrupted");
-        }
-      }
-    }
-  }
-
-  /**
-   * checks for corrupt blocks and fixes them (if any)
-   */
-  private void checkAndFixBlocks(long startTime)
-    throws IOException, InterruptedException, ClassNotFoundException {
-    checkJobs();
-
-    if (pendingFiles >= maxPendingFiles) {
-      return;
-    }
-
-    List<Path> corruptFiles = getCorruptFiles();
-    filterUnfixableSourceFiles(corruptFiles.iterator());
-
-    String startTimeStr = dateFormat.format(new Date(startTime));
-
-    LOG.info("found " + corruptFiles.size() + " corrupt files");
-
-    if (corruptFiles.size() > 0) {
-      String jobName = "blockfixer." + startTime;
-      startJob(jobName, corruptFiles);
-    }
-  }
-
-  /**
-   * Handle a failed job.
-   */
-  private void failJob(Job job) throws IOException {
-    // assume no files have been fixed
-    LOG.error("DistBlockFixer job " + job.getJobID() + "(" + job.getJobName() +
-      ") finished (failed)");
-    for (CorruptFileInfo fileInfo: jobIndex.get(job)) {
-      fileInfo.fail();
-    }
-    numJobsRunning--;
-  }
-
-  /**
-   * Handle a successful job.
-   */ 
-  private void succeedJob(Job job, long filesSucceeded, long filesFailed)
-    throws IOException {
-    LOG.info("DistBlockFixer job " + job.getJobID() + "(" + job.getJobName() +
-      ") finished (succeeded)");
-
-    if (filesFailed == 0) {
-      // no files have failed
-      for (CorruptFileInfo fileInfo: jobIndex.get(job)) {
-        fileInfo.succeed();
-      }
-    } else {
-      // we have to look at the output to check which files have failed
-      Set<String> failedFiles = getFailedFiles(job);
-      
-      for (CorruptFileInfo fileInfo: jobIndex.get(job)) {
-        if (failedFiles.contains(fileInfo.getFile().toString())) {
-          fileInfo.fail();
-        } else {
-          // call succeed for files that have succeeded or for which no action
-          // was taken
-          fileInfo.succeed();
-        }
-      }
-    }
-    // report succeeded files to metrics
-    incrFilesFixed(filesSucceeded);
-    numJobsRunning--;
-  }
-
-  /**
-   * checks if jobs have completed and updates job and file index
-   * returns a list of failed files for restarting
-   */
-  private void checkJobs() throws IOException {
-    Iterator<Job> jobIter = jobIndex.keySet().iterator();
-    while(jobIter.hasNext()) {
-      Job job = jobIter.next();
-
-      try {
-        if (job.isComplete()) {
-          long filesSucceeded =
-            job.getCounters().findCounter(Counter.FILES_SUCCEEDED).getValue();
-          long filesFailed =
-            job.getCounters().findCounter(Counter.FILES_FAILED).getValue();
-          long filesNoAction =
-            job.getCounters().findCounter(Counter.FILES_NOACTION).getValue();
-          int files = jobIndex.get(job).size();
-          if (job.isSuccessful() && 
-              (filesSucceeded + filesFailed + filesNoAction == 
-               ((long) files))) {
-            // job has processed all files
-            succeedJob(job, filesSucceeded, filesFailed);
-          } else {
-            failJob(job);
-          }
-          jobIter.remove();
-        } else {
-          LOG.info("job " + job.getJobName() + " still running");
-        }
-      } catch (Exception e) {
-        LOG.error(StringUtils.stringifyException(e));
-        failJob(job);
-        try {
-          job.killJob();
-        } catch (Exception ee) {
-          LOG.error(StringUtils.stringifyException(ee));
-        }
-        jobIter.remove();
-      }
-    }
-    purgeFileIndex();
-  }
-
-  /**
-   * determines which files have failed for a given job
-   */
-  private Set<String> getFailedFiles(Job job) throws IOException {
-    Set<String> failedFiles = new HashSet<String>();
-
-    Path outDir = SequenceFileOutputFormat.getOutputPath(job);
-    FileSystem fs  = outDir.getFileSystem(getConf());
-    if (!fs.getFileStatus(outDir).isDir()) {
-      throw new IOException(outDir.toString() + " is not a directory");
-    }
-
-    FileStatus[] files = fs.listStatus(outDir);
-
-    for (FileStatus f: files) {
-      Path fPath = f.getPath();
-      if ((!f.isDir()) && (fPath.getName().startsWith(PART_PREFIX))) {
-        LOG.info("opening " + fPath.toString());
-        SequenceFile.Reader reader = 
-          new SequenceFile.Reader(fs, fPath, getConf());
-
-        Text key = new Text();
-        Text value = new Text();
-        while (reader.next(key, value)) {
-          failedFiles.add(key.toString());
-        }
-        reader.close();
-      }
-    }
-    return failedFiles;
-  }
-
-
-  /**
-   * purge expired jobs from the file index
-   */
-  private void purgeFileIndex() {
-    Iterator<String> fileIter = fileIndex.keySet().iterator();
-    while(fileIter.hasNext()) {
-      String file = fileIter.next();
-      if (fileIndex.get(file).isExpired()) {
-        fileIter.remove();
-      }
-    }
-    
-  }
-
-  /**
-   * creates and submits a job, updates file index and job index
-   */
-  private Job startJob(String jobName, List<Path> corruptFiles) 
-    throws IOException, InterruptedException, ClassNotFoundException {
-    Path inDir = new Path(WORK_DIR_PREFIX + "/in/" + jobName);
-    Path outDir = new Path(WORK_DIR_PREFIX + "/out/" + jobName);
-    List<Path> filesInJob = createInputFile(jobName, inDir, corruptFiles);
-
-    Configuration jobConf = new Configuration(getConf());
-    Job job = new Job(jobConf, jobName);
-    job.setJarByClass(getClass());
-    job.setMapperClass(DistBlockFixerMapper.class);
-    job.setNumReduceTasks(0);
-    job.setInputFormatClass(DistBlockFixerInputFormat.class);
-    job.setOutputFormatClass(SequenceFileOutputFormat.class);
-    job.setOutputKeyClass(Text.class);
-    job.setOutputValueClass(Text.class);
-
-    DistBlockFixerInputFormat.setInputPaths(job, inDir);
-    SequenceFileOutputFormat.setOutputPath(job, outDir);
-
-    job.submit();
-    LOG.info("DistBlockFixer job " + job.getJobID() + "(" + job.getJobName() +
-          ") started");
-
-    // submit the job before inserting it into the index
-    // this way, if submit fails, we won't have added anything to the index
-    insertJob(job, filesInJob);
-    return job;
-  }
-
-  /**
-   * inserts new job into file index and job index
-   */
-  private void insertJob(Job job, List<Path> corruptFiles) {
-    List<CorruptFileInfo> fileInfos = new LinkedList<CorruptFileInfo>();
-
-    for (Path file: corruptFiles) {
-      CorruptFileInfo fileInfo = new CorruptFileInfo(file, job);
-      fileInfos.add(fileInfo);
-      fileIndex.put(file.toString(), fileInfo);
-    }
-
-    jobIndex.put(job, fileInfos);
-    numJobsRunning++;
-  }
-    
-  /**
-   * creates the input file (containing the names of the files to be fixed
-   */
-  private List<Path> createInputFile(String jobName, Path inDir, 
-                                     List<Path> corruptFiles) 
-    throws IOException {
-
-    Path file = new Path(inDir, jobName + IN_FILE_SUFFIX);
-    FileSystem fs = file.getFileSystem(getConf());
-    SequenceFile.Writer fileOut = SequenceFile.createWriter(fs, getConf(), file,
-                                                            LongWritable.class,
-                                                            Text.class);
-    long index = 0L;
-
-    List<Path> filesAdded = new LinkedList<Path>();
-
-    for (Path corruptFile: corruptFiles) {
-      if (pendingFiles >= maxPendingFiles) {
-        break;
-      }
-
-      String corruptFileName = corruptFile.toString();
-      fileOut.append(new LongWritable(index++), new Text(corruptFileName));
-      filesAdded.add(corruptFile);
-      pendingFiles++;
-
-      if (index % filesPerTask == 0) {
-        fileOut.sync(); // create sync point to make sure we can split here
-      }
-    }
-
-    fileOut.close();
-    return filesAdded;
-  }
-
-  /**
-   * gets a list of corrupt files from the name node
-   * and filters out files that are currently being fixed or 
-   * that were recently fixed
-   */
-  private List<Path> getCorruptFiles() throws IOException {
-    DistributedFileSystem dfs = (DistributedFileSystem) 
-      (new Path("/")).getFileSystem(getConf());
-
-    String[] files = RaidDFSUtil.getCorruptFiles(dfs);
-    List<Path> corruptFiles = new LinkedList<Path>();
-
-    for (String f: files) {
-      Path p = new Path(f);
-      // filter out files that are being fixed or that were recently fixed
-      if (!fileIndex.containsKey(p.toString())) {
-        corruptFiles.add(p);
-      }
-    }
-    RaidUtils.filterTrash(getConf(), corruptFiles);
-
-    return corruptFiles;
-  }
-
-  /**
-   * returns the number of map reduce jobs running
-   */
-  public int jobsRunning() {
-    return numJobsRunning;
-  }
-
-  /**
-   * hold information about a corrupt file that is being fixed
-   */
-  class CorruptFileInfo {
-
-    private Path file;
-    private Job job;
-    private boolean done;
-    private long time;
-
-    public CorruptFileInfo(Path file, Job job) {
-      this.file = file;
-      this.job = job;
-      this.done = false;
-      this.time = 0;
-    }
-
-    public boolean isDone() {
-      return done;
-    }
-    
-    public boolean isExpired() {
-      return done && ((Time.now() - time) > historyInterval);
-    }
-
-    public Path getFile() {
-      return file;
-    }
-    
-    /**
-     * updates file index to record a failed attempt at fixing a file,
-     * immediately removes the entry from the file index 
-     * (instead of letting it expire)
-     * so that we can retry right away
-     */
-    public void fail() {
-      // remove this file from the index
-      CorruptFileInfo removed = fileIndex.remove(file.toString());
-      if (removed == null) {
-        LOG.error("trying to remove file not in file index: " +
-                  file.toString());
-      } else {
-        LOG.error("fixing " + file.toString() + " failed");
-      }
-      pendingFiles--;
-    }
-
-    /**
-     * marks a file as fixed successfully
-     * and sets time stamp for expiry after specified interval
-     */
-    public void succeed() {
-      // leave the file in the index,
-      // will be pruged later
-      job = null;
-      done = true;
-      time = Time.now();
-      LOG.info("fixing " + file.toString() + " succeeded");
-      pendingFiles--;
-    }
-  }
-
-  static class DistBlockFixerInputFormat
-    extends SequenceFileInputFormat<LongWritable, Text> {
-
-    protected static final Log LOG = 
-      LogFactory.getLog(DistBlockFixerMapper.class);
-    
-    /**
-     * splits the input files into tasks handled by a single node
-     * we have to read the input files to do this based on a number of 
-     * items in a sequence
-     */
-    @Override
-    public List <InputSplit> getSplits(JobContext job) 
-      throws IOException {
-      long filesPerTask = DistBlockFixer.filesPerTask(job.getConfiguration());
-
-      Path[] inPaths = getInputPaths(job);
-
-      List<InputSplit> splits = new LinkedList<InputSplit>();
-
-      long fileCounter = 0;
-
-      for (Path inPath: inPaths) {
-        
-        FileSystem fs = inPath.getFileSystem(job.getConfiguration());      
-
-        if (!fs.getFileStatus(inPath).isDir()) {
-          throw new IOException(inPath.toString() + " is not a directory");
-        }
-
-        FileStatus[] inFiles = fs.listStatus(inPath);
-
-        for (FileStatus inFileStatus: inFiles) {
-          Path inFile = inFileStatus.getPath();
-          
-          if (!inFileStatus.isDir() &&
-              (inFile.getName().equals(job.getJobName() + IN_FILE_SUFFIX))) {
-
-            fileCounter++;
-            SequenceFile.Reader inFileReader = 
-              new SequenceFile.Reader(fs, inFile, job.getConfiguration());
-            
-            long startPos = inFileReader.getPosition();
-            long counter = 0;
-            
-            // create an input split every filesPerTask items in the sequence
-            LongWritable key = new LongWritable();
-            Text value = new Text();
-            try {
-              while (inFileReader.next(key, value)) {
-                if (counter % filesPerTask == filesPerTask - 1L) {
-                  splits.add(new FileSplit(inFile, startPos, 
-                                           inFileReader.getPosition() - 
-                                           startPos,
-                                           null));
-                  startPos = inFileReader.getPosition();
-                }
-                counter++;
-              }
-              
-              // create input split for remaining items if necessary
-              // this includes the case where no splits were created by the loop
-              if (startPos != inFileReader.getPosition()) {
-                splits.add(new FileSplit(inFile, startPos,
-                                         inFileReader.getPosition() - startPos,
-                                         null));
-              }
-            } finally {
-              inFileReader.close();
-            }
-          }
-        }
-      }
-
-      LOG.info("created " + splits.size() + " input splits from " +
-               fileCounter + " files");
-      
-      return splits;
-    }
-
-    /**
-     * indicates that input file can be split
-     */
-    @Override
-    public boolean isSplitable (JobContext job, Path file) {
-      return true;
-    }
-  }
-
-
-  /**
-   * mapper for fixing stripes with corrupt blocks
-   */
-  static class DistBlockFixerMapper
-    extends Mapper<LongWritable, Text, Text, Text> {
-
-    protected static final Log LOG = 
-      LogFactory.getLog(DistBlockFixerMapper.class);
-
-    /**
-     * fix a stripe
-     */
-    @Override
-    public void map(LongWritable key, Text fileText, Context context) 
-      throws IOException, InterruptedException {
-      
-      BlockFixerHelper helper = 
-        new BlockFixerHelper(context.getConfiguration());
-
-      String fileStr = fileText.toString();
-      LOG.info("fixing " + fileStr);
-
-      Path file = new Path(fileStr);
-      boolean success = false;
-
-      try {
-        boolean fixed = helper.fixFile(file, context);
-        
-        if (fixed) {
-          context.getCounter(Counter.FILES_SUCCEEDED).increment(1L);
-        } else {
-          context.getCounter(Counter.FILES_NOACTION).increment(1L);
-        }
-      } catch (Exception e) {
-        LOG.error(StringUtils.stringifyException(e));
-
-        // report file as failed
-        context.getCounter(Counter.FILES_FAILED).increment(1L);
-        String outkey = fileStr;
-        String outval = "failed";
-        context.write(new Text(outkey), new Text(outval));
-      }
-      
-      context.progress();
-    }
-  }
-
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaid.java

@@ -1,374 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.Date;
-import java.text.SimpleDateFormat;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.SequenceFile.Writer;
-import org.apache.hadoop.io.SequenceFile.Reader;
-
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.raid.RaidNode.Statistics;
-import org.apache.hadoop.raid.protocol.PolicyInfo;
-import org.apache.hadoop.util.StringUtils;
-
-public class DistRaid extends Configured {
-
-  protected static final Log LOG = LogFactory.getLog(DistRaid.class);
-
-  static final String NAME = "distRaid";
-  static final String JOB_DIR_LABEL = NAME + ".job.dir";
-  static final int   OP_LIST_BLOCK_SIZE = 32 * 1024 * 1024; // block size of control file
-  static final short OP_LIST_REPLICATION = 10; // replication factor of control file
-
-  public static final String OPS_PER_TASK = "raid.distraid.opspertask";
-  private static final int DEFAULT_OPS_PER_TASK = 100;
-  private static final int SYNC_FILE_MAX = 10;
-  private static final SimpleDateFormat dateForm = new SimpleDateFormat("yyyy-MM-dd HH:mm");
-
-  static enum Counter {
-    FILES_SUCCEEDED, FILES_FAILED, PROCESSED_BLOCKS, PROCESSED_SIZE, META_BLOCKS, META_SIZE
-  }
-
-  public DistRaid(Configuration conf) {
-    super(conf);
-  }
-
-  private static final Random RANDOM = new Random();
-
-  protected static String getRandomId() {
-    return Integer.toString(RANDOM.nextInt(Integer.MAX_VALUE), 36);
-  }
-
-  /**
-   *
-   * helper class which holds the policy and paths
-   *
-   */
-  public static class RaidPolicyPathPair {
-    public PolicyInfo policy;
-    public List<FileStatus> srcPaths;
-
-    RaidPolicyPathPair(PolicyInfo policy, List<FileStatus> srcPaths) {
-      this.policy = policy;
-      this.srcPaths = srcPaths;
-    }
-  }
-
-  List<RaidPolicyPathPair> raidPolicyPathPairList = new ArrayList<RaidPolicyPathPair>();
-
-  private Job runningJob;
-  private String lastReport = null;
-
-  /** Responsible for generating splits of the src file list. */
-  static class DistRaidInputFormat extends 
-		  SequenceFileInputFormat<Text, PolicyInfo> {
-    /**
-     * Produce splits such that each is no greater than the quotient of the
-     * total size and the number of splits requested.
-     *
-     * @param job
-     *          The handle to the Configuration object
-     * @param numSplits
-     *          Number of splits requested
-     */
-    public List<InputSplit> getSplits(JobContext job) throws IOException {
-      Configuration conf = job.getConfiguration();
-
-      // We create only one input file. So just get the first file in the first
-      // input directory.
-      Path inDir = getInputPaths(job)[0];
-      FileSystem fs = inDir.getFileSystem(conf);
-      FileStatus[] inputFiles = fs.listStatus(inDir);
-      Path inputFile = inputFiles[0].getPath();
-
-      List<InputSplit> splits = new ArrayList<InputSplit>();
-      SequenceFile.Reader in =
-        new SequenceFile.Reader(conf, Reader.file(inputFile));
-      long prev = 0L;
-      final int opsPerTask = conf.getInt(OPS_PER_TASK, DEFAULT_OPS_PER_TASK);
-      try {
-        Text key = new Text();
-        PolicyInfo value = new PolicyInfo();
-        int count = 0; // count src
-        while (in.next(key, value)) {
-          long curr = in.getPosition();
-          long delta = curr - prev;
-          if (++count > opsPerTask) {
-            count = 0;
-            splits.add(new FileSplit(inputFile, prev, delta, (String[]) null));
-            prev = curr;
-          }
-        }
-      } finally {
-        in.close();
-      }
-      long remaining = fs.getFileStatus(inputFile).getLen() - prev;
-      if (remaining != 0) {
-        splits.add(new FileSplit(inputFile, prev, remaining, (String[]) null));
-      }
-      return splits;
-    }
-  }
-
-  /** The mapper for raiding files. */
-  static class DistRaidMapper extends Mapper<Text, PolicyInfo, Text, Text> {
-    private boolean ignoreFailures = false;
-
-    private int failcount = 0;
-    private int succeedcount = 0;
-    private Statistics st = new Statistics();
-
-    private String getCountString() {
-      return "Succeeded: " + succeedcount + " Failed: " + failcount;
-    }
-
-    /** Run a FileOperation
-     * @throws IOException
-     * @throws InterruptedException */
-    public void map(Text key, PolicyInfo policy, Context context)
-        throws IOException, InterruptedException {
-      try {
-        Configuration jobConf = context.getConfiguration();
-        LOG.info("Raiding file=" + key.toString() + " policy=" + policy);
-        Path p = new Path(key.toString());
-        FileStatus fs = p.getFileSystem(jobConf).getFileStatus(p);
-        st.clear();
-        RaidNode.doRaid(jobConf, policy, fs, st, context);
-
-        ++succeedcount;
-
-        context.getCounter(Counter.PROCESSED_BLOCKS).increment(st.numProcessedBlocks);
-        context.getCounter(Counter.PROCESSED_SIZE).increment(st.processedSize);
-        context.getCounter(Counter.META_BLOCKS).increment(st.numMetaBlocks);
-        context.getCounter(Counter.META_SIZE).increment(st.metaSize);
-        context.getCounter(Counter.FILES_SUCCEEDED).increment(1);
-      } catch (IOException e) {
-        ++failcount;
-        context.getCounter(Counter.FILES_FAILED).increment(1);
-
-        String s = "FAIL: " + policy + ", " + key + " "
-            + StringUtils.stringifyException(e);
-        context.write(new Text(key), new Text(s));
-        LOG.error(s);
-      } finally {
-        context.setStatus(getCountString());
-      }
-    }
-
-    /** {@inheritDoc} */
-    public void close() throws IOException {
-      if (failcount == 0 || ignoreFailures) {
-        return;
-      }
-      throw new IOException(getCountString());
-    }
-  }
-
-  /**
-   * Set options specified in raid.scheduleroption.
-   * The string should be formatted as key:value[,key:value]*
-   */
-  static void setSchedulerOption(Configuration conf) {
-    String schedulerOption = conf.get("raid.scheduleroption");
-    if (schedulerOption != null) {
-      // Parse the scheduler option to get key:value pairs.
-      String[] keyValues = schedulerOption.trim().split(",");
-      for (String keyValue: keyValues) {
-        String[] fields = keyValue.trim().split(":");
-        String key = fields[0].trim();
-        String value = fields[1].trim();
-        conf.set(key, value);
-      }
-    }
-  }
-
-  /**
-   * Creates a new Job object.
-   * @param conf
-   * @return a Job object
-   * @throws IOException
-   */
-  static Job createJob(Configuration jobConf) throws IOException {
-    String jobName = NAME + " " + dateForm.format(new Date(RaidNode.now()));
-
-    setSchedulerOption(jobConf);
-
-    Job job = Job.getInstance(jobConf, jobName);
-    job.setSpeculativeExecution(false);
-    job.setJarByClass(DistRaid.class);
-    job.setInputFormatClass(DistRaidInputFormat.class);
-    job.setOutputKeyClass(Text.class);
-    job.setOutputValueClass(Text.class);
-
-    job.setMapperClass(DistRaidMapper.class);
-    job.setNumReduceTasks(0);
-
-    return job;
-  }
-
-  /** Add paths to be raided */
-  public void addRaidPaths(PolicyInfo info, List<FileStatus> paths) {
-    raidPolicyPathPairList.add(new RaidPolicyPathPair(info, paths));
-  }
-
-  /** Invokes a map-reduce job do parallel raiding.
-   *  @return true if the job was started, false otherwise
-   * @throws InterruptedException
-   */
-  public boolean startDistRaid() throws IOException {
-    assert(raidPolicyPathPairList.size() > 0);
-    Job job = createJob(getConf());
-    createInputFile(job);
-    try {
-      job.submit();
-      this.runningJob = job;
-      LOG.info("Job Started: " + runningJob.getJobID());
-      return true;
-    } catch (ClassNotFoundException e) {
-      throw new IOException(e);
-    } catch (InterruptedException e) {
-      return false;
-    }
-  }
-
-   /** Checks if the map-reduce job has completed.
-    *
-    * @return true if the job completed, false otherwise.
-    * @throws IOException
-    */
-   public boolean checkComplete() throws IOException {
-     JobID jobID = runningJob.getJobID();
-     LOG.info("Checking job " + jobID);
-     try {
-      if (runningJob.isComplete()) {
-         // delete job directory
-         Configuration jobConf = runningJob.getConfiguration();
-         final String jobdir = jobConf.get(JOB_DIR_LABEL);
-         if (jobdir != null) {
-           final Path jobpath = new Path(jobdir);
-           jobpath.getFileSystem(jobConf).delete(jobpath, true);
-         }
-         if (runningJob.isSuccessful()) {
-           LOG.info("Job Complete(Succeeded): " + jobID);
-         } else {
-           LOG.error("Job Complete(Failed): " + jobID);
-         }
-         raidPolicyPathPairList.clear();
-         return true;
-       } else {
-         String report =  (" job " + jobID +
-           " map " + StringUtils.formatPercent(runningJob.mapProgress(), 0)+
-           " reduce " + StringUtils.formatPercent(runningJob.reduceProgress(), 0));
-         if (!report.equals(lastReport)) {
-           LOG.info(report);
-           lastReport = report;
-         }
-         return false;
-       }
-    } catch (InterruptedException e) {
-      return false;
-    }
-   }
-
-   public boolean successful() throws IOException {
-     try {
-      return runningJob.isSuccessful();
-    } catch (InterruptedException e) {
-      return false;
-    }
-   }
-
-  /**
-   * set up input file which has the list of input files.
-   *
-   * @return boolean
-   * @throws IOException
-   */
-  private void createInputFile(Job job) throws IOException {
-    Configuration jobConf = job.getConfiguration();
-    Path jobDir = new Path(JOB_DIR_LABEL + getRandomId());
-    Path inDir = new Path(jobDir, "in");
-    Path outDir = new Path(jobDir, "out");
-    FileInputFormat.setInputPaths(job, inDir);
-    FileOutputFormat.setOutputPath(job, outDir);
-    Path opList = new Path(inDir, NAME);
-
-    Configuration tmp = new Configuration(jobConf);
-    // The control file should have small size blocks. This helps
-    // in spreading out the load from mappers that will be spawned.
-    tmp.setInt("dfs.blocks.size",  OP_LIST_BLOCK_SIZE);
-    FileSystem fs = opList.getFileSystem(tmp);
-
-    int opCount = 0, synCount = 0;
-    SequenceFile.Writer opWriter = null;
-    try {
-      opWriter = SequenceFile.createWriter(
-          jobConf, Writer.file(opList), Writer.keyClass(Text.class),
-          Writer.valueClass(PolicyInfo.class),
-          Writer.compression(SequenceFile.CompressionType.NONE));
-      for (RaidPolicyPathPair p : raidPolicyPathPairList) {
-        // If a large set of files are Raided for the first time, files
-        // in the same directory that tend to have the same size will end up
-        // with the same map. This shuffle mixes things up, allowing a better
-        // mix of files.
-        java.util.Collections.shuffle(p.srcPaths);
-        for (FileStatus st : p.srcPaths) {
-          opWriter.append(new Text(st.getPath().toString()), p.policy);
-          opCount++;
-          if (++synCount > SYNC_FILE_MAX) {
-            opWriter.sync();
-            synCount = 0;
-          }
-        }
-      }
-
-    } finally {
-      if (opWriter != null) {
-        opWriter.close();
-      }
-      // increase replication for control file
-      fs.setReplication(opList, OP_LIST_REPLICATION);
-    }
-    raidPolicyPathPairList.clear();
-    LOG.info("Number of files=" + opCount);
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/DistRaidNode.java

@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.conf.Configuration;
-
-import org.apache.hadoop.util.Daemon;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileStatus;
-
-import org.apache.hadoop.raid.protocol.PolicyInfo;
-
-/**
- * Implementation of {@link RaidNode} that uses map reduce jobs to raid files.
- */
-public class DistRaidNode extends RaidNode {
-
-  public static final Log LOG = LogFactory.getLog(DistRaidNode.class);
-
-  /** Daemon thread to monitor raid job progress */
-  JobMonitor jobMonitor = null;
-  Daemon jobMonitorThread = null;
-
-  public DistRaidNode(Configuration conf) throws IOException {
-    super(conf);
-    this.jobMonitor = new JobMonitor(conf);
-    this.jobMonitorThread = new Daemon(this.jobMonitor);
-    this.jobMonitorThread.start();
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public void join() {
-    super.join();
-    try {
-      if (jobMonitorThread != null) jobMonitorThread.join();
-    } catch (InterruptedException ie) {
-      // do nothing
-    }
-  }
-  
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public void stop() {
-    if (stopRequested) {
-      return;
-    }
-    super.stop();
-    if (jobMonitor != null) jobMonitor.running = false;
-    if (jobMonitorThread != null) jobMonitorThread.interrupt();
-  }
-
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  void raidFiles(PolicyInfo info, List<FileStatus> paths) throws IOException {
-    // We already checked that no job for this policy is running
-    // So we can start a new job.
-    DistRaid dr = new DistRaid(conf);
-    //add paths for distributed raiding
-    dr.addRaidPaths(info, paths);
-    boolean started = dr.startDistRaid();
-    if (started) {
-      jobMonitor.monitorJob(info.getName(), dr);
-    }
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  int getRunningJobsForPolicy(String policyName) {
-    return jobMonitor.runningJobsCount(policyName);
-  }
-
-
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/Encoder.java

@@ -1,350 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.Random;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Progressable;
-
-/**
- * Represents a generic encoder that can generate a parity file for a source
- * file.
- * This is an abstract class, concrete subclasses need to implement
- * encodeFileImpl.
- */
-public abstract class Encoder {
-  public static final Log LOG = LogFactory.getLog(
-                                  "org.apache.hadoop.raid.Encoder");
-  protected Configuration conf;
-  protected int stripeSize;
-  protected int paritySize;
-  protected Random rand;
-  protected int bufSize;
-  protected byte[][] readBufs;
-  protected byte[][] writeBufs;
-
-  /**
-   * A class that acts as a sink for data, similar to /dev/null.
-   */
-  static class NullOutputStream extends OutputStream {
-    public void write(byte[] b) throws IOException {}
-    public void write(int b) throws IOException {}
-    public void write(byte[] b, int off, int len) throws IOException {}
-  }
-
-  Encoder(
-    Configuration conf, int stripeSize, int paritySize) {
-    this.conf = conf;
-    this.stripeSize = stripeSize;
-    this.paritySize = paritySize;
-    this.rand = new Random();
-    this.bufSize = conf.getInt("raid.encoder.bufsize", 1024 * 1024);
-    this.readBufs = new byte[stripeSize][];
-    this.writeBufs = new byte[paritySize][];
-    allocateBuffers();
-  }
-
-  private void allocateBuffers() {
-    for (int i = 0; i < stripeSize; i++) {
-      readBufs[i] = new byte[bufSize];
-    }
-    for (int i = 0; i < paritySize; i++) {
-      writeBufs[i] = new byte[bufSize];
-    }
-  }
-
-  private void configureBuffers(long blockSize) {
-    if ((long)bufSize > blockSize) {
-      bufSize = (int)blockSize;
-      allocateBuffers();
-    } else if (blockSize % bufSize != 0) {
-      bufSize = (int)(blockSize / 256L); // heuristic.
-      if (bufSize == 0) {
-        bufSize = 1024;
-      }
-      bufSize = Math.min(bufSize, 1024 * 1024);
-      allocateBuffers();
-    }
-  }
-
-  /**
-   * The interface to use to generate a parity file.
-   * This method can be called multiple times with the same Encoder object,
-   * thus allowing reuse of the buffers allocated by the Encoder object.
-   *
-   * @param fs The filesystem containing the source file.
-   * @param srcFile The source file.
-   * @param parityFile The parity file to be generated.
-   */
-  public void encodeFile(
-    FileSystem fs, Path srcFile, FileSystem parityFs, Path parityFile,
-    short parityRepl, Progressable reporter) throws IOException {
-    FileStatus srcStat = fs.getFileStatus(srcFile);
-    long srcSize = srcStat.getLen();
-    long blockSize = srcStat.getBlockSize();
-
-    configureBuffers(blockSize);
-
-    // Create a tmp file to which we will write first.
-    Path tmpDir = getParityTempPath();
-    if (!parityFs.mkdirs(tmpDir)) {
-      throw new IOException("Could not create tmp dir " + tmpDir);
-    }
-    Path parityTmp = new Path(tmpDir,
-                            parityFile.getName() + rand.nextLong());
-    FSDataOutputStream out = parityFs.create(
-                               parityTmp,
-                               true,
-                               conf.getInt("io.file.buffer.size", 64 * 1024),
-                               parityRepl,
-                               blockSize);
-
-    try {
-      encodeFileToStream(fs, srcFile, srcSize, blockSize, out, reporter);
-      out.close();
-      out = null;
-      LOG.info("Wrote temp parity file " + parityTmp);
-
-      // delete destination if exists
-      if (parityFs.exists(parityFile)){
-        parityFs.delete(parityFile, false);
-      }
-      parityFs.mkdirs(parityFile.getParent());
-      if (!parityFs.rename(parityTmp, parityFile)) {
-        String msg = "Unable to rename file " + parityTmp + " to " + parityFile;
-        throw new IOException (msg);
-      }
-      LOG.info("Wrote parity file " + parityFile);
-    } finally {
-      if (out != null) {
-        out.close();
-      }
-      parityFs.delete(parityTmp, false);
-    }
-  }
-
-  /**
-   * Recovers a corrupt block in a parity file to a local file.
-   *
-   * The encoder generates paritySize parity blocks for a source file stripe.
-   * Since we want only one of the parity blocks, this function creates
-   * null outputs for the blocks to be discarded.
-   *
-   * @param fs The filesystem in which both srcFile and parityFile reside.
-   * @param srcFile The source file.
-   * @param srcSize The size of the source file.
-   * @param blockSize The block size for the source/parity files.
-   * @param corruptOffset The location of corruption in the parity file.
-   * @param localBlockFile The destination for the reovered block.
-   */
-  public void recoverParityBlockToFile(
-    FileSystem fs,
-    Path srcFile, long srcSize, long blockSize,
-    Path parityFile, long corruptOffset,
-    File localBlockFile) throws IOException {
-    OutputStream out = new FileOutputStream(localBlockFile);
-    try {
-      recoverParityBlockToStream(fs, srcFile, srcSize, blockSize, parityFile,
-        corruptOffset, out);
-    } finally {
-      out.close();
-    }
-  }
-
-  /**
-   * Recovers a corrupt block in a parity file to a local file.
-   *
-   * The encoder generates paritySize parity blocks for a source file stripe.
-   * Since we want only one of the parity blocks, this function creates
-   * null outputs for the blocks to be discarded.
-   *
-   * @param fs The filesystem in which both srcFile and parityFile reside.
-   * @param srcFile The source file.
-   * @param srcSize The size of the source file.
-   * @param blockSize The block size for the source/parity files.
-   * @param corruptOffset The location of corruption in the parity file.
-   * @param out The destination for the reovered block.
-   */
-  public void recoverParityBlockToStream(
-    FileSystem fs,
-    Path srcFile, long srcSize, long blockSize,
-    Path parityFile, long corruptOffset,
-    OutputStream out) throws IOException {
-    LOG.info("Recovering parity block" + parityFile + ":" + corruptOffset);
-    // Get the start offset of the corrupt block.
-    corruptOffset = (corruptOffset / blockSize) * blockSize;
-    // Output streams to each block in the parity file stripe.
-    OutputStream[] outs = new OutputStream[paritySize];
-    long indexOfCorruptBlockInParityStripe =
-      (corruptOffset / blockSize) % paritySize;
-    LOG.info("Index of corrupt block in parity stripe: " +
-              indexOfCorruptBlockInParityStripe);
-    // Create a real output stream for the block we want to recover,
-    // and create null streams for the rest.
-    for (int i = 0; i < paritySize; i++) {
-      if (indexOfCorruptBlockInParityStripe == i) {
-        outs[i] = out;
-      } else {
-        outs[i] = new NullOutputStream();
-      }
-    }
-    // Get the stripe index and start offset of stripe.
-    long stripeIdx = corruptOffset / (paritySize * blockSize);
-    long stripeStart = stripeIdx * blockSize * stripeSize;
-
-    // Get input streams to each block in the source file stripe.
-    InputStream[] blocks = stripeInputs(fs, srcFile, stripeStart,
-        srcSize, blockSize);
-    LOG.info("Starting recovery by using source stripe " +
-              srcFile + ":" + stripeStart);
-    // Read the data from the blocks and write to the parity file.
-    encodeStripe(blocks, stripeStart, blockSize, outs,
-                 new RaidUtils.DummyProgressable());
-  }
-
-  /**
-   * Recovers a corrupt block in a parity file to an output stream.
-   *
-   * The encoder generates paritySize parity blocks for a source file stripe.
-   * Since there is only one output provided, some blocks are written out to
-   * files before being written out to the output.
-   *
-   * @param fs The filesystem in which both srcFile and parityFile reside.
-   * @param srcFile The source file.
-   * @param srcSize The size of the source file.
-   * @param blockSize The block size for the source/parity files.
-   * @param out The destination for the reovered block.
-   */
-  private void encodeFileToStream(FileSystem fs, Path srcFile, long srcSize,
-    long blockSize, OutputStream out, Progressable reporter) throws IOException {
-    OutputStream[] tmpOuts = new OutputStream[paritySize];
-    // One parity block can be written directly to out, rest to local files.
-    tmpOuts[0] = out;
-    File[] tmpFiles = new File[paritySize - 1];
-    for (int i = 0; i < paritySize - 1; i++) {
-      tmpFiles[i] = File.createTempFile("parity", "_" + i);
-      LOG.info("Created tmp file " + tmpFiles[i]);
-      tmpFiles[i].deleteOnExit();
-    }
-    try {
-      // Loop over stripes in the file.
-      for (long stripeStart = 0; stripeStart < srcSize;
-          stripeStart += blockSize * stripeSize) {
-        reporter.progress();
-        LOG.info("Starting encoding of stripe " + srcFile + ":" + stripeStart);
-        // Create input streams for blocks in the stripe.
-        InputStream[] blocks = stripeInputs(fs, srcFile, stripeStart,
-          srcSize, blockSize);
-        // Create output streams to the temp files.
-        for (int i = 0; i < paritySize - 1; i++) {
-          tmpOuts[i + 1] = new FileOutputStream(tmpFiles[i]);
-        }
-        // Call the implementation of encoding.
-        encodeStripe(blocks, stripeStart, blockSize, tmpOuts, reporter);
-        // Close output streams to the temp files and write the temp files
-        // to the output provided.
-        for (int i = 0; i < paritySize - 1; i++) {
-          tmpOuts[i + 1].close();
-          tmpOuts[i + 1] = null;
-          InputStream in  = new FileInputStream(tmpFiles[i]);
-          RaidUtils.copyBytes(in, out, writeBufs[i], blockSize);
-          reporter.progress();
-        }
-      }
-    } finally {
-      for (int i = 0; i < paritySize - 1; i++) {
-        if (tmpOuts[i + 1] != null) {
-          tmpOuts[i + 1].close();
-        }
-        tmpFiles[i].delete();
-        LOG.info("Deleted tmp file " + tmpFiles[i]);
-      }
-    }
-  }
-
-  /**
-   * Return input streams for each block in a source file's stripe.
-   * @param fs The filesystem where the file resides.
-   * @param srcFile The source file.
-   * @param stripeStartOffset The start offset of the stripe.
-   * @param srcSize The size of the source file.
-   * @param blockSize The block size for the source file.
-   */
-  protected InputStream[] stripeInputs(
-    FileSystem fs,
-    Path srcFile,
-    long stripeStartOffset,
-    long srcSize,
-    long blockSize
-    ) throws IOException {
-    InputStream[] blocks = new InputStream[stripeSize];
-    for (int i = 0; i < stripeSize; i++) {
-      long seekOffset = stripeStartOffset + i * blockSize;
-      if (seekOffset < srcSize) {
-        FSDataInputStream in = fs.open(
-                   srcFile, conf.getInt("io.file.buffer.size", 64 * 1024));
-        in.seek(seekOffset);
-        LOG.info("Opening stream at " + srcFile + ":" + seekOffset);
-        blocks[i] = in;
-      } else {
-        LOG.info("Using zeros at offset " + seekOffset);
-        // We have no src data at this offset.
-        blocks[i] = new RaidUtils.ZeroInputStream(
-                          seekOffset + blockSize);
-      }
-    }
-    return blocks;
-  }
-
-  /**
-   * The implementation of generating parity data for a stripe.
-   *
-   * @param blocks The streams to blocks in the stripe.
-   * @param stripeStartOffset The start offset of the stripe
-   * @param blockSize The maximum size of a block.
-   * @param outs output streams to the parity blocks.
-   * @param reporter progress indicator.
-   */
-  protected abstract void encodeStripe(
-    InputStream[] blocks,
-    long stripeStartOffset,
-    long blockSize,
-    OutputStream[] outs,
-    Progressable reporter) throws IOException;
-
-  /**
-   * Return the temp path for the parity file
-   */
-  protected abstract Path getParityTempPath();
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ErasureCode.java

@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-public interface ErasureCode {
-  /**
-   * Encodes the given message.
-   * @param message The data of the message. The data is present in the least
-   *                significant bits of each int. The number of data bits is
-   *                symbolSize(). The number of elements of message is
-   *                stripeSize().
-   * @param parity  (out) The information is present in the least
-   *                significant bits of each int. The number of parity bits is
-   *                symbolSize(). The number of elements in the code is
-   *                paritySize().
-   */
-  public void encode(int[] message, int[] parity);
-
-  /**
-   * Generates missing portions of data.
-   * @param data The message and parity. The parity should be placed in the
-   *             first part of the array. In each integer, the relevant portion
-   *             is present in the least significant bits of each int.
-   *             The number of elements in data is stripeSize() + paritySize().
-   * @param erasedLocations The indexes in data which are not available.
-   * @param erasedValues    (out)The decoded values corresponding to erasedLocations.
-   */
-  public void decode(int[] data, int[] erasedLocations, int[] erasedValues);
-
-  /**
-   * The number of elements in the message.
-   */
-  public int stripeSize();
-
-  /**
-   * The number of elements in the code.
-   */
-  public int paritySize();
-
-  /**
-   * Number of bits for each symbol.
-   */
-  public int symbolSize();
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/GaloisField.java

@@ -1,350 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Implementation of Galois field arithmetics with 2^p elements.
- * The input must be unsigned integers.
- */
-public class GaloisField {
-  
-  private final int[] logTable;
-  private final int[] powTable;
-  private final int[][] mulTable;
-  private final int[][] divTable;
-  private final int fieldSize;
-  private final int primitivePeriod;
-  private final int primitivePolynomial;
-
-  // Field size 256 is good for byte based system
-  private static final int DEFAULT_FIELD_SIZE = 256;
-  // primitive polynomial 1 + X^2 + X^3 + X^4 + X^8
-  private static final int DEFAULT_PRIMITIVE_POLYNOMIAL = 285;
-
-  static private final Map<Integer, GaloisField> instances =
-    new HashMap<Integer, GaloisField>();
-
-  /**
-   * Get the object performs Galois field arithmetics
-   * @param fieldSize size of the field
-   * @param primitivePolynomial a primitive polynomial corresponds to the size
-   */
-  public static GaloisField getInstance(int fieldSize,
-      int primitivePolynomial) {
-    int key = ((fieldSize << 16) & 0xFFFF0000) + (primitivePolynomial & 0x0000FFFF);
-    GaloisField gf;
-    synchronized (instances) {
-      gf = instances.get(key);
-      if (gf == null) {
-        gf = new GaloisField(fieldSize, primitivePolynomial);
-        instances.put(key, gf);
-      }
-    }
-    return gf;
-  }
-
-  /**
-   * Get the object performs Galois field arithmetics with default setting
-   */
-  public static GaloisField getInstance() {
-    return getInstance(DEFAULT_FIELD_SIZE, DEFAULT_PRIMITIVE_POLYNOMIAL);
-  }
-
-  private GaloisField(int fieldSize, int primitivePolynomial) {
-    assert fieldSize > 0;
-    assert primitivePolynomial > 0;
-
-    this.fieldSize = fieldSize;
-    this.primitivePeriod = fieldSize - 1;
-    this.primitivePolynomial = primitivePolynomial;
-    logTable = new int[fieldSize];
-    powTable = new int[fieldSize];
-    mulTable = new int[fieldSize][fieldSize];
-    divTable = new int[fieldSize][fieldSize];
-    int value = 1;
-    for (int pow = 0; pow < fieldSize - 1; pow++) {
-      powTable[pow] = value;
-      logTable[value] = pow;
-      value = value * 2;
-      if (value >= fieldSize) {
-        value = value ^ primitivePolynomial;
-      }
-    }
-    // building multiplication table
-    for (int i = 0; i < fieldSize; i++) {
-      for (int j = 0; j < fieldSize; j++) {
-        if (i == 0 || j == 0) {
-          mulTable[i][j] = 0;
-          continue;
-        }
-        int z = logTable[i] + logTable[j];
-        z = z >= primitivePeriod ? z - primitivePeriod : z;
-        z = powTable[z];
-        mulTable[i][j] = z;
-      }
-    }
-    // building division table
-    for (int i = 0; i < fieldSize; i++) {
-      for (int j = 1; j < fieldSize; j++) {
-        if (i == 0) {
-          divTable[i][j] = 0;
-          continue;
-        }
-        int z = logTable[i] - logTable[j];
-        z = z < 0 ? z + primitivePeriod : z;
-        z = powTable[z];
-        divTable[i][j] = z;
-      }
-    }
-  }
-
-  /**
-   * Return number of elements in the field
-   * @return number of elements in the field
-   */
-  public int getFieldSize() {
-    return fieldSize;
-  }
-
-  /**
-   * Return the primitive polynomial in GF(2)
-   * @return primitive polynomial as a integer
-   */
-  public int getPrimitivePolynomial() {
-    return primitivePolynomial;
-  }
-
-  /**
-   * Compute the sum of two fields
-   * @param x input field
-   * @param y input field
-   * @return result of addition
-   */
-  public int add(int x, int y) {
-    assert(x >= 0 && x < getFieldSize() && y >= 0 && y < getFieldSize());
-    return x ^ y;
-  }
-
-  /**
-   * Compute the multiplication of two fields
-   * @param x input field
-   * @param y input field
-   * @return result of multiplication
-   */
-  public int multiply(int x, int y) {
-    assert(x >= 0 && x < getFieldSize() && y >= 0 && y < getFieldSize());
-    return mulTable[x][y];
-  }
-
-  /**
-   * Compute the division of two fields
-   * @param x input field
-   * @param y input field
-   * @return x/y
-   */
-  public int divide(int x, int y) {
-    assert(x >= 0 && x < getFieldSize() && y > 0 && y < getFieldSize());
-    return divTable[x][y];
-  }
-
-  /**
-   * Compute power n of a field
-   * @param x input field
-   * @param n power
-   * @return x^n
-   */
-  public int power(int x, int n) {
-    assert(x >= 0 && x < getFieldSize());
-    if (n == 0) {
-      return 1;
-    }
-    if (x == 0) {
-      return 0;
-    }
-    x = logTable[x] * n;
-    if (x < primitivePeriod) {
-      return powTable[x];
-    }
-    x = x % primitivePeriod;
-    return powTable[x];
-  }
-
-  /**
-   * Given a Vandermonde matrix V[i][j]=x[j]^i and vector y, solve for z such
-   * that Vz=y. The output z will be placed in y.
-   * @param x the vector which describe the Vandermonde matrix
-   * @param y right-hand side of the Vandermonde system equation.
-   *          will be replaced the output in this vector
-   */
-  public void solveVandermondeSystem(int[] x, int[] y) {
-    solveVandermondeSystem(x, y, x.length);
-  }
-
-  /**
-   * Given a Vandermonde matrix V[i][j]=x[j]^i and vector y, solve for z such
-   * that Vz=y. The output z will be placed in y.
-   * @param x the vector which describe the Vandermonde matrix
-   * @param y right-hand side of the Vandermonde system equation.
-   *          will be replaced the output in this vector
-   * @param len consider x and y only from 0...len-1
-   */
-  public void solveVandermondeSystem(int[] x, int[] y, int len) {
-    assert(y.length <= len);
-    for (int i = 0; i < len - 1; i++) {
-      for (int j = len - 1; j > i; j--) {
-        y[j] = y[j] ^ mulTable[x[i]][y[j - 1]];
-      }
-    }
-    for (int i = len - 1; i >= 0; i--) {
-      for (int j = i + 1; j < len; j++) {
-        y[j] = divTable[y[j]][x[j] ^ x[j - i - 1]];
-      }
-      for (int j = i; j < len - 1; j++) {
-        y[j] = y[j] ^ y[j + 1];
-      }
-    }
-  }
-
-  /**
-   * Compute the multiplication of two polynomials. The index in the
-   * array corresponds to the power of the entry. For example p[0] is the
-   * constant term of the polynomial p. 
-   * @param p input polynomial
-   * @param q input polynomial
-   * @return polynomial represents p*q
-   */
-  public int[] multiply(int[] p, int[] q) {
-    int len = p.length + q.length - 1;
-    int[] result = new int[len];
-    for (int i = 0; i < len; i++) {
-      result[i] = 0;
-    }
-    for (int i = 0; i < p.length; i++) {
-      for (int j = 0; j < q.length; j++) {
-        result[i + j] = add(result[i + j], multiply(p[i], q[j]));
-      }
-    }
-    return result;
-  }
-
-  /**
-   * Compute the remainder of a dividend and divisor pair. The index in the
-   * array corresponds to the power of the entry. For example p[0] is the
-   * constant term of the polynomial p. 
-   * @param dividend dividend polynomial, the remainder will be placed here when return
-   * @param divisor divisor polynomial
-   */
-  public void remainder(int[] dividend, int[] divisor) {
-    for (int i = dividend.length - divisor.length; i >= 0; i--) {
-      int ratio =
-        divTable[dividend[i + divisor.length - 1]][divisor[divisor.length - 1]];
-      for (int j = 0; j < divisor.length; j++) {
-        int k = j + i;
-        dividend[k] = dividend[k] ^ mulTable[ratio][divisor[j]];
-      }
-    }
-  }
-
-  /**
-   * Compute the sum of two polynomials. The index in the
-   * array corresponds to the power of the entry. For example p[0] is the
-   * constant term of the polynomial p. 
-   * @param p input polynomial
-   * @param q input polynomial
-   * @return polynomial represents p+q
-   */
-  public int[] add(int[] p, int[] q) {
-    int len = Math.max(p.length, q.length);
-    int[] result = new int[len];
-    for (int i = 0; i < len; i++) {
-      if (i < p.length && i < q.length) {
-        result[i] = add(p[i], q[i]);
-      } else if (i < p.length) {
-        result[i] = p[i];
-      } else {
-        result[i] = q[i];
-      }
-    }
-    return result;
-  }
-
-  /**
-   * Substitute x into polynomial p(x). 
-   * @param p input polynomial
-   * @param x input field
-   * @return p(x)
-   */
-  public int substitute(int[] p, int x) {
-    int result = 0;
-    int y = 1;
-    for (int i = 0; i < p.length; i++) {
-      result = result ^ mulTable[p[i]][y];
-      y = mulTable[x][y];
-    }
-    return result;
-  }
-
-  /**
-   * Perform Gaussian elimination on the given matrix. This matrix has to be a
-   * fat matrix (number of rows > number of columns).
-   */
-  public void gaussianElimination(int[][] matrix) {
-    assert(matrix != null && matrix.length > 0 && matrix[0].length > 0
-           && matrix.length < matrix[0].length);
-    int height = matrix.length;
-    int width = matrix[0].length;
-    for (int i = 0; i < height; i++) {
-      boolean pivotFound = false;
-      // scan the column for a nonzero pivot and swap it to the diagonal
-      for (int j = i; j < height; j++) {
-        if (matrix[i][j] != 0) {
-          int[] tmp = matrix[i];
-          matrix[i] = matrix[j];
-          matrix[j] = tmp;
-          pivotFound = true;
-          break;
-        }
-      }
-      if (!pivotFound) {
-        continue;
-      }
-      int pivot = matrix[i][i];
-      for (int j = i; j < width; j++) {
-        matrix[i][j] = divide(matrix[i][j], pivot);
-      }
-      for (int j = i + 1; j < height; j++) {
-        int lead = matrix[j][i];
-        for (int k = i; k < width; k++) {
-          matrix[j][k] = add(matrix[j][k], multiply(lead, matrix[i][k]));
-        }
-      }
-    }
-    for (int i = height - 1; i >=0; i--) {
-      for (int j = 0; j < i; j++) {
-        int lead = matrix[j][i];
-        for (int k = i; k < width; k++) {
-          matrix[j][k] = add(matrix[j][k], multiply(lead, matrix[i][k]));
-        }
-      }
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/HarIndex.java

@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.InputStream;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
-import java.util.List;
-import java.util.LinkedList;
-
-import org.apache.hadoop.util.LineReader;
-import org.apache.hadoop.io.Text;
-
-/**
- * Represents the contents of a HAR Index file. The HAR is assumed to be
- * comprising of RAID parity files only and no directories.
- */
-public class HarIndex {
-  public static final String indexFileName = "_index";
-  private List<IndexEntry> entries = new LinkedList<IndexEntry>();
-
-  /**
-   * Represents information in a single line of the HAR index file.
-   */
-  public static class IndexEntry {
-    String fileName; // Name of the file in the part file.
-    long startOffset; // Start offset within the part file.
-    long length; // Length of this file within the part file.
-    long mtime; // Modification time of the file.
-    String partFileName; // Name of the part file.
-
-    IndexEntry(String fileName, long startOffset, long length,
-                long mtime, String partFileName) {
-      this.fileName = fileName;
-      this.startOffset = startOffset;
-      this.length = length;
-      this.mtime = mtime;
-      this.partFileName = partFileName;
-    }
-
-    public String toString() {
-      return "fileName=" + fileName +
-             ", startOffset=" + startOffset +
-             ", length=" + length +
-             ", mtime=" + mtime +
-             ", partFileName=" + partFileName;
-    }
-  }
-
-  /**
-   * Constructor that reads the contents of the index file.
-   * @param in An input stream to the index file.
-   * @param max The size of the index file.
-   * @throws IOException
-   */
-  public HarIndex(InputStream in, long max) throws IOException {
-    LineReader lineReader = new LineReader(in);
-    Text text = new Text();
-    long nread = 0;
-    while (nread < max) {
-      int n = lineReader.readLine(text);
-      nread += n;
-      String line = text.toString();
-      try {
-        parseLine(line);
-      } catch (UnsupportedEncodingException e) {
-        throw new IOException("UnsupportedEncodingException after reading " +
-                              nread + "bytes");
-      }
-    }
-  }
-
-  /**
-   * Parses each line and extracts relevant information.
-   * @param line
-   * @throws UnsupportedEncodingException
-   */
-  void parseLine(String line) throws UnsupportedEncodingException {
-    String[] splits = line.split(" ");
-
-    boolean isDir = "dir".equals(splits[1]) ? true: false;
-    if (!isDir && splits.length >= 6) {
-      String name = URLDecoder.decode(splits[0], "UTF-8");
-      String partName = URLDecoder.decode(splits[2], "UTF-8");
-      long startIndex = Long.parseLong(splits[3]);
-      long length = Long.parseLong(splits[4]);
-      String[] newsplits = URLDecoder.decode(splits[5],"UTF-8").split(" ");
-      if (newsplits != null && newsplits.length >= 4) {
-        long mtime = Long.parseLong(newsplits[0]);
-        IndexEntry entry = new IndexEntry(
-          name, startIndex, length, mtime, partName);
-        entries.add(entry);
-      }
-    }
-  }
-
-  /**
-   * Finds the index entry corresponding to a HAR partFile at an offset.
-   * @param partName The name of the part file (part-*).
-   * @param partFileOffset The offset into the part file.
-   * @return The entry corresponding to partName:partFileOffset.
-   */
-  public IndexEntry findEntry(String partName, long partFileOffset) {
-    for (IndexEntry e: entries) {
-      boolean nameMatch = partName.equals(e.partFileName);
-      boolean inRange = (partFileOffset >= e.startOffset) &&
-                        (partFileOffset < e.startOffset + e.length);
-      if (nameMatch && inRange) {
-        return e;
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Finds the index entry corresponding to a file in the archive
-   */
-  public IndexEntry findEntryByFileName(String fileName) {
-    for (IndexEntry e: entries) {
-      if (fileName.equals(e.fileName)) {
-        return e;
-      }
-    }
-    return null;
-  }
-
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/JobMonitor.java

@@ -1,211 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-import java.util.LinkedList;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.StringUtils;
-
-/**
- * Periodically monitors the status of jobs registered with it.
- *
- * Jobs that are submitted for the same policy name are kept in the same list,
- * and the list itself is kept in a map that has the policy name as the key and
- * the list as value.
- */
-class JobMonitor implements Runnable {
-  public static final Log LOG = LogFactory.getLog(
-                                  "org.apache.hadoop.raid.JobMonitor");
-
-  volatile boolean running = true;
-
-  private Map<String, List<DistRaid>> jobs;
-  public static final String JOBMONITOR_INTERVAL_KEY = "raid.jobmonitor.interval";
-  private long jobMonitorInterval;
-  private volatile long jobsMonitored = 0;
-  private volatile long jobsSucceeded = 0;
-
-  public JobMonitor(Configuration conf) {
-    jobMonitorInterval = conf.getLong(JOBMONITOR_INTERVAL_KEY, 60000);
-    jobs = new java.util.HashMap<String, List<DistRaid>>();
-  }
-
-  public void run() {
-    while (running) {
-      try {
-        LOG.info("JobMonitor thread continuing to run...");
-        doMonitor();
-      } catch (Throwable e) {
-        LOG.error("JobMonitor encountered exception " +
-          StringUtils.stringifyException(e));
-        // All expected exceptions are caught by doMonitor(). It is better
-        // to exit now, this will prevent RaidNode from submitting more jobs
-        // since the number of running jobs will never decrease.
-        return;
-      }
-    }
-  }
-
-  /**
-   * Periodically checks status of running map-reduce jobs.
-   */
-  public void doMonitor() {
-    while (running) {
-      String[] keys = null;
-      // Make a copy of the names of the current jobs.
-      synchronized(jobs) {
-        keys = jobs.keySet().toArray(new String[0]);
-      }
-
-      // Check all the jobs. We do not want to block access to `jobs`
-      // because that will prevent new jobs from being added.
-      // This is safe because JobMonitor.run is the only code that can
-      // remove a job from `jobs`. Thus all elements in `keys` will have
-      // valid values.
-      Map<String, List<DistRaid>> finishedJobs =
-        new HashMap<String, List<DistRaid>>();
-
-      for (String key: keys) {
-        // For each policy being monitored, get the list of jobs running.
-        DistRaid[] jobListCopy = null;
-        synchronized(jobs) {
-          List<DistRaid> jobList = jobs.get(key);
-          synchronized(jobList) {
-            jobListCopy = jobList.toArray(new DistRaid[jobList.size()]);
-          }
-        }
-        // The code that actually contacts the JobTracker is not synchronized,
-        // it uses copies of the list of jobs.
-        for (DistRaid job: jobListCopy) {
-          // Check each running job.
-          try {
-            boolean complete = job.checkComplete();
-            if (complete) {
-              addJob(finishedJobs, key, job);
-              if (job.successful()) {
-                jobsSucceeded++;
-              }
-            }
-          } catch (IOException ioe) {
-            // If there was an error, consider the job finished.
-            addJob(finishedJobs, key, job);
-            LOG.error("JobMonitor exception", ioe);
-          }
-        }
-      }
-
-      if (finishedJobs.size() > 0) {
-        for (String key: finishedJobs.keySet()) {
-          List<DistRaid> finishedJobList = finishedJobs.get(key);
-          // Iterate through finished jobs and remove from jobs.
-          // removeJob takes care of locking.
-          for (DistRaid job: finishedJobList) {
-            removeJob(jobs, key, job);
-          }
-        }
-      }
-
-      try {
-        Thread.sleep(jobMonitorInterval);
-      } catch (InterruptedException ie) {
-      }
-    }
-  }
-
-  public int runningJobsCount(String key) {
-    int count = 0;
-    synchronized(jobs) {
-      if (jobs.containsKey(key)) {
-        List<DistRaid> jobList = jobs.get(key);
-        synchronized(jobList) {
-          count = jobList.size();
-        }
-      }
-    }
-    return count;
-  }
-
-  public void monitorJob(String key, DistRaid job) {
-    addJob(jobs, key, job);
-    jobsMonitored++;
-  }
-
-  public long jobsMonitored() {
-    return this.jobsMonitored;
-  }
-
-  public long jobsSucceeded() {
-    return this.jobsSucceeded;
-  }
-  
-  // For test code
-  int runningJobsCount() {
-    int total = 0;
-    synchronized(jobs) {
-      for (String key: jobs.keySet()) {
-        total += jobs.get(key).size();
-      }
-    }
-    return total;
-  }
-
-  private static void addJob(Map<String, List<DistRaid>> jobsMap,
-                              String jobName, DistRaid job) {
-    synchronized(jobsMap) {
-      List<DistRaid> list = null;
-      if (jobsMap.containsKey(jobName)) {
-        list = jobsMap.get(jobName);
-      } else {
-        list = new LinkedList<DistRaid>();
-        jobsMap.put(jobName, list);
-      }
-      synchronized(list) {
-        list.add(job);
-      }
-    }
-  }
-
-  private static void removeJob(Map<String, List<DistRaid>> jobsMap,
-                                  String jobName, DistRaid job) {
-    synchronized(jobsMap) {
-      if (jobsMap.containsKey(jobName)) {
-        List<DistRaid> list = jobsMap.get(jobName);
-        synchronized(list) {
-          for (Iterator<DistRaid> it = list.iterator(); it.hasNext(); ) {
-            DistRaid val = it.next();
-            if (val == job) {
-              it.remove();
-            }
-          }
-          if (list.size() == 0) {
-            jobsMap.remove(jobName);
-          }
-        }
-      }
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/LocalBlockFixer.java

@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.RaidDFSUtil;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Time;
-
-import org.apache.hadoop.net.NetUtils;
-
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-
-import org.apache.hadoop.raid.RaidNode;
-import org.apache.hadoop.raid.RaidUtils;
-import org.apache.hadoop.raid.protocol.PolicyInfo.ErasureCodeType;
-
-/**
- * This class fixes source file blocks using the parity file,
- * and parity file blocks using the source file.
- * It periodically fetches the list of corrupt files from the namenode,
- * and figures out the location of the bad block by reading through
- * the corrupt file.
- */
-public class LocalBlockFixer extends BlockFixer {
-  public static final Log LOG = LogFactory.getLog(LocalBlockFixer.class);
-
-  private java.util.HashMap<String, java.util.Date> history;
-  
-  private BlockFixerHelper helper;
-
-  public LocalBlockFixer(Configuration conf) throws IOException {
-    super(conf);
-    history = new java.util.HashMap<String, java.util.Date>();
-    helper = new BlockFixerHelper(getConf());
-  }
-
-  public void run() {
-    while (running) {
-      try {
-        LOG.info("LocalBlockFixer continuing to run...");
-        doFix();
-      } catch (Exception e) {
-        LOG.error(StringUtils.stringifyException(e));
-      } catch (Error err) {
-        LOG.error("Exiting after encountering " +
-                    StringUtils.stringifyException(err));
-        throw err;
-      }
-    }
-  }
-
-  void doFix() throws InterruptedException, IOException {
-    while (running) {
-      // Sleep before proceeding to fix files.
-      Thread.sleep(blockFixInterval);
-
-      // Purge history older than the history interval.
-      purgeHistory();
-
-      List<Path> corruptFiles = getCorruptFiles();
-
-      filterUnfixableSourceFiles(corruptFiles.iterator());
-
-      if (corruptFiles.isEmpty()) {
-        // If there are no corrupt files, retry after some time.
-        continue;
-      }
-      LOG.info("Found " + corruptFiles.size() + " corrupt files.");
-
-      helper.sortCorruptFiles(corruptFiles);
-
-      for (Path srcPath: corruptFiles) {
-        if (!running) break;
-        try {
-          boolean fixed = helper.fixFile(srcPath);
-          LOG.info("Adding " + srcPath + " to history");
-          history.put(srcPath.toString(), new java.util.Date());
-          if (fixed) {
-            incrFilesFixed();
-          }
-        } catch (IOException ie) {
-          LOG.error("Hit error while processing " + srcPath +
-            ": " + StringUtils.stringifyException(ie));
-          // Do nothing, move on to the next file.
-        }
-      }
-    }
-  }
-
-
-  /**
-   * We maintain history of fixed files because a fixed file may appear in
-   * the list of corrupt files if we loop around too quickly.
-   * This function removes the old items in the history so that we can
-   * recognize files that have actually become corrupt since being fixed.
-   */
-  void purgeHistory() {
-    java.util.Date cutOff = new java.util.Date(Time.now() -
-                                               historyInterval);
-    List<String> toRemove = new java.util.ArrayList<String>();
-
-    for (String key: history.keySet()) {
-      java.util.Date item = history.get(key);
-      if (item.before(cutOff)) {
-        toRemove.add(key);
-      }
-    }
-    for (String key: toRemove) {
-      LOG.info("Removing " + key + " from history");
-      history.remove(key);
-    }
-  }
-
-  /**
-   * @return A list of corrupt files as obtained from the namenode
-   */
-  List<Path> getCorruptFiles() throws IOException {
-    DistributedFileSystem dfs = helper.getDFS(new Path("/"));
-
-    String[] files = RaidDFSUtil.getCorruptFiles(dfs);
-    List<Path> corruptFiles = new LinkedList<Path>();
-    for (String f: files) {
-      Path p = new Path(f);
-      if (!history.containsKey(p.toString())) {
-        corruptFiles.add(p);
-      }
-    }
-    RaidUtils.filterTrash(getConf(), corruptFiles);
-    return corruptFiles;
-  }
-
-}
-

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/LocalRaidNode.java

@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.conf.Configuration;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileStatus;
-
-import org.apache.hadoop.raid.protocol.PolicyInfo;
-
-/**
- * Implementation of {@link RaidNode} that performs raiding locally.
- */
-public class LocalRaidNode extends RaidNode {
-
-  public static final Log LOG = LogFactory.getLog(LocalRaidNode.class);
-
-  public LocalRaidNode(Configuration conf) throws IOException {
-    super(conf);
-  }
-
-  /**
-   * {@inheritDocs}
-   */
-  @Override
-  void raidFiles(PolicyInfo info, List<FileStatus> paths) throws IOException {
-    doRaid(conf, info, paths);
-  }
-
-  /**
-   * {@inheritDocs}
-   */
-  @Override
-  int getRunningJobsForPolicy(String policyName) {
-    return 0;
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ParityInputStream.java

@@ -1,151 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.hadoop.util.Progressable;
-
-/**
- * Wraps over multiple input streams and provides an input stream that is
- * an XOR of the streams.
- */
-class ParityInputStream extends InputStream {
-  private static final int DEFAULT_BUFSIZE = 5*1024*1024;
-  private InputStream[] streams;
-  private byte[] xor;
-  private byte[] buf;
-  private int bufSize;
-  private long remaining;
-  private int available = 0;
-  private int readPos = 0;
-
-  public ParityInputStream(
-      InputStream[] streams, long parityBlockSize, byte[] buf, byte[] xor) {
-    assert buf.length == xor.length;
-    bufSize = buf.length;
-    this.streams = streams;
-    remaining = parityBlockSize;
-    this.buf = buf;
-    this.xor = xor;
-  }
-  
-  @Override
-  public int read() throws IOException {
-    makeAvailable();
-    if (available == 0) {
-      return -1;
-    }
-    int ret = xor[readPos];
-    readPos++;
-    available--;
-    return ret;
-  }
-  
-  @Override
-  public int read(byte b[], int off, int len) throws IOException {
-    makeAvailable();
-    if (available == 0) {
-      return -1;
-    }
-    int ret = Math.min(len, available);
-    for (int i = 0; i < ret; ++i) {
-      b[off+i] = xor[readPos+i];
-    }
-    readPos += ret;
-    available -= ret;
-    return ret;
-  }
-
-  public void close() throws IOException {
-    for (InputStream i: streams) {
-      i.close();
-    }
-  }
-  
-  /**
-   * Send the contents of the stream to the sink.
-   * @param sink
-   * @param reporter
-   * @throws IOException
-   */
-  public void drain(OutputStream sink, Progressable reporter)
-          throws IOException {
-    
-    while (true) {
-      makeAvailable();
-      if (available == 0) {
-        break;
-      }
-      sink.write(xor, readPos, available);
-      available = 0;
-      if (reporter != null) {
-        reporter.progress();
-      }
-    }
-  }
-
-  /**
-   * Make some bytes available for reading in the internal buffer.
-   * @throws IOException
-   */
-  private void makeAvailable() throws IOException {
-    if (available > 0 || remaining <= 0) {
-      return;
-    }
-    // Read some bytes from the first stream.
-    int xorlen = (int)Math.min(remaining, bufSize);
-    readExact(streams[0], xor, xorlen);
-
-    // Read bytes from all the other streams and xor them.
-    for (int i = 1; i < streams.length; i++) {
-      readExact(streams[i], buf, xorlen);
-
-      for (int j = 0; j < xorlen; j++) {
-        xor[j] ^= buf[j];
-      }
-    }
-    
-    remaining -= xorlen;
-    available = xorlen;
-    readPos = 0;
-    readPos = 0;
-  }
-
-  private static void readExact(InputStream in, byte[] bufs, int toRead)
-  throws IOException {
-    int tread = 0;
-    while (tread < toRead) {
-      int read = in.read(bufs, tread, toRead - tread);
-      if (read == -1) {
-        // If the stream ends, fill in zeros.
-        Arrays.fill(bufs, tread, toRead, (byte)0);
-        tread = toRead;
-      } else {
-        tread += read;
-      }
-    }
-    assert tread == toRead;
-  }
-
-}
-

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidConfigurationException.java

@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-/**
- * Thrown when the config file for {@link RaidNode} is malformed.
- */
-public class RaidConfigurationException extends Exception {
-  private static final long serialVersionUID = 4046516718965587999L;
-  
-  public RaidConfigurationException(String message) {
-    super(message);
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidFilter.java

@@ -1,259 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.raid.protocol.PolicyInfo;
-
-public class RaidFilter {
-  static class Statistics {
-    long numRaided = 0;
-    long numTooNew = 0;
-    long sizeTooNew = 0;
-    long numTooSmall = 0;
-    long sizeTooSmall = 0;
-
-    public void aggregate(Statistics other) {
-      this.numRaided += other.numRaided;
-      this.numTooNew += other.numTooNew;
-      this.sizeTooNew += other.sizeTooNew;
-      this.numTooSmall += other.numTooSmall;
-      this.sizeTooSmall += other.sizeTooSmall;
-    }
-
-    public String toString() {
-      return "numRaided = " + numRaided +
-             ", numTooNew = " + numTooNew +
-             ", sizeTooNew = " + sizeTooNew +
-             ", numTooSmall = " + numTooSmall +
-             ", sizeTooSmall = " + sizeTooSmall;
-    }
-  }
-
-  static class TimeBasedFilter extends Configured
-    implements DirectoryTraversal.FileFilter {
-    int targetRepl;
-    Path raidDestPrefix;
-    long modTimePeriod;
-    long startTime;
-    Statistics stats = new Statistics();
-    String currentSrcPath = null;
-    long[] modTimePeriods = new long[0];
-    String[] otherSrcPaths = new String[0];
-
-    TimeBasedFilter(Configuration conf, Path destPrefix, int targetRepl,
-      long startTime, long modTimePeriod) {
-      super(conf);
-      this.raidDestPrefix = destPrefix;
-      this.targetRepl = targetRepl;
-      this.startTime = startTime;
-      this.modTimePeriod = modTimePeriod;
-    }
-
-    TimeBasedFilter(Configuration conf,
-      Path destPrefix, PolicyInfo info,
-      List<PolicyInfo> allPolicies, long startTime, Statistics stats) {
-      super(conf);
-      this.raidDestPrefix = destPrefix;
-      this.targetRepl = Integer.parseInt(info.getProperty("targetReplication"));
-      this.modTimePeriod = Long.parseLong(info.getProperty("modTimePeriod"));
-      this.startTime = startTime;
-      this.stats = stats;
-      this.currentSrcPath = info.getSrcPath().toUri().getPath();
-      initializeOtherPaths(allPolicies);
-    }
-
-    private void initializeOtherPaths(List<PolicyInfo> allPolicies) {
-      ArrayList<PolicyInfo> tmp = new ArrayList<PolicyInfo>(allPolicies);
-      // Remove all policies where srcPath <= currentSrcPath or
-      // matchingPrefixLength is < length(currentSrcPath)
-      // The policies remaining are the only ones that could better
-      // select a file chosen by the current policy.
-      for (Iterator<PolicyInfo> it = tmp.iterator(); it.hasNext(); ) {
-        String src = it.next().getSrcPath().toUri().getPath();
-        if (src.compareTo(currentSrcPath) <= 0) {
-          it.remove();
-          continue;
-        }
-        int matchLen = matchingPrefixLength(src, currentSrcPath);
-        if (matchLen < currentSrcPath.length()) {
-          it.remove();
-        }
-      }
-      // Sort in reverse lexicographic order.
-      Collections.sort(tmp, new Comparator() {
-        public int compare(Object o1, Object o2) {
-          return 0 -
-            ((PolicyInfo)o1).getSrcPath().toUri().getPath().compareTo(
-              ((PolicyInfo)o1).getSrcPath().toUri().getPath());
-        }
-      });
-      otherSrcPaths = new String[tmp.size()];
-      modTimePeriods = new long[otherSrcPaths.length];
-      for (int i = 0; i < otherSrcPaths.length; i++) {
-        otherSrcPaths[i] = tmp.get(i).getSrcPath().toUri().getPath();
-        modTimePeriods[i] = Long.parseLong(
-          tmp.get(i).getProperty("modTimePeriod"));
-      }
-    }
-
-    public boolean check(FileStatus f) throws IOException {
-      if (!canChooseForCurrentPolicy(f)) {
-        return false;
-      }
-
-      // If the source file has fewer than or equal to 2 blocks, then skip it.
-      long blockSize = f.getBlockSize();
-      if (2 * blockSize >= f.getLen()) {
-        stats.numTooSmall++;
-        stats.sizeTooSmall += f.getLen();
-        return false;
-      }
-
-      boolean select = false;
-      try {
-        Object ppair = RaidNode.getParityFile(
-            raidDestPrefix, f.getPath(), getConf());
-        // Is there is a valid parity file?
-        if (ppair != null) {
-          // Is the source at the target replication?
-          if (f.getReplication() != targetRepl) {
-            // Select the file so that its replication can be set.
-            select = true;
-          } else {
-            stats.numRaided++;
-            // Nothing to do, don't select the file.
-            select = false;
-          }
-        } else {
-          // No parity file.
-          if (f.getModificationTime() + modTimePeriod < startTime) {
-            // If the file is not too new, choose it for raiding.
-            select = true;
-          } else {
-            select = false;
-            stats.numTooNew++;
-            stats.sizeTooNew += f.getLen();
-          }
-        }
-      } catch (java.io.FileNotFoundException e) {
-        select = true; // destination file does not exist
-      } catch (java.io.IOException e) {
-        // If there is a problem with the har path, this will let us continue.
-        DirectoryTraversal.LOG.error(
-          "Error while selecting " + StringUtils.stringifyException(e));
-      }
-      return select;
-    }
-
-    /**
-     * Checks if a file can be chosen for the current policy.
-     */
-    boolean canChooseForCurrentPolicy(FileStatus stat) {
-      boolean choose = true;
-      if (otherSrcPaths.length > 0) {
-        String fileStr = stat.getPath().toUri().getPath();
-
-        // For a given string, find the best matching srcPath.
-        int matchWithCurrent = matchingPrefixLength(fileStr, currentSrcPath);
-        for (int i = 0; i < otherSrcPaths.length; i++) {
-          // If the file is too new, move to the next.
-          if (stat.getModificationTime() > startTime - modTimePeriods[i]) {
-            continue;
-          }
-          int matchLen = matchingPrefixLength(fileStr, otherSrcPaths[i]);
-          if (matchLen > 0 &&
-              fileStr.charAt(matchLen - 1) == Path.SEPARATOR_CHAR) {
-            matchLen--;
-          }
-          if (matchLen > matchWithCurrent) {
-            choose = false;
-            break;
-          }
-        }
-      }
-      return choose;
-    }
-
-    int matchingPrefixLength(final String s1, final String s2) {
-      int len = 0;
-      for (int j = 0; j < s1.length() && j < s2.length(); j++) {
-        if (s1.charAt(j) == s2.charAt(j)) {
-          len++;
-        } else {
-          break;
-        }
-      }
-      return len;
-    }
-  }
-
-  static class PreferenceFilter extends Configured
-    implements DirectoryTraversal.FileFilter {
-    Path firstChoicePrefix;
-    DirectoryTraversal.FileFilter secondChoiceFilter;
-
-    PreferenceFilter(Configuration conf,
-      Path firstChoicePrefix, Path secondChoicePrefix,
-      int targetRepl, long startTime, long modTimePeriod) {
-      super(conf);
-      this.firstChoicePrefix = firstChoicePrefix;
-      this.secondChoiceFilter = new TimeBasedFilter(conf,
-        secondChoicePrefix, targetRepl, startTime, modTimePeriod);
-    }
-
-    PreferenceFilter(Configuration conf,
-      Path firstChoicePrefix, Path secondChoicePrefix,
-      PolicyInfo info, List<PolicyInfo> allPolicies, long startTime,
-        Statistics stats) {
-      super(conf);
-      this.firstChoicePrefix = firstChoicePrefix;
-      this.secondChoiceFilter = new TimeBasedFilter(
-        conf, secondChoicePrefix, info, allPolicies, startTime, stats);
-    }
-
-    public boolean check(FileStatus f) throws IOException {
-      Object firstChoicePPair =
-        RaidNode.getParityFile(firstChoicePrefix, f.getPath(), getConf());
-      if (firstChoicePPair == null) {
-        // The decision is upto the the second choice filter.
-        return secondChoiceFilter.check(f);
-      } else {
-        // There is already a parity file under the first choice path.
-        // We dont want to choose this file.
-        return false;
-      }
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidNode.java

@@ -1,1630 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.IOException;
-import java.io.FileNotFoundException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.LinkedList;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.lang.Thread;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-
-import org.xml.sax.SAXException;
-import javax.xml.parsers.ParserConfigurationException;
-
-import org.apache.hadoop.ipc.*;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.tools.HadoopArchives;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Daemon;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.fs.HarFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.util.Progressable;
-
-import org.apache.hadoop.raid.protocol.PolicyInfo;
-import org.apache.hadoop.raid.protocol.PolicyList;
-import org.apache.hadoop.raid.protocol.RaidProtocol;
-import org.apache.hadoop.raid.protocol.PolicyInfo.ErasureCodeType;
-
-/**
- * A base class that implements {@link RaidProtocol}.
- *
- * use raid.classname to specify which implementation to use
- */
-public abstract class RaidNode implements RaidProtocol {
-
-  static{
-    Configuration.addDefaultResource("hdfs-default.xml");
-    Configuration.addDefaultResource("hdfs-site.xml");
-    Configuration.addDefaultResource("mapred-default.xml");
-    Configuration.addDefaultResource("mapred-site.xml");
-  }
-  public static final Log LOG = LogFactory.getLog( "org.apache.hadoop.raid.RaidNode");
-  public static final long SLEEP_TIME = 10000L; // 10 seconds
-  public static final String TRIGGER_MONITOR_SLEEP_TIME_KEY = 
-      "hdfs.raid.trigger.monitor.sleep.time";
-  public static final int DEFAULT_PORT = 60000;
-  // Default stripe length = 5, parity length for RS code = 3
-  public static final int DEFAULT_STRIPE_LENGTH = 5;
-  public static final int RS_PARITY_LENGTH_DEFAULT = 3;
-
-  public static final String RS_PARITY_LENGTH_KEY = "hdfs.raidrs.paritylength";
-  public static final String STRIPE_LENGTH_KEY = "hdfs.raid.stripeLength";
-
-  public static final String DEFAULT_RAID_LOCATION = "/raid";
-  public static final String RAID_LOCATION_KEY = "hdfs.raid.locations";
-  public static final String DEFAULT_RAID_TMP_LOCATION = "/tmp/raid";
-  public static final String RAID_TMP_LOCATION_KEY = "fs.raid.tmpdir";
-  public static final String DEFAULT_RAID_HAR_TMP_LOCATION = "/tmp/raid_har";
-  public static final String RAID_HAR_TMP_LOCATION_KEY = "fs.raid.hartmpdir";
-
-  public static final String DEFAULT_RAIDRS_LOCATION = "/raidrs";
-  public static final String RAIDRS_LOCATION_KEY = "hdfs.raidrs.locations";
-  public static final String DEFAULT_RAIDRS_TMP_LOCATION = "/tmp/raidrs";
-  public static final String RAIDRS_TMP_LOCATION_KEY = "fs.raidrs.tmpdir";
-  public static final String DEFAULT_RAIDRS_HAR_TMP_LOCATION = "/tmp/raidrs_har";
-  public static final String RAIDRS_HAR_TMP_LOCATION_KEY = "fs.raidrs.hartmpdir";
-
-  public static final String HAR_SUFFIX = "_raid.har";
-  public static final Pattern PARITY_HAR_PARTFILE_PATTERN =
-    Pattern.compile(".*" + HAR_SUFFIX + "/part-.*");
-
-  public static final String RAIDNODE_CLASSNAME_KEY = "raid.classname";
-
-  /** RPC server */
-  private Server server;
-  /** RPC server address */
-  private InetSocketAddress serverAddress = null;
-  /** only used for testing purposes  */
-  protected boolean stopRequested = false;
-
-  /** Configuration Manager */
-  private ConfigManager configMgr;
-
-  /** hadoop configuration */
-  protected Configuration conf;
-
-  protected boolean initialized;  // Are we initialized?
-  protected volatile boolean running; // Are we running?
-
-  /** Deamon thread to trigger policies */
-  Daemon triggerThread = null;
-  public static long triggerMonitorSleepTime = SLEEP_TIME;
-
-  /** Deamon thread to delete obsolete parity files */
-  PurgeMonitor purgeMonitor = null;
-  Daemon purgeThread = null;
-  
-  /** Deamon thread to har raid directories */
-  Daemon harThread = null;
-
-  /** Daemon thread to fix corrupt files */
-  BlockFixer blockFixer = null;
-  Daemon blockFixerThread = null;
-
-  // statistics about RAW hdfs blocks. This counts all replicas of a block.
-  public static class Statistics {
-    long numProcessedBlocks; // total blocks encountered in namespace
-    long processedSize;   // disk space occupied by all blocks
-    long remainingSize;      // total disk space post RAID
-    
-    long numMetaBlocks;      // total blocks in metafile
-    long metaSize;           // total disk space for meta files
-
-    public void clear() {
-      numProcessedBlocks = 0;
-      processedSize = 0;
-      remainingSize = 0;
-      numMetaBlocks = 0;
-      metaSize = 0;
-    }
-    public String toString() {
-      long save = processedSize - (remainingSize + metaSize);
-      long savep = 0;
-      if (processedSize > 0) {
-        savep = (save * 100)/processedSize;
-      }
-      String msg = " numProcessedBlocks = " + numProcessedBlocks +
-                   " processedSize = " + processedSize +
-                   " postRaidSize = " + remainingSize +
-                   " numMetaBlocks = " + numMetaBlocks +
-                   " metaSize = " + metaSize +
-                   " %save in raw disk space = " + savep;
-      return msg;
-    }
-  }
-
-  // Startup options
-  static public enum StartupOption{
-    TEST ("-test"),
-    REGULAR ("-regular");
-
-    private String name = null;
-    private StartupOption(String arg) {this.name = arg;}
-    public String getName() {return name;}
-  }
-  
-  /**
-   * Start RaidNode.
-   * <p>
-   * The raid-node can be started with one of the following startup options:
-   * <ul> 
-   * <li>{@link StartupOption#REGULAR REGULAR} - normal raid node startup</li>
-   * </ul>
-   * The option is passed via configuration field: 
-   * <tt>fs.raidnode.startup</tt>
-   * 
-   * The conf will be modified to reflect the actual ports on which 
-   * the RaidNode is up and running if the user passes the port as
-   * <code>zero</code> in the conf.
-   * 
-   * @param conf  confirguration
-   * @throws IOException
-   */
-
-  RaidNode(Configuration conf) throws IOException {
-    try {
-      initialize(conf);
-    } catch (IOException e) {
-      LOG.error(StringUtils.stringifyException(e));
-      this.stop();
-      throw e;
-    } catch (Exception e) {
-      this.stop();
-      throw new IOException(e);
-    }
-  }
-
-  public long getProtocolVersion(String protocol,
-                                 long clientVersion) throws IOException {
-    if (protocol.equals(RaidProtocol.class.getName())) {
-      return RaidProtocol.versionID;
-    } else {
-      throw new IOException("Unknown protocol to name node: " + protocol);
-    }
-  }
-
-  @Override
-  public ProtocolSignature getProtocolSignature(String protocol,
-      long clientVersion, int clientMethodsHash) throws IOException {
-    return ProtocolSignature.getProtocolSignature(
-        this, protocol, clientVersion, clientMethodsHash);
-  }
-
-  /**
-   * Wait for service to finish.
-   * (Normally, it runs forever.)
-   */
-  public void join() {
-    try {
-      if (server != null) server.join();
-      if (triggerThread != null) triggerThread.join();
-      if (blockFixerThread != null) blockFixerThread.join();
-      if (purgeThread != null) purgeThread.join();
-    } catch (InterruptedException ie) {
-      // do nothing
-    }
-  }
-  
-  /**
-   * Stop all RaidNode threads and wait for all to finish.
-   */
-  public void stop() {
-    if (stopRequested) {
-      return;
-    }
-    stopRequested = true;
-    running = false;
-    if (server != null) server.stop();
-    if (triggerThread != null) triggerThread.interrupt();
-    if (blockFixer != null) blockFixer.running = false;
-    if (blockFixerThread != null) blockFixerThread.interrupt();
-    if (purgeThread != null) purgeThread.interrupt();
-  }
-
-  private static InetSocketAddress getAddress(String address) {
-    return NetUtils.createSocketAddr(address);
-  }
-
-  public static InetSocketAddress getAddress(Configuration conf) {
-    String nodeport = conf.get("raid.server.address");
-    if (nodeport == null) {
-      nodeport = "localhost:" + DEFAULT_PORT;
-    }
-    return getAddress(nodeport);
-  }
-
-  public InetSocketAddress getListenerAddress() {
-    return server.getListenerAddress();
-  }
-  
-  private void initialize(Configuration conf) 
-    throws IOException, SAXException, InterruptedException, RaidConfigurationException,
-           ClassNotFoundException, ParserConfigurationException {
-    this.conf = conf;
-    InetSocketAddress socAddr = RaidNode.getAddress(conf);
-    int handlerCount = conf.getInt("fs.raidnode.handler.count", 10);
-
-    // read in the configuration
-    configMgr = new ConfigManager(conf);
-
-    // create rpc server 
-    this.server = RPC.getServer(this, socAddr.getHostName(), socAddr.getPort(),
-                                handlerCount, false, conf);
-
-    // The rpc-server port can be ephemeral... ensure we have the correct info
-    this.serverAddress = this.server.getListenerAddress();
-    LOG.info("RaidNode up at: " + this.serverAddress);
-
-    initialized = true;
-    running = true;
-    this.server.start(); // start RPC server
-
-    this.blockFixer = BlockFixer.createBlockFixer(conf);
-    this.blockFixerThread = new Daemon(this.blockFixer);
-    this.blockFixerThread.start();
- // start the deamon thread to fire polcies appropriately
-    RaidNode.triggerMonitorSleepTime = conf.getLong(
-        TRIGGER_MONITOR_SLEEP_TIME_KEY, 
-        SLEEP_TIME);
-
-    // start the deamon thread to fire polcies appropriately
-    this.triggerThread = new Daemon(new TriggerMonitor());
-    this.triggerThread.start();
-
-    // start the thread that deletes obsolete parity files
-    this.purgeMonitor = new PurgeMonitor();
-    this.purgeThread = new Daemon(purgeMonitor);
-    this.purgeThread.start();
-
-    // start the thread that creates HAR files
-    this.harThread = new Daemon(new HarMonitor());
-    this.harThread.start();
-  }
-
-  
-  /**
-   * Implement RaidProtocol methods
-   */
-
-  /** {@inheritDoc} */
-  public PolicyList[] getAllPolicies() throws IOException {
-    Collection<PolicyList> list = configMgr.getAllPolicies();
-    return list.toArray(new PolicyList[list.size()]);
-  }
-
-  /** {@inheritDoc} */
-  public String recoverFile(String inStr, long corruptOffset) throws IOException {
-
-    LOG.info("Recover File for " + inStr + " for corrupt offset " + corruptOffset);
-    Path inputPath = new Path(inStr);
-    Path srcPath = inputPath.makeQualified(inputPath.getFileSystem(conf));
-    // find stripe length from config
-    int stripeLength = getStripeLength(conf);
-
-    // first try decode using XOR code
-    Path destPref = xorDestinationPath(conf);
-    Decoder decoder = new XORDecoder(conf, RaidNode.getStripeLength(conf));
-    Path unraided = unRaid(conf, srcPath, destPref, decoder,
-        stripeLength, corruptOffset);
-    if (unraided != null) {
-      return unraided.toString();
-    }
-
-    // try decode using ReedSolomon code
-    destPref = rsDestinationPath(conf);
-    decoder = new ReedSolomonDecoder(conf, RaidNode.getStripeLength(conf),
-        RaidNode.rsParityLength(conf));
-    unraided = unRaid(conf, srcPath, destPref, decoder,
-        stripeLength, corruptOffset);
-    if (unraided != null) {
-      return unraided.toString();
-    }
-    return null;
-  }
-
-  /**
-   * returns the number of raid jobs running for a particular policy
-   */
-  abstract int getRunningJobsForPolicy(String policyName);
-
-  /**
-   * Periodically checks to see which policies should be fired.
-   */
-  class TriggerMonitor implements Runnable {
-    class ScanState {
-      long fullScanStartTime;
-      DirectoryTraversal pendingTraversal;
-      RaidFilter.Statistics stats;
-      ScanState() {
-        fullScanStartTime = 0;
-        pendingTraversal = null;
-        stats = new RaidFilter.Statistics();
-      }
-    }
-
-    private Map<String, ScanState> scanStateMap =
-      new HashMap<String, ScanState>();
-
-    /**
-     */
-    public void run() {
-      while (running) {
-        try {
-          doProcess();
-        } catch (Exception e) {
-          LOG.error(StringUtils.stringifyException(e));
-        } finally {
-          LOG.info("Trigger thread continuing to run...");
-        }
-      }
-    }
-
-    /**
-     * Should we select more files for a policy.
-     */
-    private boolean shouldSelectFiles(PolicyInfo info) {
-      String policyName = info.getName();
-      ScanState scanState = scanStateMap.get(policyName);
-      int runningJobsCount = getRunningJobsForPolicy(policyName);
-      // Is there a scan in progress for this policy?
-      if (scanState.pendingTraversal != null) {
-        int maxJobsPerPolicy = configMgr.getMaxJobsPerPolicy();
-
-        // If there is a scan in progress for this policy, we can have
-        // upto maxJobsPerPolicy running jobs.
-        return (runningJobsCount < maxJobsPerPolicy);
-      } else {
-        // If there isn't a scan in progress for this policy, we don't
-        // want to start a fresh scan if there is even one running job.
-        if (runningJobsCount >= 1) {
-          return false;
-        }
-        // Check the time of the last full traversal before starting a fresh
-        // traversal.
-        long lastScan = scanState.fullScanStartTime;
-        return (now() > lastScan + configMgr.getPeriodicity());
-      }
-    }
-
-   /**
-    * Returns a list of pathnames that needs raiding.
-    * The list of paths could be obtained by resuming a previously suspended
-    * traversal.
-    * The number of paths returned is limited by raid.distraid.max.jobs.
-    */
-    private List<FileStatus> selectFiles(
-      PolicyInfo info, ArrayList<PolicyInfo> allPolicies) throws IOException {
-      Path destPrefix = getDestinationPath(info.getErasureCode(), conf);
-      String policyName = info.getName();
-      Path srcPath = info.getSrcPath();
-      long modTimePeriod = Long.parseLong(info.getProperty("modTimePeriod"));
-
-      // Max number of files returned.
-      int selectLimit = configMgr.getMaxFilesPerJob();
-      int targetRepl = Integer.parseInt(info.getProperty("targetReplication"));
-
-      long selectStartTime = Time.now();
-
-      ScanState scanState = scanStateMap.get(policyName);
-      // If we have a pending traversal, resume it.
-      if (scanState.pendingTraversal != null) {
-        DirectoryTraversal dt = scanState.pendingTraversal;
-        LOG.info("Resuming traversal for policy " + policyName);
-        DirectoryTraversal.FileFilter filter =
-          filterForPolicy(selectStartTime, info, allPolicies, scanState.stats);
-        List<FileStatus> returnSet = dt.getFilteredFiles(filter, selectLimit);
-        if (dt.doneTraversal()) {
-          scanState.pendingTraversal = null;
-        }
-        return returnSet;
-      }
-
-      // Expand destination prefix path.
-      String destpstr = destPrefix.toString();
-      if (!destpstr.endsWith(Path.SEPARATOR)) {
-        destpstr += Path.SEPARATOR;
-      }
-
-      List<FileStatus> returnSet = new LinkedList<FileStatus>();
-
-      FileSystem fs = srcPath.getFileSystem(conf);
-      FileStatus[] gpaths = fs.globStatus(srcPath);
-      if (gpaths != null) {
-        List<FileStatus> selectedPaths = new LinkedList<FileStatus>();
-        for (FileStatus onepath: gpaths) {
-          String pathstr = onepath.getPath().makeQualified(fs).toString();
-          if (!pathstr.endsWith(Path.SEPARATOR)) {
-            pathstr += Path.SEPARATOR;
-          }
-          if (pathstr.startsWith(destpstr) || destpstr.startsWith(pathstr)) {
-            LOG.info("Skipping source " + pathstr +
-                     " because it conflicts with raid directory " + destpstr);
-          } else {
-            selectedPaths.add(onepath);
-          }
-        }
-
-        // Set the time for a new traversal.
-        scanState.fullScanStartTime = now();
-        DirectoryTraversal dt = new DirectoryTraversal(fs, selectedPaths,
-          conf.getInt("raid.directorytraversal.threads", 4));
-        DirectoryTraversal.FileFilter filter =
-          filterForPolicy(selectStartTime, info, allPolicies, scanState.stats);
-        returnSet = dt.getFilteredFiles(filter, selectLimit);
-        if (!dt.doneTraversal()) {
-          scanState.pendingTraversal = dt;
-        }
-      }
-      return returnSet;
-    }
-
-    /**
-     * Keep processing policies.
-     * If the config file has changed, then reload config file and start afresh.
-     */
-    private void doProcess() throws IOException, InterruptedException {
-      ArrayList<PolicyInfo> allPolicies = new ArrayList<PolicyInfo>();
-      for (PolicyList category : configMgr.getAllPolicies()) {
-        for (PolicyInfo info: category.getAll()) {
-          allPolicies.add(info);
-        }
-      }
-      while (running) {
-        Thread.sleep(RaidNode.triggerMonitorSleepTime);
-
-        boolean reloaded = configMgr.reloadConfigsIfNecessary();
-        if (reloaded) {
-          allPolicies.clear();
-          for (PolicyList category : configMgr.getAllPolicies()) {
-            for (PolicyInfo info: category.getAll()) {
-              allPolicies.add(info);
-            }
-          }
-        }
-        for (PolicyInfo info: allPolicies) {
-          if (!scanStateMap.containsKey(info.getName())) {
-            scanStateMap.put(info.getName(), new ScanState());
-          }
-
-          if (!shouldSelectFiles(info)) {
-            continue;
-          }
-
-          LOG.info("Triggering Policy Filter " + info.getName() +
-                   " " + info.getSrcPath());
-          List<FileStatus> filteredPaths = null;
-          try {
-            filteredPaths = selectFiles(info, allPolicies);
-          } catch (Exception e) {
-            LOG.error("Exception while invoking filter on policy " + info.getName() +
-                     " srcPath " + info.getSrcPath() + 
-                     " exception " + StringUtils.stringifyException(e));
-            continue;
-          }
-
-          if (filteredPaths == null || filteredPaths.size() == 0) {
-            LOG.info("No filtered paths for policy " + info.getName());
-             continue;
-          }
-
-          // Apply the action on accepted paths
-          LOG.info("Triggering Policy Action " + info.getName() +
-                   " " + info.getSrcPath() + " raid " + filteredPaths.size() + " files");
-          try {
-            raidFiles(info, filteredPaths);
-          } catch (Exception e) {
-            LOG.error("Exception while invoking action on policy " + info.getName() +
-                     " srcPath " + info.getSrcPath() + 
-                     " exception " + StringUtils.stringifyException(e));
-            continue;
-          }
-        }
-      }
-    }
-
-    DirectoryTraversal.FileFilter filterForPolicy(
-      long startTime, PolicyInfo info, List<PolicyInfo> allPolicies,
-      RaidFilter.Statistics stats)
-      throws IOException {
-      switch (info.getErasureCode()) {
-      case XOR:
-        // Return a preference-based filter that prefers RS parity files
-        // over XOR parity files.
-        return new RaidFilter.PreferenceFilter(
-          conf, rsDestinationPath(conf), xorDestinationPath(conf),
-          info, allPolicies, startTime, stats);
-      case RS:
-        return new RaidFilter.TimeBasedFilter(conf, rsDestinationPath(conf),
-          info, allPolicies, startTime, stats);
-      default:
-        return null;
-      }
-    }
-  }
-
-  /**
-   * raid a list of files, this will be overridden by subclasses of RaidNode
-   */
-  abstract void raidFiles(PolicyInfo info, List<FileStatus> paths) 
-    throws IOException;
-
-  static private Path getOriginalParityFile(Path destPathPrefix, Path srcPath) {
-    return new Path(destPathPrefix, makeRelative(srcPath));
-  }
-  
-  static class ParityFilePair {
-    private Path path;
-    private FileSystem fs;
-    
-    public ParityFilePair( Path path, FileSystem fs) {
-      this.path = path;
-      this.fs = fs;
-    }
-    
-    public Path getPath() {
-      return this.path;
-    }
-    
-    public FileSystem getFileSystem() {
-      return this.fs;
-    }
-    
-  }
-  
-  
-  /**
-   * Returns the Path to the parity file of a given file
-   * 
-   * @param destPathPrefix Destination prefix defined by some policy
-   * @param srcPath Path to the original source file
-   * @param create Boolean value telling whether a new parity file should be created
-   * @return Path object representing the parity file of the source
-   * @throws IOException
-   */
-  static ParityFilePair getParityFile(Path destPathPrefix, Path srcPath, Configuration conf) throws IOException {
-    Path srcParent = srcPath.getParent();
-
-    FileSystem fsDest = destPathPrefix.getFileSystem(conf);
-    FileSystem fsSrc = srcPath.getFileSystem(conf);
-    
-    FileStatus srcStatus = null;
-    try {
-      srcStatus = fsSrc.getFileStatus(srcPath);
-    } catch (java.io.FileNotFoundException e) {
-      return null;
-    }
-    
-    Path outDir = destPathPrefix;
-    if (srcParent != null) {
-      if (srcParent.getParent() == null) {
-        outDir = destPathPrefix;
-      } else {
-        outDir = new Path(destPathPrefix, makeRelative(srcParent));
-      }
-    }
-
-    
-    //CASE 1: CHECK HAR - Must be checked first because har is created after
-    // parity file and returning the parity file could result in error while
-    // reading it.
-    Path outPath =  getOriginalParityFile(destPathPrefix, srcPath);
-    String harDirName = srcParent.getName() + HAR_SUFFIX; 
-    Path HarPath = new Path(outDir,harDirName);
-    if (fsDest.exists(HarPath)) {  
-      URI HarPathUri = HarPath.toUri();
-      Path inHarPath = new Path("har://",HarPathUri.getPath()+"/"+outPath.toUri().getPath());
-      FileSystem fsHar = new HarFileSystem(fsDest);
-      fsHar.initialize(inHarPath.toUri(), conf);
-      if (fsHar.exists(inHarPath)) {
-        FileStatus inHar = fsHar.getFileStatus(inHarPath);
-        if (inHar.getModificationTime() == srcStatus.getModificationTime()) {
-          return new ParityFilePair(inHarPath,fsHar);
-        }
-      }
-    }
-    
-    //CASE 2: CHECK PARITY
-    try {
-      FileStatus outHar = fsDest.getFileStatus(outPath);
-      if (outHar.getModificationTime() == srcStatus.getModificationTime()) {
-        return new ParityFilePair(outPath,fsDest);
-      }
-    } catch (java.io.FileNotFoundException e) {
-    }
-
-    return null; // NULL if no parity file
-  }
-
-  static ParityFilePair xorParityForSource(Path srcPath, Configuration conf)
-      throws IOException {
-    try {
-      Path destPath = xorDestinationPath(conf);
-      return getParityFile(destPath, srcPath, conf);
-    } catch (FileNotFoundException e) {
-    }
-    return null;
-  }
-
-  static ParityFilePair rsParityForSource(Path srcPath, Configuration conf)
-      throws IOException {
-    try {
-      Path destPath = rsDestinationPath(conf);
-      return getParityFile(destPath, srcPath, conf);
-    } catch (FileNotFoundException e) {
-    }
-    return null;
-  }
-
-  private ParityFilePair getParityFile(Path destPathPrefix, Path srcPath)
-      throws IOException {
-    return getParityFile(destPathPrefix, srcPath, conf);
-  }
-
-  /**
-   * RAID a list of files.
-   */
-  void doRaid(Configuration conf, PolicyInfo info, List<FileStatus> paths)
-      throws IOException {
-    int targetRepl = Integer.parseInt(info.getProperty("targetReplication"));
-    int metaRepl = Integer.parseInt(info.getProperty("metaReplication"));
-    int stripeLength = getStripeLength(conf);
-    Path destPref = getDestinationPath(info.getErasureCode(), conf);
-    String simulate = info.getProperty("simulate");
-    boolean doSimulate = simulate == null ? false : Boolean
-        .parseBoolean(simulate);
-
-    Statistics statistics = new Statistics();
-    int count = 0;
-
-    for (FileStatus s : paths) {
-      doRaid(conf, s, destPref, info.getErasureCode(), statistics,
-        new RaidUtils.DummyProgressable(), doSimulate, targetRepl, metaRepl,
-        stripeLength);
-      if (count % 1000 == 0) {
-        LOG.info("RAID statistics " + statistics.toString());
-      }
-      count++;
-    }
-    LOG.info("RAID statistics " + statistics.toString());
-  }
-
-  
-  /**
-   * RAID an individual file
-   */
-
-  static public void doRaid(Configuration conf, PolicyInfo info,
-      FileStatus src, Statistics statistics, Progressable reporter)
-      throws IOException {
-    int targetRepl = Integer.parseInt(info.getProperty("targetReplication"));
-    int metaRepl = Integer.parseInt(info.getProperty("metaReplication"));
-    int stripeLength = getStripeLength(conf);
-
-    Path destPref = getDestinationPath(info.getErasureCode(), conf);
-    String simulate = info.getProperty("simulate");
-    boolean doSimulate = simulate == null ? false : Boolean
-        .parseBoolean(simulate);
-
-    doRaid(conf, src, destPref, info.getErasureCode(), statistics,
-                  reporter, doSimulate, targetRepl, metaRepl, stripeLength);
-  }
-
-  /**
-   * RAID an individual file
-   */
-  static public void doRaid(Configuration conf, FileStatus stat, Path destPath,
-                      PolicyInfo.ErasureCodeType code, Statistics statistics,
-                      Progressable reporter, boolean doSimulate,
-                      int targetRepl, int metaRepl, int stripeLength)
-    throws IOException {
-    Path p = stat.getPath();
-    FileSystem srcFs = p.getFileSystem(conf);
-
-    // extract block locations from File system
-    BlockLocation[] locations = srcFs.getFileBlockLocations(stat, 0, stat.getLen());
-    
-    // if the file has fewer than 2 blocks, then nothing to do
-    if (locations.length <= 2) {
-      return;
-    }
-
-    // add up the raw disk space occupied by this file
-    long diskSpace = 0;
-    for (BlockLocation l: locations) {
-      diskSpace += (l.getLength() * stat.getReplication());
-    }
-    statistics.numProcessedBlocks += locations.length;
-    statistics.processedSize += diskSpace;
-
-    // generate parity file
-    generateParityFile(conf, stat, reporter, srcFs, destPath, code,
-        locations, metaRepl, stripeLength);
-
-    // reduce the replication factor of the source file
-    if (!doSimulate) {
-      if (srcFs.setReplication(p, (short)targetRepl) == false) {
-        LOG.error("Error in reducing relication factor of file " + p + " to " + targetRepl);
-        statistics.remainingSize += diskSpace;  // no change in disk space usage
-        return;
-      }
-    }
-
-    diskSpace = 0;
-    for (BlockLocation l: locations) {
-      diskSpace += (l.getLength() * targetRepl);
-    }
-    statistics.remainingSize += diskSpace;
-
-    // the metafile will have this many number of blocks
-    int numMeta = locations.length / stripeLength;
-    if (locations.length % stripeLength != 0) {
-      numMeta++;
-    }
-
-    // we create numMeta for every file. This metablock has metaRepl # replicas.
-    // the last block of the metafile might not be completely filled up, but we
-    // ignore that for now.
-    statistics.numMetaBlocks += (numMeta * metaRepl);
-    statistics.metaSize += (numMeta * metaRepl * stat.getBlockSize());
-  }
-
-  /**
-   * Create the parity file.
-   */
-  static private void generateParityFile(Configuration conf, FileStatus stat,
-                                  Progressable reporter,
-                                  FileSystem inFs,
-                                  Path destPathPrefix,
-                                  ErasureCodeType code,
-                                  BlockLocation[] locations,
-                                  int metaRepl, int stripeLength) throws IOException {
-
-    Path inpath = stat.getPath();
-    Path outpath =  getOriginalParityFile(destPathPrefix, inpath);
-    FileSystem outFs = outpath.getFileSystem(conf);
-
-    // If the parity file is already upto-date, then nothing to do
-    try {
-      FileStatus stmp = outFs.getFileStatus(outpath);
-      if (stmp.getModificationTime() == stat.getModificationTime()) {
-        LOG.info("Parity file for " + inpath + "(" + locations.length +
-              ") is " + outpath + " already upto-date. Nothing more to do.");
-        return;
-      }
-    } catch (IOException e) {
-      // ignore errors because the raid file might not exist yet.
-    }
-
-    Encoder encoder = encoderForCode(conf, code);
-    encoder.encodeFile(inFs, inpath, outFs, outpath, (short)metaRepl, reporter);
-
-    // set the modification time of the RAID file. This is done so that the modTime of the
-    // RAID file reflects that contents of the source file that it has RAIDed. This should
-    // also work for files that are being appended to. This is necessary because the time on
-    // on the destination namenode may not be synchronised with the timestamp of the 
-    // source namenode.
-    outFs.setTimes(outpath, stat.getModificationTime(), -1);
-    inFs.setTimes(inpath, stat.getModificationTime(), stat.getAccessTime());
-
-    FileStatus outstat = outFs.getFileStatus(outpath);
-    FileStatus inStat = inFs.getFileStatus(inpath);
-    LOG.info("Source file " + inpath + " of size " + inStat.getLen() +
-             " Parity file " + outpath + " of size " + outstat.getLen() +
-             " src mtime " + stat.getModificationTime()  +
-             " parity mtime " + outstat.getModificationTime());
-  }
-
-  /**
-   * Extract a good block from the parity block. This assumes that the
-   * corruption is in the main file and the parity file is always good.
-   */
-  public static Path unRaid(Configuration conf, Path srcPath,
-      Path destPathPrefix, Decoder decoder, int stripeLength,
-      long corruptOffset) throws IOException {
-
-    // Test if parity file exists
-    ParityFilePair ppair = getParityFile(destPathPrefix, srcPath, conf);
-    if (ppair == null) {
-      return null;
-    }
-
-    final Path recoveryDestination = new Path(
-      RaidNode.unraidTmpDirectory(conf));
-    FileSystem destFs = recoveryDestination.getFileSystem(conf);
-    final Path recoveredPrefix = 
-      destFs.makeQualified(new Path(recoveryDestination, makeRelative(srcPath)));
-    final Path recoveredPath = 
-      new Path(recoveredPrefix + "." + new Random().nextLong() + ".recovered");
-    LOG.info("Creating recovered file " + recoveredPath);
-
-    FileSystem srcFs = srcPath.getFileSystem(conf);
-    decoder.decodeFile(srcFs, srcPath, ppair.getFileSystem(),
-        ppair.getPath(), corruptOffset, recoveredPath);
-
-    return recoveredPath;
-  }
-
-  public static Path unRaidCorruptBlock(Configuration conf, Path srcPath,
-    Path destPathPrefix, Decoder decoder, int stripeLength,
-      long corruptOffset) throws IOException {
-    // Test if parity file exists
-    ParityFilePair ppair = getParityFile(destPathPrefix, srcPath, conf);
-    if (ppair == null) {
-      LOG.error("Could not find parity file for " + srcPath);
-      return null;
-    }
-
-    final Path recoveryDestination = new Path(RaidNode.xorTempPrefix(conf));
-    FileSystem destFs = recoveryDestination.getFileSystem(conf);
-    final Path recoveredPrefix = 
-      destFs.makeQualified(new Path(recoveryDestination, makeRelative(srcPath)));
-    final Path recoveredBlock = 
-      new Path(recoveredPrefix + "." + new Random().nextLong() + ".recovered");
-    LOG.info("Creating recovered Block " + recoveredBlock);
-
-    FileSystem srcFs = srcPath.getFileSystem(conf);
-    FileStatus stat = srcFs.getFileStatus(srcPath);
-    long limit = Math.min(stat.getBlockSize(), stat.getLen() - corruptOffset);
-    java.io.OutputStream out = ppair.getFileSystem().create(recoveredBlock);
-    decoder.fixErasedBlock(srcFs, srcPath,
-        ppair.getFileSystem(), ppair.getPath(),
-        stat.getBlockSize(), corruptOffset, 0, limit, out);
-    out.close();
-    return recoveredBlock;
-  }
-
-  /**
-   * Periodically delete orphaned parity files.
-   */
-  class PurgeMonitor implements Runnable {
-    /**
-     */
-    public void run() {
-      while (running) {
-        try {
-          doPurge();
-        } catch (Exception e) {
-          LOG.error(StringUtils.stringifyException(e));
-        } finally {
-          LOG.info("Purge parity files thread continuing to run...");
-        }
-      }
-    }
-
-    /**
-     * Traverse the parity destination directory, removing directories that
-     * no longer existing in the source.
-     * @throws IOException
-     */
-    private void purgeDirectories(FileSystem fs, Path root) throws IOException {
-      String prefix = root.toUri().getPath();
-      List<FileStatus> startPaths = new LinkedList<FileStatus>();
-      try {
-        startPaths.add(fs.getFileStatus(root));
-      } catch (FileNotFoundException e) {
-        return;
-      }
-      DirectoryTraversal dt = new DirectoryTraversal(fs, startPaths);
-      FileStatus dir = dt.getNextDirectory();
-      for (; dir != null; dir = dt.getNextDirectory()) {
-        Path dirPath = dir.getPath();
-        if (dirPath.toUri().getPath().endsWith(HAR_SUFFIX)) {
-          continue;
-        }
-        String dirStr = dirPath.toUri().getPath();
-        if (!dirStr.startsWith(prefix)) {
-          continue;
-        }
-        String src = dirStr.replaceFirst(prefix, "");
-        if (src.length() == 0) continue;
-        Path srcPath = new Path(src);
-        if (!fs.exists(srcPath)) {
-          LOG.info("Purging directory " + dirPath);
-          boolean done = fs.delete(dirPath, true);
-          if (!done) {
-            LOG.error("Could not purge " + dirPath);
-          }
-        }
-      }
-    }
-
-    /**
-     * Delete orphaned files. The reason this is done by a separate thread 
-     * is to not burden the TriggerMonitor with scanning the 
-     * destination directories.
-     */
-    private void doPurge() throws IOException, InterruptedException {
-      long prevExec = 0;
-      while (running) {
-
-        // The config may be reloaded by the TriggerMonitor. 
-        // This thread uses whatever config is currently active.
-        while(now() < prevExec + configMgr.getPeriodicity()){
-          Thread.sleep(SLEEP_TIME);
-        }
-
-        LOG.info("Started purge scan");
-        prevExec = now();
-
-        // expand destination prefix path
-        Path destPref = xorDestinationPath(conf);
-        FileSystem destFs = destPref.getFileSystem(conf);
-        purgeDirectories(destFs, destPref);
-
-        destPref = rsDestinationPath(conf);
-        destFs = destPref.getFileSystem(conf);
-        purgeDirectories(destFs, destPref);
-
-        // fetch all categories
-        for (PolicyList category : configMgr.getAllPolicies()) {
-          for (PolicyInfo info: category.getAll()) {
-
-            try {
-              // expand destination prefix path
-              destPref = getDestinationPath(info.getErasureCode(), conf);
-              destFs = destPref.getFileSystem(conf);
-
-              //get srcPaths
-              Path[] srcPaths = info.getSrcPathExpanded();
-
-              if (srcPaths != null) {
-                for (Path srcPath: srcPaths) {
-                  // expand destination prefix
-                  Path destPath = getOriginalParityFile(destPref, srcPath);
-
-                  FileSystem srcFs = info.getSrcPath().getFileSystem(conf);
-                  FileStatus stat = null;
-                  try {
-                    stat = destFs.getFileStatus(destPath);
-                  } catch (FileNotFoundException e) {
-                    // do nothing, leave stat = null;
-                  }
-                  if (stat != null) {
-                    LOG.info("Purging obsolete parity files for policy " + 
-                              info.getName() + " " + destPath);
-                    recursePurge(info.getErasureCode(), srcFs, destFs,
-                              destPref.toUri().getPath(), stat);
-                  }
-
-                }
-              }
-            } catch (Exception e) {
-              LOG.warn("Ignoring Exception while processing policy " + 
-                       info.getName() + " " + 
-                       StringUtils.stringifyException(e));
-            }
-          }
-        }
-      }
-    }
-
-    /**
-     * The destPrefix is the absolute pathname of the destinationPath
-     * specified in the policy (without the host:port)
-     */ 
-    void recursePurge(ErasureCodeType code,
-                              FileSystem srcFs, FileSystem destFs,
-                              String destPrefix, FileStatus dest) 
-      throws IOException {
-
-      Path destPath = dest.getPath(); // pathname, no host:port
-      String destStr = destPath.toUri().getPath();
-      LOG.debug("Checking " + destPath + " prefix " + destPrefix);
-
-      // Verify if it is a har file
-      if (dest.isDirectory() && destStr.endsWith(HAR_SUFFIX)) {
-        try {
-          int harUsedPercent =
-            usefulHar(code, srcFs, destFs, destPath, destPrefix, conf);
-          LOG.info("Useful percentage of " + destStr + " " + harUsedPercent);
-          // Delete the har if its usefulness reaches a threshold.
-          if (harUsedPercent <= conf.getInt("raid.har.usage.threshold", 0)) {
-            LOG.info("Purging " + destStr + " at usage " + harUsedPercent);
-            boolean done = destFs.delete(destPath, true);
-            if (!done) {
-              LOG.error("Unable to purge directory " + destPath);
-            }
-          }
-        } catch (IOException e) {
-          LOG.warn("Error during purging " + destStr + " " +
-              StringUtils.stringifyException(e));
-        }
-        return;
-      }
-      
-      // Verify the destPrefix is a prefix of the destPath
-      if (!destStr.startsWith(destPrefix)) {
-        LOG.error("Destination path " + destStr + " should have " + 
-                  destPrefix + " as its prefix.");
-        return;
-      }
-      
-      if (dest.isDirectory()) {
-        FileStatus[] files = null;
-        files = destFs.listStatus(destPath);
-        if (files == null || files.length == 0){
-          boolean done = destFs.delete(destPath,true); // ideal is false, but
-                  // DFSClient only deletes directories if it is recursive
-          if (done) {
-            LOG.info("Purged directory " + destPath );
-          }
-          else {
-            LOG.error("Unable to purge directory " + destPath);
-          }
-        }
-        if (files != null) {
-          for (FileStatus one:files) {
-            recursePurge(code, srcFs, destFs, destPrefix, one);
-          }
-        }
-        // If the directory is empty now, it will be purged the next time this
-        // thread runs.
-        return; // the code below does the file checking
-      }
-      
-      String src = destStr.replaceFirst(destPrefix, "");
-      
-      Path srcPath = new Path(src);
-      boolean shouldDelete = false;
-
-      if (!srcFs.exists(srcPath)) {
-        shouldDelete = true;
-      } else {
-        try {
-          // If there is a RS parity file, the XOR parity can be deleted.
-          if (code == ErasureCodeType.XOR) {
-            ParityFilePair ppair = getParityFile(
-               getDestinationPath(ErasureCodeType.RS, conf), srcPath, conf);
-            if (ppair != null) {
-              shouldDelete = true;
-            }
-          }
-          if (!shouldDelete) {
-            Path dstPath = (new Path(destPrefix.trim())).makeQualified(destFs);
-            ParityFilePair ppair = getParityFile(dstPath,srcPath);
-            // If the parity file is not the appropriate one for the source or
-            // the parityFs is not the same as this file's filesystem
-            // (it is a HAR), this file can be deleted.
-            if ( ppair == null ||
-                 !destFs.equals(ppair.getFileSystem()) ||
-                 !destPath.equals(ppair.getPath())) {
-              shouldDelete = true;
-            }
-          }
-        } catch (IOException e) {
-          LOG.warn("Error during purging " + src + " " +
-                   StringUtils.stringifyException(e));
-        }
-      }
-
-      if (shouldDelete) {
-        boolean done = destFs.delete(destPath, false);
-        if (done) {
-          LOG.info("Purged file " + destPath );
-        } else {
-          LOG.error("Unable to purge file " + destPath );
-        }
-      }
-    } 
-  }
-
-  //
-  // Returns the number of up-to-date files in the har as a percentage of the
-  // total number of files in the har.
-  //
-  protected static int usefulHar(
-    ErasureCodeType code,
-    FileSystem srcFs, FileSystem destFs,
-    Path harPath, String destPrefix, Configuration conf) throws IOException {
-
-    FileSystem fsHar = new HarFileSystem(destFs);
-    String harURIPath = harPath.toUri().getPath();
-    Path qualifiedPath = new Path("har://", harURIPath +
-      Path.SEPARATOR + harPath.getParent().toUri().getPath());
-    fsHar.initialize(qualifiedPath.toUri(), conf);
-    FileStatus[] filesInHar = fsHar.listStatus(qualifiedPath);
-    if (filesInHar.length == 0) {
-      return 0;
-    }
-    int numUseless = 0;
-    for (FileStatus one: filesInHar) {
-      Path parityPath = one.getPath();
-      String parityStr = parityPath.toUri().getPath();
-      if (parityStr.startsWith("har:/")) {
-        LOG.error("Unexpected prefix har:/ for " + parityStr);
-        continue;
-      }
-      String prefixToReplace = harURIPath + destPrefix;
-      if (!parityStr.startsWith(prefixToReplace)) {
-        continue;
-      }
-      String src = parityStr.substring(prefixToReplace.length());
-      if (code == ErasureCodeType.XOR) {
-        ParityFilePair ppair = getParityFile(
-          getDestinationPath(ErasureCodeType.RS, conf), new Path(src), conf);
-        if (ppair != null) {
-          // There is a valid RS parity file, so the XOR one is useless.
-          numUseless++;
-          continue;
-        }
-      }
-      try {
-        FileStatus srcStatus = srcFs.getFileStatus(new Path(src));
-        if (srcStatus == null) {
-          numUseless++;
-        } else if (one.getModificationTime() !=
-                  srcStatus.getModificationTime()) {
-          numUseless++;
-        }
-      } catch (FileNotFoundException e) {
-        LOG.info("File not found: " + e);
-        numUseless++;
-      }
-    }
-    int uselessPercent = numUseless * 100 / filesInHar.length;
-    return 100 - uselessPercent;
-  }
-
-  private void doHar() throws IOException, InterruptedException {
-    long prevExec = 0;
-    while (running) {
-
-      // The config may be reloaded by the TriggerMonitor. 
-      // This thread uses whatever config is currently active.
-      while(now() < prevExec + configMgr.getPeriodicity()){
-        Thread.sleep(SLEEP_TIME);
-      }
-
-      LOG.info("Started archive scan");
-      prevExec = now();
-
-      // fetch all categories
-      for (PolicyList category : configMgr.getAllPolicies()) {
-        for (PolicyInfo info: category.getAll()) {
-          String tmpHarPath = tmpHarPathForCode(conf, info.getErasureCode());
-          String str = info.getProperty("time_before_har");
-          if (str != null) {
-            try {
-              long cutoff = now() - ( Long.parseLong(str) * 24L * 3600000L );
-
-              Path destPref = getDestinationPath(info.getErasureCode(), conf);
-              FileSystem destFs = destPref.getFileSystem(conf); 
-
-              //get srcPaths
-              Path[] srcPaths = info.getSrcPathExpanded();
-              
-              if ( srcPaths != null ){
-                for (Path srcPath: srcPaths) {
-                  // expand destination prefix
-                  Path destPath = getOriginalParityFile(destPref, srcPath);
-
-                  FileStatus stat = null;
-                  try {
-                    stat = destFs.getFileStatus(destPath);
-                  } catch (FileNotFoundException e) {
-                    // do nothing, leave stat = null;
-                  }
-                  if (stat != null) {
-                    LOG.info("Haring parity files for policy " + 
-                        info.getName() + " " + destPath);
-                    recurseHar(info, destFs, stat, destPref.toUri().getPath(),
-                        srcPath.getFileSystem(conf), cutoff, tmpHarPath);
-                  }
-                }
-              }
-            } catch (Exception e) {
-              LOG.warn("Ignoring Exception while processing policy " + 
-                  info.getName() + " " + 
-                  StringUtils.stringifyException(e));
-            }
-          }
-        }
-      }
-    }
-    return;
-  }
-  
-  void recurseHar(PolicyInfo info, FileSystem destFs, FileStatus dest,
-    String destPrefix, FileSystem srcFs, long cutoff, String tmpHarPath)
-    throws IOException {
-
-    if (dest.isFile()) {
-      return;
-    }
-    
-    Path destPath = dest.getPath(); // pathname, no host:port
-    String destStr = destPath.toUri().getPath();
-
-    // Verify if it already contains a HAR directory
-    if ( destFs.exists(new Path(destPath, destPath.getName()+HAR_SUFFIX)) ) {
-      return;
-    }
-
-    FileStatus[] files = null;
-    files = destFs.listStatus(destPath);
-    boolean shouldHar = false;
-    if (files != null) {
-      shouldHar = files.length > 0;
-      for (FileStatus one: files) {
-        if (one.isDirectory()){
-          recurseHar(info, destFs, one, destPrefix, srcFs, cutoff, tmpHarPath);
-          shouldHar = false;
-        } else if (one.getModificationTime() > cutoff ) {
-          if (shouldHar) {
-            LOG.info("Cannot archive " + destPath + 
-                   " because " + one.getPath() + " was modified after cutoff");
-            shouldHar = false;
-          }
-        }
-      }
-
-      if (shouldHar) {
-        String src = destStr.replaceFirst(destPrefix, "");
-        Path srcPath = new Path(src);
-        FileStatus[] statuses = srcFs.listStatus(srcPath);
-        Path destPathPrefix = new Path(destPrefix).makeQualified(destFs);
-        if (statuses != null) {
-          for (FileStatus status : statuses) {
-            if (getParityFile(destPathPrefix, 
-                              status.getPath().makeQualified(srcFs)) == null ) {
-              LOG.info("Cannot archive " + destPath + 
-                  " because it doesn't contain parity file for " +
-                  status.getPath().makeQualified(srcFs) + " on destination " +
-                  destPathPrefix);
-              shouldHar = false;
-              break;
-            }
-          }
-        }
-      }
-    }
-
-    if ( shouldHar ) {
-      LOG.info("Archiving " + dest.getPath() + " to " + tmpHarPath );
-      singleHar(info, destFs, dest, tmpHarPath);
-    }
-  } 
-
-  
-  private void singleHar(PolicyInfo info, FileSystem destFs, FileStatus dest,
-    String tmpHarPath) throws IOException {
-
-    Random rand = new Random();
-    Path root = new Path("/");
-    Path qualifiedPath = dest.getPath().makeQualified(destFs);
-    String harFileDst = qualifiedPath.getName() + HAR_SUFFIX;
-    String harFileSrc = qualifiedPath.getName() + "-" + 
-                                rand.nextLong() + "-" + HAR_SUFFIX;
-    short metaReplication =
-      (short) Integer.parseInt(info.getProperty("metaReplication"));
-    // HadoopArchives.HAR_PARTFILE_LABEL is private, so hard-coding the label.
-    conf.setLong("har.partfile.size", configMgr.getHarPartfileSize());
-    HadoopArchives har = new HadoopArchives(conf);
-    String[] args = new String[7];
-    args[0] = "-Ddfs.replication=" + metaReplication;
-    args[1] = "-archiveName";
-    args[2] = harFileSrc;
-    args[3] = "-p"; 
-    args[4] = root.makeQualified(destFs).toString();
-    args[5] = qualifiedPath.toUri().getPath().substring(1);
-    args[6] = tmpHarPath.toString();
-    int ret = 0;
-    try {
-      ret = ToolRunner.run(har, args);
-      if (ret == 0 && !destFs.rename(new Path(tmpHarPath+"/"+harFileSrc), 
-                                     new Path(qualifiedPath, harFileDst))) {
-        LOG.error("HAR rename didn't succeed from " + tmpHarPath+"/"+harFileSrc +
-            " to " + qualifiedPath + "/" + harFileDst);
-        ret = -2;
-      }
-    } catch (Exception exc) {
-      throw new IOException("Error while creating archive " + ret, exc);
-    }
-    
-    if (ret != 0){
-      throw new IOException("Error while creating archive " + ret);
-    }
-    return;
-  }
-  
-  /**
-   * Periodically generates HAR files
-   */
-  class HarMonitor implements Runnable {
-
-    public void run() {
-      while (running) {
-        try {
-          doHar();
-        } catch (Exception e) {
-          LOG.error(StringUtils.stringifyException(e));
-        } finally {
-          LOG.info("Har parity files thread continuing to run...");
-        }
-      }
-      LOG.info("Leaving Har thread.");
-    }
-    
-  }
-
-  /**
-   * Return the temp path for XOR parity files
-   */
-  public static String unraidTmpDirectory(Configuration conf) {
-    return conf.get(RAID_TMP_LOCATION_KEY, DEFAULT_RAID_TMP_LOCATION);
-  }
-
-  /**
-   * Return the temp path for ReedSolomonEncoder parity files
-   */
-  public static String rsTempPrefix(Configuration conf) {
-    return conf.get(RAIDRS_TMP_LOCATION_KEY, DEFAULT_RAIDRS_TMP_LOCATION);
-  }
-
-  /**
-   * Return the temp path for XOR parity files
-   */
-  public static String xorTempPrefix(Configuration conf) {
-    return conf.get(RAID_TMP_LOCATION_KEY, DEFAULT_RAID_TMP_LOCATION);
-  }
-
-  /**
-   * Return the temp path for XOR parity files
-   */
-  public static String xorHarTempPrefix(Configuration conf) {
-    return conf.get(RAID_HAR_TMP_LOCATION_KEY, DEFAULT_RAID_HAR_TMP_LOCATION);
-  }
-
-  /**
-   * Return the temp path for ReedSolomonEncoder parity files
-   */
-  public static String rsHarTempPrefix(Configuration conf) {
-    return conf.get(RAIDRS_HAR_TMP_LOCATION_KEY,
-        DEFAULT_RAIDRS_HAR_TMP_LOCATION);
-  }
-
-  /**
-   * Return the destination path for ReedSolomon parity files
-   */
-  public static Path rsDestinationPath(Configuration conf, FileSystem fs) {
-    String loc = conf.get(RAIDRS_LOCATION_KEY, DEFAULT_RAIDRS_LOCATION);
-    Path p = new Path(loc.trim());
-    p = p.makeQualified(fs);
-    return p;
-  }
-
-  /**
-   * Return the destination path for ReedSolomon parity files
-   */
-  public static Path rsDestinationPath(Configuration conf)
-    throws IOException {
-    String loc = conf.get(RAIDRS_LOCATION_KEY, DEFAULT_RAIDRS_LOCATION);
-    Path p = new Path(loc.trim());
-    FileSystem fs = FileSystem.get(p.toUri(), conf);
-    p = p.makeQualified(fs);
-    return p;
-  }
-
-  /**
-   * Return the destination path for XOR parity files
-   */
-  public static Path xorDestinationPath(Configuration conf, FileSystem fs) {
-    String loc = conf.get(RAID_LOCATION_KEY, DEFAULT_RAID_LOCATION);
-    Path p = new Path(loc.trim());
-    p = p.makeQualified(fs);
-    return p;
-  }
-
-  /**
-   * Return the destination path for XOR parity files
-   */
-  public static Path xorDestinationPath(Configuration conf)
-    throws IOException {
-    String loc = conf.get(RAID_LOCATION_KEY, DEFAULT_RAID_LOCATION);
-    Path p = new Path(loc.trim());
-    FileSystem fs = FileSystem.get(p.toUri(), conf);
-    p = p.makeQualified(fs);
-    return p;
-  }
-
-  /**
-   * Return the path prefix that stores the parity files
-   */
-  static Path getDestinationPath(ErasureCodeType code, Configuration conf)
-    throws IOException {
-    switch (code) {
-      case XOR:
-        return xorDestinationPath(conf);
-      case RS:
-        return rsDestinationPath(conf);
-      default:
-        return null;
-    }
-  }
-
-  static Encoder encoderForCode(Configuration conf, ErasureCodeType code) {
-    int stripeLength = getStripeLength(conf);
-    switch (code) {
-      case XOR:
-        return new XOREncoder(conf, stripeLength);
-      case RS:
-        return new ReedSolomonEncoder(conf, stripeLength, rsParityLength(conf));
-      default:
-        return null;
-    }
-  }
-
-  static String tmpHarPathForCode(Configuration conf, ErasureCodeType code) {
-    switch (code) {
-      case XOR:
-        return xorHarTempPrefix(conf);
-      case RS:
-        return rsHarTempPrefix(conf);
-      default:
-        return null;
-    }
-  }
-
-  /**
-   * Obtain stripe length from configuration
-   */
-  public static int getStripeLength(Configuration conf) {
-    return conf.getInt(STRIPE_LENGTH_KEY, DEFAULT_STRIPE_LENGTH);
-  }
-
-  /**
-   * Obtain stripe length from configuration
-   */
-  public static int rsParityLength(Configuration conf) {
-    return conf.getInt(RS_PARITY_LENGTH_KEY, RS_PARITY_LENGTH_DEFAULT);
-  }
-
-  static boolean isParityHarPartFile(Path p) {
-    Matcher m = PARITY_HAR_PARTFILE_PATTERN.matcher(p.toUri().getPath());
-    return m.matches();
-  }
-
-  /**
-   * Returns current time.
-   */
-  static long now() {
-    return Time.now();
-  }
-
-  /**                       
-   * Make an absolute path relative by stripping the leading /
-   */   
-  static private Path makeRelative(Path path) {
-    if (!path.isAbsolute()) {
-      return path;
-    }          
-    String p = path.toUri().getPath();
-    String relative = p.substring(1, p.length());
-    return new Path(relative);
-  } 
-
-  private static void printUsage() {
-    System.err.println("Usage: java RaidNode ");
-  }
-
-  private static StartupOption parseArguments(String args[]) {
-    int argsLen = (args == null) ? 0 : args.length;
-    StartupOption startOpt = StartupOption.REGULAR;
-    for(int i=0; i < argsLen; i++) {
-      String cmd = args[i]; // We have to parse command line args in future.
-    }
-    return startOpt;
-  }
-
-
-  /**
-   * Convert command line options to configuration parameters
-   */
-  private static void setStartupOption(Configuration conf, StartupOption opt) {
-    conf.set("fs.raidnode.startup", opt.toString());
-  }
-
-  /**
-   * Create an instance of the appropriate subclass of RaidNode 
-   */
-  public static RaidNode createRaidNode(Configuration conf)
-    throws ClassNotFoundException {
-    try {
-      // default to distributed raid node
-      Class<?> raidNodeClass =
-        conf.getClass(RAIDNODE_CLASSNAME_KEY, DistRaidNode.class);
-      if (!RaidNode.class.isAssignableFrom(raidNodeClass)) {
-        throw new ClassNotFoundException("not an implementation of RaidNode");
-      }
-      Constructor<?> constructor =
-        raidNodeClass.getConstructor(new Class[] {Configuration.class} );
-      return (RaidNode) constructor.newInstance(conf);
-    } catch (NoSuchMethodException e) {
-      throw new ClassNotFoundException("cannot construct blockfixer", e);
-    } catch (InstantiationException e) {
-      throw new ClassNotFoundException("cannot construct blockfixer", e);
-    } catch (IllegalAccessException e) {
-      throw new ClassNotFoundException("cannot construct blockfixer", e);
-    } catch (InvocationTargetException e) {
-      throw new ClassNotFoundException("cannot construct blockfixer", e);
-    }
-  }
-
-  /**
-   * Create an instance of the RaidNode 
-   */
-  public static RaidNode createRaidNode(String argv[], Configuration conf)
-    throws IOException, ClassNotFoundException {
-    if (conf == null) {
-      conf = new Configuration();
-    }
-    StartupOption startOpt = parseArguments(argv);
-    if (startOpt == null) {
-      printUsage();
-      return null;
-    }
-    setStartupOption(conf, startOpt);
-    RaidNode node = createRaidNode(conf);
-    return node;
-  }
-
-
-  /**
-   */
-  public static void main(String argv[]) throws Exception {
-    try {
-      StringUtils.startupShutdownMessage(RaidNode.class, argv, LOG);
-      RaidNode raid = createRaidNode(argv, null);
-      if (raid != null) {
-        raid.join();
-      }
-    } catch (Throwable e) {
-      LOG.error(StringUtils.stringifyException(e));
-      System.exit(-1);
-    }
-  }
-
-  
-
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidShell.java

@@ -1,682 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.IOException;
-import java.io.FileNotFoundException;
-import java.util.Collection;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.LinkedHashMap;
-import java.util.HashMap;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.net.InetSocketAddress;
-import javax.security.auth.login.LoginException;
-
-import org.apache.hadoop.ipc.*;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryProxy;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.HarFileSystem;
-
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.DistributedRaidFileSystem;
-import org.apache.hadoop.hdfs.RaidDFSUtil;
-
-import org.apache.hadoop.raid.protocol.PolicyInfo;
-import org.apache.hadoop.raid.protocol.PolicyList;
-import org.apache.hadoop.raid.protocol.RaidProtocol;
-
-/**
- * A {@link RaidShell} that allows browsing configured raid policies.
- */
-public class RaidShell extends Configured implements Tool {
-  static {
-    Configuration.addDefaultResource("hdfs-default.xml");
-    Configuration.addDefaultResource("hdfs-site.xml");
-  }
-  public static final Log LOG = LogFactory.getLog( "org.apache.hadoop.RaidShell");
-  public RaidProtocol raidnode;
-  RaidProtocol rpcRaidnode;
-  private UserGroupInformation ugi;
-  volatile boolean clientRunning = true;
-  private Configuration conf;
-
-  /**
-   * Start RaidShell.
-   * <p>
-   * The RaidShell connects to the specified RaidNode and performs basic
-   * configuration options.
-   * @throws IOException
-   */
-  public RaidShell(Configuration conf) throws IOException {
-    super(conf);
-    this.conf = conf;
-  }
-
-  void initializeRpc(Configuration conf, InetSocketAddress address) throws IOException {
-    this.ugi = UserGroupInformation.getCurrentUser();
-    this.rpcRaidnode = createRPCRaidnode(address, conf, ugi);
-    this.raidnode = createRaidnode(rpcRaidnode);
-  }
-
-  void initializeLocal(Configuration conf) throws IOException {
-    this.ugi = UserGroupInformation.getCurrentUser();
-  }
-
-  public static RaidProtocol createRaidnode(Configuration conf) throws IOException {
-    return createRaidnode(RaidNode.getAddress(conf), conf);
-  }
-
-  public static RaidProtocol createRaidnode(InetSocketAddress raidNodeAddr,
-      Configuration conf) throws IOException {
-    return createRaidnode(createRPCRaidnode(raidNodeAddr, conf,
-      UserGroupInformation.getCurrentUser()));
-  }
-
-  private static RaidProtocol createRPCRaidnode(InetSocketAddress raidNodeAddr,
-      Configuration conf, UserGroupInformation ugi)
-    throws IOException {
-    LOG.debug("RaidShell connecting to " + raidNodeAddr);
-    return (RaidProtocol)RPC.getProxy(RaidProtocol.class,
-        RaidProtocol.versionID, raidNodeAddr, ugi, conf,
-        NetUtils.getSocketFactory(conf, RaidProtocol.class));
-  }
-
-  private static RaidProtocol createRaidnode(RaidProtocol rpcRaidnode)
-    throws IOException {
-    RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
-        5, 5000, TimeUnit.MILLISECONDS);
-
-    Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
-      new HashMap<Class<? extends Exception>, RetryPolicy>();
-
-    Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
-      new HashMap<Class<? extends Exception>, RetryPolicy>();
-    exceptionToPolicyMap.put(RemoteException.class,
-        RetryPolicies.retryByRemoteException(
-            RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
-    RetryPolicy methodPolicy = RetryPolicies.retryByException(
-        RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
-    Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
-
-    methodNameToPolicyMap.put("create", methodPolicy);
-
-    return (RaidProtocol) RetryProxy.create(RaidProtocol.class,
-        rpcRaidnode, methodNameToPolicyMap);
-  }
-
-  private void checkOpen() throws IOException {
-    if (!clientRunning) {
-      IOException result = new IOException("RaidNode closed");
-      throw result;
-    }
-  }
-
-  /**
-   * Close the connection to the raidNode.
-   */
-  public synchronized void close() throws IOException {
-    if(clientRunning) {
-      clientRunning = false;
-      RPC.stopProxy(rpcRaidnode);
-    }
-  }
-
-  /**
-   * Displays format of commands.
-   */
-  private static void printUsage(String cmd) {
-    String prefix = "Usage: java " + RaidShell.class.getSimpleName();
-    if ("-showConfig".equals(cmd)) {
-      System.err.println("Usage: java RaidShell" + 
-                         " [-showConfig]"); 
-    } else if ("-recover".equals(cmd)) {
-      System.err.println("Usage: java RaidShell" +
-                         " [-recover srcPath1 corruptOffset]");
-    } else if ("-recoverBlocks".equals(cmd)) {
-      System.err.println("Usage: java RaidShell" +
-                         " [-recoverBlocks path1 path2...]");
-    } else {
-      System.err.println("Usage: java RaidShell");
-      System.err.println("           [-showConfig ]");
-      System.err.println("           [-help [cmd]]");
-      System.err.println("           [-recover srcPath1 corruptOffset]");
-      System.err.println("           [-recoverBlocks path1 path2...]");
-      System.err.println("           [-fsck [path]]");
-      System.err.println();
-      ToolRunner.printGenericCommandUsage(System.err);
-    }
-  }
-
-  /**
-   * run
-   */
-  public int run(String argv[]) throws Exception {
-
-    if (argv.length < 1) {
-      printUsage("");
-      return -1;
-    }
-
-    int exitCode = -1;
-    int i = 0;
-    String cmd = argv[i++];
-    //
-    // verify that we have enough command line parameters
-    //
-    if ("-showConfig".equals(cmd)) {
-      if (argv.length < 1) {
-        printUsage(cmd);
-        return exitCode;
-      }
-    } else if ("-recover".equals(cmd)) {
-      if (argv.length < 3) {
-        printUsage(cmd);
-        return exitCode;
-      }
-    } else if ("-fsck".equals(cmd)) {
-      if ((argv.length < 1) || (argv.length > 2)) {
-        printUsage(cmd);
-        return exitCode;
-      }
-    }
-
-    try {
-      if ("-showConfig".equals(cmd)) {
-        initializeRpc(conf, RaidNode.getAddress(conf));
-        exitCode = showConfig(cmd, argv, i);
-      } else if ("-recover".equals(cmd)) {
-        initializeRpc(conf, RaidNode.getAddress(conf));
-        exitCode = recoverAndPrint(cmd, argv, i);
-      } else if ("-recoverBlocks".equals(cmd)) {
-        initializeLocal(conf);
-        recoverBlocks(argv, i);
-        exitCode = 0;
-      } else if ("-fsck".equals(cmd)) {
-        if (argv.length == 1) {
-          // if there are no args, check the whole file system
-          exitCode = fsck("/");
-        } else {
-          // argv.length == 2
-          // otherwise, check the path passed
-          exitCode = fsck(argv[1]);
-        }
-      } else {
-        exitCode = -1;
-        System.err.println(cmd.substring(1) + ": Unknown command");
-        printUsage("");
-      }
-    } catch (IllegalArgumentException arge) {
-      exitCode = -1;
-      System.err.println(cmd.substring(1) + ": " + arge.getLocalizedMessage());
-      printUsage(cmd);
-    } catch (RemoteException e) {
-      //
-      // This is a error returned by raidnode server. Print
-      // out the first line of the error mesage, ignore the stack trace.
-      exitCode = -1;
-      try {
-        String[] content;
-        content = e.getLocalizedMessage().split("\n");
-        System.err.println(cmd.substring(1) + ": " +
-                           content[0]);
-      } catch (Exception ex) {
-        System.err.println(cmd.substring(1) + ": " +
-                           ex.getLocalizedMessage());
-      }
-    } catch (IOException e) {
-      //
-      // IO exception encountered locally.
-      // 
-      exitCode = -1;
-      System.err.println(cmd.substring(1) + ": " +
-                         e.getLocalizedMessage());
-    } catch (Exception re) {
-      exitCode = -1;
-      System.err.println(cmd.substring(1) + ": " + re.getLocalizedMessage());
-    } finally {
-    }
-    return exitCode;
-  }
-
-  /**
-   * Apply operation specified by 'cmd' on all parameters
-   * starting from argv[startindex].
-   */
-  private int showConfig(String cmd, String argv[], int startindex) throws IOException {
-    int exitCode = 0;
-    int i = startindex;
-    PolicyList[] all = raidnode.getAllPolicies();
-    for (PolicyList list: all) {
-      for (PolicyInfo p : list.getAll()) {
-        System.out.println(p);
-      }
-    }
-    return exitCode;
-  }
-
-  /**
-   * Recovers the specified path from the parity file
-   */
-  public Path[] recover(String cmd, String argv[], int startindex)
-    throws IOException {
-    Path[] paths = new Path[(argv.length - startindex) / 2];
-    int j = 0;
-    for (int i = startindex; i < argv.length; i = i + 2) {
-      String path = argv[i];
-      long corruptOffset = Long.parseLong(argv[i+1]);
-      LOG.info("RaidShell recoverFile for " + path + " corruptOffset " + corruptOffset);
-      Path recovered = new Path("/tmp/recovered." + Time.now());
-      FileSystem fs = recovered.getFileSystem(conf);
-      DistributedFileSystem dfs = (DistributedFileSystem)fs;
-      Configuration raidConf = new Configuration(conf);
-      raidConf.set("fs.hdfs.impl",
-                     "org.apache.hadoop.hdfs.DistributedRaidFileSystem");
-      raidConf.set("fs.raid.underlyingfs.impl",
-                     "org.apache.hadoop.hdfs.DistributedFileSystem");
-      raidConf.setBoolean("fs.hdfs.impl.disable.cache", true);
-      java.net.URI dfsUri = dfs.getUri();
-      FileSystem raidFs = FileSystem.get(dfsUri, raidConf);
-      FileUtil.copy(raidFs, new Path(path), fs, recovered, false, conf);
-
-      paths[j] = recovered;
-      LOG.info("Raidshell created recovery file " + paths[j]);
-      j++;
-    }
-    return paths;
-  }
-
-  public int recoverAndPrint(String cmd, String argv[], int startindex)
-    throws IOException {
-    int exitCode = 0;
-    for (Path p : recover(cmd,argv,startindex)) {
-      System.out.println(p);
-    }
-    return exitCode;
-  }
-
-  public void recoverBlocks(String[] args, int startIndex)
-    throws IOException {
-    LOG.debug("Recovering blocks for " + (args.length - startIndex) + " files");
-    BlockFixer.BlockFixerHelper fixer = new BlockFixer.BlockFixerHelper(conf);
-    for (int i = startIndex; i < args.length; i++) {
-      String path = args[i];
-      fixer.fixFile(new Path(path));
-    }
-  }
-
-  /**
-   * checks whether a file has more than the allowable number of
-   * corrupt blocks and must therefore be considered corrupt
-   */
-  private boolean isFileCorrupt(final DistributedFileSystem dfs,
-                                final Path filePath)
-    throws IOException {
-    // corruptBlocksPerStripe:
-    // map stripe # -> # of corrupt blocks in that stripe (data + parity)
-    HashMap<Integer, Integer> corruptBlocksPerStripe =
-      new LinkedHashMap<Integer, Integer>();
-
-    // read conf
-    final int stripeBlocks = RaidNode.getStripeLength(conf);
-
-    // figure out which blocks are missing/corrupted
-    final FileStatus fileStatus = dfs.getFileStatus(filePath);
-    final long blockSize = fileStatus.getBlockSize();
-    final long fileLength = fileStatus.getLen();
-    final long fileLengthInBlocks = (fileLength / blockSize) +
-      (((fileLength % blockSize) == 0) ? 0L : 1L);
-    final long fileStripes = (fileLengthInBlocks / stripeBlocks) +
-      (((fileLengthInBlocks % stripeBlocks) == 0) ? 0L : 1L);
-    final BlockLocation[] fileBlocks =
-      dfs.getFileBlockLocations(fileStatus, 0, fileLength);
-
-    // figure out which stripes these corrupted blocks belong to
-    for (BlockLocation fileBlock: fileBlocks) {
-      int blockNo = (int) (fileBlock.getOffset() / blockSize);
-      final int stripe = (int) (blockNo / stripeBlocks);
-      if (fileBlock.isCorrupt() ||
-          (fileBlock.getNames().length == 0 && fileBlock.getLength() > 0)) {
-        if (corruptBlocksPerStripe.get(stripe) == null) {
-          corruptBlocksPerStripe.put(stripe, 1);
-        } else {
-          corruptBlocksPerStripe.put(stripe, corruptBlocksPerStripe.
-                                     get(stripe) + 1);
-        }
-        LOG.debug("file " + filePath.toString() + " corrupt in block " +
-                 blockNo + "/" + fileLengthInBlocks + ", stripe " + stripe +
-                 "/" + fileStripes);
-      } else {
-        LOG.debug("file " + filePath.toString() + " OK in block " + blockNo +
-                 "/" + fileLengthInBlocks + ", stripe " + stripe + "/" +
-                 fileStripes);
-      }
-    }
-
-    RaidInfo raidInfo = getFileRaidInfo(dfs, filePath);
-
-    // now check parity blocks
-    if (raidInfo.raidType != RaidType.NONE) {
-      checkParityBlocks(filePath, corruptBlocksPerStripe, blockSize,
-                        fileStripes, raidInfo);
-    }
-
-    final int maxCorruptBlocksPerStripe = raidInfo.parityBlocksPerStripe;
-
-    for (int corruptBlocksInStripe: corruptBlocksPerStripe.values()) {
-      if (corruptBlocksInStripe > maxCorruptBlocksPerStripe) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  /**
-   * holds the type of raid used for a particular file
-   */
-  private enum RaidType {
-    XOR,
-    RS,
-    NONE
-  }
-
-  /**
-   * holds raid type and parity file pair
-   */
-  private class RaidInfo {
-    public RaidInfo(final RaidType raidType,
-                    final RaidNode.ParityFilePair parityPair,
-                    final int parityBlocksPerStripe) {
-      this.raidType = raidType;
-      this.parityPair = parityPair;
-      this.parityBlocksPerStripe = parityBlocksPerStripe;
-    }
-    public final RaidType raidType;
-    public final RaidNode.ParityFilePair parityPair;
-    public final int parityBlocksPerStripe;
-  }
-
-  /**
-   * returns the raid for a given file
-   */
-  private RaidInfo getFileRaidInfo(final DistributedFileSystem dfs,
-                                   final Path filePath)
-    throws IOException {
-    // now look for the parity file
-    Path destPath = null;
-    RaidNode.ParityFilePair ppair = null;
-    try {
-      // look for xor parity file first
-      destPath = RaidNode.xorDestinationPath(conf);
-      ppair = RaidNode.getParityFile(destPath, filePath, conf);
-    } catch (FileNotFoundException ignore) {
-    }
-    if (ppair != null) {
-      return new RaidInfo(RaidType.XOR, ppair, 1);
-    } else {
-      // failing that, look for rs parity file
-      try {
-        destPath = RaidNode.rsDestinationPath(conf);
-        ppair = RaidNode.getParityFile(destPath, filePath, conf);
-      } catch (FileNotFoundException ignore) {
-      }
-      if (ppair != null) {
-        return new RaidInfo(RaidType.RS, ppair, RaidNode.rsParityLength(conf));
-      } else {
-        return new RaidInfo(RaidType.NONE, null, 0);
-      }
-    }
-  }
-
-  /**
-   * gets the parity blocks corresponding to file
-   * returns the parity blocks in case of DFS
-   * and the part blocks containing parity blocks
-   * in case of HAR FS
-   */
-  private BlockLocation[] getParityBlocks(final Path filePath,
-                                          final long blockSize,
-                                          final long fileStripes,
-                                          final RaidInfo raidInfo)
-    throws IOException {
-
-
-    final String parityPathStr = raidInfo.parityPair.getPath().toUri().
-      getPath();
-    FileSystem parityFS = raidInfo.parityPair.getFileSystem();
-
-    // get parity file metadata
-    FileStatus parityFileStatus = parityFS.
-      getFileStatus(new Path(parityPathStr));
-    long parityFileLength = parityFileStatus.getLen();
-
-    if (parityFileLength != fileStripes * raidInfo.parityBlocksPerStripe *
-        blockSize) {
-      throw new IOException("expected parity file of length" +
-                            (fileStripes * raidInfo.parityBlocksPerStripe *
-                             blockSize) +
-                            " but got parity file of length " +
-                            parityFileLength);
-    }
-
-    BlockLocation[] parityBlocks =
-      parityFS.getFileBlockLocations(parityFileStatus, 0L, parityFileLength);
-
-    if (parityFS instanceof DistributedFileSystem ||
-        parityFS instanceof DistributedRaidFileSystem) {
-      long parityBlockSize = parityFileStatus.getBlockSize();
-      if (parityBlockSize != blockSize) {
-        throw new IOException("file block size is " + blockSize +
-                              " but parity file block size is " +
-                              parityBlockSize);
-      }
-    } else if (parityFS instanceof HarFileSystem) {
-      LOG.debug("HAR FS found");
-    } else {
-      LOG.warn("parity file system is not of a supported type");
-    }
-
-    return parityBlocks;
-  }
-
-  /**
-   * checks the parity blocks for a given file and modifies
-   * corruptBlocksPerStripe accordingly
-   */
-  private void checkParityBlocks(final Path filePath,
-                                 final HashMap<Integer, Integer>
-                                 corruptBlocksPerStripe,
-                                 final long blockSize,
-                                 final long fileStripes,
-                                 final RaidInfo raidInfo)
-    throws IOException {
-
-    // get the blocks of the parity file
-    // because of har, multiple blocks may be returned as one container block
-    BlockLocation[] containerBlocks = getParityBlocks(filePath, blockSize,
-                                                      fileStripes, raidInfo);
-
-    long parityStripeLength = blockSize *
-      ((long) raidInfo.parityBlocksPerStripe);
-
-    long parityFileLength = parityStripeLength * fileStripes;
-
-    long parityBlocksFound = 0L;
-
-    for (BlockLocation cb: containerBlocks) {
-      if (cb.getLength() % blockSize != 0) {
-        throw new IOException("container block size is not " +
-                              "multiple of parity block size");
-      }
-      int blocksInContainer = (int) (cb.getLength() / blockSize);
-      LOG.debug("found container with offset " + cb.getOffset() +
-               ", length " + cb.getLength());
-
-      for (long offset = cb.getOffset();
-           offset < cb.getOffset() + cb.getLength();
-           offset += blockSize) {
-        long block = offset / blockSize;
-
-        int stripe = (int) (offset / parityStripeLength);
-
-        if (stripe < 0) {
-          // before the beginning of the parity file
-          continue;
-        }
-        if (stripe >= fileStripes) {
-          // past the end of the parity file
-          break;
-        }
-
-        parityBlocksFound++;
-
-        if (cb.isCorrupt() ||
-            (cb.getNames().length == 0 && cb.getLength() > 0)) {
-          LOG.debug("parity file for " + filePath.toString() +
-                   " corrupt in block " + block +
-                   ", stripe " + stripe + "/" + fileStripes);
-
-          if (corruptBlocksPerStripe.get(stripe) == null) {
-            corruptBlocksPerStripe.put(stripe, 1);
-          } else {
-            corruptBlocksPerStripe.put(stripe,
-                                       corruptBlocksPerStripe.get(stripe) +
-                                       1);
-          }
-        } else {
-          LOG.debug("parity file for " + filePath.toString() +
-                   " OK in block " + block +
-                   ", stripe " + stripe + "/" + fileStripes);
-        }
-      }
-    }
-
-    long parityBlocksExpected = raidInfo.parityBlocksPerStripe * fileStripes;
-    if (parityBlocksFound != parityBlocksExpected ) {
-      throw new IOException("expected " + parityBlocksExpected +
-                            " parity blocks but got " + parityBlocksFound);
-    }
-  }
-
-
-  /**
-   * checks the raided file system, prints a list of corrupt files to
-   * System.out and returns the number of corrupt files
-   */
-  public int fsck(final String path) throws IOException {
-
-    FileSystem fs = (new Path(path)).getFileSystem(conf);
-
-    // if we got a raid fs, get the underlying fs
-    if (fs instanceof DistributedRaidFileSystem) {
-      fs = ((DistributedRaidFileSystem) fs).getFileSystem();
-    }
-
-    // check that we have a distributed fs
-    if (!(fs instanceof DistributedFileSystem)) {
-      throw new IOException("expected DistributedFileSystem but got " +
-                fs.getClass().getName());
-    }
-    final DistributedFileSystem dfs = (DistributedFileSystem) fs;
-
-    // get conf settings
-    String xorPrefix = RaidNode.xorDestinationPath(conf).toUri().getPath();
-    String rsPrefix = RaidNode.rsDestinationPath(conf).toUri().getPath();
-    if (!xorPrefix.endsWith("/")) {
-      xorPrefix = xorPrefix + "/";
-    }
-    if (!rsPrefix.endsWith("/")) {
-      rsPrefix = rsPrefix + "/";
-    }
-    LOG.debug("prefixes: " + xorPrefix + ", " + rsPrefix);
-
-    // get a list of corrupted files (not considering parity blocks just yet)
-    // from the name node
-    // these are the only files we need to consider:
-    // if a file has no corrupted data blocks, it is OK even if some
-    // of its parity blocks are corrupted, so no further checking is
-    // necessary
-    final String[] files = RaidDFSUtil.getCorruptFiles(dfs);
-    final List<Path> corruptFileCandidates = new LinkedList<Path>();
-    for (final String f: files) {
-      final Path p = new Path(f);
-      // if this file is a parity file
-      // or if it does not start with the specified path,
-      // ignore it
-      if (!p.toString().startsWith(xorPrefix) &&
-          !p.toString().startsWith(rsPrefix) &&
-          p.toString().startsWith(path)) {
-        corruptFileCandidates.add(p);
-      }
-    }
-    // filter files marked for deletion
-    RaidUtils.filterTrash(conf, corruptFileCandidates);
-
-    int numberOfCorruptFiles = 0;
-
-    for (final Path corruptFileCandidate: corruptFileCandidates) {
-      if (isFileCorrupt(dfs, corruptFileCandidate)) {
-        System.out.println(corruptFileCandidate.toString());
-        numberOfCorruptFiles++;
-      }
-    }
-
-    return numberOfCorruptFiles;
-  }
-
-  /**
-   * main() has some simple utility methods
-   */
-  public static void main(String argv[]) throws Exception {
-    RaidShell shell = null;
-    try {
-      shell = new RaidShell(new Configuration());
-      int res = ToolRunner.run(shell, argv);
-      System.exit(res);
-    } catch (RPC.VersionMismatch v) {
-      System.err.println("Version Mismatch between client and server" +
-                         "... command aborted.");
-      System.exit(-1);
-    } catch (IOException e) {
-      System.err.
-        println("Bad connection to RaidNode or NameNode. command aborted.");
-      System.err.println(e.getMessage());
-      System.exit(-1);
-    } finally {
-      shell.close();
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/RaidUtils.java

@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-import java.util.regex.Pattern;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PositionedReadable;
-import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.Progressable;
-
-public class RaidUtils {
-  /**
-   * A {@link Progressable} that does nothing.
-   *
-   * We could have used Reporter.NULL here but that would introduce
-   * a dependency on mapreduce.
-   */
-  public static class DummyProgressable implements Progressable {
-    /**
-     * Do nothing.
-     */
-    @Override
-    public void progress() {
-    }
-  }
-
-  /**
-   * Removes files matching the trash file pattern.
-   */
-  public static void filterTrash(Configuration conf, List<Path> files) {
-    // Remove files under Trash.
-    String trashPattern = conf.get("raid.blockfixer.trash.pattern",
-                                   "^/user/.*/\\.Trash.*");
-    for (Iterator<Path> it = files.iterator(); it.hasNext(); ) {
-      String pathStr = it.next().toString();
-      if (Pattern.matches(trashPattern, pathStr)) {
-        it.remove();
-      }
-    }
-  }
-
-  public static void readTillEnd(InputStream in, byte[] buf, boolean eofOK)
-    throws IOException {
-    int toRead = buf.length;
-    int numRead = 0;
-    while (numRead < toRead) {
-      int nread = in.read(buf, numRead, toRead - numRead);
-      if (nread < 0) {
-        if (eofOK) {
-          // EOF hit, fill with zeros
-          Arrays.fill(buf, numRead, toRead, (byte)0);
-          numRead = toRead;
-        } else {
-          // EOF hit, throw.
-          throw new IOException("Premature EOF");
-        }
-      } else {
-        numRead += nread;
-      }
-    }
-  }
-
-  public static void copyBytes(
-    InputStream in, OutputStream out, byte[] buf, long count)
-    throws IOException {
-    for (long bytesRead = 0; bytesRead < count; ) {
-      int toRead = Math.min(buf.length, (int)(count - bytesRead));
-      IOUtils.readFully(in, buf, 0, toRead);
-      bytesRead += toRead;
-      out.write(buf, 0, toRead);
-    }
-  }
-
-  public static class ZeroInputStream extends InputStream
-	    implements Seekable, PositionedReadable {
-    private long endOffset;
-    private long pos;
-
-    public ZeroInputStream(long endOffset) {
-      this.endOffset = endOffset;
-      this.pos = 0;
-    }
-
-    @Override
-    public int read() throws IOException {
-      if (pos < endOffset) {
-        pos++;
-        return 0;
-      }
-      return -1;
-    }
-
-    @Override
-    public int available() throws IOException {
-      return (int)(endOffset - pos);
-    }
-
-    @Override
-    public long getPos() throws IOException {
-      return pos;
-    }
-
-    @Override
-    public void seek(long seekOffset) throws IOException {
-      if (seekOffset < endOffset) {
-        pos = seekOffset;
-      } else {
-        throw new IOException("Illegal Offset" + pos);
-      }
-    }
-
-    @Override
-    public boolean seekToNewSource(long targetPos) throws IOException {
-      return false;
-    }
-
-    @Override
-    public int read(long position, byte[] buffer, int offset, int length)
-        throws IOException {
-      int count = 0;
-      for (; position < endOffset && count < length; position++) {
-        buffer[offset + count] = 0;
-        count++;
-      }
-      return count;
-    }
-
-    @Override
-    public void readFully(long position, byte[] buffer, int offset, int length)
-        throws IOException {
-      int count = 0;
-      for (; position < endOffset && count < length; position++) {
-        buffer[offset + count] = 0;
-        count++;
-      }
-      if (count < length) {
-        throw new IOException("Premature EOF");
-      }
-    }
-
-    @Override
-    public void readFully(long position, byte[] buffer) throws IOException {
-      readFully(position, buffer, 0, buffer.length);
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonCode.java

@@ -1,183 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-import java.util.Set;
-
-
-public class ReedSolomonCode implements ErasureCode {
-
-  private final int stripeSize;
-  private final int paritySize;
-  private final int[] generatingPolynomial;
-  private final int PRIMITIVE_ROOT = 2;
-  private final int[] primitivePower;
-  private final GaloisField GF = GaloisField.getInstance();
-  private int[] errSignature;
-  private final int[] paritySymbolLocations;
-  private final int[] dataBuff;
-
-  public ReedSolomonCode(int stripeSize, int paritySize) {
-    assert(stripeSize + paritySize < GF.getFieldSize());
-    this.stripeSize = stripeSize;
-    this.paritySize = paritySize;
-    this.errSignature = new int[paritySize];
-    this.paritySymbolLocations = new int[paritySize];
-    this.dataBuff = new int[paritySize + stripeSize];
-    for (int i = 0; i < paritySize; i++) {
-      paritySymbolLocations[i] = i;
-    }
-
-    this.primitivePower = new int[stripeSize + paritySize];
-    // compute powers of the primitive root
-    for (int i = 0; i < stripeSize + paritySize; i++) {
-      primitivePower[i] = GF.power(PRIMITIVE_ROOT, i);
-    }
-    // compute generating polynomial
-    int[] gen = {1};
-    int[] poly = new int[2];
-    for (int i = 0; i < paritySize; i++) {
-      poly[0] = primitivePower[i];
-      poly[1] = 1;
-      gen = GF.multiply(gen, poly);
-    }
-    // generating polynomial has all generating roots
-    generatingPolynomial = gen;
-  }
-
-  @Override
-  public void encode(int[] message, int[] parity) {
-    assert(message.length == stripeSize && parity.length == paritySize);
-    for (int i = 0; i < paritySize; i++) {
-      dataBuff[i] = 0;
-    }
-    for (int i = 0; i < stripeSize; i++) {
-      dataBuff[i + paritySize] = message[i];
-    }
-    GF.remainder(dataBuff, generatingPolynomial);
-    for (int i = 0; i < paritySize; i++) {
-      parity[i] = dataBuff[i];
-    }
-  }
-
-  @Override
-  public void decode(int[] data, int[] erasedLocation, int[] erasedValue) {
-    if (erasedLocation.length == 0) {
-      return;
-    }
-    assert(erasedLocation.length == erasedValue.length);
-    for (int i = 0; i < erasedLocation.length; i++) {
-      data[erasedLocation[i]] = 0;
-    }
-    for (int i = 0; i < erasedLocation.length; i++) {
-      errSignature[i] = primitivePower[erasedLocation[i]];
-      erasedValue[i] = GF.substitute(data, primitivePower[i]);
-    }
-    GF.solveVandermondeSystem(errSignature, erasedValue, erasedLocation.length);
-  }
-
-  @Override
-  public int stripeSize() {
-    return this.stripeSize;
-  }
-
-  @Override
-  public int paritySize() {
-    return this.paritySize;
-  }
-
-  @Override
-  public int symbolSize() {
-    return (int) Math.round(Math.log(GF.getFieldSize()) / Math.log(2));
-  }
-
-  /**
-   * Given parity symbols followed by message symbols, return the locations of
-   * symbols that are corrupted. Can resolve up to (parity length / 2) error
-   * locations.
-   * @param data The message and parity. The parity should be placed in the
-   *             first part of the array. In each integer, the relevant portion
-   *             is present in the least significant bits of each int.
-   *             The number of elements in data is stripeSize() + paritySize().
-   *             <b>Note that data may be changed after calling this method.</b>
-   * @param errorLocations The set to put the error location results
-   * @return true If the locations can be resolved, return true.
-   */
-  public boolean computeErrorLocations(int[] data,
-      Set<Integer> errorLocations) {
-    assert(data.length == paritySize + stripeSize && errorLocations != null);
-    errorLocations.clear();
-    int maxError = paritySize / 2;
-    int[][] syndromeMatrix = new int[maxError][];
-    for (int i = 0; i < syndromeMatrix.length; ++i) {
-      syndromeMatrix[i] = new int[maxError + 1];
-    }
-    int[] syndrome = new int[paritySize];
-
-    if (computeSyndrome(data, syndrome)) {
-      // Parity check OK. No error location added.
-      return true;
-    }
-    for (int i = 0; i < maxError; ++i) {
-      for (int j = 0; j < maxError + 1; ++j) {
-        syndromeMatrix[i][j] = syndrome[i + j];
-      }
-    }
-    GF.gaussianElimination(syndromeMatrix);
-    int[] polynomial = new int[maxError + 1];
-    polynomial[0] = 1;
-    for (int i = 0; i < maxError; ++i) {
-      polynomial[i + 1] = syndromeMatrix[maxError - 1 - i][maxError];
-    }
-    for (int i = 0; i < paritySize + stripeSize; ++i) {
-      int possibleRoot = GF.divide(1, primitivePower[i]);
-      if (GF.substitute(polynomial, possibleRoot) == 0) {
-        errorLocations.add(i);
-      }
-    }
-    // Now recover with error locations and check the syndrome again
-    int[] locations = new int[errorLocations.size()];
-    int k = 0;
-    for (int loc : errorLocations) {
-      locations[k++] = loc;
-    }
-    int [] erasedValue = new int[locations.length];
-    decode(data, locations, erasedValue);
-    for (int i = 0; i < locations.length; ++i) {
-      data[locations[i]] = erasedValue[i];
-    }
-    return computeSyndrome(data, syndrome);
-  }
-
-  /**
-   * Compute the syndrome of the input [parity, message]
-   * @param data [parity, message]
-   * @param syndrome The syndromes (checksums) of the data
-   * @return true If syndromes are all zeros
-   */
-  private boolean computeSyndrome(int[] data, int [] syndrome) {
-    boolean corruptionFound = false;
-    for (int i = 0; i < paritySize; i++) {
-      syndrome[i] = GF.substitute(data, primitivePower[i]);
-      if (syndrome[i] != 0) {
-        corruptionFound = true;
-      }
-    }
-    return !corruptionFound;
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonDecoder.java

@@ -1,226 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.OutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.ChecksumException;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.BlockMissingException;
-
-public class ReedSolomonDecoder extends Decoder {
-  public static final Log LOG = LogFactory.getLog(
-                                  "org.apache.hadoop.raid.ReedSolomonDecoder");
-  private ErasureCode reedSolomonCode;
-
-  public ReedSolomonDecoder(
-    Configuration conf, int stripeSize, int paritySize) {
-    super(conf, stripeSize, paritySize);
-    this.reedSolomonCode = new ReedSolomonCode(stripeSize, paritySize);
-  }
-
-  @Override
-  protected void fixErasedBlock(
-      FileSystem fs, Path srcFile,
-      FileSystem parityFs, Path parityFile,
-      long blockSize, long errorOffset, long bytesToSkip, long limit,
-      OutputStream out) throws IOException {
-    FSDataInputStream[] inputs = new FSDataInputStream[stripeSize + paritySize];
-    int[] erasedLocations = buildInputs(fs, srcFile, parityFs, parityFile,
-                                        errorOffset, inputs);
-    int blockIdxInStripe = ((int)(errorOffset/blockSize)) % stripeSize;
-    int erasedLocationToFix = paritySize + blockIdxInStripe;
-    writeFixedBlock(inputs, erasedLocations, erasedLocationToFix,
-                    bytesToSkip, limit, out);
-  }
-
-  protected int[] buildInputs(FileSystem fs, Path srcFile,
-                              FileSystem parityFs, Path parityFile,
-                              long errorOffset, FSDataInputStream[] inputs)
-      throws IOException {
-    LOG.info("Building inputs to recover block starting at " + errorOffset);
-    FileStatus srcStat = fs.getFileStatus(srcFile);
-    long blockSize = srcStat.getBlockSize();
-    long blockIdx = (int)(errorOffset / blockSize);
-    long stripeIdx = blockIdx / stripeSize;
-    LOG.info("FileSize = " + srcStat.getLen() + ", blockSize = " + blockSize +
-             ", blockIdx = " + blockIdx + ", stripeIdx = " + stripeIdx);
-    ArrayList<Integer> erasedLocations = new ArrayList<Integer>();
-    // First open streams to the parity blocks.
-    for (int i = 0; i < paritySize; i++) {
-      long offset = blockSize * (stripeIdx * paritySize + i);
-      FSDataInputStream in = parityFs.open(
-        parityFile, conf.getInt("io.file.buffer.size", 64 * 1024));
-      in.seek(offset);
-      LOG.info("Adding " + parityFile + ":" + offset + " as input " + i);
-      inputs[i] = in;
-    }
-    // Now open streams to the data blocks.
-    for (int i = paritySize; i < paritySize + stripeSize; i++) {
-      long offset = blockSize * (stripeIdx * stripeSize + i - paritySize);
-      if (offset == errorOffset) {
-        LOG.info(srcFile + ":" + offset +
-            " is known to have error, adding zeros as input " + i);
-        inputs[i] = new FSDataInputStream(new RaidUtils.ZeroInputStream(
-            offset + blockSize));
-        erasedLocations.add(i);
-      } else if (offset > srcStat.getLen()) {
-        LOG.info(srcFile + ":" + offset +
-                 " is past file size, adding zeros as input " + i);
-        inputs[i] = new FSDataInputStream(new RaidUtils.ZeroInputStream(
-            offset + blockSize));
-      } else {
-        FSDataInputStream in = fs.open(
-          srcFile, conf.getInt("io.file.buffer.size", 64 * 1024));
-        in.seek(offset);
-        LOG.info("Adding " + srcFile + ":" + offset + " as input " + i);
-        inputs[i] = in;
-      }
-    }
-    if (erasedLocations.size() > paritySize) {
-      String msg = "Too many erased locations: " + erasedLocations.size();
-      LOG.error(msg);
-      throw new IOException(msg);
-    }
-    int[] locs = new int[erasedLocations.size()];
-    for (int i = 0; i < locs.length; i++) {
-      locs[i] = erasedLocations.get(i);
-    }
-    return locs;
-  }
-
-  /**
-   * Decode the inputs provided and write to the output.
-   * @param inputs array of inputs.
-   * @param erasedLocations indexes in the inputs which are known to be erased.
-   * @param erasedLocationToFix index in the inputs which needs to be fixed.
-   * @param skipBytes number of bytes to skip before writing to output.
-   * @param limit maximum number of bytes to be written/skipped.
-   * @param out the output.
-   * @throws IOException
-   */
-  void writeFixedBlock(
-          FSDataInputStream[] inputs,
-          int[] erasedLocations,
-          int erasedLocationToFix,
-          long skipBytes,
-          long limit,
-          OutputStream out) throws IOException {
-
-    LOG.info("Need to write " + (limit - skipBytes) +
-             " bytes for erased location index " + erasedLocationToFix);
-    int[] tmp = new int[inputs.length];
-    int[] decoded = new int[erasedLocations.length];
-    long toDiscard = skipBytes;
-    // Loop while the number of skipped + written bytes is less than the max.
-    for (long written = 0; skipBytes + written < limit; ) {
-      erasedLocations = readFromInputs(inputs, erasedLocations, limit);
-      if (decoded.length != erasedLocations.length) {
-        decoded = new int[erasedLocations.length];
-      }
-
-      int toWrite = (int)Math.min((long)bufSize, limit - (skipBytes + written));
-      if (toDiscard >= toWrite) {
-        toDiscard -= toWrite;
-        continue;
-      }
-
-      // Decoded bufSize amount of data.
-      for (int i = 0; i < bufSize; i++) {
-        performDecode(readBufs, writeBufs, i, tmp, erasedLocations, decoded);
-      }
-
-      for (int i = 0; i < erasedLocations.length; i++) {
-        if (erasedLocations[i] == erasedLocationToFix) {
-          toWrite -= toDiscard;
-          out.write(writeBufs[i], (int)toDiscard, toWrite);
-          toDiscard = 0;
-          written += toWrite;
-          LOG.debug("Wrote " + toWrite + " bytes for erased location index " +
-                    erasedLocationToFix);
-          break;
-        }
-      }
-    }
-  }
-
-  int[] readFromInputs(
-          FSDataInputStream[] inputs,
-          int[] erasedLocations,
-          long limit) throws IOException {
-    // For every input, read some data = bufSize.
-    for (int i = 0; i < inputs.length; i++) {
-      long curPos = inputs[i].getPos();
-      try {
-        RaidUtils.readTillEnd(inputs[i], readBufs[i], true);
-        continue;
-      } catch (BlockMissingException e) {
-        LOG.error("Encountered BlockMissingException in stream " + i);
-      } catch (ChecksumException e) {
-        LOG.error("Encountered ChecksumException in stream " + i);
-      }
-
-      // Found a new erased location.
-      if (erasedLocations.length == paritySize) {
-        String msg = "Too many read errors";
-        LOG.error(msg);
-        throw new IOException(msg);
-      }
-
-      // Add this stream to the set of erased locations.
-      int[] newErasedLocations = new int[erasedLocations.length + 1];
-      for (int j = 0; j < erasedLocations.length; j++) {
-        newErasedLocations[j] = erasedLocations[j];
-      }
-      newErasedLocations[newErasedLocations.length - 1] = i;
-      erasedLocations = newErasedLocations;
-
-      LOG.info("Using zeros for stream " + i);
-      inputs[i] = new FSDataInputStream(
-        new RaidUtils.ZeroInputStream(curPos + limit));
-      inputs[i].seek(curPos);
-      RaidUtils.readTillEnd(inputs[i], readBufs[i], true);
-    }
-    return erasedLocations;
-  }
-
-  void performDecode(byte[][] readBufs, byte[][] writeBufs,
-                     int idx, int[] inputs,
-                     int[] erasedLocations, int[] decoded) {
-    for (int i = 0; i < decoded.length; i++) {
-      decoded[i] = 0;
-    }
-    for (int i = 0; i < inputs.length; i++) {
-      inputs[i] = readBufs[i][idx] & 0x000000FF;
-    }
-    reedSolomonCode.decode(inputs, erasedLocations, decoded);
-    for (int i = 0; i < decoded.length; i++) {
-      writeBufs[i][idx] = (byte)decoded[i];
-    }
-  }
-
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/ReedSolomonEncoder.java

@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.Progressable;
-
-public class ReedSolomonEncoder extends Encoder {
-  public static final Log LOG = LogFactory.getLog(
-                                  "org.apache.hadoop.raid.ReedSolomonEncoder");
-  private ErasureCode reedSolomonCode;
-
-  public ReedSolomonEncoder(
-    Configuration conf, int stripeSize, int paritySize) {
-    super(conf, stripeSize, paritySize);
-    this.reedSolomonCode = new ReedSolomonCode(stripeSize, paritySize);
-  }
-
-  protected void encodeStripe(
-    InputStream[] blocks,
-    long stripeStartOffset,
-    long blockSize,
-    OutputStream[] outs,
-    Progressable reporter) throws IOException {
-
-    int[] data = new int[stripeSize];
-    int[] code = new int[paritySize];
-
-    for (long encoded = 0; encoded < blockSize; encoded += bufSize) {
-      // Read some data from each block = bufSize.
-      for (int i = 0; i < blocks.length; i++) {
-        RaidUtils.readTillEnd(blocks[i], readBufs[i], true);
-      }
-
-      // Encode the data read.
-      for (int j = 0; j < bufSize; j++) {
-        performEncode(readBufs, writeBufs, j, data, code);
-      }
-
-      // Now that we have some data to write, send it to the temp files.
-      for (int i = 0; i < paritySize; i++) {
-        outs[i].write(writeBufs[i], 0, bufSize);
-      }
-
-      if (reporter != null) {
-        reporter.progress();
-      }
-    }
-  }
-
-  void performEncode(byte[][] readBufs, byte[][] writeBufs, int idx,
-                          int[] data, int[] code) {
-    for (int i = 0; i < paritySize; i++) {
-      code[i] = 0;
-    }
-    for (int i = 0; i < stripeSize; i++) {
-      data[i] = readBufs[i][idx] & 0x000000FF;
-    }
-    reedSolomonCode.encode(data, code);
-    for (int i = 0; i < paritySize; i++) {
-      writeBufs[i][idx] = (byte)code[i];
-    }
-  }
-
-  @Override
-  public Path getParityTempPath() {
-    return new Path(RaidNode.rsTempPrefix(conf));
-  }
-
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/XORDecoder.java

@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.OutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.Path;
-
-public class XORDecoder extends Decoder {
-  public static final Log LOG = LogFactory.getLog(
-                                  "org.apache.hadoop.raid.XORDecoder");
-
-  public XORDecoder(
-    Configuration conf, int stripeSize) {
-    super(conf, stripeSize, 1);
-  }
-
-  @Override
-  protected void fixErasedBlock(
-      FileSystem fs, Path srcFile, FileSystem parityFs, Path parityFile,
-      long blockSize, long errorOffset, long bytesToSkip, long limit,
-      OutputStream out) throws IOException {
-    LOG.info("Fixing block at " + srcFile + ":" + errorOffset +
-             ", skipping " + bytesToSkip + ", limit " + limit);
-    FileStatus srcStat = fs.getFileStatus(srcFile);
-    ArrayList<FSDataInputStream> xorinputs = new ArrayList<FSDataInputStream>();
-
-    FSDataInputStream parityFileIn = parityFs.open(parityFile);
-    parityFileIn.seek(parityOffset(errorOffset, blockSize));
-    xorinputs.add(parityFileIn);
-
-    long errorBlockOffset = (errorOffset / blockSize) * blockSize;
-    long[] srcOffsets = stripeOffsets(errorOffset, blockSize);
-    for (int i = 0; i < srcOffsets.length; i++) {
-      if (srcOffsets[i] == errorBlockOffset) {
-        LOG.info("Skipping block at " + srcFile + ":" + errorBlockOffset);
-        continue;
-      }
-      if (srcOffsets[i] < srcStat.getLen()) {
-        FSDataInputStream in = fs.open(srcFile);
-        in.seek(srcOffsets[i]);
-        xorinputs.add(in);
-      }
-    }
-    FSDataInputStream[] inputs = xorinputs.toArray(
-                                    new FSDataInputStream[]{null});
-    ParityInputStream recovered =
-      new ParityInputStream(inputs, limit, readBufs[0], writeBufs[0]);
-    recovered.skip(bytesToSkip);
-    recovered.drain(out, null);
-  }
-
-  protected long[] stripeOffsets(long errorOffset, long blockSize) {
-    long[] offsets = new long[stripeSize];
-    long stripeIdx = errorOffset / (blockSize * stripeSize);
-    long startOffsetOfStripe = stripeIdx * stripeSize * blockSize;
-    for (int i = 0; i < stripeSize; i++) {
-      offsets[i] = startOffsetOfStripe + i * blockSize;
-    }
-    return offsets;
-  }
-
-  protected long parityOffset(long errorOffset, long blockSize) {
-    long stripeIdx = errorOffset / (blockSize * stripeSize);
-    return stripeIdx * blockSize;
-  }
-
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/XOREncoder.java

@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import java.io.OutputStream;
-import java.io.InputStream;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.Progressable;
-
-public class XOREncoder extends Encoder {
-  public static final Log LOG = LogFactory.getLog(
-                                  "org.apache.hadoop.raid.XOREncoder");
-  public XOREncoder(
-    Configuration conf, int stripeSize) {
-    super(conf, stripeSize, 1);
-  }
-
-  @Override
-  protected void encodeStripe(
-    InputStream[] blocks,
-    long stripeStartOffset,
-    long blockSize,
-    OutputStream[] outs,
-    Progressable reporter) throws IOException {
-    LOG.info("Peforming XOR ");
-    ParityInputStream parityIn =
-      new ParityInputStream(blocks, blockSize, readBufs[0], writeBufs[0]);
-    try {
-      parityIn.drain(outs[0], reporter);
-    } finally {
-      parityIn.close();
-    }
-  }
-
-  @Override
-  public Path getParityTempPath() {
-    return new Path(RaidNode.unraidTmpDirectory(conf));
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/PolicyInfo.java

@@ -1,256 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid.protocol;
-
-import java.io.IOException;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.util.Properties;
-import java.util.Enumeration;
-import java.lang.Math;
-import java.text.SimpleDateFormat;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileStatus;
-
-/**
- * Maintains information about one policy
- */
-public class PolicyInfo implements Writable {
-  public static final Log LOG = LogFactory.getLog(
-    "org.apache.hadoop.raid.protocol.PolicyInfo");
-  protected static final SimpleDateFormat dateFormat =
-    new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-
-  private Path srcPath;            // the specified src path
-  private String policyName;       // name of policy
-  private ErasureCodeType codeType;// the erasure code used
-  private String description;      // A verbose description of this policy
-  private Configuration conf;      // Hadoop configuration
-
-  private Properties properties;   // Policy-dependent properties
-
-  private ReentrantReadWriteLock plock; // protects policy operations.
-  public static enum ErasureCodeType {
-    XOR, RS;
-    public static ErasureCodeType fromString(String s) {
-      if (XOR.toString().equalsIgnoreCase(s)) {
-        return XOR;
-      }
-      if (RS.toString().equalsIgnoreCase(s)) {
-        return RS;
-      }
-      return null;
-    }
-  }
-
-  /**
-   * Create the empty object
-   */
-  public PolicyInfo() {
-    this.conf = null;
-    this.policyName = "";
-    this.description = "";
-    this.srcPath = null;
-    this.properties = new Properties();
-    this.plock = new ReentrantReadWriteLock();
-  }
-
-  /**
-   * Create the metadata that describes a policy
-   */
-  public PolicyInfo(String  policyName, Configuration conf) {
-    this.conf = conf;
-    this.policyName = policyName;
-    this.description = "";
-    this.srcPath = null;
-    this.properties = new Properties();
-    this.plock = new ReentrantReadWriteLock();
-  }
-
-  /**
-   * Copy fields from another PolicyInfo
-   */
-  public void copyFrom(PolicyInfo other) {
-    if (other.conf != null) {
-      this.conf = other.conf;
-    }
-    if (other.policyName != null && other.policyName.length() > 0) {
-      this.policyName = other.policyName;
-    }
-    if (other.description != null && other.description.length() > 0) {
-      this.description = other.description;
-    }
-    if (other.codeType != null) {
-      this.codeType = other.codeType;
-    }
-    if (other.srcPath != null) {
-      this.srcPath = other.srcPath;
-    }
-    for (Object key : other.properties.keySet()) {
-      String skey = (String) key;
-      this.properties.setProperty(skey, other.properties.getProperty(skey));
-    }
-  }
-
-  /**
-   * Sets the input path on which this policy has to be applied
-   */
-  public void setSrcPath(String in) throws IOException {
-    srcPath = new Path(in);
-    srcPath = srcPath.makeQualified(srcPath.getFileSystem(conf));
-  }
-
-  /**
-   * Set the erasure code type used in this policy
-   */
-  public void setErasureCode(String code) {
-    this.codeType = ErasureCodeType.fromString(code);
-  }
-
-  /**
-   * Set the description of this policy.
-   */
-  public void setDescription(String des) {
-    this.description = des;
-  }
-
-  /**
-   * Sets an internal property.
-   * @param name property name.
-   * @param value property value.
-   */
-  public void setProperty(String name, String value) {
-    properties.setProperty(name, value);
-  }
-  
-  /**
-   * Returns the value of an internal property.
-   * @param name property name.
-   */
-  public String getProperty(String name) {
-    return properties.getProperty(name);
-  }
-  
-  /**
-   * Get the name of this policy.
-   */
-  public String getName() {
-    return this.policyName;
-  }
-
-  /**
-   * Get the destination path of this policy.
-   */
-  public ErasureCodeType getErasureCode() {
-    return this.codeType;
-  }
-
-  /**
-   * Get the srcPath
-   */
-  public Path getSrcPath() {
-    return srcPath;
-  }
-
-  /**
-   * Get the expanded (unglobbed) forms of the srcPaths
-   */
-  public Path[] getSrcPathExpanded() throws IOException {
-    FileSystem fs = srcPath.getFileSystem(conf);
-
-    // globbing on srcPath
-    FileStatus[] gpaths = fs.globStatus(srcPath);
-    if (gpaths == null) {
-      return null;
-    }
-    Path[] values = new Path[gpaths.length];
-    for (int i = 0; i < gpaths.length; i++) {
-      Path p = gpaths[i].getPath();
-      values[i] = p.makeQualified(fs);
-    }
-    return values;
-  }
-
-  /**
-   * Convert this policy into a printable form
-   */
-  public String toString() {
-    StringBuffer buff = new StringBuffer();
-    buff.append("Policy Name:\t" + policyName + " --------------------\n");
-    buff.append("Source Path:\t" + srcPath + "\n");
-    buff.append("Erasure Code:\t" + codeType + "\n");
-    for (Enumeration<?> e = properties.propertyNames(); e.hasMoreElements();) {
-      String name = (String) e.nextElement(); 
-      buff.append( name + ":\t" + properties.getProperty(name) + "\n");
-    }
-    if (description.length() > 0) {
-      int len = Math.min(description.length(), 80);
-      String sub = description.substring(0, len).trim();
-      sub = sub.replaceAll("\n", " ");
-      buff.append("Description:\t" + sub + "...\n");
-    }
-    return buff.toString();
-  }
-
-  //////////////////////////////////////////////////
-  // Writable
-  //////////////////////////////////////////////////
-  static {                                      // register a ctor
-    WritableFactories.setFactory
-      (PolicyInfo.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new PolicyInfo(); }
-       });
-  }
-
-  public void write(DataOutput out) throws IOException {
-    Text.writeString(out, srcPath.toString());
-    Text.writeString(out, policyName);
-    Text.writeString(out, codeType.toString());
-    Text.writeString(out, description);
-    out.writeInt(properties.size());
-    for (Enumeration<?> e = properties.propertyNames(); e.hasMoreElements();) {
-      String name = (String) e.nextElement(); 
-      Text.writeString(out, name);
-      Text.writeString(out, properties.getProperty(name));
-    }
-  }
-
-  public void readFields(DataInput in) throws IOException {
-    this.srcPath = new Path(Text.readString(in));
-    this.policyName = Text.readString(in);
-    this.codeType = ErasureCodeType.fromString(Text.readString(in));
-    this.description = Text.readString(in);
-    for (int n = in.readInt(); n>0; n--) {
-      String name = Text.readString(in);
-      String value = Text.readString(in);
-      properties.setProperty(name,value);
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/PolicyList.java

@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid.protocol;
-
-import java.io.IOException;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.util.Collection;
-import java.util.List;
-import java.util.LinkedList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Maintains informatiom about all policies that belong to a category.
- * These policies have to be applied one-at-a-time and cannot be run
- * simultaneously.
- */
-public class PolicyList implements Writable {
-  public static final Log LOG = LogFactory.getLog(
-    "org.apache.hadoop.raid.protocol.PolicyList");
-
-  private List<PolicyInfo> category; // list of policies
-  private Path srcPath;
-  
-  /**
-   * Create a new category of policies.
-   */
-  public PolicyList() {
-    this.category = new LinkedList<PolicyInfo>();
-    this.srcPath = null;
-  }
-
-  /**
-   * Add a new policy to this category.
-   */
-  public void add(PolicyInfo info) {
-    category.add(info); 
-  }
-
-  public void setSrcPath(Configuration conf, String src) throws IOException {
-    srcPath = new Path(src);
-    srcPath = srcPath.makeQualified(srcPath.getFileSystem(conf));
-  }
-  
-  public Path getSrcPath() {
-    return srcPath;
-  }
-  
-  /**
-   * Returns the policies in this category
-   */
-  public Collection<PolicyInfo> getAll() {
-    return category;
-  }
-
-  //////////////////////////////////////////////////
-  // Writable
-  //////////////////////////////////////////////////
-  static {                                      // register a ctor
-    WritableFactories.setFactory
-      (PolicyList.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new PolicyList(); }
-       });
-  }
-
-  public void write(DataOutput out) throws IOException {
-    out.writeInt(category.size());
-    for (PolicyInfo p : category) {
-      p.write(out);
-    }
-  }
-
-  public void readFields(DataInput in) throws IOException {
-    int count = in.readInt();
-    for (int i = 0; i < count; i++) {
-      PolicyInfo p = new PolicyInfo();
-      p.readFields(in);
-      add(p);
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/java/org/apache/hadoop/raid/protocol/RaidProtocol.java

@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid.protocol;
-
-import java.util.Collection;
-import java.io.IOException;
-
-import org.apache.hadoop.ipc.VersionedProtocol;
-import org.apache.hadoop.fs.Path;
-
-/**********************************************************************
- * RaidProtocol is used by user code 
- * {@link org.apache.hadoop.raid.RaidShell} class to communicate 
- * with the RaidNode.  User code can manipulate the configured policies.
- *
- **********************************************************************/
-public interface RaidProtocol extends VersionedProtocol {
-
-  /**
-   * Compared to the previous version the following changes have been introduced:
-   * Only the latest change is reflected.
-   * 1: new protocol introduced
-   */
-  public static final long versionID = 1L;
-
-  /**
-   * Get a listing of all configured policies
-   * @throws IOException
-   * return all categories of configured policies
-   */
-  public PolicyList[] getAllPolicies() throws IOException;
-
-  /**
-   * Unraid the specified input path. This is called when the specified file
-   * is corrupted. This call will move the specified file to file.old
-   * and then recover it from the RAID subsystem.
-   *
-   * @param inputPath The absolute pathname of the file to be recovered.
-   * @param corruptOffset The offset that has the corruption
-   */
-  public String recoverFile(String inputPath, long corruptOffset) throws IOException;
-
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/start-raidnode-remote.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Start hadoop RaidNode process on machine specified on file conf/raidnode
-
-usage="Usage: start-raidnode-remote.sh"
-
-params=$#
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-
-# get arguments
-if [ $# -ge 1 ]; then
-    echo $usage
-    exit
-fi
-
-if [ -f "${HADOOP_CONF_DIR}/raidnode" ]; then
-  export HADOOP_SLAVES="${HADOOP_CONF_DIR}/raidnode"
-  echo "Starting raidnode at "`cat ${HADOOP_SLAVES}`
-  "$bin"/slaves.sh --config $HADOOP_CONF_DIR cd "$HADOOP_PREFIX" \; "$bin/start-raidnode.sh"
-else
-  echo "No raidnode file in ${HADOOP_CONF_DIR}/raidnode"
-fi
-
-

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/start-raidnode.sh

@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Start hadoop RaidNode process
-# Run this on RaidNode machine
-
-usage="Usage: start-raidnode.sh"
-
-params=$#
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-
-# get arguments
-if [ $# -ge 1 ]; then
-    echo $usage
-fi
-
-if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then
-  . "${HADOOP_CONF_DIR}/hadoop-env.sh"
-fi
-export HADOOP_OPTS="$HADOOP_OPTS $HADOOP_RAIDNODE_OPTS"
-
-"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR start org.apache.hadoop.raid.RaidNode

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/stop-raidnode-remote.sh

@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Stop hadoop RaidNode process on machine specified on file conf/raidnode
-
-usage="Usage: stop-raidnode-remote.sh"
-
-params=$#
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-
-# get arguments
-if [ $# -ge 1 ]; then
-    echo $usage
-fi
-
-if [ -f "${HADOOP_CONF_DIR}/raidnode" ]; then
-  export HADOOP_SLAVES="${HADOOP_CONF_DIR}/raidnode"
-  echo "Stopping raidnode at "`cat ${HADOOP_SLAVES}`
-  "$bin"/slaves.sh --config $HADOOP_CONF_DIR cd "$HADOOP_PREFIX" \; "$bin/stop-raidnode.sh"
-else
-  echo "No raidnode file in ${HADOOP_CONF_DIR}/raidnode"
-fi

hadoop-hdfs-project/hadoop-hdfs-raid/src/main/sbin/stop-raidnode.sh

@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Stop hadoop RaidNode process
-# Run this on RaidNode machine.
-
-usage="Usage: stop-raidnode.sh"
-
-params=$#
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-
-# get arguments
-if [ $# -ge 1 ]; then
-    echo $usage
-fi
-
-export HADOOP_OPTS="$HADOOP_OPTS $HADOOP_RAIDNODE_OPTS"
-
-"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR stop org.apache.hadoop.raid.RaidNode

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/TestRaidDfs.java

@@ -1,501 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.RandomAccessFile;
-import java.net.URI;
-import java.util.Random;
-import java.util.regex.Pattern;
-import java.util.zip.CRC32;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.raid.RaidNode;
-import org.apache.hadoop.raid.RaidUtils;
-import org.apache.hadoop.raid.protocol.PolicyInfo.ErasureCodeType;
-import org.apache.hadoop.util.StringUtils;
-import org.junit.Test;
-
-public class TestRaidDfs {
-  final static String TEST_DIR = new File(System.getProperty("test.build.data",
-      "target/test-data")).getAbsolutePath();
-  final static String LOG_DIR = "target/raidlog";
-  final static long RELOAD_INTERVAL = 1000;
-  final static Log LOG = LogFactory.getLog("org.apache.hadoop.raid.TestRaidDfs");
-  final static int NUM_DATANODES = 3;
-
-  Configuration conf;
-  String namenode = null;
-  String hftp = null;
-  MiniDFSCluster dfs = null;
-  FileSystem fileSys = null;
-  String jobTrackerName = null;
-  ErasureCodeType code;
-  int stripeLength;
-
-  private void mySetup(
-      String erasureCode, int rsParityLength) throws Exception {
-
-    new File(TEST_DIR).mkdirs(); // Make sure data directory exists
-    conf = new Configuration();
-
-    conf.set("fs.raid.recoverylogdir", LOG_DIR);
-    conf.setInt(RaidNode.RS_PARITY_LENGTH_KEY, rsParityLength);
-
-    // scan all policies once every 5 second
-    conf.setLong("raid.policy.rescan.interval", 5000);
-
-    // make all deletions not go through Trash
-    conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient");
-
-    // do not use map-reduce cluster for Raiding
-    conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode");
-
-    conf.set("raid.server.address", "localhost:0");
-    conf.setInt("hdfs.raid.stripeLength", stripeLength);
-    conf.set("xor".equals(erasureCode) ? RaidNode.RAID_LOCATION_KEY :
-             RaidNode.RAIDRS_LOCATION_KEY, "/destraid");
-
-    dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build();
-    dfs.waitActive();
-    fileSys = dfs.getFileSystem();
-    namenode = fileSys.getUri().toString();
-    hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort();
-
-    FileSystem.setDefaultUri(conf, namenode);
-  }
-
-  private void myTearDown() throws Exception {
-    if (dfs != null) { dfs.shutdown(); }
-  }
-  
-  private LocatedBlocks getBlockLocations(Path file, long length)
-    throws IOException {
-    DistributedFileSystem dfs = (DistributedFileSystem) fileSys;
-    return RaidDFSUtil.getBlockLocations(
-      dfs, file.toUri().getPath(), 0, length);
-  }
-
-  private LocatedBlocks getBlockLocations(Path file)
-    throws IOException {
-    FileStatus stat = fileSys.getFileStatus(file);
-    return getBlockLocations(file, stat.getLen());
-  }
-
-  private DistributedRaidFileSystem getRaidFS() throws IOException {
-    DistributedFileSystem dfs = (DistributedFileSystem)fileSys;
-    Configuration clientConf = new Configuration(conf);
-    clientConf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedRaidFileSystem");
-    clientConf.set("fs.raid.underlyingfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
-    clientConf.setBoolean("fs.hdfs.impl.disable.cache", true);
-    URI dfsUri = dfs.getUri();
-    return (DistributedRaidFileSystem)FileSystem.get(dfsUri, clientConf);
-  }
-
-  public static void waitForFileRaided(
-    Log logger, FileSystem fileSys, Path file, Path destPath)
-  throws IOException, InterruptedException {
-    FileStatus parityStat = null;
-    String fileName = file.getName().toString();
-    // wait till file is raided
-    while (parityStat == null) {
-      logger.info("Waiting for files to be raided.");
-      try {
-        FileStatus[] listPaths = fileSys.listStatus(destPath);
-        if (listPaths != null) {
-          for (FileStatus f : listPaths) {
-            logger.info("File raided so far : " + f.getPath());
-            String found = f.getPath().getName().toString();
-            if (fileName.equals(found)) {
-              parityStat = f;
-              break;
-            }
-          }
-        }
-      } catch (FileNotFoundException e) {
-        //ignore
-      }
-      Thread.sleep(1000);                  // keep waiting
-    }
-
-    while (true) {
-      LocatedBlocks locations = null;
-      DistributedFileSystem dfs = (DistributedFileSystem) fileSys;
-      locations = RaidDFSUtil.getBlockLocations(
-        dfs, file.toUri().getPath(), 0, parityStat.getLen());
-      if (!locations.isUnderConstruction()) {
-        break;
-      }
-      Thread.sleep(1000);
-    }
-
-    while (true) {
-      FileStatus stat = fileSys.getFileStatus(file);
-      if (stat.getReplication() == 1) break;
-      Thread.sleep(1000);
-    }
-  }
-
-  private void corruptBlockAndValidate(Path srcFile, Path destPath,
-    int[] listBlockNumToCorrupt, long blockSize, int numBlocks)
-  throws IOException, InterruptedException {
-    int repl = 1;
-    long crc = createTestFilePartialLastBlock(fileSys, srcFile, repl,
-                  numBlocks, blockSize);
-    long length = fileSys.getFileStatus(srcFile).getLen();
-
-    RaidNode.doRaid(conf, fileSys.getFileStatus(srcFile),
-      destPath, code, new RaidNode.Statistics(), new RaidUtils.DummyProgressable(),
-      false, repl, repl, stripeLength);
-
-    // Delete first block of file
-    for (int blockNumToCorrupt : listBlockNumToCorrupt) {
-      LOG.info("Corrupt block " + blockNumToCorrupt + " of file " + srcFile);
-      LocatedBlocks locations = getBlockLocations(srcFile);
-      corruptBlock(dfs, srcFile, locations.get(blockNumToCorrupt).getBlock(),
-            NUM_DATANODES, true);
-    }
-
-    // Validate
-    DistributedRaidFileSystem raidfs = getRaidFS();
-    assertTrue(validateFile(raidfs, srcFile, length, crc));
-    validateLogFile(getRaidFS(), new Path(LOG_DIR));
-  }
-
-  /**
-   * Create a file, corrupt several blocks in it and ensure that the file can be
-   * read through DistributedRaidFileSystem by ReedSolomon coding.
-   */
-  @Test
-  public void testRaidDfsRs() throws Exception {
-    LOG.info("Test testRaidDfs started.");
-
-    code = ErasureCodeType.RS;
-    long blockSize = 8192L;
-    int numBlocks = 8;
-    stripeLength = 3;
-    mySetup("rs", 3);
-
-    int[][] corrupt = {{1, 2, 3}, {1, 4, 7}, {3, 6, 7}};
-    try {
-      for (int i = 0; i < corrupt.length; i++) {
-        Path file = new Path("/user/dhruba/raidtest/file" + i);
-        corruptBlockAndValidate(
-            file, new Path("/destraid"), corrupt[i], blockSize, numBlocks);
-      }
-    } catch (Exception e) {
-      LOG.info("testRaidDfs Exception " + e +
-                StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      myTearDown();
-    }
-    LOG.info("Test testRaidDfs completed.");
-  }
-
-  /**
-   * Test DistributedRaidFileSystem.readFully()
-   */
-  @Test
-  public void testReadFully() throws Exception {
-    code = ErasureCodeType.XOR;
-    stripeLength = 3;
-    mySetup("xor", 1);
-
-    try {
-      Path file = new Path("/user/raid/raidtest/file1");
-      long crc = createTestFile(fileSys, file, 1, 8, 8192L);
-      FileStatus stat = fileSys.getFileStatus(file);
-      LOG.info("Created " + file + ", crc=" + crc + ", len=" + stat.getLen());
-
-      byte[] filebytes = new byte[(int)stat.getLen()];
-      // Test that readFully returns the correct CRC when there are no errors.
-      DistributedRaidFileSystem raidfs = getRaidFS();
-      FSDataInputStream stm = raidfs.open(file);
-      stm.readFully(0, filebytes);
-      assertEquals(crc, bufferCRC(filebytes));
-      stm.close();
-
-      // Generate parity.
-      RaidNode.doRaid(conf, fileSys.getFileStatus(file),
-        new Path("/destraid"), code, new RaidNode.Statistics(),
-        new RaidUtils.DummyProgressable(),
-        false, 1, 1, stripeLength);
-      int[] corrupt = {0, 4, 7}; // first, last and middle block
-      for (int blockIdx : corrupt) {
-        LOG.info("Corrupt block " + blockIdx + " of file " + file);
-        LocatedBlocks locations = getBlockLocations(file);
-        corruptBlock(dfs, file, locations.get(blockIdx).getBlock(),
-            NUM_DATANODES, true);
-      }
-      // Test that readFully returns the correct CRC when there are errors.
-      stm = raidfs.open(file);
-      stm.readFully(0, filebytes);
-      assertEquals(crc, bufferCRC(filebytes));
-    } finally {
-      myTearDown();
-    }
-  }
-
-  /**
-   * Test that access time and mtime of a source file do not change after
-   * raiding.
-   */
-  @Test
-  public void testAccessTime() throws Exception {
-    LOG.info("Test testAccessTime started.");
-
-    code = ErasureCodeType.XOR;
-    long blockSize = 8192L;
-    int numBlocks = 8;
-    int repl = 1;
-    stripeLength = 3;
-    mySetup("xor", 1);
-
-    Path file = new Path("/user/dhruba/raidtest/file");
-    createTestFilePartialLastBlock(fileSys, file, repl, numBlocks, blockSize);
-    FileStatus stat = fileSys.getFileStatus(file);
-
-    try {
-      RaidNode.doRaid(conf, fileSys.getFileStatus(file),
-        new Path("/destraid"), code, new RaidNode.Statistics(),
-        new RaidUtils.DummyProgressable(), false, repl, repl, stripeLength);
-
-      FileStatus newStat = fileSys.getFileStatus(file);
-
-      assertEquals(stat.getModificationTime(), newStat.getModificationTime());
-      assertEquals(stat.getAccessTime(), newStat.getAccessTime());
-    } finally {
-      myTearDown();
-    }
-  }
-
-  /**
-   * Create a file, corrupt a block in it and ensure that the file can be
-   * read through DistributedRaidFileSystem by XOR code.
-   */
-  @Test
-  public void testRaidDfsXor() throws Exception {
-    LOG.info("Test testRaidDfs started.");
-
-    code = ErasureCodeType.XOR;
-    long blockSize = 8192L;
-    int numBlocks = 8;
-    stripeLength = 3;
-    mySetup("xor", 1);
-
-    int[][] corrupt = {{0}, {4}, {7}}; // first, last and middle block
-    try {
-      for (int i = 0; i < corrupt.length; i++) {
-        Path file = new Path("/user/dhruba/raidtest/" + i);
-        corruptBlockAndValidate(
-            file, new Path("/destraid"), corrupt[i], blockSize, numBlocks);
-      }
-    } catch (Exception e) {
-      LOG.info("testRaidDfs Exception " + e +
-                StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      myTearDown();
-    }
-    LOG.info("Test testRaidDfs completed.");
-  }
-
-  //
-  // creates a file and populate it with random data. Returns its crc.
-  //
-  public static long createTestFile(FileSystem fileSys, Path name, int repl,
-                        int numBlocks, long blocksize)
-    throws IOException {
-    CRC32 crc = new CRC32();
-    Random rand = new Random();
-    FSDataOutputStream stm = fileSys.create(name, true,
-                                            fileSys.getConf().getInt("io.file.buffer.size", 4096),
-                                            (short)repl, blocksize);
-    // fill random data into file
-    final byte[] b = new byte[(int)blocksize];
-    for (int i = 0; i < numBlocks; i++) {
-      rand.nextBytes(b);
-      stm.write(b);
-      crc.update(b);
-    }
-    stm.close();
-    return crc.getValue();
-  }
-
-  //
-  // Creates a file with partially full last block. Populate it with random
-  // data. Returns its crc.
-  //
-  public static long createTestFilePartialLastBlock(
-      FileSystem fileSys, Path name, int repl, int numBlocks, long blocksize)
-    throws IOException {
-    CRC32 crc = new CRC32();
-    Random rand = new Random();
-    FSDataOutputStream stm = fileSys.create(name, true,
-                                            fileSys.getConf().getInt("io.file.buffer.size", 4096),
-                                            (short)repl, blocksize);
-    // Write whole blocks.
-    byte[] b = new byte[(int)blocksize];
-    for (int i = 1; i < numBlocks; i++) {
-      rand.nextBytes(b);
-      stm.write(b);
-      crc.update(b);
-    }
-    // Write partial block.
-    b = new byte[(int)blocksize/2 - 1];
-    rand.nextBytes(b);
-    stm.write(b);
-    crc.update(b);
-
-    stm.close();
-    return crc.getValue();
-  }
-
-  static long bufferCRC(byte[] buf) {
-    CRC32 crc = new CRC32();
-    crc.update(buf, 0, buf.length);
-    return crc.getValue();
-  }
-
-  //
-  // validates that file matches the crc.
-  //
-  public static boolean validateFile(FileSystem fileSys, Path name, long length,
-                                  long crc) 
-    throws IOException {
-
-    long numRead = 0;
-    CRC32 newcrc = new CRC32();
-    FSDataInputStream stm = fileSys.open(name);
-    final byte[] b = new byte[4192];
-    int num = 0;
-    while (num >= 0) {
-      num = stm.read(b);
-      if (num < 0) {
-        break;
-      }
-      numRead += num;
-      newcrc.update(b, 0, num);
-    }
-    stm.close();
-
-    if (numRead != length) {
-      LOG.info("Number of bytes read " + numRead +
-               " does not match file size " + length);
-      return false;
-    }
-
-    LOG.info(" Newcrc " + newcrc.getValue() + " old crc " + crc);
-    if (newcrc.getValue() != crc) {
-      LOG.info("CRC mismatch of file " + name + ": " + newcrc + " vs. " + crc);
-      return false;
-    }
-    return true;
-  }
-  //
-  // validates the contents of raid recovery log file
-  //
-  public static void validateLogFile(FileSystem fileSys, Path logDir)
-      throws IOException {
-    FileStatus f = fileSys.listStatus(logDir)[0];
-    FSDataInputStream stm = fileSys.open(f.getPath());
-    try {
-      BufferedReader reader = new BufferedReader(new InputStreamReader(stm));
-      assertEquals("Recovery attempt log", reader.readLine());
-      assertTrue(Pattern.matches("Source path : /user/dhruba/raidtest/.*",
-                 reader.readLine()));
-      assertTrue(Pattern.matches("Alternate path : .*/destraid",
-                 reader.readLine()));
-      assertEquals("Stripe lentgh : 3", reader.readLine());
-      assertTrue(Pattern.matches("Corrupt offset : \\d*", reader.readLine()));
-      assertTrue(Pattern.matches("Output from unRaid : " +
-          "hdfs://.*/tmp/raid/user/dhruba/raidtest/.*recovered",
-          reader.readLine()));
-    } finally {
-      stm.close();
-    }
-    LOG.info("Raid HDFS Recovery log verified");
-  }
-
-  //
-  // Delete/Corrupt specified block of file
-  //
-  public static void corruptBlock(MiniDFSCluster dfs, Path file, ExtendedBlock blockNum,
-                    int numDataNodes, boolean delete) throws IOException {
-    // Now deliberately remove/truncate replicas of blocks
-    int numDeleted = 0;
-    int numCorrupted = 0;
-    for (int i = 0; i < numDataNodes; i++) {
-      File block = MiniDFSCluster.getBlockFile(i, blockNum);
-      if (block == null || !block.exists()) {
-        continue;
-      }
-      if (delete) {
-        block.delete();
-        LOG.info("Deleted block " + block);
-        numDeleted++;
-      } else {
-        // Corrupt
-        long seekPos = block.length()/2;
-        RandomAccessFile raf = new RandomAccessFile(block, "rw");
-        raf.seek(seekPos);
-        int data = raf.readInt();
-        raf.seek(seekPos);
-        raf.writeInt(data+1);
-        LOG.info("Corrupted block " + block);
-        numCorrupted++;
-      }
-    }
-    assertTrue("Nothing corrupted or deleted",
-              (numCorrupted + numDeleted) > 0);
-  }
-
-  public static void corruptBlock(Path file, ExtendedBlock blockNum,
-                    int numDataNodes, long offset) throws IOException {
-    // Now deliberately corrupt replicas of the the block.
-    for (int i = 0; i < numDataNodes; i++) {
-      File block = MiniDFSCluster.getBlockFile(i, blockNum);
-      if (block == null || !block.exists()) {
-        continue;
-      }
-      RandomAccessFile raf = new RandomAccessFile(block, "rw");
-      raf.seek(offset);
-      int data = raf.readInt();
-      raf.seek(offset);
-      raf.writeInt(data+1);
-      LOG.info("Corrupted block " + block);
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockPlacementPolicyRaid.java

@@ -1,518 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.server.blockmanagement;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSTestUtil;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRaid.CachedFullPathNames;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRaid.CachedLocatedBlocks;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRaid.FileType;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.INodeFile;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeRaidTestUtil;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeRaidUtil;
-import org.apache.hadoop.net.NetworkTopology;
-import org.apache.hadoop.raid.RaidNode;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TestBlockPlacementPolicyRaid {
-  private Configuration conf = null;
-  private MiniDFSCluster cluster = null;
-  private FSNamesystem namesystem = null;
-  private BlockManager blockManager;
-  private NetworkTopology networktopology;
-  private BlockPlacementPolicyRaid policy = null;
-  private FileSystem fs = null;
-  String[] rack1 = {"/rack1"};
-  String[] rack2 = {"/rack2"};
-  String[] host1 = {"host1.rack1.com"};
-  String[] host2 = {"host2.rack2.com"};
-  String xorPrefix = null;
-  String raidTempPrefix = null;
-  String raidrsTempPrefix = null;
-  String raidrsHarTempPrefix = null;
-
-  final static Log LOG =
-      LogFactory.getLog(TestBlockPlacementPolicyRaid.class);
-
-  protected void setupCluster() throws IOException {
-    conf = new Configuration();
-    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L);
-    conf.set("dfs.replication.pending.timeout.sec", "2");
-    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1L);
-    conf.set("dfs.block.replicator.classname",
-        BlockPlacementPolicyRaid.class.getName());
-    conf.set(RaidNode.STRIPE_LENGTH_KEY, "2");
-    conf.set(RaidNode.RS_PARITY_LENGTH_KEY, "3");
-    conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1);
-    // start the cluster with one datanode first
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).
-        format(true).racks(rack1).hosts(host1).build();
-    cluster.waitActive();
-    namesystem = cluster.getNameNode().getNamesystem();
-    blockManager = namesystem.getBlockManager();
-    networktopology = blockManager.getDatanodeManager().getNetworkTopology();
-
-    Assert.assertTrue("BlockPlacementPolicy type is not correct.",
-      blockManager.getBlockPlacementPolicy() instanceof BlockPlacementPolicyRaid);
-    policy = (BlockPlacementPolicyRaid)blockManager.getBlockPlacementPolicy();
-    fs = cluster.getFileSystem();
-    xorPrefix = RaidNode.xorDestinationPath(conf).toUri().getPath();
-    raidTempPrefix = RaidNode.xorTempPrefix(conf);
-    raidrsTempPrefix = RaidNode.rsTempPrefix(conf);
-    raidrsHarTempPrefix = RaidNode.rsHarTempPrefix(conf);
-  }
-
-  /**
-   * Test that the parity files will be placed at the good locations when we
-   * create them.
-   */
-  @Test
-  public void testChooseTargetForRaidFile() throws IOException {
-    setupCluster();
-    try {
-      String src = "/dir/file";
-      String parity = raidrsTempPrefix + src;
-      DFSTestUtil.createFile(fs, new Path(src), 4, (short)1, 0L);
-      DFSTestUtil.waitReplication(fs, new Path(src), (short)1);
-      refreshPolicy();
-      setBlockPlacementPolicy(namesystem, policy);
-      // start 3 more datanodes
-      String[] racks = {"/rack2", "/rack2", "/rack2",
-                        "/rack2", "/rack2", "/rack2"};
-      String[] hosts =
-        {"host2.rack2.com", "host3.rack2.com", "host4.rack2.com",
-         "host5.rack2.com", "host6.rack2.com", "host7.rack2.com"};
-      cluster.startDataNodes(conf, 6, true, null, racks, hosts, null);
-      int numBlocks = 6;
-      DFSTestUtil.createFile(fs, new Path(parity), numBlocks, (short)2, 0L);
-      DFSTestUtil.waitReplication(fs, new Path(parity), (short)2);
-      FileStatus srcStat = fs.getFileStatus(new Path(src));
-      BlockLocation[] srcLoc =
-        fs.getFileBlockLocations(srcStat, 0, srcStat.getLen());
-      FileStatus parityStat = fs.getFileStatus(new Path(parity));
-      BlockLocation[] parityLoc =
-          fs.getFileBlockLocations(parityStat, 0, parityStat.getLen());
-      int parityLen = RaidNode.rsParityLength(conf);
-      for (int i = 0; i < numBlocks / parityLen; i++) {
-        Set<String> locations = new HashSet<String>();
-        for (int j = 0; j < srcLoc.length; j++) {
-          String [] names = srcLoc[j].getNames();
-          for (int k = 0; k < names.length; k++) {
-            LOG.info("Source block location: " + names[k]);
-            locations.add(names[k]);
-          }
-        }
-        for (int j = 0 ; j < parityLen; j++) {
-          String[] names = parityLoc[j + i * parityLen].getNames();
-          for (int k = 0; k < names.length; k++) {
-            LOG.info("Parity block location: " + names[k]);
-            Assert.assertTrue(locations.add(names[k]));
-          }
-        }
-      }
-    } finally {
-      if (cluster != null) {
-        cluster.shutdown();
-      }
-    }
-  }
-
-  /**
-   * Test that the har parity files will be placed at the good locations when we
-   * create them.
-   */
-  @Test
-  public void testChooseTargetForHarRaidFile() throws IOException {
-    setupCluster();
-    try {
-      String[] racks = {"/rack2", "/rack2", "/rack2",
-                        "/rack2", "/rack2", "/rack2"};
-      String[] hosts =
-        {"host2.rack2.com", "host3.rack2.com", "host4.rack2.com",
-         "host5.rack2.com", "host6.rack2.com", "host7.rack2.com"};
-      cluster.startDataNodes(conf, 6, true, null, racks, hosts, null);
-      String harParity = raidrsHarTempPrefix + "/dir/file";
-      int numBlocks = 11;
-      DFSTestUtil.createFile(fs, new Path(harParity), numBlocks, (short)1, 0L);
-      DFSTestUtil.waitReplication(fs, new Path(harParity), (short)1);
-      FileStatus stat = fs.getFileStatus(new Path(harParity));
-      BlockLocation[] loc = fs.getFileBlockLocations(stat, 0, stat.getLen());
-      int rsParityLength = RaidNode.rsParityLength(conf);
-      for (int i = 0; i < numBlocks - rsParityLength; i++) {
-        Set<String> locations = new HashSet<String>();
-        for (int j = 0; j < rsParityLength; j++) {
-          for (int k = 0; k < loc[i + j].getNames().length; k++) {
-            // verify that every adjacent 4 blocks are on differnt nodes
-            String name = loc[i + j].getNames()[k];
-            LOG.info("Har Raid block location: " + name);
-            Assert.assertTrue(locations.add(name));
-          }
-        }
-      }
-    } finally {
-      if (cluster != null) {
-        cluster.shutdown();
-      }
-    }
-  }
-
-  /**
-   * Test BlockPlacementPolicyRaid.CachedLocatedBlocks
-   * Verify that the results obtained from cache is the same as
-   * the results obtained directly
-   */
-  @Test
-  public void testCachedBlocks() throws IOException {
-    setupCluster();
-    try {
-      String file1 = "/dir/file1";
-      String file2 = "/dir/file2";
-      DFSTestUtil.createFile(fs, new Path(file1), 3, (short)1, 0L);
-      DFSTestUtil.createFile(fs, new Path(file2), 4, (short)1, 0L);
-      // test blocks cache
-      CachedLocatedBlocks cachedBlocks = new CachedLocatedBlocks(namesystem);
-      verifyCachedBlocksResult(cachedBlocks, namesystem, file1);
-      verifyCachedBlocksResult(cachedBlocks, namesystem, file1);
-      verifyCachedBlocksResult(cachedBlocks, namesystem, file2);
-      verifyCachedBlocksResult(cachedBlocks, namesystem, file2);
-      try {
-        Thread.sleep(1200L);
-      } catch (InterruptedException e) {
-      }
-      verifyCachedBlocksResult(cachedBlocks, namesystem, file2);
-      verifyCachedBlocksResult(cachedBlocks, namesystem, file1);
-    } finally {
-      if (cluster != null) {
-        cluster.shutdown();
-      }
-    }
-  }
-
-  /**
-   * Test BlockPlacementPolicyRaid.CachedFullPathNames
-   * Verify that the results obtained from cache is the same as
-   * the results obtained directly
-   */
-  @Test
-  public void testCachedPathNames() throws IOException {
-    setupCluster();
-    try {
-      String file1 = "/dir/file1";
-      String file2 = "/dir/file2";
-      DFSTestUtil.createFile(fs, new Path(file1), 3, (short)1, 0L);
-      DFSTestUtil.createFile(fs, new Path(file2), 4, (short)1, 0L);
-      // test full path cache
-      CachedFullPathNames cachedFullPathNames =
-          new CachedFullPathNames(namesystem);
-      final BlockCollection[] bcs = NameNodeRaidTestUtil.getBlockCollections(
-          namesystem, file1, file2);
-
-      verifyCachedFullPathNameResult(cachedFullPathNames, bcs[0]);
-      verifyCachedFullPathNameResult(cachedFullPathNames, bcs[0]);
-      verifyCachedFullPathNameResult(cachedFullPathNames, bcs[1]);
-      verifyCachedFullPathNameResult(cachedFullPathNames, bcs[1]);
-      try {
-        Thread.sleep(1200L);
-      } catch (InterruptedException e) {
-      }
-      verifyCachedFullPathNameResult(cachedFullPathNames, bcs[1]);
-      verifyCachedFullPathNameResult(cachedFullPathNames, bcs[0]);
-    } finally {
-      if (cluster != null) {
-        cluster.shutdown();
-      }
-    }
-  }
-  /**
-   * Test the result of getCompanionBlocks() on the unraided files
-   */
-  @Test
-  public void testGetCompanionBLocks() throws IOException {
-    setupCluster();
-    try {
-      String file1 = "/dir/file1";
-      String file2 = "/raid/dir/file2";
-      String file3 = "/raidrs/dir/file3";
-      // Set the policy to default policy to place the block in the default way
-      setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(
-          conf, namesystem, networktopology));
-      DFSTestUtil.createFile(fs, new Path(file1), 3, (short)1, 0L);
-      DFSTestUtil.createFile(fs, new Path(file2), 4, (short)1, 0L);
-      DFSTestUtil.createFile(fs, new Path(file3), 8, (short)1, 0L);
-      Collection<LocatedBlock> companionBlocks;
-
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, getBlocks(namesystem, file1).get(0).getBlock());
-      Assert.assertTrue(companionBlocks == null || companionBlocks.size() == 0);
-
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, getBlocks(namesystem, file1).get(2).getBlock());
-      Assert.assertTrue(companionBlocks == null || companionBlocks.size() == 0);
-
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, getBlocks(namesystem, file2).get(0).getBlock());
-      Assert.assertEquals(1, companionBlocks.size());
-
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, getBlocks(namesystem, file2).get(3).getBlock());
-      Assert.assertEquals(1, companionBlocks.size());
-
-      int rsParityLength = RaidNode.rsParityLength(conf);
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, getBlocks(namesystem, file3).get(0).getBlock());
-      Assert.assertEquals(rsParityLength, companionBlocks.size());
-
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, getBlocks(namesystem, file3).get(4).getBlock());
-      Assert.assertEquals(rsParityLength, companionBlocks.size());
-
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, getBlocks(namesystem, file3).get(6).getBlock());
-      Assert.assertEquals(2, companionBlocks.size());
-    } finally {
-      if (cluster != null) {
-        cluster.shutdown();
-      }
-    }
-  }
-
-  static void setBlockPlacementPolicy(
-      FSNamesystem namesystem, BlockPlacementPolicy policy) {
-    namesystem.writeLock();
-    try {
-      namesystem.getBlockManager().setBlockPlacementPolicy(policy);
-    } finally {
-      namesystem.writeUnlock();
-    }
-  }
-
-  /**
-   * Test BlockPlacementPolicyRaid actually deletes the correct replica.
-   * Start 2 datanodes and create 1 source file and its parity file.
-   * 1) Start host1, create the parity file with replication 1
-   * 2) Start host2, create the source file with replication 2
-   * 3) Set repliation of source file to 1
-   * Verify that the policy should delete the block with more companion blocks.
-   */
-  @Test
-  public void testDeleteReplica() throws IOException {
-    setupCluster();
-    try {
-      // Set the policy to default policy to place the block in the default way
-      setBlockPlacementPolicy(namesystem, new BlockPlacementPolicyDefault(
-          conf, namesystem, networktopology));
-      DatanodeDescriptor datanode1 = blockManager.getDatanodeManager(
-          ).getDatanodeCyclicIteration("").iterator().next().getValue();
-      String source = "/dir/file";
-      String parity = xorPrefix + source;
-
-      final Path parityPath = new Path(parity);
-      DFSTestUtil.createFile(fs, parityPath, 3, (short)1, 0L);
-      DFSTestUtil.waitReplication(fs, parityPath, (short)1);
-
-      // start one more datanode
-      cluster.startDataNodes(conf, 1, true, null, rack2, host2, null);
-      DatanodeDescriptor datanode2 = null;
-      for(Map.Entry<String, DatanodeDescriptor> e : blockManager.getDatanodeManager(
-          ).getDatanodeCyclicIteration("")) {
-        final DatanodeDescriptor d = e.getValue();
-        if (!d.getName().equals(datanode1.getName())) {
-          datanode2 = d;
-        }
-      }
-      Assert.assertTrue(datanode2 != null);
-      cluster.waitActive();
-      final Path sourcePath = new Path(source);
-      DFSTestUtil.createFile(fs, sourcePath, 5, (short)2, 0L);
-      DFSTestUtil.waitReplication(fs, sourcePath, (short)2);
-
-      refreshPolicy();
-      Assert.assertEquals(parity,
-                          policy.getParityFile(source));
-      Assert.assertEquals(source,
-                          policy.getSourceFile(parity, xorPrefix));
-
-      List<LocatedBlock> sourceBlocks = getBlocks(namesystem, source);
-      List<LocatedBlock> parityBlocks = getBlocks(namesystem, parity);
-      Assert.assertEquals(5, sourceBlocks.size());
-      Assert.assertEquals(3, parityBlocks.size());
-
-      // verify the result of getCompanionBlocks()
-      Collection<LocatedBlock> companionBlocks;
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, sourceBlocks.get(0).getBlock());
-      verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks,
-                            new int[]{0, 1}, new int[]{0});
-
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, sourceBlocks.get(1).getBlock());
-      verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks,
-                            new int[]{0, 1}, new int[]{0});
-
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, sourceBlocks.get(2).getBlock());
-      verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks,
-                            new int[]{2, 3}, new int[]{1});
-
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, sourceBlocks.get(3).getBlock());
-      verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks,
-                            new int[]{2, 3}, new int[]{1});
-
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, sourceBlocks.get(4).getBlock());
-      verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks,
-                            new int[]{4}, new int[]{2});
-
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, parityBlocks.get(0).getBlock());
-      verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks,
-                            new int[]{0, 1}, new int[]{0});
-
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, parityBlocks.get(1).getBlock());
-      verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks,
-                            new int[]{2, 3}, new int[]{1});
-
-      companionBlocks = getCompanionBlocks(
-          namesystem, policy, parityBlocks.get(2).getBlock());
-      verifyCompanionBlocks(companionBlocks, sourceBlocks, parityBlocks,
-                            new int[]{4}, new int[]{2});
-
-      // Set the policy back to raid policy. We have to create a new object
-      // here to clear the block location cache
-      refreshPolicy();
-      setBlockPlacementPolicy(namesystem, policy);
-      // verify policy deletes the correct blocks. companion blocks should be
-      // evenly distributed.
-      fs.setReplication(sourcePath, (short)1);
-      DFSTestUtil.waitReplication(fs, sourcePath, (short)1);
-      Map<String, Integer> counters = new HashMap<String, Integer>();
-      refreshPolicy();
-      for (int i = 0; i < parityBlocks.size(); i++) {
-        companionBlocks = getCompanionBlocks(
-            namesystem, policy, parityBlocks.get(i).getBlock());
-
-        counters = BlockPlacementPolicyRaid.countCompanionBlocks(
-            companionBlocks, false);
-        Assert.assertTrue(counters.get(datanode1.getName()) >= 1 &&
-                          counters.get(datanode1.getName()) <= 2);
-        Assert.assertTrue(counters.get(datanode1.getName()) +
-                          counters.get(datanode2.getName()) ==
-                          companionBlocks.size());
-
-        counters = BlockPlacementPolicyRaid.countCompanionBlocks(
-            companionBlocks, true);
-        Assert.assertTrue(counters.get(datanode1.getParent().getName()) >= 1 &&
-                          counters.get(datanode1.getParent().getName()) <= 2);
-        Assert.assertTrue(counters.get(datanode1.getParent().getName()) +
-                          counters.get(datanode2.getParent().getName()) ==
-                          companionBlocks.size());
-      }
-    } finally {
-      if (cluster != null) {
-        cluster.shutdown();
-      }
-    }
-  }
-
-  // create a new BlockPlacementPolicyRaid to clear the cache
-  private void refreshPolicy() {
-      policy = new BlockPlacementPolicyRaid();
-      policy.initialize(conf, namesystem, networktopology);
-  }
-
-  private void verifyCompanionBlocks(Collection<LocatedBlock> companionBlocks,
-      List<LocatedBlock> sourceBlocks, List<LocatedBlock> parityBlocks,
-      int[] sourceBlockIndexes, int[] parityBlockIndexes) {
-    Set<ExtendedBlock> blockSet = new HashSet<ExtendedBlock>();
-    for (LocatedBlock b : companionBlocks) {
-      blockSet.add(b.getBlock());
-    }
-    Assert.assertEquals(sourceBlockIndexes.length + parityBlockIndexes.length,
-                        blockSet.size());
-    for (int index : sourceBlockIndexes) {
-      Assert.assertTrue(blockSet.contains(sourceBlocks.get(index).getBlock()));
-    }
-    for (int index : parityBlockIndexes) {
-      Assert.assertTrue(blockSet.contains(parityBlocks.get(index).getBlock()));
-    }
-  }
-
-  private void verifyCachedFullPathNameResult(
-      CachedFullPathNames cachedFullPathNames, BlockCollection bc)
-  throws IOException {
-    String res1 = bc.getName();
-    String res2 = cachedFullPathNames.get(bc);
-    LOG.info("Actual path name: " + res1);
-    LOG.info("Cached path name: " + res2);
-    Assert.assertEquals(cachedFullPathNames.get(bc),
-                        bc.getName());
-  }
-
-  private void verifyCachedBlocksResult(CachedLocatedBlocks cachedBlocks,
-      FSNamesystem namesystem, String file) throws IOException{
-    long len = NameNodeRaidUtil.getFileInfo(namesystem, file, true).getLen();
-    List<LocatedBlock> res1 = NameNodeRaidUtil.getBlockLocations(namesystem,
-        file, 0L, len, false, false).getLocatedBlocks();
-    List<LocatedBlock> res2 = cachedBlocks.get(file);
-    for (int i = 0; i < res1.size(); i++) {
-      LOG.info("Actual block: " + res1.get(i).getBlock());
-      LOG.info("Cached block: " + res2.get(i).getBlock());
-      Assert.assertEquals(res1.get(i).getBlock(), res2.get(i).getBlock());
-    }
-  }
-
-  private Collection<LocatedBlock> getCompanionBlocks(
-      FSNamesystem namesystem, BlockPlacementPolicyRaid policy,
-      ExtendedBlock block) throws IOException {
-    INodeFile inode = (INodeFile)blockManager.blocksMap.getBlockCollection(block
-        .getLocalBlock());
-    FileType type = policy.getFileType(inode.getFullPathName());
-    return policy.getCompanionBlocks(inode.getFullPathName(), type,
-        block.getLocalBlock());
-  }
-
-  private List<LocatedBlock> getBlocks(FSNamesystem namesystem, String file) 
-      throws IOException {
-    long len = NameNodeRaidUtil.getFileInfo(namesystem, file, true).getLen();
-    return NameNodeRaidUtil.getBlockLocations(namesystem,
-               file, 0, len, false, false).getLocatedBlocks();
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRaidTestUtil.java

@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.namenode;
-
-import org.apache.hadoop.fs.UnresolvedLinkException;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
-
-public class NameNodeRaidTestUtil {
-  public static BlockCollection[] getBlockCollections(final FSNamesystem namesystem,
-      final String... files) throws UnresolvedLinkException {
-    final BlockCollection[] inodes = new BlockCollection[files.length];
-    final FSDirectory dir = namesystem.dir; 
-    dir.readLock();
-    try {
-      for(int i = 0; i < files.length; i++) {
-        inodes[i] = (BlockCollection)dir.rootDir.getNode(files[i], true);
-      }
-      return inodes;
-    } finally {
-      dir.readUnlock();
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixer.java

@@ -1,671 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.net.URI;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Random;
-import java.util.zip.CRC32;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.RaidDFSUtil;
-import org.apache.hadoop.hdfs.TestRaidDfs;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobContext;
-import org.apache.hadoop.mapred.MiniMRCluster;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.util.JarFinder;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Time;
-import org.junit.Test;
-
-
-public class TestBlockFixer {
-  final static Log LOG = LogFactory.getLog(
-                            "org.apache.hadoop.raid.TestBlockFixer");
-  final static String TEST_DIR = new File(System.getProperty("test.build.data",
-      "target/test-data")).getAbsolutePath();
-  final static String CONFIG_FILE = new File(TEST_DIR, 
-      "test-raid.xml").getAbsolutePath();
-  public static final String DistBlockFixer_JAR =
-      JarFinder.getJar(DistBlockFixer.class);
-  final static long RELOAD_INTERVAL = 1000;
-  final static int NUM_DATANODES = 3;
-  Configuration conf;
-  String namenode = null;
-  MiniDFSCluster dfs = null;
-  String hftp = null;
-  MiniMRCluster mr = null;
-  FileSystem fileSys = null;
-  RaidNode cnode = null;
-  String jobTrackerName = null;
-  Random rand = new Random();
-
-  /**
-   * Tests isXorParityFile and isRsParityFile
-   */
-  @Test
-  public void testIsParityFile() throws IOException {
-    Configuration testConf = new Configuration();
-    testConf.set("hdfs.raid.locations", "/raid");
-    testConf.set("hdfs.raidrs.locations", "/raidrs");
-
-    BlockFixer.BlockFixerHelper helper =
-      new BlockFixer.BlockFixerHelper(testConf);
-
-    assertFalse("incorrectly identified rs parity file as xor parity file",
-                helper.isXorParityFile(new Path("/raidrs/test/test")));
-    assertTrue("could not identify rs parity file",
-               helper.isRsParityFile(new Path("/raidrs/test/test")));
-    assertTrue("could not identify xor parity file",
-               helper.isXorParityFile(new Path("/raid/test/test")));
-    assertFalse("incorrectly identified xor parity file as rs parity file",
-                helper.isRsParityFile(new Path("/raid/test/test")));
-  }
-
-
-  /**
-   * Test the filtering of trash files from the list of corrupt files.
-   */
-  @Test
-  public void testTrashFilter() {
-    List<Path> files = new LinkedList<Path>();
-    // Paths that do not match the trash pattern.
-    Path p1 = new Path("/user/raid/raidtest/f1");
-    Path p2 = new Path("/user/.Trash/"); 
-    // Paths that match the trash pattern.
-    Path p3 = new Path("/user/raid/.Trash/raidtest/f1");
-    Path p4 = new Path("/user/raid/.Trash/");
-    files.add(p1);
-    files.add(p3);
-    files.add(p4);
-    files.add(p2);
-
-    Configuration conf = new Configuration();
-    RaidUtils.filterTrash(conf, files);
-
-    assertEquals("expected 2 non-trash files but got " + files.size(),
-                 2, files.size());
-    for (Path p: files) {
-      assertTrue("wrong file returned by filterTrash",
-                 p == p1 || p == p2);
-    }
-  }
-
-  @Test
-  public void testBlockFixLocal() throws Exception {
-    implBlockFix(true);
-  }
-
-  /**
-   * Create a file with three stripes, corrupt a block each in two stripes,
-   * and wait for the the file to be fixed.
-   */
-  protected void implBlockFix(boolean local) throws Exception {
-    LOG.info("Test testBlockFix started.");
-    long blockSize = 8192L;
-    int stripeLength = 3;
-    mySetup(stripeLength, -1); // never har
-    Path file1 = new Path("/user/dhruba/raidtest/file1");
-    Path destPath = new Path("/destraid/user/dhruba/raidtest");
-    long crc1 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1,
-                                                          1, 7, blockSize);
-    long file1Len = fileSys.getFileStatus(file1).getLen();
-    LOG.info("Test testBlockFix created test files");
-
-    // create an instance of the RaidNode
-    Configuration localConf = new Configuration(conf);
-    localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
-    localConf.setInt("raid.blockfix.interval", 1000);
-    if (local) {
-      localConf.set("raid.blockfix.classname",
-                    "org.apache.hadoop.raid.LocalBlockFixer");
-    } else {
-      localConf.set("raid.blockfix.classname",
-                    "org.apache.hadoop.raid.DistBlockFixer");
-    }
-    localConf.setLong("raid.blockfix.filespertask", 2L);
-
-    try {
-      cnode = RaidNode.createRaidNode(null, localConf);
-      TestRaidDfs.waitForFileRaided(LOG, fileSys, file1, destPath);
-      cnode.stop(); cnode.join();
-      
-      FileStatus srcStat = fileSys.getFileStatus(file1);
-      DistributedFileSystem dfs = (DistributedFileSystem)fileSys;
-      LocatedBlocks locs = RaidDFSUtil.getBlockLocations(
-        dfs, file1.toUri().getPath(), 0, srcStat.getLen());
-
-      String[] corruptFiles = RaidDFSUtil.getCorruptFiles(dfs);
-      assertEquals("no corrupt files expected", 0, corruptFiles.length);
-      assertEquals("filesFixed() should return 0 before fixing files",
-                   0, cnode.blockFixer.filesFixed());
-      
-      // Corrupt blocks in two different stripes. We can fix them.
-      int[] corruptBlockIdxs = new int[]{0, 4, 6};
-      for (int idx: corruptBlockIdxs)
-        corruptBlock(locs.get(idx).getBlock());
-      reportCorruptBlocks(dfs, file1, corruptBlockIdxs, blockSize);
-      
-      corruptFiles = RaidDFSUtil.getCorruptFiles(dfs);
-      assertEquals("file not corrupted", 1, corruptFiles.length);
-      assertEquals("wrong file corrupted",
-                   corruptFiles[0], file1.toUri().getPath());
-      assertEquals("wrong number of corrupt blocks", 3,
-        RaidDFSUtil.corruptBlocksInFile(dfs, file1.toUri().getPath(), 0,
-          srcStat.getLen()).size());
-
-      cnode = RaidNode.createRaidNode(null, localConf);
-      long start = Time.now();
-      while (cnode.blockFixer.filesFixed() < 1 &&
-             Time.now() - start < 120000) {
-        LOG.info("Test testBlockFix waiting for files to be fixed.");
-        Thread.sleep(1000);
-      }
-      assertEquals("file not fixed", 1, cnode.blockFixer.filesFixed());
-      
-      dfs = getDFS(conf, dfs);
-      assertTrue("file not fixed",
-                 TestRaidDfs.validateFile(dfs, file1, file1Len, crc1));
-
-    } catch (Exception e) {
-      LOG.info("Test testBlockFix Exception " + e +
-               StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      myTearDown();
-    }
-    LOG.info("Test testBlockFix completed.");
-  }
-
-  /**
-   * Tests integrity of generated block.
-   * Create a file and delete a block entirely. Wait for the block to be
-   * regenerated. Now stop RaidNode and corrupt the generated block.
-   * Test that corruption in the generated block can be detected by clients.
-   */
-  protected void generatedBlockTestCommon(String testName, int blockToCorrupt,
-                                        boolean local) throws Exception {
-    LOG.info("Test " + testName + " started.");
-    long blockSize = 8192L;
-    int stripeLength = 3;
-    mySetup(stripeLength, -1); // never har
-    Path file1 = new Path("/user/dhruba/raidtest/file1");
-    Path destPath = new Path("/destraid/user/dhruba/raidtest");
-    long crc1 = TestRaidDfs.createTestFile(fileSys, file1, 1, 7, blockSize);
-    long file1Len = fileSys.getFileStatus(file1).getLen();
-    LOG.info("Test " + testName + " created test files");
-
-    // create an instance of the RaidNode
-    Configuration localConf = new Configuration(conf);
-    localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
-    localConf.setInt("raid.blockfix.interval", 1000);
-    if (local) {
-      localConf.set("raid.blockfix.classname",
-                    "org.apache.hadoop.raid.LocalBlockFixer");
-    } else {
-      localConf.set("raid.blockfix.classname",
-                    "org.apache.hadoop.raid.DistBlockFixer");
-    }
-    localConf.setLong("raid.blockfix.filespertask", 2L);
-    try {
-      cnode = RaidNode.createRaidNode(null, localConf);
-      TestRaidDfs.waitForFileRaided(LOG, fileSys, file1, destPath);
-      cnode.stop(); cnode.join();
-      
-      FileStatus srcStat = fileSys.getFileStatus(file1);
-      DistributedFileSystem dfs = (DistributedFileSystem)fileSys;
-      LocatedBlocks locs = RaidDFSUtil.getBlockLocations(
-        dfs, file1.toUri().getPath(), 0, srcStat.getLen());
-
-      String[] corruptFiles = RaidDFSUtil.getCorruptFiles(dfs);
-      assertEquals("no corrupt files expected", 0, corruptFiles.length);
-      assertEquals("filesFixed() should return 0 before fixing files",
-                   0, cnode.blockFixer.filesFixed());
-      
-      corruptBlock(locs.get(0).getBlock());
-      reportCorruptBlocks(dfs, file1, new int[]{0}, blockSize);
-      
-      corruptFiles = RaidDFSUtil.getCorruptFiles(dfs);
-      assertEquals("file not corrupted",
-                   1, corruptFiles.length);
-      assertEquals("wrong file corrupted",
-                   corruptFiles[0], file1.toUri().getPath());
-      
-      cnode = RaidNode.createRaidNode(null, localConf);
-      long start = Time.now();
-      while (cnode.blockFixer.filesFixed() < 1 &&
-             Time.now() - start < 120000) {
-        LOG.info("Test " + testName + " waiting for files to be fixed.");
-        Thread.sleep(1000);
-      }
-      assertEquals("file not fixed",
-                   1, cnode.blockFixer.filesFixed());
-      
-      // Stop RaidNode
-      cnode.stop(); cnode.join(); cnode = null;
-
-      // The block has successfully been reconstructed.
-      dfs = getDFS(conf, dfs);
-      assertTrue("file not fixed",
-                 TestRaidDfs.validateFile(dfs, file1, file1Len, crc1));
-
-      // Now corrupt the generated block.
-      locs = RaidDFSUtil.getBlockLocations(
-        dfs, file1.toUri().getPath(), 0, srcStat.getLen());
-      corruptBlock(locs.get(0).getBlock());
-      reportCorruptBlocks(dfs, file1, new int[]{0}, blockSize);
-
-      try {
-        Thread.sleep(5*1000);
-      } catch (InterruptedException ignore) {
-      }
-      try {
-        TestRaidDfs.validateFile(dfs, file1, file1Len, crc1);
-        fail("Expected exception not thrown");
-      } catch (org.apache.hadoop.fs.ChecksumException ce) {
-      } catch (org.apache.hadoop.hdfs.BlockMissingException bme) {
-      }
-    } catch (Exception e) {
-      LOG.info("Test " + testName + " Exception " + e +
-               StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      myTearDown();
-    }
-    LOG.info("Test " + testName + " completed.");
-  }
-
-  /**
-   * Tests integrity of generated block.
-   * Create a file and delete a block entirely. Wait for the block to be
-   * regenerated. Now stop RaidNode and corrupt the generated block.
-   * Test that corruption in the generated block can be detected by clients.
-   */
-  @Test
-  public void testGeneratedBlockLocal() throws Exception {
-    generatedBlockTestCommon("testGeneratedBlock", 3, true);
-  }
-
-  /**
-   * Tests integrity of generated last block.
-   * Create a file and delete a block entirely. Wait for the block to be
-   * regenerated. Now stop RaidNode and corrupt the generated block.
-   * Test that corruption in the generated block can be detected by clients.
-   */
-  @Test
-  public void testGeneratedLastBlockLocal() throws Exception {
-    generatedBlockTestCommon("testGeneratedLastBlock", 6, true);
-  }
-
-  @Test
-  public void testParityBlockFixLocal() throws Exception {
-    implParityBlockFix("testParityBlockFixLocal", true);
-  }
-
-  /**
-   * Corrupt a parity file and wait for it to get fixed.
-   */
-  protected void implParityBlockFix(String testName, boolean local)
-    throws Exception {
-    LOG.info("Test " + testName + " started.");
-    long blockSize = 8192L;
-    int stripeLength = 3;
-    mySetup(stripeLength, -1); // never har
-    Path file1 = new Path("/user/dhruba/raidtest/file1");
-    Path destPath = new Path("/destraid/user/dhruba/raidtest");
-    Path parityFile = new Path("/destraid/user/dhruba/raidtest/file1");
-    TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1,
-                                                          1, 7, blockSize);
-    LOG.info("Test " + testName + " created test files");
-
-    // create an instance of the RaidNode
-    Configuration localConf = new Configuration(conf);
-    localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
-    localConf.setInt("raid.blockfix.interval", 1000);
-    if (local) {
-      localConf.set("raid.blockfix.classname",
-                    "org.apache.hadoop.raid.LocalBlockFixer");
-    } else {
-      localConf.set("raid.blockfix.classname",
-                    "org.apache.hadoop.raid.DistBlockFixer");
-    }
-    localConf.setLong("raid.blockfix.filespertask", 2L);
-
-    try {
-      cnode = RaidNode.createRaidNode(null, localConf);
-      TestRaidDfs.waitForFileRaided(LOG, fileSys, file1, destPath);
-      cnode.stop(); cnode.join();
-
-      long parityCRC = getCRC(fileSys, parityFile);
-
-      FileStatus parityStat = fileSys.getFileStatus(parityFile);
-      DistributedFileSystem dfs = (DistributedFileSystem)fileSys;
-      LocatedBlocks locs = RaidDFSUtil.getBlockLocations(
-        dfs, parityFile.toUri().getPath(), 0, parityStat.getLen());
-
-      String[] corruptFiles = RaidDFSUtil.getCorruptFiles(dfs);
-      assertEquals("no corrupt files expected", 0, corruptFiles.length);
-      assertEquals("filesFixed() should return 0 before fixing files",
-                   0, cnode.blockFixer.filesFixed());
-
-      // Corrupt parity blocks for different stripes.
-      int[] corruptBlockIdxs = new int[]{0, 1, 2};
-      for (int idx: corruptBlockIdxs)
-        corruptBlock(locs.get(idx).getBlock());
-      reportCorruptBlocks(dfs, parityFile, corruptBlockIdxs, blockSize);
-
-      corruptFiles = RaidDFSUtil.getCorruptFiles(dfs);
-      assertEquals("file not corrupted",
-                   1, corruptFiles.length);
-      assertEquals("wrong file corrupted",
-                   corruptFiles[0], parityFile.toUri().getPath());
-
-      cnode = RaidNode.createRaidNode(null, localConf);
-      long start = Time.now();
-      while (cnode.blockFixer.filesFixed() < 1 &&
-             Time.now() - start < 120000) {
-        LOG.info("Test " + testName + " waiting for files to be fixed.");
-        Thread.sleep(1000);
-      }
-      assertEquals("file not fixed",
-                   1, cnode.blockFixer.filesFixed());
-
-      long checkCRC = getCRC(fileSys, parityFile);
-
-      assertEquals("file not fixed",
-                   parityCRC, checkCRC);
-
-    } catch (Exception e) {
-      LOG.info("Test " + testName + " Exception " + e +
-               StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      myTearDown();
-    }
-    LOG.info("Test " + testName + " completed.");
-  }
-
-  @Test
-  public void testParityHarBlockFixLocal() throws Exception {
-    implParityHarBlockFix("testParityHarBlockFixLocal", true);
-  }
-
-  protected void implParityHarBlockFix(String testName, boolean local)
-    throws Exception {
-    LOG.info("Test " + testName + " started.");
-    long blockSize = 8192L;
-    int stripeLength = 3;
-    mySetup(stripeLength, 0); // Time before har = 0 days.
-    Path file1 = new Path("/user/dhruba/raidtest/file1");
-    // Parity file will have 7 blocks.
-    TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1,
-                                               1, 20, blockSize);
-    LOG.info("Test " + testName + " created test files");
-
-    // create an instance of the RaidNode
-    // HAR block size = 2 * src block size = 2 * parity block size.
-    Configuration localConf = new Configuration(conf);
-    localConf.setLong("har.block.size", blockSize * 2);
-    localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
-    localConf.setInt("raid.blockfix.interval", 1000);
-    if (local) {
-      localConf.set("raid.blockfix.classname",
-                    "org.apache.hadoop.raid.LocalBlockFixer");
-    } else {
-      localConf.set("raid.blockfix.classname",
-                    "org.apache.hadoop.raid.DistBlockFixer");
-    }
-    localConf.setLong("raid.blockfix.filespertask", 2L);
-
-    try {
-      cnode = RaidNode.createRaidNode(null, localConf);
-      Path harDirectory =
-        new Path("/destraid/user/dhruba/raidtest/raidtest" +
-                 RaidNode.HAR_SUFFIX);
-      long start = Time.now();
-      while (Time.now() - start < 1000 * 120) {
-        if (fileSys.exists(harDirectory)) {
-          break;
-        }
-        LOG.info("Test " + testName + " waiting for har");
-        Thread.sleep(1000);
-      }
-
-      Path partFile = new Path(harDirectory, "part-0");
-      long partCRC = getCRC(fileSys, partFile);
-      FileStatus partStat = fileSys.getFileStatus(partFile);
-      DistributedFileSystem dfs = (DistributedFileSystem)fileSys;
-      LocatedBlocks locs = RaidDFSUtil.getBlockLocations(
-        dfs, partFile.toUri().getPath(), 0, partStat.getLen());
-      // 7 parity blocks => 4 har blocks.
-      assertEquals("wrong number of har blocks",
-                   4, locs.getLocatedBlocks().size());
-      cnode.stop(); cnode.join();
-
-      String[] corruptFiles = RaidDFSUtil.getCorruptFiles(dfs);
-      assertEquals("no corrupt files expected", 0, corruptFiles.length);
-      assertEquals("filesFixed() should return 0 before fixing files",
-                   0, cnode.blockFixer.filesFixed());
-
-      // Corrupt parity blocks for different stripes.
-      int[] corruptBlockIdxs = new int[]{0, 3};
-      for (int idx: corruptBlockIdxs)
-        corruptBlock(locs.get(idx).getBlock());
-      reportCorruptBlocks(dfs, partFile, corruptBlockIdxs,
-        partStat.getBlockSize());
-
-      corruptFiles = RaidDFSUtil.getCorruptFiles(dfs);
-      assertEquals("file not corrupted", 1, corruptFiles.length);
-      assertEquals("wrong file corrupted",
-                   corruptFiles[0], partFile.toUri().getPath());
-
-      cnode = RaidNode.createRaidNode(null, localConf);
-      start = Time.now();
-      while (cnode.blockFixer.filesFixed() < 1 &&
-             Time.now() - start < 120000) {
-        LOG.info("Test " + testName + " waiting for files to be fixed.");
-        Thread.sleep(1000);
-      }
-      assertEquals("file not fixed",
-                   1, cnode.blockFixer.filesFixed());
-
-      long checkCRC = getCRC(fileSys, partFile);
-
-      assertEquals("file not fixed",
-                   partCRC, checkCRC);
-    } catch (Exception e) {
-      LOG.info("Test " + testName + " Exception " + e +
-               StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      myTearDown();
-    }
-    LOG.info("Test " + testName + " completed.");
-  }
-
-
-  protected static DistributedFileSystem getDFS(
-        Configuration conf, FileSystem dfs) throws IOException {
-    Configuration clientConf = new Configuration(conf);
-    clientConf.set("fs.hdfs.impl",
-                   "org.apache.hadoop.hdfs.DistributedFileSystem");
-    clientConf.setBoolean("fs.hdfs.impl.disable.cache", true);
-    URI dfsUri = dfs.getUri();
-    FileSystem.closeAll();
-    return (DistributedFileSystem) FileSystem.get(dfsUri, clientConf);
-  }
-
-  protected void mySetup(int stripeLength, int timeBeforeHar) throws Exception {
-
-    new File(TEST_DIR).mkdirs(); // Make sure data directory exists
-    conf = new Configuration();
-
-    conf.set("raid.config.file", CONFIG_FILE);
-    conf.setBoolean("raid.config.reload", true);
-    conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL);
-
-    // scan all policies once every 5 second
-    conf.setLong("raid.policy.rescan.interval", 5000);
-
-    // make all deletions not go through Trash
-    conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient");
-
-    // do not use map-reduce cluster for Raiding
-    conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode");
-    conf.set("raid.server.address", "localhost:0");
-    conf.setInt("hdfs.raid.stripeLength", stripeLength);
-    conf.set("hdfs.raid.locations", "/destraid");
-
-    conf.setBoolean("dfs.permissions", false);
-
-    conf.set("mapreduce.framework.name", "yarn");
-
-    dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build();
-    dfs.waitActive();
-    fileSys = dfs.getFileSystem();
-    namenode = fileSys.getUri().toString();
-
-    FileSystem.setDefaultUri(conf, namenode);
-    mr = new MiniMRCluster(4, namenode, 3);
-    JobConf jobConf = mr.createJobConf();
-    jobTrackerName = "localhost:" + jobConf.get(JTConfig.JT_IPC_ADDRESS);
-    hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort();
-
-    FileSystem.setDefaultUri(conf, namenode);
-    conf.set("mapred.job.tracker", jobTrackerName);
-    conf.set("mapreduce.framework.name", "yarn");
-    String rmAdress = jobConf.get("yarn.resourcemanager.address");
-    if (rmAdress != null) {
-      conf.set("yarn.resourcemanager.address", rmAdress);
-    }
-    String schedulerAdress =
-      jobConf.get("yarn.resourcemanager.scheduler.address");
-    if (schedulerAdress != null) {
-      conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress);
-    }
-    String jobHistoryAddress = 
-        jobConf.get("mapreduce.jobhistory.address");
-    if (jobHistoryAddress != null) {
-      conf.set("mapreduce.jobhistory.address", jobHistoryAddress);
-    }
-    conf.set(JobContext.JAR, TestBlockFixer.DistBlockFixer_JAR);
-    
-    FileWriter fileWriter = new FileWriter(CONFIG_FILE);
-    fileWriter.write("<?xml version=\"1.0\"?>\n");
-    String str = "<configuration> " +
-                   "<srcPath prefix=\"/user/dhruba/raidtest\"> " +
-                     "<policy name = \"RaidTest1\"> " +
-                        "<erasureCode>xor</erasureCode> " +
-                        "<destPath> /destraid</destPath> " +
-                        "<property> " +
-                          "<name>targetReplication</name> " +
-                          "<value>1</value> " + 
-                          "<description>after RAIDing, decrease the replication factor of a file to this value." +
-                          "</description> " + 
-                        "</property> " +
-                        "<property> " +
-                          "<name>metaReplication</name> " +
-                          "<value>1</value> " + 
-                          "<description> replication factor of parity file" +
-                          "</description> " + 
-                        "</property> " +
-                        "<property> " +
-                          "<name>modTimePeriod</name> " +
-                          "<value>2000</value> " + 
-                          "<description> time (milliseconds) after a file is modified to make it " +
-                                         "a candidate for RAIDing " +
-                          "</description> " + 
-                        "</property> ";
-    if (timeBeforeHar >= 0) {
-      str +=
-                        "<property> " +
-                          "<name>time_before_har</name> " +
-                          "<value>" + timeBeforeHar + "</value> " +
-                          "<description> amount of time waited before har'ing parity files" +
-                          "</description> " + 
-                        "</property> ";
-    }
-
-    str +=
-                     "</policy>" +
-                   "</srcPath>" +
-                 "</configuration>";
-    fileWriter.write(str);
-    fileWriter.close();
-  }
-
-  protected void myTearDown() throws Exception {
-    if (cnode != null) { cnode.stop(); cnode.join(); }
-    if (mr != null) { mr.shutdown(); }
-    if (dfs != null) { dfs.shutdown(); }
-  }
-
-  public static long getCRC(FileSystem fs, Path p) throws IOException {
-    CRC32 crc = new CRC32();
-    FSDataInputStream stm = fs.open(p);
-    int b;
-    while ((b = stm.read())>=0) {
-      crc.update(b);
-    }
-    stm.close();
-    return crc.getValue();
-  }
-
-  void corruptBlock(ExtendedBlock block) throws IOException {
-    assertTrue("Could not corrupt block",
-        dfs.corruptBlockOnDataNodes(block) > 0);
-  }
-  
-  static void reportCorruptBlocks(FileSystem fs, Path file, int[] idxs,
-    long blockSize) throws IOException {
-
-    FSDataInputStream in = fs.open(file);
-    for (int idx: idxs) {
-      long offset = idx * blockSize;
-      LOG.info("Reporting corrupt block " + file + ":" + offset);
-      in.seek(offset);
-      try {
-        in.readFully(new byte[(int)blockSize]);
-        fail("Expected exception not thrown for " + file + ":" + offset);
-      } catch (org.apache.hadoop.fs.ChecksumException e) {
-      } catch (org.apache.hadoop.hdfs.BlockMissingException bme) {
-      }
-    }
-  }
-}
-

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerBlockFixDist.java

@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-import org.junit.Test;
-
-public class TestBlockFixerBlockFixDist extends TestBlockFixer {
-  @Test
-  public void testBlockFixDist() throws Exception {
-    implBlockFix(false);
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerDistConcurrency.java

@@ -1,245 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.RaidDFSUtil;
-import org.apache.hadoop.hdfs.TestRaidDfs;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Time;
-import org.junit.Test;
-
-public class TestBlockFixerDistConcurrency extends TestBlockFixer {
-  /**
-   * tests that we can have 2 concurrent jobs fixing files 
-   * (dist block fixer)
-   */
-  @Test
-  public void testConcurrentJobs() throws Exception {
-    LOG.info("Test testConcurrentJobs started.");
-    long blockSize = 8192L;
-    int stripeLength = 3;
-    mySetup(stripeLength, -1); // never har
-    Path file1 = new Path("/user/dhruba/raidtest/file1");
-    Path file2 = new Path("/user/dhruba/raidtest/file2");
-    Path destPath = new Path("/destraid/user/dhruba/raidtest");
-    long crc1 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1,
-                                                          1, 20, blockSize);
-    long crc2 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file2,
-                                                          1, 20, blockSize);
-    long file1Len = fileSys.getFileStatus(file1).getLen();
-    long file2Len = fileSys.getFileStatus(file2).getLen();
-    LOG.info("Test testConcurrentJobs created test files");
-
-    // create an instance of the RaidNode
-    Configuration localConf = new Configuration(conf);
-    localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
-    localConf.setInt("raid.blockfix.interval", 1000);
-    localConf.set("raid.blockfix.classname", 
-                  "org.apache.hadoop.raid.DistBlockFixer");
-    localConf.setLong("raid.blockfix.filespertask", 2L);
-
-    try {
-      cnode = RaidNode.createRaidNode(null, localConf);
-      TestRaidDfs.waitForFileRaided(LOG, fileSys, file1, destPath);
-      TestRaidDfs.waitForFileRaided(LOG, fileSys, file2, destPath);
-      cnode.stop(); cnode.join();
-
-      FileStatus file1Stat = fileSys.getFileStatus(file1);
-      FileStatus file2Stat = fileSys.getFileStatus(file2);
-      DistributedFileSystem dfs = (DistributedFileSystem)fileSys;
-      LocatedBlocks file1Loc =
-        RaidDFSUtil.getBlockLocations(dfs, file1.toUri().getPath(),
-                                      0, file1Stat.getLen());
-      LocatedBlocks file2Loc =
-        RaidDFSUtil.getBlockLocations(dfs, file2.toUri().getPath(),
-                                      0, file2Stat.getLen());
-      
-      String[] corruptFiles = RaidDFSUtil.getCorruptFiles(dfs);
-      assertEquals("no corrupt files expected", 0, corruptFiles.length);
-      assertEquals("filesFixed() should return 0 before fixing files",
-                   0, cnode.blockFixer.filesFixed());
-
-      // corrupt file1
-      int[] corruptBlockIdxs = new int[]{0, 4, 6};
-      for (int idx: corruptBlockIdxs)
-        corruptBlock(file1Loc.get(idx).getBlock());
-      reportCorruptBlocks(dfs, file1, corruptBlockIdxs, blockSize);
-
-      cnode = RaidNode.createRaidNode(null, localConf);
-      DistBlockFixer blockFixer = (DistBlockFixer) cnode.blockFixer;
-      long start = Time.now();
-
-      while (blockFixer.jobsRunning() < 1 &&
-             Time.now() - start < 240000) {
-        LOG.info("Test testBlockFix waiting for fixing job 1 to start");
-        Thread.sleep(10);
-      }
-      assertEquals("job 1 not running", 1, blockFixer.jobsRunning());
-
-      // corrupt file2
-      for (int idx: corruptBlockIdxs)
-        corruptBlock(file2Loc.get(idx).getBlock());
-      reportCorruptBlocks(dfs, file2, corruptBlockIdxs, blockSize);
-      
-      while (blockFixer.jobsRunning() < 2 &&
-             Time.now() - start < 240000) {
-        LOG.info("Test testBlockFix waiting for fixing job 2 to start");
-        Thread.sleep(10);
-      }
-      assertEquals("2 jobs not running", 2, blockFixer.jobsRunning());
-
-      while (blockFixer.filesFixed() < 2 &&
-             Time.now() - start < 240000) {
-        LOG.info("Test testBlockFix waiting for files to be fixed.");
-        Thread.sleep(10);
-      }
-      assertEquals("files not fixed", 2, blockFixer.filesFixed());
-
-      dfs = getDFS(conf, dfs);
-      
-      try {
-        Thread.sleep(5*1000);
-      } catch (InterruptedException ignore) {
-      }
-      assertTrue("file not fixed",
-                 TestRaidDfs.validateFile(dfs, file1, file1Len, crc1));
-      assertTrue("file not fixed",
-                 TestRaidDfs.validateFile(dfs, file2, file2Len, crc2));
-    } catch (Exception e) {
-      LOG.info("Test testConcurrentJobs exception " + e +
-               StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      myTearDown();
-    }
-
-  }
-
-  /**
-   * tests that the distributed block fixer obeys
-   * the limit on how many files to fix simultaneously
-   */
-  @Test
-  public void testMaxPendingFiles() throws Exception {
-    LOG.info("Test testMaxPendingFiles started.");
-    long blockSize = 8192L;
-    int stripeLength = 3;
-    mySetup(stripeLength, -1); // never har
-    Path file1 = new Path("/user/dhruba/raidtest/file1");
-    Path file2 = new Path("/user/dhruba/raidtest/file2");
-    Path destPath = new Path("/destraid/user/dhruba/raidtest");
-    long crc1 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1,
-                                                          1, 20, blockSize);
-    long crc2 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file2,
-                                                          1, 20, blockSize);
-    long file1Len = fileSys.getFileStatus(file1).getLen();
-    long file2Len = fileSys.getFileStatus(file2).getLen();
-    LOG.info("Test testMaxPendingFiles created test files");
-
-    // create an instance of the RaidNode
-    Configuration localConf = new Configuration(conf);
-    localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
-    localConf.setInt("raid.blockfix.interval", 1000);
-    localConf.set("raid.blockfix.classname", 
-                  "org.apache.hadoop.raid.DistBlockFixer");
-    localConf.setLong("raid.blockfix.filespertask", 2L);
-    localConf.setLong("raid.blockfix.maxpendingfiles", 1L);
-
-    try {
-      cnode = RaidNode.createRaidNode(null, localConf);
-      TestRaidDfs.waitForFileRaided(LOG, fileSys, file1, destPath);
-      TestRaidDfs.waitForFileRaided(LOG, fileSys, file2, destPath);
-      cnode.stop(); cnode.join();
-
-      FileStatus file1Stat = fileSys.getFileStatus(file1);
-      FileStatus file2Stat = fileSys.getFileStatus(file2);
-      DistributedFileSystem dfs = (DistributedFileSystem)fileSys;
-      LocatedBlocks file1Loc =
-        RaidDFSUtil.getBlockLocations(dfs, file1.toUri().getPath(),
-                                      0, file1Stat.getLen());
-      LocatedBlocks file2Loc =
-        RaidDFSUtil.getBlockLocations(dfs, file2.toUri().getPath(),
-                                      0, file2Stat.getLen());
-
-      String[] corruptFiles = RaidDFSUtil.getCorruptFiles(dfs);
-      assertEquals("no corrupt files expected", 0, corruptFiles.length);
-      assertEquals("filesFixed() should return 0 before fixing files",
-                   0, cnode.blockFixer.filesFixed());
-
-      // corrupt file1
-      int[] corruptBlockIdxs = new int[]{0, 4, 6};
-      for (int idx: corruptBlockIdxs)
-        corruptBlock(file1Loc.get(idx).getBlock());
-      reportCorruptBlocks(dfs, file1, corruptBlockIdxs, blockSize);
-      corruptFiles = RaidDFSUtil.getCorruptFiles(dfs);
-
-      cnode = RaidNode.createRaidNode(null, localConf);
-      DistBlockFixer blockFixer = (DistBlockFixer) cnode.blockFixer;
-      long start = Time.now();
-
-      while (blockFixer.jobsRunning() < 1 &&
-             Time.now() - start < 240000) {
-        LOG.info("Test testBlockFix waiting for fixing job 1 to start");
-        Thread.sleep(10);
-      }
-      assertEquals("job not running", 1, blockFixer.jobsRunning());
-
-      // corrupt file2
-      for (int idx: corruptBlockIdxs)
-        corruptBlock(file2Loc.get(idx).getBlock());
-      reportCorruptBlocks(dfs, file2, corruptBlockIdxs, blockSize);
-      corruptFiles = RaidDFSUtil.getCorruptFiles(dfs);
-      
-      // wait until both files are fixed
-      while (blockFixer.filesFixed() < 2 &&
-             Time.now() - start < 240000) {
-        // make sure the block fixer does not start a second job while
-        // the first one is still running
-        assertTrue("too many jobs running", blockFixer.jobsRunning() <= 1);
-        Thread.sleep(10);
-      }
-      assertEquals("files not fixed", 2, blockFixer.filesFixed());
-
-      dfs = getDFS(conf, dfs);
-      
-      try {
-        Thread.sleep(5*1000);
-      } catch (InterruptedException ignore) {
-      }
-      assertTrue("file not fixed",
-                 TestRaidDfs.validateFile(dfs, file1, file1Len, crc1));
-      assertTrue("file not fixed",
-                 TestRaidDfs.validateFile(dfs, file2, file2Len, crc2));
-    } catch (Exception e) {
-      LOG.info("Test testMaxPendingFiles exception " + e +
-               StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      myTearDown();
-    }
-
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerGeneratedBlockDist.java

@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-
-import org.junit.Test;
-
-public class TestBlockFixerGeneratedBlockDist extends TestBlockFixer {
-  /**
-   * Tests integrity of generated block.
-   * Create a file and delete a block entirely. Wait for the block to be
-   * regenerated. Now stop RaidNode and corrupt the generated block.
-   * Test that corruption in the generated block can be detected by clients.
-   */
-  @Test
-  public void testGeneratedBlockDist() throws Exception {
-    generatedBlockTestCommon("testGeneratedBlock", 3, false);
-  }
-
-  /**
-   * Tests integrity of generated last block.
-   * Create a file and delete a block entirely. Wait for the block to be
-   * regenerated. Now stop RaidNode and corrupt the generated block.
-   * Test that corruption in the generated block can be detected by clients.
-   */
-  @Test
-  public void testGeneratedLastBlockDist() throws Exception {
-    generatedBlockTestCommon("testGeneratedLastBlock", 6, false);
-  }
-
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestBlockFixerParityBlockFixDist.java

@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-
-import org.junit.Test;
-
-public class TestBlockFixerParityBlockFixDist extends TestBlockFixer {
-  @Test
-  public void testParityBlockFixDist() throws Exception {
-    implParityBlockFix("testParityBlockFixDist", false);
-  }
-
-  @Test
-  public void testParityHarBlockFixDist() throws Exception {
-    implParityHarBlockFix("testParityHarBlockFixDist", false);
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestDirectoryTraversal.java

@@ -1,228 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.raid.protocol.PolicyInfo;
-import org.apache.hadoop.util.Time;
-import org.junit.Test;
-
-public class TestDirectoryTraversal {
-  final static Log LOG = LogFactory.getLog(
-                            "org.apache.hadoop.raid.TestDirectoryTraversal");
-  final static String TEST_DIR = new File(System.getProperty("test.build.data",
-      "target/test-data")).getAbsolutePath();
-
-  MiniDFSCluster dfs = null;
-  FileSystem fs = null;
-  Configuration conf = null;
-
-  /**
-   * Test basic enumeration.
-   */
-  @Test
-  public void testEnumeration() throws IOException {
-    mySetup();
-
-    try {
-      Path topDir = new Path(TEST_DIR + "/testenumeration");
-
-      createTestTree(topDir);
-
-      LOG.info("Enumerating files");
-      List<FileStatus> startPaths = new LinkedList<FileStatus>();
-      startPaths.add(fs.getFileStatus(topDir));
-      DirectoryTraversal dt = new DirectoryTraversal(fs, startPaths, 2);
-
-      List<FileStatus> selected = new LinkedList<FileStatus>();
-      while (true) {
-        FileStatus f = dt.getNextFile();
-        if (f == null) break;
-        assertEquals(false, f.isDir());
-        LOG.info(f.getPath());
-        selected.add(f);
-      }
-      assertEquals(5, selected.size());
-
-      LOG.info("Enumerating directories");
-      startPaths.clear();
-      startPaths.add(fs.getFileStatus(topDir));
-      dt = new DirectoryTraversal(fs, startPaths);
-      selected.clear();
-      while (true) {
-        FileStatus dir = dt.getNextDirectory();
-        if (dir == null) break;
-        assertEquals(true, dir.isDir());
-        LOG.info(dir.getPath());
-        selected.add(dir);
-      }
-      assertEquals(4, selected.size());
-    } finally {
-      myTearDown();
-    }
-  }
-
-  @Test
-  public void testSuspension() throws IOException {
-    LOG.info("Starting testSuspension");
-    mySetup();
-
-    try {
-      Path topDir = new Path(TEST_DIR + "/testenumeration");
-
-      createTestTree(topDir);
-
-      String top = topDir.toString();
-      List<FileStatus> startPaths = new LinkedList<FileStatus>();
-      startPaths.add(fs.getFileStatus(new Path(top + "/a")));
-      startPaths.add(fs.getFileStatus(new Path(top + "/b")));
-      DirectoryTraversal dt = new DirectoryTraversal(fs, startPaths);
-
-      int limit = 2;
-      short targetRepl = 1;
-      Path raid = new Path("/raid");
-      DirectoryTraversal.FileFilter filter =
-        new RaidFilter.TimeBasedFilter(conf,
-          RaidNode.xorDestinationPath(conf), 1, Time.now(), 0);
-      List<FileStatus> selected = dt.getFilteredFiles(filter, limit);
-      for (FileStatus f: selected) {
-        LOG.info(f.getPath());
-      }
-      assertEquals(limit, selected.size());
-
-      selected = dt.getFilteredFiles(filter, limit);
-      for (FileStatus f: selected) {
-        LOG.info(f.getPath());
-      }
-      assertEquals(limit, selected.size());
-    } finally {
-      myTearDown();
-    }
-  }
-
-  @Test
-  public void testFileFilter() throws IOException {
-    mySetup();
-
-    try {
-      Path topDir = new Path(TEST_DIR + "/testFileFilter");
-      int targetRepl = 1;
-      createTestTree(topDir);
-      Path file = new Path(topDir.toString() + "/a/f1");
-      FileStatus stat = fs.getFileStatus(file);
-      PolicyInfo info = new PolicyInfo("testFileFilter", conf);
-      info.setSrcPath(topDir.toString());
-      info.setErasureCode("rs");
-      info.setDescription("test policy");
-      info.setProperty("targetReplication", "1");
-      info.setProperty("metaReplication", "1");
-
-      DirectoryTraversal.FileFilter timeBasedXORFilter =
-        new RaidFilter.TimeBasedFilter(conf,
-          RaidNode.xorDestinationPath(conf), targetRepl,
-            Time.now(), 0);
-      DirectoryTraversal.FileFilter timeBasedRSFilter =
-        new RaidFilter.TimeBasedFilter(conf,
-          RaidNode.rsDestinationPath(conf), targetRepl,
-            Time.now(), 0);
-      DirectoryTraversal.FileFilter preferenceForRSFilter =
-        new RaidFilter.PreferenceFilter(
-          conf, RaidNode.rsDestinationPath(conf),
-          RaidNode.xorDestinationPath(conf), 1, Time.now(), 0);
-
-      assertTrue(timeBasedXORFilter.check(stat));
-      assertTrue(timeBasedRSFilter.check(stat));
-      assertTrue(preferenceForRSFilter.check(stat));
-
-      RaidNode.doRaid(
-        conf, info, stat, new RaidNode.Statistics(), Reporter.NULL);
-
-      assertTrue(timeBasedXORFilter.check(stat));
-      assertFalse(timeBasedRSFilter.check(stat));
-      assertFalse(preferenceForRSFilter.check(stat));
-    } finally {
-      myTearDown();
-    }
-  }
-
-  /**
-   * Creates a test directory tree.
-   *            top
-   *           / | \
-   *          /  |  f5
-   *         a   b___
-   *        / \  |\  \
-   *       f1 f2 f3f4 c
-   */
-  private void createTestTree(Path topDir) throws IOException {
-    String top = topDir.toString();
-    fs.delete(topDir, true);
-
-    fs.mkdirs(topDir);
-    fs.create(new Path(top + "/f5")).close();
-
-    fs.mkdirs(new Path(top + "/a"));
-    createTestFile(new Path(top + "/a/f1"));
-    createTestFile(new Path(top + "/a/f2"));
-
-    fs.mkdirs(new Path(top + "/b"));
-    fs.mkdirs(new Path(top + "/b/c"));
-    createTestFile(new Path(top + "/b/f3"));
-    createTestFile(new Path(top + "/b/f4"));
-  }
-
-  private void createTestFile(Path file) throws IOException {
-    long blockSize = 8192;
-    byte[] bytes = new byte[(int)blockSize];
-    FSDataOutputStream stm = fs.create(file, false, 4096, (short)1, blockSize);
-    stm.write(bytes);
-    stm.write(bytes);
-    stm.write(bytes);
-    stm.close();
-    FileStatus stat = fs.getFileStatus(file);
-    assertEquals(blockSize, stat.getBlockSize());
-  }
-
-  private void mySetup() throws IOException {
-    conf = new Configuration();
-    dfs = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
-    dfs.waitActive();
-    fs = dfs.getFileSystem();
-  }
-
-  private void myTearDown() {
-    if (dfs != null) { dfs.shutdown(); }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestErasureCodes.java

@@ -1,245 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.hadoop.util.Time;
-import org.junit.Test;
-
-public class TestErasureCodes {
-  final int TEST_CODES = 100;
-  final int TEST_TIMES = 1000;
-  final Random RAND = new Random();
-
-  @Test
-  public void testEncodeDecode() {
-    for (int n = 0; n < TEST_CODES; n++) {
-      int stripeSize = RAND.nextInt(99) + 1; // 1, 2, 3, ... 100
-      int paritySize = RAND.nextInt(9) + 1; //1, 2, 3, 4, ... 10
-      ErasureCode ec = new ReedSolomonCode(stripeSize, paritySize);
-      for (int m = 0; m < TEST_TIMES; m++) {
-        int symbolMax = (int) Math.pow(2, ec.symbolSize());
-        int[] message = new int[stripeSize];
-        for (int i = 0; i < stripeSize; i++) {
-          message[i] = RAND.nextInt(symbolMax);
-        }
-        int[] parity = new int[paritySize];
-        ec.encode(message, parity);
-        int[] data = new int[stripeSize + paritySize];
-        int[] copy = new int[data.length];
-        for (int i = 0; i < paritySize; i++) {
-          data[i] = parity[i];
-          copy[i] = parity[i];
-        }
-        for (int i = 0; i < stripeSize; i++) {
-          data[i + paritySize] = message[i];
-          copy[i + paritySize] = message[i];
-        }
-        int erasedLen = paritySize == 1 ? 1 : RAND.nextInt(paritySize - 1) + 1;
-        int[] erasedLocations = randomErasedLocation(erasedLen, data.length);
-        for (int i = 0; i < erasedLocations.length; i++) {
-          data[erasedLocations[i]] = 0;
-        }
-        int[] erasedValues = new int[erasedLen];
-        ec.decode(data, erasedLocations, erasedValues);
-        for (int i = 0; i < erasedLen; i++) {
-          assertEquals("Decode failed", copy[erasedLocations[i]], erasedValues[i]);
-        }
-      }
-    }
-  }
-
-  @Test
-  public void testRSPerformance() {
-    int stripeSize = 10;
-    int paritySize = 4;
-    ErasureCode ec = new ReedSolomonCode(stripeSize, paritySize);
-    int symbolMax = (int) Math.pow(2, ec.symbolSize());
-    byte[][] message = new byte[stripeSize][];
-    int bufsize = 1024 * 1024 * 10;
-    for (int i = 0; i < stripeSize; i++) {
-      message[i] = new byte[bufsize];
-      for (int j = 0; j < bufsize; j++) {
-        message[i][j] = (byte) RAND.nextInt(symbolMax);
-      }
-    }
-    byte[][] parity = new byte[paritySize][];
-    for (int i = 0; i < paritySize; i++) {
-      parity[i] = new byte[bufsize];
-    }
-    long encodeStart = Time.now();
-    int[] tmpIn = new int[stripeSize];
-    int[] tmpOut = new int[paritySize];
-    for (int i = 0; i < bufsize; i++) {
-      // Copy message.
-      for (int j = 0; j < stripeSize; j++) tmpIn[j] = 0x000000FF & message[j][i];
-      ec.encode(tmpIn, tmpOut);
-      // Copy parity.
-      for (int j = 0; j < paritySize; j++) parity[j][i] = (byte)tmpOut[j];
-    }
-    long encodeEnd = Time.now();
-    float encodeMSecs = (encodeEnd - encodeStart);
-    System.out.println("Time to encode rs = " + encodeMSecs +
-      "msec (" + message[0].length / (1000 * encodeMSecs) + " MB/s)");
-
-    // Copy erased array.
-    int[] data = new int[paritySize + stripeSize];
-    // 4th location is the 0th symbol in the message
-    int[] erasedLocations = new int[]{4, 1, 5, 7};
-    int[] erasedValues = new int[erasedLocations.length];
-    byte[] copy = new byte[bufsize];
-    for (int j = 0; j < bufsize; j++) {
-      copy[j] = message[0][j];
-      message[0][j] = 0;
-    }
-
-    long decodeStart = Time.now();
-    for (int i = 0; i < bufsize; i++) {
-      // Copy parity first.
-      for (int j = 0; j < paritySize; j++) {
-        data[j] = 0x000000FF & parity[j][i];
-      }
-      // Copy message. Skip 0 as the erased symbol
-      for (int j = 1; j < stripeSize; j++) {
-        data[j + paritySize] = 0x000000FF & message[j][i];
-      }
-      // Use 0, 2, 3, 6, 8, 9, 10, 11, 12, 13th symbol to reconstruct the data
-      ec.decode(data, erasedLocations, erasedValues);
-      message[0][i] = (byte)erasedValues[0];
-    }
-    long decodeEnd = Time.now();
-    float decodeMSecs = (decodeEnd - decodeStart);
-    System.out.println("Time to decode = " + decodeMSecs +
-      "msec (" + message[0].length / (1000 * decodeMSecs) + " MB/s)");
-    assertTrue("Decode failed", java.util.Arrays.equals(copy, message[0]));
-  }
-
-  @Test
-  public void testXorPerformance() {
-    java.util.Random RAND = new java.util.Random();
-    int stripeSize = 10;
-    byte[][] message = new byte[stripeSize][];
-    int bufsize = 1024 * 1024 * 10;
-    for (int i = 0; i < stripeSize; i++) {
-      message[i] = new byte[bufsize];
-      for (int j = 0; j < bufsize; j++) {
-        message[i][j] = (byte)RAND.nextInt(256);
-      }
-    }
-    byte[] parity = new byte[bufsize];
-
-    long encodeStart = Time.now();
-    for (int i = 0; i < bufsize; i++) {
-      for (int j = 0; j < stripeSize; j++) parity[i] ^= message[j][i];
-    }
-    long encodeEnd = Time.now();
-    float encodeMSecs = encodeEnd - encodeStart;
-    System.out.println("Time to encode xor = " + encodeMSecs +
-      " msec (" + message[0].length / (1000 * encodeMSecs) + "MB/s)");
-
-    byte[] copy = new byte[bufsize];
-    for (int j = 0; j < bufsize; j++) {
-      copy[j] = message[0][j];
-      message[0][j] = 0;
-    }
-
-    long decodeStart = Time.now();
-    for (int i = 0; i < bufsize; i++) {
-      for (int j = 1; j < stripeSize; j++) message[0][i] ^= message[j][i];
-      message[0][i] ^= parity[i];
-    }
-    long decodeEnd = Time.now();
-    float decodeMSecs = decodeEnd - decodeStart;
-    System.out.println("Time to decode xor = " + decodeMSecs +
-      " msec (" + message[0].length / (1000 * decodeMSecs) + "MB/s)");
-    assertTrue("Decode failed", java.util.Arrays.equals(copy, message[0]));
-  }
-
-  @Test
-  public void testComputeErrorLocations() {
-    for (int i = 0; i < TEST_TIMES; ++i) {
-      verifyErrorLocations(10, 4, 1);
-      verifyErrorLocations(10, 4, 2);
-    }
-  }
-
-  public void verifyErrorLocations(int stripeSize, int paritySize, int errors) {
-    int[] message = new int[stripeSize];
-    int[] parity = new int[paritySize];
-    Set<Integer> errorLocations = new HashSet<Integer>();
-    for (int i = 0; i < message.length; ++i) {
-      message[i] = RAND.nextInt(256);
-    }
-    while (errorLocations.size() < errors) {
-      int loc = RAND.nextInt(stripeSize + paritySize);
-      errorLocations.add(loc);
-    }
-    ReedSolomonCode codec = new ReedSolomonCode(stripeSize, paritySize);
-    codec.encode(message, parity);
-    int[] data = combineArrays(parity, message);
-    for (Integer i : errorLocations) {
-      data[i] = randError(data[i]);
-    }
-    Set<Integer> recoveredLocations = new HashSet<Integer>();
-    boolean resolved = codec.computeErrorLocations(data, recoveredLocations);
-    if (resolved) {
-      assertEquals(errorLocations, recoveredLocations);
-    }
-  }
-
-  private int randError(int actual) {
-    while (true) {
-      int r = RAND.nextInt(256);
-      if (r != actual) {
-        return r;
-      }
-    }
-  }
-
-  private int[] combineArrays(int[] array1, int[] array2) {
-    int[] result = new int[array1.length + array2.length];
-    for (int i = 0; i < array1.length; ++i) {
-      result[i] = array1[i];
-    }
-    for (int i = 0; i < array2.length; ++i) {
-      result[i + array1.length] = array2[i];
-    }
-    return result;
-  }
-
-  private int[] randomErasedLocation(int erasedLen, int dataLen) {
-    int[] erasedLocations = new int[erasedLen];
-    for (int i = 0; i < erasedLen; i++) {
-      Set<Integer> s = new HashSet<Integer>();
-      while (s.size() != erasedLen) {
-        s.add(RAND.nextInt(dataLen));
-      }
-      int t = 0;
-      for (int erased : s) {
-        erasedLocations[t++] = erased;
-      }
-    }
-    return erasedLocations;
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestGaloisField.java

@@ -1,190 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-
-import org.junit.Test;
-
-public class TestGaloisField {
-
-  final int TEST_TIMES = 10000;
-  final Random RAND = new Random();
-  final static GaloisField GF = GaloisField.getInstance();
-
-  private int randGF() {
-    return 0x000000FF & RAND.nextInt(GF.getFieldSize());
-  }
-  private int[] randGFPoly(int len) {
-    int[] result = new int[len];
-    for (int i = 0; i < len; i++) {
-      result[i] = randGF();
-    }
-    return result;
-  }
-
-  @Test
-  public void testGetInstance() {
-    GaloisField gf1 = GaloisField.getInstance(256, 285);
-    GaloisField gf2 = GaloisField.getInstance();
-    GaloisField gf3 = GaloisField.getInstance(128, 137);
-    GaloisField gf4 = GaloisField.getInstance(128, 137);
-    GaloisField gf5 = GaloisField.getInstance(512, 529);
-    GaloisField gf6 = GaloisField.getInstance(512, 529);
-    assertTrue(gf1 == gf2);
-    assertTrue(gf3 == gf4);
-    assertTrue(gf5 == gf6);
-  }
-
-  @Test
-  public void testDistributivity() {
-    for (int i = 0; i < TEST_TIMES; i++) {
-      int a = RAND.nextInt(GF.getFieldSize());
-      int b = RAND.nextInt(GF.getFieldSize());
-      int c = RAND.nextInt(GF.getFieldSize());
-      int result1 = GF.multiply(a, GF.add(b, c));
-      int result2 = GF.add(GF.multiply(a, b), GF.multiply(a, c));
-      assertTrue("Distributivity test #" + i + " failed: " + a + ", " + b + ", "
-                 + c, result1 == result2);
-    }
-  }
-
-  @Test
-  public void testDevision() {
-    for (int i = 0; i < TEST_TIMES; i++) {
-      int a = RAND.nextInt(GF.getFieldSize());
-      int b = RAND.nextInt(GF.getFieldSize());
-      if (b == 0) {
-        continue;
-      }
-      int c = GF.divide(a, b);
-      assertTrue("Division test #" + i + " failed: " + a + "/" + b + " = " + c,
-                 a == GF.multiply(c, b));
-    }
-  }
-
-  @Test
-  public void testPower() {
-    for (int i = 0; i < TEST_TIMES; i++) {
-      int a = randGF();
-      int n = RAND.nextInt(10);
-      int result1 = GF.power(a, n);
-      int result2 = 1;
-      for (int j = 0; j < n; j++) {
-        result2 = GF.multiply(result2, a);
-      }
-      assert(result1 == result2);
-    }
-  }
-
-  @Test
-  public void testPolynomialDistributivity() {
-    final int TEST_LEN = 15;
-    for (int i = 0; i < TEST_TIMES; i++) {
-      int[] a = randGFPoly(RAND.nextInt(TEST_LEN - 1) + 1);
-      int[] b = randGFPoly(RAND.nextInt(TEST_LEN - 1) + 1);
-      int[] c = randGFPoly(RAND.nextInt(TEST_LEN - 1) + 1);
-      int[] result1 = GF.multiply(a, GF.add(b, c));
-      int[] result2 = GF.add(GF.multiply(a, b), GF.multiply(a, c));
-      assertTrue("Distributivity test on polynomials failed",
-                 java.util.Arrays.equals(result1, result2));
-    }
-  }
-
-  @Test
-  public void testSubstitute() {
-    final int TEST_LEN = 15;
-    for (int i = 0; i < TEST_TIMES; i++) {
-      int[] a = randGFPoly(RAND.nextInt(TEST_LEN - 1) + 1);
-      int[] b = randGFPoly(RAND.nextInt(TEST_LEN - 1) + 1);
-      int[] c = randGFPoly(RAND.nextInt(TEST_LEN - 1) + 1);
-      int x = randGF();
-      // (a * b * c)(x)
-      int result1 = GF.substitute(GF.multiply(GF.multiply(a, b), c), x);
-      // a(x) * b(x) * c(x)
-      int result2 =
-        GF.multiply(GF.multiply(GF.substitute(a, x), GF.substitute(b, x)),
-                    GF.substitute(c, x));
-      assertTrue("Substitute test on polynomial failed",
-                 result1 == result2);
-    }
-  }
-
-  @Test
-  public void testSolveVandermondeSystem() {
-    final int TEST_LEN = 15;
-    for (int i = 0; i < TEST_TIMES; i++) {
-      int[] z = randGFPoly(RAND.nextInt(TEST_LEN - 1) + 1);
-      // generate distinct values for x
-      int[] x = new int[z.length];
-      Set<Integer> s = new HashSet<Integer>();
-      while (s.size() != z.length) {
-        s.add(randGF());
-      }
-      int t = 0;
-      for (int v : s) {
-        x[t++] = v;
-      }
-      // compute the output for the Vandermonde system
-      int[] y = new int[x.length];
-      for (int j = 0; j < x.length; j++) {
-        y[j] = 0;
-        for (int k = 0; k < x.length; k++) {
-          //y[j] = y[j] + z[k] * pow(x[k], j);
-          y[j] = GF.add(y[j], GF.multiply(GF.power(x[k], j), z[k]));
-        }
-      }
-
-      GF.solveVandermondeSystem(x, y);
-      assertTrue("Solving Vandermonde system failed",
-                 java.util.Arrays.equals(y, z));
-    }
-  }
-
-  @Test
-  public void testRemainder() {
-    final int TEST_LEN = 15;
-    for (int i = 0; i < TEST_TIMES; i++) {
-      int[] quotient = null;
-      int[] divisor = null;
-      int[] remainder = null;
-      int[] dividend = null;
-      while (true) {
-        quotient = randGFPoly(RAND.nextInt(TEST_LEN - 3) + 3);
-        divisor = randGFPoly(RAND.nextInt(quotient.length - 2) + 2);
-        remainder = randGFPoly(RAND.nextInt(divisor.length - 1) + 1);
-        dividend = GF.add(remainder, GF.multiply(quotient, divisor));
-        if (quotient[quotient.length - 1] != 0 &&
-            divisor[divisor.length - 1] != 0 &&
-            remainder[remainder.length - 1] != 0) {
-          // make sure all the leading terms are not zero
-          break;
-        }
-      }
-      GF.remainder(dividend, divisor);
-      for (int j = 0; j < remainder.length; j++) {
-        assertTrue("Distributivity test on polynomials failed",
-                   dividend[j] == remainder[j]);
-      }
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestHarIndexParser.java

@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.UnsupportedEncodingException;
-import java.nio.charset.Charset;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestHarIndexParser {
-  final static Log LOG = LogFactory.getLog(TestHarIndexParser.class);
-  File indexFile = null;
-
-  @Before
-  public void setUp() throws FileNotFoundException, IOException {
-    LOG.info("TestHarIndexParser.setUp()");
-    indexFile = File.createTempFile("harindex", ".tmp");
-    indexFile.deleteOnExit();
-    OutputStreamWriter out = new OutputStreamWriter(
-      new FileOutputStream(indexFile),
-      Charset.forName("UTF-8"));
-    out.write("%2F dir 1282018162460+0+493+hadoop+hadoop 0 0 f1 f2 f3 f4\n");
-    out.write("%2Ff1 file part-0 0 1024 1282018141145+1282018140822+420+hadoop+hadoop\n");
-    out.write("%2Ff3 file part-0 2048 1024 1282018148590+1282018148255+420+hadoop+hadoop\n");
-    out.write("%2Ff2 file part-0 1024 1024 1282018144198+1282018143852+420+hadoop+hadoop\n");
-    out.write("%2Ff4 file part-1 0 1024000 1282018162959+1282018162460+420+hadoop+hadoop\n");
-    out.flush();
-    out.close();
-  }
-
-  @After
-  public void tearDown() {
-    LOG.info("TestHarIndexParser.tearDown()");
-    if (indexFile != null)
-      indexFile.delete();
-  }
-
-  @Test
-  public void testHarIndexParser()
-    throws UnsupportedEncodingException, IOException {
-    LOG.info("testHarIndexParser started.");
-    InputStream in = new FileInputStream(indexFile);
-    long size = indexFile.length();
-    HarIndex parser = new HarIndex(in, size);
-
-    HarIndex.IndexEntry entry = parser.findEntry("part-0", 2100);
-    assertEquals("/f3", entry.fileName);
-
-    LOG.info("testHarIndexParser finished.");
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidFilter.java

@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.raid.protocol.PolicyInfo;
-import org.apache.hadoop.util.Time;
-import org.junit.Test;
-
-public class TestRaidFilter {
-  final static String TEST_DIR = new File(System.getProperty("test.build.data",
-      "target/test-data")).getAbsolutePath();
-  final static Log LOG =
-    LogFactory.getLog("org.apache.hadoop.raid.TestRaidFilter");
-
-  Configuration conf;
-  MiniDFSCluster dfs = null;
-  FileSystem fs = null;
-
-  private void mySetup() throws Exception {
-    new File(TEST_DIR).mkdirs(); // Make sure data directory exists
-    conf = new Configuration();
-    dfs = new MiniDFSCluster(conf, 2, true, null);
-    dfs.waitActive();
-    fs = dfs.getFileSystem();
-    String namenode = fs.getUri().toString();
-    FileSystem.setDefaultUri(conf, namenode);
-  }
-
-  private void myTearDown() throws Exception {
-    if (dfs != null) { dfs.shutdown(); }
-  }
-
-  @Test
-  public void testLayeredPolicies() throws Exception {
-    mySetup();
-    Path src1 = new Path("/user/foo");
-    Path src2 = new Path("/user/foo/bar");
-
-    PolicyInfo info1 = new PolicyInfo("p1", conf);
-    info1.setSrcPath(src1.toString());
-    info1.setErasureCode("xor");
-    info1.setDescription("test policy");
-    info1.setProperty("targetReplication", "1");
-    info1.setProperty("metaReplication", "1");
-    info1.setProperty("modTimePeriod", "0");
-
-    PolicyInfo info2 = new PolicyInfo("p2", conf);
-    info2.setSrcPath(src2.toString());
-    info2.setErasureCode("xor");
-    info2.setDescription("test policy");
-    info2.setProperty("targetReplication", "1");
-    info2.setProperty("metaReplication", "1");
-    info2.setProperty("modTimePeriod", "0");
-
-    ArrayList<PolicyInfo> all = new ArrayList<PolicyInfo>();
-    all.add(info1);
-    all.add(info2);
-
-    try {
-      long blockSize = 1024;
-      byte[] bytes = new byte[(int)blockSize];
-      Path f1 = new Path(src1, "f1");
-      Path f2 = new Path(src2, "f2");
-      FSDataOutputStream stm1 = fs.create(f1, false, 4096, (short)1, blockSize);
-      FSDataOutputStream stm2 = fs.create(f2, false, 4096, (short)1, blockSize);
-      FSDataOutputStream[]  stms = new FSDataOutputStream[]{stm1, stm2};
-      for (FSDataOutputStream stm: stms) {
-        stm.write(bytes);
-        stm.write(bytes);
-        stm.write(bytes);
-        stm.close();
-      }
-
-      Thread.sleep(1000);
-
-      FileStatus stat1 = fs.getFileStatus(f1);
-      FileStatus stat2 = fs.getFileStatus(f2);
-
-      RaidFilter.Statistics stats = new RaidFilter.Statistics();
-      RaidFilter.TimeBasedFilter filter = new RaidFilter.TimeBasedFilter(
-        conf, RaidNode.xorDestinationPath(conf), info1, all,
-        Time.now(), stats);
-      System.out.println("Stats " + stats);
-
-      assertTrue(filter.check(stat1));
-      assertFalse(filter.check(stat2));
-
-    } finally {
-      myTearDown();
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidHar.java

@@ -1,315 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileWriter;
-import java.util.Random;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MiniMRCluster;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.log4j.Level;
-import org.junit.Test;
-
-/**
- * If a file gets deleted, then verify that the parity file gets deleted too.
- */
-public class TestRaidHar {
-  final static String TEST_DIR = new File(System.getProperty("test.build.data",
-     "target/test-data")).getAbsolutePath();
-  final static String CONFIG_FILE = new File(TEST_DIR, 
-      "test-raid.xml").getAbsolutePath();
-  final static long RELOAD_INTERVAL = 1000;
-  final static Log LOG = LogFactory.getLog("org.apache.hadoop.raid.TestRaidNode");
-  final Random rand = new Random();
-
-  {
-    ((Log4JLogger)RaidNode.LOG).getLogger().setLevel(Level.ALL);
-  }
-
-
-  Configuration conf;
-  String namenode = null;
-  String hftp = null;
-  MiniDFSCluster dfs = null;
-  MiniMRCluster mr = null;
-  FileSystem fileSys = null;
-  String jobTrackerName = null;
-
-  /**
-   * create mapreduce and dfs clusters
-   */
-  private void createClusters(boolean local) throws Exception {
-
-    new File(TEST_DIR).mkdirs(); // Make sure data directory exists
-    conf = new Configuration();
-    conf.set("raid.config.file", CONFIG_FILE);
-    conf.setBoolean("raid.config.reload", true);
-    conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL);
-
-    // scan all policies once every 5 second
-    conf.setLong("raid.policy.rescan.interval", 5000);
-
-    // make all deletions not go through Trash
-    conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient");
-
-    // the RaidNode does the raiding inline (instead of submitting to map/reduce)
-    if (local) {
-      conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode");
-    } else {
-      conf.set("raid.classname", "org.apache.hadoop.raid.DistRaidNode");
-    }
-
-    conf.set("raid.server.address", "localhost:0");
-    conf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
-
-    // create a dfs and map-reduce cluster
-    final int taskTrackers = 4;
-
-    dfs = new MiniDFSCluster(conf, 3, true, null);
-    dfs.waitActive();
-    fileSys = dfs.getFileSystem();
-    namenode = fileSys.getUri().toString();
-    mr = new MiniMRCluster(taskTrackers, namenode, 3);
-    JobConf jobConf = mr.createJobConf();
-    jobTrackerName = "localhost:" + jobConf.get(JTConfig.JT_IPC_ADDRESS);
-    hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort();
-
-    FileSystem.setDefaultUri(conf, namenode);
-    conf.set("mapred.job.tracker", jobTrackerName);
-    conf.set("mapreduce.framework.name", "yarn");
-    String rmAdress = jobConf.get("yarn.resourcemanager.address");
-    if (rmAdress != null) {
-      conf.set("yarn.resourcemanager.address", rmAdress);
-    }
-    String schedulerAdress =
-      jobConf.get("yarn.resourcemanager.scheduler.address");
-    if (schedulerAdress != null) {
-      conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress);
-    }
-    String jobHistoryAddress = 
-        jobConf.get("mapreduce.jobhistory.address");
-    if (jobHistoryAddress != null) {
-      conf.set("mapreduce.jobhistory.address", jobHistoryAddress);
-    }
-  }
-    
-  /**
-   * create raid.xml file for RaidNode
-   */
-  private void mySetup(long targetReplication,
-                long metaReplication, long stripeLength) throws Exception {
-    FileWriter fileWriter = new FileWriter(CONFIG_FILE);
-    fileWriter.write("<?xml version=\"1.0\"?>\n");
-    String str = "<configuration> " +
-                   "<srcPath prefix=\"/user/test/raidtest\"> " +
-                     "<policy name = \"RaidTest1\"> " +
-                        "<erasureCode>xor</erasureCode> " +
-                        "<property> " +
-                          "<name>targetReplication</name> " +
-                          "<value>" + targetReplication + "</value> " +
-                          "<description>after RAIDing, decrease the replication factor of a file to this value." +
-                          "</description> " + 
-                        "</property> " +
-                        "<property> " +
-                          "<name>metaReplication</name> " +
-                          "<value>" + metaReplication + "</value> " +
-                          "<description> replication factor of parity file" +
-                          "</description> " + 
-                        "</property> " +
-                        "<property> " +
-                          "<name>stripeLength</name> " +
-                          "<value>" + stripeLength + "</value> " +
-                          "<description> the max number of blocks in a file to RAID together " +
-                          "</description> " + 
-                        "</property> " +
-                        "<property> " +
-                          "<name>time_before_har</name> " +
-                          "<value>0</value> " +
-                          "<description> amount of time waited before har'ing parity files" +
-                          "</description> " + 
-                        "</property> " +
-                        "<property> " +
-                          "<name>modTimePeriod</name> " +
-                          "<value>2000</value> " + 
-                          "<description> time (milliseconds) after a file is modified to make it " +
-                                         "a candidate for RAIDing " +
-                          "</description> " + 
-                        "</property> " +
-                     "</policy>" +
-                   "</srcPath>" +
-                 "</configuration>";
-    fileWriter.write(str);
-    fileWriter.close();
-  }
-
-  /**
-   * stop clusters created earlier
-   */
-  private void stopClusters() throws Exception {
-    if (mr != null) { mr.shutdown(); }
-    if (dfs != null) { dfs.shutdown(); }
-  }
-
-  /**
-   * Test that parity files that do not have an associated master file
-   * get deleted.
-   */
-  @Test
-  public void testRaidHar() throws Exception {
-    LOG.info("Test testRaidHar  started.");
-
-    long blockSizes    []  = {1024L};
-    long stripeLengths []  = {5};
-    long targetReplication = 1;
-    long metaReplication   = 1;
-    int  numBlock          = 9;
-    int  iter = 0;
-
-    createClusters(true);
-    try {
-      for (long blockSize : blockSizes) {
-        for (long stripeLength : stripeLengths) {
-           doTestHar(iter, targetReplication, metaReplication,
-                       stripeLength, blockSize, numBlock);
-           iter++;
-        }
-      }
-    } finally {
-      stopClusters();
-    }
-    LOG.info("Test testRaidHar completed.");
-  }
-
-  /**
-   * Create parity file, delete original file and then validate that
-   * parity file is automatically deleted.
-   */
-  private void doTestHar(int iter, long targetReplication,
-                          long metaReplication, long stripeLength,
-                          long blockSize, int numBlock) throws Exception {
-    LOG.info("doTestHar started---------------------------:" +  " iter " + iter +
-             " blockSize=" + blockSize + " stripeLength=" + stripeLength);
-    mySetup(targetReplication, metaReplication, stripeLength);
-    Path dir = new Path("/user/test/raidtest/subdir/");
-    Path file1 = new Path(dir + "/file" + iter);
-    RaidNode cnode = null;
-    try {
-      Path destPath = new Path("/destraid/user/test/raidtest/subdir");
-      fileSys.delete(dir, true);
-      fileSys.delete(destPath, true);
-      for (int i = 0; i < 10; i++) {
-        Path file = new Path(dir + "/file" + i);
-        TestRaidNode.createOldFile(fileSys, file, 1, numBlock, blockSize);
-      }
-      LOG.info("doTestHar created test files for iteration " + iter);
-
-      // create an instance of the RaidNode
-      Configuration localConf = new Configuration(conf);
-      localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
-      cnode = RaidNode.createRaidNode(null, localConf);
-      FileStatus[] listPaths = null;
-
-      int maxFilesFound = 0;
-      // wait till file is raided
-      while (true) {
-        try {
-          listPaths = fileSys.listStatus(destPath);
-          int count = 0;
-          Path harPath = null;
-          int filesFound = 0;
-          if (listPaths != null) {
-            for (FileStatus s : listPaths) {
-              LOG.info("doTestHar found path " + s.getPath());
-
-              if (!s.isDir())
-                filesFound++;
-              if (filesFound > maxFilesFound)
-                maxFilesFound = filesFound;
-
-              if (s.getPath().toString().endsWith(".har")) {
-                // If a HAR directory is found, ensure that we have seen
-                // 10 parity files. We have to keep track of the max # of
-                // files since some parity files might get deleted by the
-                // purge thread.
-                assertEquals(10, maxFilesFound);
-                harPath = s.getPath();
-                count++;
-              }
-            }
-          }
-          if (count == 1  && listPaths.length == 1) {
-            Path partfile = new Path(harPath, "part-0");
-            assertEquals(fileSys.getFileStatus(partfile).getReplication(),
-              targetReplication);
-            break;
-          }
-        } catch (FileNotFoundException e) {
-          //ignore
-        }
-        LOG.info("doTestHar waiting for files to be raided and parity files to be har'ed and deleted. Found " +
-                 (listPaths == null ? "none" : listPaths.length));
-        Thread.sleep(1000);                  // keep waiting
-
-      }
-
-      fileSys.delete(dir, true);
-      // wait till raid file is deleted
-      int count = 1;
-      while (count > 0) {
-        count = 0;
-        try {
-          listPaths = fileSys.listStatus(destPath);
-          if (listPaths != null) {
-            for (FileStatus s : listPaths) {
-              LOG.info("doTestHar found path " + s.getPath());
-              if (s.getPath().toString().endsWith(".har")) {
-                count++;
-              }
-            }
-          }
-        } catch (FileNotFoundException e) { } //ignoring
-        LOG.info("doTestHar waiting for har file to be deleted. Found " + 
-                (listPaths == null ? "none" : listPaths.length) + " files");
-        Thread.sleep(1000);
-      }
-      
-    } catch (Exception e) {
-      LOG.info("doTestHar Exception " + e +
-                                          StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      if (cnode != null) { cnode.stop(); cnode.join(); }
-    }
-    LOG.info("doTestHar completed:" + " blockSize=" + blockSize +
-             " stripeLength=" + stripeLength);
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidNode.java

@@ -1,738 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.List;
-import java.util.Random;
-import java.util.zip.CRC32;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobContext;
-import org.apache.hadoop.mapred.MiniMRCluster;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.raid.protocol.PolicyInfo;
-import org.apache.hadoop.raid.protocol.PolicyList;
-import org.apache.hadoop.util.JarFinder;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Time;
-import org.junit.Test;
-
-/**
-  * Test the generation of parity blocks for files with different block
-  * sizes. Also test that a data block can be regenerated from a raid stripe
-  * using the parity block
-  */
-public class TestRaidNode {
-  final static String TEST_DIR = new File(System.getProperty("test.build.data",
-      "target/test-data")).getAbsolutePath();
-  public static final String DistRaid_JAR = JarFinder.getJar(DistRaid.class);
-  final static String CONFIG_FILE = new File(TEST_DIR, 
-      "test-raid.xml").getAbsolutePath();
-  final static long RELOAD_INTERVAL = 1000;
-  final static Log LOG = LogFactory.getLog("org.apache.hadoop.raid.TestRaidNode");
-  final static Random rand = new Random();
-
-  Configuration conf;
-  String namenode = null;
-  String hftp = null;
-  MiniDFSCluster dfs = null;
-  MiniMRCluster mr = null;
-  FileSystem fileSys = null;
-  String jobTrackerName = null;
-
-  /**
-   * create mapreduce and dfs clusters
-   */
-  private void createClusters(boolean local) throws Exception {
-
-    new File(TEST_DIR).mkdirs(); // Make sure data directory exists
-    conf = new Configuration();
-    conf.set("raid.config.file", CONFIG_FILE);
-    conf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
-    conf.setBoolean("raid.config.reload", true);
-    conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL);
-    conf.setBoolean("dfs.permissions.enabled", true);
-    conf.setLong(JobMonitor.JOBMONITOR_INTERVAL_KEY, 20000);
-    conf.setLong(RaidNode.TRIGGER_MONITOR_SLEEP_TIME_KEY, 3000L);
-
-    // scan all policies once every 5 second
-    conf.setLong("raid.policy.rescan.interval", 5000);
-
-    // make all deletions not go through Trash
-    conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient");
-
-    // the RaidNode does the raiding inline (instead of submitting to map/reduce)
-    if (local) {
-      conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode");
-    } else {
-      conf.set("raid.classname", "org.apache.hadoop.raid.DistRaidNode");
-    }
-
-    conf.set("raid.server.address", "localhost:0");
-
-    // create a dfs and map-reduce cluster
-    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
-    builder.numDataNodes(6);
-    builder.format(true);
-    dfs = builder.build();
-    dfs.waitActive();
-    fileSys = dfs.getFileSystem();
-    
-    namenode = fileSys.getUri().toString();
-    final int taskTrackers = 4;
-    mr = new MiniMRCluster(taskTrackers, namenode, 3);
-    JobConf jobConf = mr.createJobConf();
-    jobTrackerName = "localhost:" + jobConf.get(JTConfig.JT_IPC_ADDRESS);
-    hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort();
-
-    FileSystem.setDefaultUri(conf, namenode);
-    conf.set("mapred.job.tracker", jobTrackerName);
-    conf.set("mapreduce.framework.name", "yarn");
-    String rmAdress = jobConf.get("yarn.resourcemanager.address");
-    if (rmAdress != null) {
-      conf.set("yarn.resourcemanager.address", rmAdress);
-    }
-    String schedulerAdress =
-      jobConf.get("yarn.resourcemanager.scheduler.address");
-    if (schedulerAdress != null) {
-      conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress);
-    }
-    String jobHistoryAddress = 
-        jobConf.get("mapreduce.jobhistory.address");
-    if (jobHistoryAddress != null) {
-      conf.set("mapreduce.jobhistory.address", jobHistoryAddress);
-    }
-  }
-
-  class ConfigBuilder {
-    private List<String> policies;
-
-    public ConfigBuilder() {
-      policies = new java.util.ArrayList<String>();
-    }
-
-    public void addPolicy(String name, String path, String parent) {
-      String str =
-        "<srcPath prefix=\"" + path + "\"> " +
-          "<policy name = \"" + name + "\"> " +
-             "<parentPolicy>" + parent + "</parentPolicy>" +
-          "</policy>" +
-        "</srcPath>";
-      policies.add(str);
-    }
-
-    public void addPolicy(String name, short srcReplication,
-                          long targetReplication, long metaReplication, long stripeLength) {
-      String str =
-        "<srcPath prefix=\"/user/dhruba/raidtest\"> " +
-          "<policy name = \"" + name + "\"> " +
-             "<erasureCode>xor</erasureCode> " +
-             "<property> " +
-               "<name>srcReplication</name> " +
-               "<value>" + srcReplication + "</value> " +
-               "<description> pick only files whole replFactor is greater than or equal to " +
-               "</description> " +
-             "</property> " +
-             "<property> " +
-               "<name>targetReplication</name> " +
-               "<value>" + targetReplication + "</value> " +
-               "<description>after RAIDing, decrease the replication factor of a file to this value." +
-               "</description> " +
-             "</property> " +
-             "<property> " +
-               "<name>metaReplication</name> " +
-               "<value>" + metaReplication + "</value> " +
-               "<description> replication factor of parity file" +
-               "</description> " +
-             "</property> " +
-             "<property> " +
-               "<name>stripeLength</name> " +
-               "<value>" + stripeLength + "</value> " +
-               "<description> the max number of blocks in a file to RAID together " +
-               "</description> " +
-             "</property> " +
-             "<property> " +
-               "<name>modTimePeriod</name> " +
-               "<value>2000</value> " +
-               "<description> time (milliseconds) after a file is modified to make it " +
-                              "a candidate for RAIDing " +
-               "</description> " +
-             "</property> " +
-          "</policy>" +
-        "</srcPath>";
-      policies.add(str);
-    }
-
-    public void addPolicy(String name, String path, short srcReplication,
-                          long targetReplication, long metaReplication, long stripeLength) {
-      String str =
-        "<srcPath prefix=\"" + path + "\"> " +
-          "<policy name = \"" + name + "\"> " +
-             "<erasureCode>xor</erasureCode> " +
-             "<property> " +
-               "<name>srcReplication</name> " +
-               "<value>" + srcReplication + "</value> " +
-               "<description> pick only files whole replFactor is greater than or equal to " +
-               "</description> " + 
-             "</property> " +
-             "<property> " +
-               "<name>targetReplication</name> " +
-               "<value>" + targetReplication + "</value> " +
-               "<description>after RAIDing, decrease the replication factor of a file to this value." +
-               "</description> " + 
-             "</property> " +
-             "<property> " +
-               "<name>metaReplication</name> " +
-               "<value>" + metaReplication + "</value> " +
-               "<description> replication factor of parity file" +
-               "</description> " + 
-             "</property> " +
-             "<property> " +
-               "<name>stripeLength</name> " +
-               "<value>" + stripeLength + "</value> " +
-               "<description> the max number of blocks in a file to RAID together " +
-               "</description> " + 
-             "</property> " +
-             "<property> " +
-               "<name>modTimePeriod</name> " +
-               "<value>2000</value> " + 
-               "<description> time (milliseconds) after a file is modified to make it " +
-                              "a candidate for RAIDing " +
-               "</description> " + 
-             "</property> " +
-          "</policy>" +
-        "</srcPath>";
-      policies.add(str);
-    }
-
-    public void persist() throws IOException {
-      FileWriter fileWriter = new FileWriter(CONFIG_FILE);
-      fileWriter.write("<?xml version=\"1.0\"?>\n");
-      fileWriter.write("<configuration>");
-      for (String policy: policies) {
-        fileWriter.write(policy);
-      }
-      fileWriter.write("</configuration>");
-      fileWriter.close();
-    }
-  }
-    
-  /**
-   * stop clusters created earlier
-   */
-  private void stopClusters() throws Exception {
-    if (mr != null) { mr.shutdown(); }
-    if (dfs != null) { dfs.shutdown(); }
-  }
-
-  /**
-   * Test to run a filter
-   */
-  @Test
-  public void testPathFilter() throws Exception {
-    LOG.info("Test testPathFilter started.");
-
-    long blockSizes    []  = {1024L};
-    int stripeLengths []  = {5, 6, 10, 11, 12};
-    int targetReplication = 1;
-    int metaReplication   = 1;
-    int  numBlock          = 11;
-    int  iter = 0;
-
-    createClusters(true);
-    try {
-      for (long blockSize : blockSizes) {
-        for (long stripeLength : stripeLengths) {
-           doTestPathFilter(iter, targetReplication, metaReplication,
-                                              stripeLength, blockSize, numBlock);
-           iter++;
-        }
-      }
-      doCheckPolicy();
-    } finally {
-      stopClusters();
-    }
-    LOG.info("Test testPathFilter completed.");
-  }
-
-  /**
-   * Test to run a filter
-   */
-  private void doTestPathFilter(int iter, long targetReplication,
-                          long metaReplication, long stripeLength,
-                          long blockSize, int numBlock) throws Exception {
-    LOG.info("doTestPathFilter started---------------------------:" +  " iter " + iter +
-             " blockSize=" + blockSize + " stripeLength=" + stripeLength);
-    ConfigBuilder cb = new ConfigBuilder();
-    cb.addPolicy("policy1", "/user/dhruba/raidtest", (short)1, targetReplication, metaReplication, stripeLength);
-    cb.persist();
-
-    RaidShell shell = null;
-    Path dir = new Path("/user/dhruba/raidtest/");
-    Path file1 = new Path(dir + "/file" + iter);
-    RaidNode cnode = null;
-    try {
-      Path destPath = new Path("/destraid/user/dhruba/raidtest");
-      fileSys.delete(dir, true);
-      fileSys.delete(destPath, true);
-      long crc1 = createOldFile(fileSys, file1, 1, numBlock, blockSize);
-      LOG.info("doTestPathFilter created test files for iteration " + iter);
-
-      // create an instance of the RaidNode
-      Configuration localConf = new Configuration(conf);
-      cnode = RaidNode.createRaidNode(null, localConf);
-      FileStatus[] listPaths = null;
-
-      // wait till file is raided
-      while (true) {
-        try {
-          listPaths = fileSys.listStatus(destPath);
-          int count = 0;
-          if (listPaths != null && listPaths.length == 1) {
-            for (FileStatus s : listPaths) {
-              LOG.info("doTestPathFilter found path " + s.getPath());
-              if (!s.getPath().toString().endsWith(".tmp") &&
-                  fileSys.getFileStatus(file1).getReplication() ==
-                  targetReplication) {
-                count++;
-              }
-            }
-          }
-          if (count > 0) {
-            break;
-          }
-        } catch (FileNotFoundException e) {
-          //ignore
-        }
-        LOG.info("doTestPathFilter waiting for files to be raided. Found " + 
-                 (listPaths == null ? "none" : listPaths.length));
-        Thread.sleep(1000);                  // keep waiting
-      }
-      // assertEquals(listPaths.length, 1); // all files raided
-      LOG.info("doTestPathFilter all files found in Raid.");
-
-      // check for error at beginning of file
-      shell = new RaidShell(conf);
-      shell.initializeRpc(conf, cnode.getListenerAddress());
-      if (numBlock >= 1) {
-        LOG.info("doTestPathFilter Check error at beginning of file.");
-        simulateError(shell, fileSys, file1, crc1, 0);
-      }
-
-      // check for error at the beginning of second block
-      if (numBlock >= 2) {
-        LOG.info("doTestPathFilter Check error at beginning of second block.");
-        simulateError(shell, fileSys, file1, crc1, blockSize + 1);
-      }
-
-      // check for error at the middle of third block
-      if (numBlock >= 3) {
-        LOG.info("doTestPathFilter Check error at middle of third block.");
-        simulateError(shell, fileSys, file1, crc1, 2 * blockSize + 10);
-      }
-
-      // check for error at the middle of second stripe
-      if (numBlock >= stripeLength + 1) {
-        LOG.info("doTestPathFilter Check error at middle of second stripe.");
-        simulateError(shell, fileSys, file1, crc1,
-                                            stripeLength * blockSize + 100);
-      }
-
-    } catch (Exception e) {
-      LOG.info("doTestPathFilter Exception " + e +
-                                          StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      if (shell != null) shell.close();
-      if (cnode != null) { cnode.stop(); cnode.join(); }
-      LOG.info("doTestPathFilter delete file " + file1);
-      fileSys.delete(file1, true);
-    }
-    LOG.info("doTestPathFilter completed:" + " blockSize=" + blockSize +
-                                             " stripeLength=" + stripeLength);
-  }
-
-  // Check that raid occurs only on files that have a replication factor
-  // greater than or equal to the specified value
-  private void doCheckPolicy() throws Exception {
-    LOG.info("doCheckPolicy started---------------------------:"); 
-    short srcReplication = 1;
-    long targetReplication = 2;
-    long metaReplication = 1;
-    long stripeLength = 2;
-    long blockSize = 1024;
-    int numBlock = 3;
-    ConfigBuilder cb = new ConfigBuilder();
-    cb.addPolicy("policy1", "/user/dhruba/policytest", srcReplication,
-        targetReplication, metaReplication, stripeLength);
-    cb.persist();
-    Path dir = new Path("/user/dhruba/policytest/");
-    Path file1 = new Path(dir + "/file1");
-    Path file2 = new Path(dir + "/file2");
-    RaidNode cnode = null;
-    try {
-      Path destPath = new Path("/destraid/user/dhruba/policytest");
-      fileSys.delete(dir, true);
-      fileSys.delete(destPath, true);
-
-      // create an instance of the RaidNode
-      Configuration localConf = new Configuration(conf);
-      localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
-      cnode = RaidNode.createRaidNode(null, localConf);
-
-      // this file should be picked up RaidNode
-      createOldFile(fileSys, file2, 2, numBlock, blockSize);
-      FileStatus[] listPaths = null;
-
-      long firstmodtime = 0;
-      // wait till file is raided
-      while (true) {
-        Thread.sleep(1000);                  // waiting
-        try {
-          listPaths = fileSys.listStatus(destPath);
-        } catch (FileNotFoundException e) {
-          LOG.warn("File not found " + destPath);
-          // The directory have been deleted by the purge thread.
-          continue;
-        }
-        int count = 0;
-        if (listPaths != null && listPaths.length == 1) {
-          for (FileStatus s : listPaths) {
-            LOG.info("doCheckPolicy found path " + s.getPath());
-            if (!s.getPath().toString().endsWith(".tmp") &&
-                fileSys.getFileStatus(file2).getReplication() ==
-                targetReplication) {
-              count++;
-              firstmodtime = s.getModificationTime();
-            }
-          }
-        }
-        if (count > 0) {
-          break;
-        }
-        LOG.info("doCheckPolicy waiting for files to be raided. Found " + 
-                 (listPaths == null ? "none" : listPaths.length));
-      }
-      assertEquals(listPaths.length, 1);
-
-      LOG.info("doCheckPolicy all files found in Raid the first time.");
-
-      LOG.info("doCheckPolicy: recreating source file");
-      createOldFile(fileSys, file2, 2, numBlock, blockSize);
-
-      FileStatus st = fileSys.getFileStatus(file2);
-      assertTrue(st.getModificationTime() > firstmodtime);
-      
-      // wait till file is raided
-      while (true) {
-        Thread.sleep(20000L);                  // waiting
-        listPaths = fileSys.listStatus(destPath);
-        int count = 0;
-        if (listPaths != null && listPaths.length == 1) {
-          for (FileStatus s : listPaths) {
-            LOG.info("doCheckPolicy found path " + s.getPath() + " " + s.getModificationTime());
-            if (!s.getPath().toString().endsWith(".tmp") &&
-                s.getModificationTime() > firstmodtime &&
-                fileSys.getFileStatus(file2).getReplication() ==
-                targetReplication) {
-              count++;
-            }
-          }
-        }
-        if (count > 0) {
-          break;
-        }
-        LOG.info("doCheckPolicy waiting for files to be raided. Found " + 
-                 (listPaths == null ? "none" : listPaths.length));
-      }
-      assertEquals(listPaths.length, 1);
-
-      LOG.info("doCheckPolicy: file got re-raided as expected.");
-      
-    } catch (Exception e) {
-      LOG.info("doCheckPolicy Exception " + e +
-                                          StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      if (cnode != null) { cnode.stop(); cnode.join(); }
-      LOG.info("doTestPathFilter delete file " + file1);
-      fileSys.delete(file1, false);
-    }
-    LOG.info("doCheckPolicy completed:");
-  }
-
-  static public void createTestFiles(FileSystem fileSys, 
-      String path, String destpath, int nfile,
-      int nblock) throws IOException {
-    createTestFiles(fileSys, path, destpath, nfile, nblock, (short)1);
-  }
-
-  static void createTestFiles(FileSystem fileSys, String path, String destpath, int nfile,
-      int nblock, short repl) throws IOException {
-    long blockSize         = 1024L;
-    Path dir = new Path(path);
-    Path destPath = new Path(destpath);
-    fileSys.delete(dir, true);
-    fileSys.delete(destPath, true);
-   
-    for(int i = 0 ; i < nfile; i++){
-      Path file = new Path(path + "file" + i);
-      createOldFile(fileSys, file, repl, nblock, blockSize);
-    }
-  }
-
-  /**
-   * Test dist Raid
-   */
-  @Test
-  public void testDistRaid() throws Exception {
-    LOG.info("Test testDistRaid started.");
-    long targetReplication = 2;
-    long metaReplication   = 2;
-    long stripeLength      = 3;
-    short srcReplication = 1;
-
-    createClusters(false);
-    ConfigBuilder cb = new ConfigBuilder();
-    cb.addPolicy("policy1", "/user/dhruba/raidtest",
-        srcReplication, targetReplication, metaReplication, stripeLength);
-    cb.addPolicy("policy2", "/user/dhruba/raidtest2",
-        srcReplication, targetReplication, metaReplication, stripeLength);
-    cb.persist();
-
-    RaidNode cnode = null;
-    try {
-      createTestFiles(fileSys, "/user/dhruba/raidtest/",
-          "/destraid/user/dhruba/raidtest", 5, 7);
-      createTestFiles(fileSys, "/user/dhruba/raidtest2/",
-          "/destraid/user/dhruba/raidtest2", 5, 7);
-      LOG.info("Test testDistRaid created test files");
-
-      Configuration localConf = new Configuration(conf);
-      localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
-      localConf.set(JobContext.JAR, TestRaidNode.DistRaid_JAR);
-      cnode = RaidNode.createRaidNode(null, localConf);
-      // Verify the policies are parsed correctly
-      for (PolicyList policyList : cnode.getAllPolicies()) {
-        for (PolicyInfo p : policyList.getAll()) {
-          if (p.getName().equals("policy1")) {
-            Path srcPath = new Path("/user/dhruba/raidtest");
-            FileSystem fs = srcPath.getFileSystem(conf);
-            assertTrue(p.getSrcPath().equals(
-                srcPath.makeQualified(fs.getUri(), fs.getWorkingDirectory())));
-          } else {
-            assertTrue(p.getName().equals("policy2"));
-            Path srcPath = new Path("/user/dhruba/raidtest2");
-            FileSystem fs = srcPath.getFileSystem(conf);
-            assertTrue(p.getSrcPath().equals(
-                srcPath.makeQualified(fs.getUri(), fs.getWorkingDirectory())));
-          }
-          assertEquals(targetReplication,
-                       Integer.parseInt(p.getProperty("targetReplication")));
-          assertEquals(metaReplication,
-                       Integer.parseInt(p.getProperty("metaReplication")));
-          assertEquals(stripeLength,
-                       Integer.parseInt(p.getProperty("stripeLength")));
-        }
-      }
-
-      long start = Time.now();
-      final int MAX_WAITTIME = 300000;
-      
-      assertTrue("cnode is not DistRaidNode", cnode instanceof DistRaidNode);
-      DistRaidNode dcnode = (DistRaidNode) cnode;
-
-      while (dcnode.jobMonitor.jobsMonitored() < 2 &&
-             Time.now() - start < MAX_WAITTIME) {
-        Thread.sleep(1000);
-      }
-      
-      start = Time.now();
-      while (dcnode.jobMonitor.jobsSucceeded() < 2 &&
-             Time.now() - start < MAX_WAITTIME) {
-        Thread.sleep(1000);
-      }
-      assertEquals(dcnode.jobMonitor.jobsSucceeded(), dcnode.jobMonitor.jobsMonitored());
-      LOG.info("Test testDistRaid successful.");
-      
-    } catch (Exception e) {
-      LOG.info("testDistRaid Exception " + e + StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      if (cnode != null) { cnode.stop(); cnode.join(); }
-      stopClusters();
-    }
-    LOG.info("Test testDistRaid completed.");
-  }
-  
-  //
-  // simulate a corruption at specified offset and verify that eveyrthing is good
-  //
-  void simulateError(RaidShell shell, FileSystem fileSys, Path file1, 
-                     long crc, long corruptOffset) throws IOException {
-    // recover the file assuming that we encountered a corruption at offset 0
-    String[] args = new String[3];
-    args[0] = "-recover";
-    args[1] = file1.toString();
-    args[2] = Long.toString(corruptOffset);
-    Path recover1 = shell.recover(args[0], args, 1)[0];
-
-    // compare that the recovered file is identical to the original one
-    LOG.info("Comparing file " + file1 + " with recovered file " + recover1);
-    validateFile(fileSys, file1, recover1, crc);
-    fileSys.delete(recover1, false);
-  }
-
-  //
-  // creates a file and populate it with random data. Returns its crc.
-  //
-  static long createOldFile(FileSystem fileSys, Path name, int repl, int numBlocks, long blocksize)
-    throws IOException {
-    CRC32 crc = new CRC32();
-    FSDataOutputStream stm = fileSys.create(name, true,
-                                            fileSys.getConf().getInt("io.file.buffer.size", 4096),
-                                            (short)repl, blocksize);
-    // fill random data into file
-    byte[] b = new byte[(int)blocksize];
-    for (int i = 0; i < numBlocks; i++) {
-      if (i == (numBlocks-1)) {
-        b = new byte[(int)blocksize/2]; 
-      }
-      rand.nextBytes(b);
-      stm.write(b);
-      crc.update(b);
-    }
-    
-    stm.close();
-    return crc.getValue();
-  }
-
-  //
-  // validates that file matches the crc.
-  //
-  private void validateFile(FileSystem fileSys, Path name1, Path name2, long crc) 
-    throws IOException {
-
-    FileStatus stat1 = fileSys.getFileStatus(name1);
-    FileStatus stat2 = fileSys.getFileStatus(name2);
-    assertTrue(" Length of file " + name1 + " is " + stat1.getLen() + 
-               " is different from length of file " + name1 + " " + stat2.getLen(),
-               stat1.getLen() == stat2.getLen());
-
-    CRC32 newcrc = new CRC32();
-    FSDataInputStream stm = fileSys.open(name2);
-    final byte[] b = new byte[4192];
-    int num = 0;
-    while (num >= 0) {
-      num = stm.read(b);
-      if (num < 0) {
-        break;
-      }
-      newcrc.update(b, 0, num);
-    }
-    stm.close();
-    if (newcrc.getValue() != crc) {
-      fail("CRC mismatch of files " + name1 + " with file " + name2);
-    }
-  }
-
-  @Test
-  public void testSuspendTraversal() throws Exception {
-    LOG.info("Test testSuspendTraversal started.");
-    long targetReplication = 2;
-    long metaReplication   = 2;
-    long stripeLength      = 3;
-    short srcReplication = 1;
-
-    createClusters(false);
-    ConfigBuilder cb = new ConfigBuilder();
-    cb.addPolicy("policy1", "/user/dhruba/raidtest",
-        srcReplication, targetReplication, metaReplication, stripeLength);
-    cb.persist();
-
-    RaidNode cnode = null;
-    try {
-      for(int i = 0; i < 4; i++){
-        Path file = new Path("/user/dhruba/raidtest/dir" + i + "/file" + i);
-        createOldFile(fileSys, file, 1, 7, 1024L);
-      }
-
-      LOG.info("Test testSuspendTraversal created test files");
-
-      Configuration localConf = new Configuration(conf);
-      localConf.setInt("raid.distraid.max.jobs", 2);
-      localConf.setInt("raid.distraid.max.files", 2);
-      localConf.setInt("raid.directorytraversal.threads", 1);
-      localConf.set(JobContext.JAR, TestRaidNode.DistRaid_JAR);
-      // 4 test files: 2 jobs with 2 files each.
-      final int numJobsExpected = 2;
-      cnode = RaidNode.createRaidNode(null, localConf);
-
-      long start = Time.now();
-      final int MAX_WAITTIME = 300000;
-
-      assertTrue("cnode is not DistRaidNode", cnode instanceof DistRaidNode);
-      DistRaidNode dcnode = (DistRaidNode) cnode;
-
-      start = Time.now();
-      while (dcnode.jobMonitor.jobsSucceeded() < numJobsExpected &&
-             Time.now() - start < MAX_WAITTIME) {
-        LOG.info("Waiting for num jobs succeeded " + dcnode.jobMonitor.jobsSucceeded() + 
-         " to reach " + numJobsExpected);
-        Thread.sleep(3000);
-      }
-      // Wait for any running jobs to finish.
-      start = Time.now();
-      while (dcnode.jobMonitor.runningJobsCount() > 0 &&
-             Time.now() - start < MAX_WAITTIME) {
-        LOG.info("Waiting for zero running jobs: " +
-             dcnode.jobMonitor.runningJobsCount());
-        Thread.sleep(1000);
-      }
-      assertEquals(numJobsExpected, dcnode.jobMonitor.jobsMonitored());
-      assertEquals(numJobsExpected, dcnode.jobMonitor.jobsSucceeded());
-
-      LOG.info("Test testSuspendTraversal successful.");
-
-    } catch (Exception e) {
-      LOG.info("testSuspendTraversal Exception " + e + StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      if (cnode != null) { cnode.stop(); cnode.join(); }
-      stopClusters();
-    }
-    LOG.info("Test testSuspendTraversal completed.");
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidPurge.java

@@ -1,521 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileWriter;
-import java.util.Random;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.TestRaidDfs;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MiniMRCluster;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.raid.protocol.PolicyInfo;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Time;
-import org.apache.log4j.Level;
-import org.junit.Test;
-
-/**
- * If a file gets deleted, then verify that the parity file gets deleted too.
- */
-public class TestRaidPurge {
-  final static String TEST_DIR = new File(System.getProperty("test.build.data",
-      "target/test-data")).getAbsolutePath();
-  final static String CONFIG_FILE = new File(TEST_DIR, 
-      "test-raid.xml").getAbsolutePath();
-  final static long RELOAD_INTERVAL = 1000;
-  final static Log LOG = LogFactory.getLog("org.apache.hadoop.raid.TestRaidNode");
-  final Random rand = new Random();
-
-  {
-    ((Log4JLogger)RaidNode.LOG).getLogger().setLevel(Level.ALL);
-  }
-
-
-  Configuration conf;
-  String namenode = null;
-  String hftp = null;
-  MiniDFSCluster dfs = null;
-  MiniMRCluster mr = null;
-  FileSystem fileSys = null;
-  String jobTrackerName = null;
-
-  /**
-   * create mapreduce and dfs clusters
-   */
-  private void createClusters(boolean local) throws Exception {
-
-    new File(TEST_DIR).mkdirs(); // Make sure data directory exists
-    conf = new Configuration();
-    conf.set("raid.config.file", CONFIG_FILE);
-    conf.setBoolean("raid.config.reload", true);
-    conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL);
-
-    // scan all policies once every 5 second
-    conf.setLong("raid.policy.rescan.interval", 5000);
-
-    // make all deletions not go through Trash
-    conf.set("fs.shell.delete.classname", "org.apache.hadoop.dfs.DFSClient");
-
-    // the RaidNode does the raiding inline (instead of submitting to map/reduce)
-    if (local) {
-      conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode");
-    } else {
-      conf.set("raid.classname", "org.apache.hadoop.raid.DistRaidNode");
-    }
-
-    conf.set("raid.server.address", "localhost:0");
-
-    // create a dfs and map-reduce cluster
-    final int taskTrackers = 4;
-    final int jobTrackerPort = 60050;
-
-    dfs = new MiniDFSCluster(conf, 3, true, null);
-    dfs.waitActive();
-    fileSys = dfs.getFileSystem();
-    namenode = fileSys.getUri().toString();
-    mr = new MiniMRCluster(taskTrackers, namenode, 3);
-    JobConf jobConf = mr.createJobConf();
-    jobTrackerName = "localhost:" + jobConf.get(JTConfig.JT_IPC_ADDRESS);
-    hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort();
-
-    FileSystem.setDefaultUri(conf, namenode);
-    conf.set("mapred.job.tracker", jobTrackerName);
-    conf.set("mapreduce.framework.name", "yarn");
-    String rmAdress = jobConf.get("yarn.resourcemanager.address");
-    if (rmAdress != null) {
-      conf.set("yarn.resourcemanager.address", rmAdress);
-    }
-    String schedulerAdress =
-      jobConf.get("yarn.resourcemanager.scheduler.address");
-    if (schedulerAdress != null) {
-      conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress);
-    }
-    String jobHistoryAddress = 
-        jobConf.get("mapreduce.jobhistory.address");
-    if (jobHistoryAddress != null) {
-      conf.set("mapreduce.jobhistory.address", jobHistoryAddress);
-    }
-  }
-    
-  /**
-   * create raid.xml file for RaidNode
-   */
-  private void mySetup(long targetReplication,
-    long metaReplication, long stripeLength) throws Exception {
-    int harDelay = 1; // 1 day.
-    mySetup(targetReplication, metaReplication, stripeLength, harDelay);
-  }
-
-  private void mySetup(long targetReplication,
-    long metaReplication, long stripeLength, int harDelay) throws Exception {
-    FileWriter fileWriter = new FileWriter(CONFIG_FILE);
-    fileWriter.write("<?xml version=\"1.0\"?>\n");
-    String str = "<configuration> " +
-                   "<srcPath prefix=\"/user/dhruba/raidtest\"> " +
-                     "<policy name = \"RaidTest1\"> " +
-                        "<erasureCode>xor</erasureCode> " +
-                        "<destPath> /destraid</destPath> " +
-                        "<property> " +
-                          "<name>targetReplication</name> " +
-                          "<value>" + targetReplication + "</value> " +
-                          "<description>after RAIDing, decrease the replication factor of a file to this value." +
-                          "</description> " + 
-                        "</property> " +
-                        "<property> " +
-                          "<name>metaReplication</name> " +
-                          "<value>" + metaReplication + "</value> " +
-                          "<description> replication factor of parity file" +
-                          "</description> " + 
-                        "</property> " +
-                        "<property> " +
-                          "<name>stripeLength</name> " +
-                          "<value>" + stripeLength + "</value> " +
-                          "<description> the max number of blocks in a file to RAID together " +
-                          "</description> " + 
-                        "</property> " +
-                        "<property> " +
-                          "<name>modTimePeriod</name> " +
-                          "<value>2000</value> " + 
-                          "<description> time (milliseconds) after a file is modified to make it " +
-                                         "a candidate for RAIDing " +
-                          "</description> " + 
-                        "</property> " +
-                        "<property> " +
-                          "<name>time_before_har</name> " +
-                          "<value> " + harDelay + "</value> " +
-                          "<description> amount of time waited before har'ing parity files" +
-                          "</description> " +
-                        "</property> " +
-                     "</policy>" +
-                   "</srcPath>" +
-                 "</configuration>";
-    fileWriter.write(str);
-    fileWriter.close();
-  }
-
-  /**
-   * stop clusters created earlier
-   */
-  private void stopClusters() throws Exception {
-    if (mr != null) { mr.shutdown(); }
-    if (dfs != null) { dfs.shutdown(); }
-  }
-
-  /**
-   * Test that parity files that do not have an associated master file
-   * get deleted.
-   */
-  @Test
-  public void testPurge() throws Exception {
-    LOG.info("Test testPurge  started.");
-
-    long blockSizes    []  = {1024L};
-    long stripeLengths []  = {5};
-    long targetReplication = 1;
-    long metaReplication   = 1;
-    int  numBlock          = 9;
-    int  iter = 0;
-
-    createClusters(true);
-    try {
-      for (long blockSize : blockSizes) {
-        for (long stripeLength : stripeLengths) {
-           doTestPurge(iter, targetReplication, metaReplication,
-                       stripeLength, blockSize, numBlock);
-           iter++;
-        }
-      }
-    } finally {
-      stopClusters();
-    }
-    LOG.info("Test testPurge completed.");
-  }
-
-  /**
-   * Create parity file, delete original file and then validate that
-   * parity file is automatically deleted.
-   */
-  private void doTestPurge(int iter, long targetReplication,
-                          long metaReplication, long stripeLength,
-                          long blockSize, int numBlock) throws Exception {
-    LOG.info("doTestPurge started---------------------------:" +  " iter " + iter +
-             " blockSize=" + blockSize + " stripeLength=" + stripeLength);
-    mySetup(targetReplication, metaReplication, stripeLength);
-    Path dir = new Path("/user/dhruba/raidtest/");
-    Path file1 = new Path(dir + "/file" + iter);
-    RaidNode cnode = null;
-    try {
-      Path destPath = new Path("/destraid/user/dhruba/raidtest");
-      fileSys.delete(dir, true);
-      fileSys.delete(destPath, true);
-      TestRaidNode.createOldFile(fileSys, file1, 1, numBlock, blockSize);
-      LOG.info("doTestPurge created test files for iteration " + iter);
-
-      // create an instance of the RaidNode
-      Configuration localConf = new Configuration(conf);
-      
-      localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
-      cnode = RaidNode.createRaidNode(null, localConf);
-      FileStatus[] listPaths = null;
-
-      // wait till file is raided
-      while (true) {
-        try {
-          listPaths = fileSys.listStatus(destPath);
-          int count = 0;
-          if (listPaths != null && listPaths.length == 1) {
-            for (FileStatus s : listPaths) {
-              LOG.info("doTestPurge found path " + s.getPath());
-              if (!s.getPath().toString().endsWith(".tmp")) {
-                count++;
-              }
-            }
-          }
-          if (count > 0) {
-            break;
-          }
-        } catch (FileNotFoundException e) {
-          //ignore
-        }
-        LOG.info("doTestPurge waiting for files to be raided. Found " + 
-                 (listPaths == null ? "none" : listPaths.length));
-        Thread.sleep(1000);                  // keep waiting
-      }
-      // assertEquals(listPaths.length, 1); // all files raided
-      LOG.info("doTestPurge all files found in Raid.");
-
-      // delete original file
-      assertTrue("Unable to delete original file " + file1 ,
-                 fileSys.delete(file1, true));
-      LOG.info("deleted file " + file1);
-
-      // wait till parity file and directory are automatically deleted
-      while (fileSys.exists(destPath)) {
-        LOG.info("doTestPurge waiting for parity files to be removed.");
-        Thread.sleep(1000);                  // keep waiting
-      }
-
-    } catch (Exception e) {
-      LOG.info("doTestPurge Exception " + e +
-                                          StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      if (cnode != null) { cnode.stop(); cnode.join(); }
-      LOG.info("doTestPurge delete file " + file1);
-      fileSys.delete(file1, true);
-    }
-    LOG.info("doTestPurge completed:" + " blockSize=" + blockSize +
-             " stripeLength=" + stripeLength);
-  }
-
-  /**
-   * Create a file, wait for parity file to get HARed. Then modify the file,
-   * wait for the HAR to get purged.
-   */
-  @Test
-  public void testPurgeHar() throws Exception {
-    LOG.info("testPurgeHar started");
-    int harDelay = 0;
-    createClusters(true);
-    mySetup(1, 1, 5, harDelay);
-    Path dir = new Path("/user/dhruba/raidtest/");
-    Path destPath = new Path("/raid/user/dhruba/raidtest");
-    Path file1 = new Path(dir + "/file");
-    RaidNode cnode = null;
-    try {
-      TestRaidNode.createOldFile(fileSys, file1, 1, 8, 8192L);
-      LOG.info("testPurgeHar created test files");
-
-      // create an instance of the RaidNode
-      Configuration localConf = new Configuration(conf);
-      cnode = RaidNode.createRaidNode(null, localConf);
-
-      // Wait till har is created.
-      while (true) {
-        try {
-          FileStatus[] listPaths = listPaths = fileSys.listStatus(destPath);
-          if (listPaths != null && listPaths.length == 1) {
-            FileStatus s = listPaths[0];
-            LOG.info("testPurgeHar found path " + s.getPath());
-            if (s.getPath().toString().endsWith(".har")) {
-              break;
-            }
-          }
-        } catch (FileNotFoundException e) {
-          //ignore
-        }
-        Thread.sleep(1000);                  // keep waiting
-      }
-
-      // Set an old timestamp.
-      fileSys.setTimes(file1, 0, 0);
-
-      boolean found = false;
-      FileStatus[] listPaths = null;
-      while (!found || listPaths == null || listPaths.length > 1) {
-        listPaths = fileSys.listStatus(destPath);
-        if (listPaths != null) {
-          for (FileStatus s: listPaths) {
-            LOG.info("testPurgeHar waiting for parity file to be recreated" +
-              " and har to be deleted found " + s.getPath());
-            if (s.getPath().toString().endsWith("file") &&
-                s.getModificationTime() == 0) {
-              found = true;
-            }
-          }
-        }
-        Thread.sleep(1000);
-      }
-    } catch (Exception e) {
-      LOG.info("testPurgeHar Exception " + e +
-          StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      if (cnode != null) { cnode.stop(); cnode.join(); }
-      fileSys.delete(dir, true);
-      fileSys.delete(destPath, true);
-      stopClusters();
-    }
-  }
-
-  /**
-   * Create parity file, delete original file's directory and then validate that
-   * parity directory is automatically deleted.
-   */
-  @Test
-  public void testPurgeDirectory() throws Exception {
-    long stripeLength = 5;
-    long blockSize = 8192;
-    long targetReplication = 1;
-    long metaReplication   = 1;
-    int  numBlock          = 9;
-
-    createClusters(true);
-    mySetup(targetReplication, metaReplication, stripeLength);
-    Path dir = new Path("/user/dhruba/raidtest/");
-    Path file1 = new Path(dir + "/file1");
-    RaidNode cnode = null;
-    try {
-      TestRaidNode.createOldFile(fileSys, file1, 1, numBlock, blockSize);
-
-      // create an instance of the RaidNode
-      Configuration localConf = new Configuration(conf);
-      localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
-      cnode = RaidNode.createRaidNode(null, localConf);
-
-      Path destPath = new Path("/destraid/user/dhruba/raidtest");
-      TestRaidDfs.waitForFileRaided(LOG, fileSys, file1, destPath);
-
-      // delete original directory.
-      assertTrue("Unable to delete original directory " + file1 ,
-                 fileSys.delete(file1.getParent(), true));
-      LOG.info("deleted file " + file1);
-
-      // wait till parity file and directory are automatically deleted
-      long start = Time.now();
-      while (fileSys.exists(destPath) &&
-            Time.now() - start < 120000) {
-        LOG.info("testPurgeDirectory waiting for parity files to be removed.");
-        Thread.sleep(1000);                  // keep waiting
-      }
-      assertFalse(fileSys.exists(destPath));
-
-    } catch (Exception e) {
-      LOG.info("testPurgeDirectory Exception " + e +
-                                          StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      if (cnode != null) { cnode.stop(); cnode.join(); }
-      LOG.info("testPurgeDirectory delete file " + file1);
-      fileSys.delete(file1, true);
-      stopClusters();
-    }
-  }
-
-  /**
-   * Test that an XOR parity file is removed when a RS parity file is detected.
-   */
-  @Test
-  public void testPurgePreference() throws Exception {
-    createClusters(true);
-    Path dir = new Path("/user/test/raidtest/");
-    Path file1 = new Path(dir + "/file1");
-
-    PolicyInfo infoXor = new PolicyInfo("testPurgePreference", conf);
-    infoXor.setSrcPath("/user/test/raidtest");
-    infoXor.setErasureCode("xor");
-    infoXor.setDescription("test policy");
-    infoXor.setProperty("targetReplication", "2");
-    infoXor.setProperty("metaReplication", "2");
-
-    PolicyInfo infoRs = new PolicyInfo("testPurgePreference", conf);
-    infoRs.setSrcPath("/user/test/raidtest");
-    infoRs.setErasureCode("rs");
-    infoRs.setDescription("test policy");
-    infoRs.setProperty("targetReplication", "1");
-    infoRs.setProperty("metaReplication", "1");
-    try {
-      TestRaidNode.createOldFile(fileSys, file1, 1, 9, 8192L);
-      FileStatus stat = fileSys.getFileStatus(file1);
-
-      // Create the parity files.
-      RaidNode.doRaid(
-        conf, infoXor, stat, new RaidNode.Statistics(), Reporter.NULL);
-      RaidNode.doRaid(
-        conf, infoRs, stat, new RaidNode.Statistics(), Reporter.NULL);
-      Path xorParity =
-        new Path(RaidNode.DEFAULT_RAID_LOCATION, "user/test/raidtest/file1");
-      Path rsParity =
-        new Path(RaidNode.DEFAULT_RAIDRS_LOCATION, "user/test/raidtest/file1");
-      assertTrue(fileSys.exists(xorParity));
-      assertTrue(fileSys.exists(rsParity));
-
-      // Check purge of a single parity file.
-      RaidNode cnode = RaidNode.createRaidNode(conf);
-      FileStatus raidRsStat =
-        fileSys.getFileStatus(new Path(RaidNode.DEFAULT_RAIDRS_LOCATION));
-      cnode.purgeMonitor.recursePurge(infoRs.getErasureCode(), fileSys, fileSys,
-         RaidNode.DEFAULT_RAIDRS_LOCATION, raidRsStat);
-
-      // Calling purge under the RS path has no effect.
-      assertTrue(fileSys.exists(xorParity));
-      assertTrue(fileSys.exists(rsParity));
-
-      FileStatus raidStat =
-         fileSys.getFileStatus(new Path(RaidNode.DEFAULT_RAID_LOCATION));
-      cnode.purgeMonitor.recursePurge(infoXor.getErasureCode(), fileSys, fileSys,
-         RaidNode.DEFAULT_RAID_LOCATION, raidStat);
-      // XOR parity must have been purged by now.
-      assertFalse(fileSys.exists(xorParity));
-      assertTrue(fileSys.exists(rsParity));
-
-      // Now check the purge of a parity har.
-      // Delete the RS parity for now.
-      fileSys.delete(rsParity);
-      // Recreate the XOR parity.
-      Path xorHar =
-        new Path(RaidNode.DEFAULT_RAID_LOCATION, "user/test/raidtest/raidtest" +
-          RaidNode.HAR_SUFFIX);
-      RaidNode.doRaid(
-        conf, infoXor, stat, new RaidNode.Statistics(), Reporter.NULL);
-      assertTrue(fileSys.exists(xorParity));
-      assertFalse(fileSys.exists(xorHar));
-
-      // Create the har.
-      long cutoff = Time.now();
-      cnode.recurseHar(infoXor, fileSys, raidStat,
-        RaidNode.DEFAULT_RAID_LOCATION, fileSys, cutoff,
-        RaidNode.tmpHarPathForCode(conf, infoXor.getErasureCode()));
-
-      // Call purge to get rid of the parity file. The har should remain.
-      cnode.purgeMonitor.recursePurge(infoXor.getErasureCode(), fileSys, fileSys,
-         RaidNode.DEFAULT_RAID_LOCATION, raidStat);
-      // XOR har should exist but xor parity file should have been purged.
-      assertFalse(fileSys.exists(xorParity));
-      assertTrue(fileSys.exists(xorHar));
-
-      // Now create the RS parity.
-      RaidNode.doRaid(
-        conf, infoRs, stat, new RaidNode.Statistics(), Reporter.NULL);
-      cnode.purgeMonitor.recursePurge(infoXor.getErasureCode(), fileSys, fileSys,
-         RaidNode.DEFAULT_RAID_LOCATION, raidStat);
-      // XOR har should get deleted.
-      assertTrue(fileSys.exists(rsParity));
-      assertFalse(fileSys.exists(xorParity));
-      assertFalse(fileSys.exists(xorHar));
-
-    } finally {
-      stopClusters();
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidShell.java

@@ -1,267 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.Random;
-import java.util.zip.CRC32;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.RaidDFSUtil;
-import org.apache.hadoop.hdfs.TestRaidDfs;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.Test;
-
-
-public class TestRaidShell {
-  final static Log LOG = LogFactory.getLog(
-                            "org.apache.hadoop.raid.TestRaidShell");
-  final static String TEST_DIR = new File(System.getProperty("test.build.data",
-      "target/test-data")).getAbsolutePath();
-  final static String CONFIG_FILE = new File(TEST_DIR,
-      "test-raid.xml").getAbsolutePath();
-  final static long RELOAD_INTERVAL = 1000;
-  final static int NUM_DATANODES = 3;
-  Configuration conf;
-  String namenode = null;
-  MiniDFSCluster dfs = null;
-  String hftp = null;
-  FileSystem fileSys = null;
-  RaidNode cnode = null;
-  Random rand = new Random();
-
-  /**
-   * Create a file with three stripes, corrupt a block each in two stripes,
-   * and wait for the the file to be fixed.
-   */
-  @Test
-  public void testBlockFix() throws Exception {
-    LOG.info("Test testBlockFix started.");
-    long blockSize = 8192L;
-    int stripeLength = 3;
-    mySetup(stripeLength, -1);
-    Path file1 = new Path("/user/dhruba/raidtest/file1");
-    Path destPath = new Path("/destraid/user/dhruba/raidtest");
-    Path parityFile = new Path(destPath, "file1");
-    long crc1 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1,
-                                                          1, 7, blockSize);
-    long file1Len = fileSys.getFileStatus(file1).getLen();
-    LOG.info("Test testBlockFix created test files");
-
-    // create an instance of the RaidNode
-    Configuration localConf = new Configuration(conf);
-    localConf.set(RaidNode.RAID_LOCATION_KEY, "/destraid");
-    localConf.setInt("raid.blockfix.interval", 1000);
-    // the RaidNode does the raiding inline (instead of submitting to map/reduce)
-    conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode");
-    conf.set("raid.blockfix.classname",
-             "org.apache.hadoop.raid.LocalBlockFixer");
-    cnode = RaidNode.createRaidNode(null, localConf);
-
-    try {
-      TestRaidDfs.waitForFileRaided(LOG, fileSys, file1, destPath);
-      cnode.stop();
-      cnode.join();
-      cnode = null;
-
-      FileStatus srcStat = fileSys.getFileStatus(file1);
-      LocatedBlocks locations = RaidDFSUtil.getBlockLocations(
-        (DistributedFileSystem) fileSys, file1.toUri().getPath(),
-        0, srcStat.getLen());
-
-      DistributedFileSystem dfs = (DistributedFileSystem)fileSys;
-
-      // Corrupt blocks in different stripes. We can fix them.
-      int[] corruptBlockIdxs = new int[]{0, 4, 6};
-      for (int idx: corruptBlockIdxs) {
-        LOG.info("Corrupting block " + locations.get(idx).getBlock());
-        corruptBlock(locations.get(idx).getBlock());
-      }
-      TestBlockFixer.reportCorruptBlocks(fileSys, file1, corruptBlockIdxs,
-        srcStat.getBlockSize());
-
-      waitForCorruptBlocks(corruptBlockIdxs.length, dfs, file1);
-
-      // Create RaidShell and fix the file.
-      RaidShell shell = new RaidShell(conf);
-      String[] args = new String[2];
-      args[0] = "-recoverBlocks";
-      args[1] = file1.toUri().getPath();
-      ToolRunner.run(shell, args);
-
-      waitForCorruptBlocks(0, dfs, file1);
-
-      assertTrue(TestRaidDfs.validateFile(dfs, file1, file1Len, crc1));
-
-      // Now corrupt and fix the parity file.
-      FileStatus parityStat = fileSys.getFileStatus(parityFile);
-      long parityCrc = getCRC(fileSys, parityFile);
-      locations = RaidDFSUtil.getBlockLocations(
-        dfs, parityFile.toUri().getPath(), 0, parityStat.getLen());
-      corruptBlock(locations.get(0).getBlock());
-      TestBlockFixer.reportCorruptBlocks(fileSys, parityFile, new int[]{0},
-        srcStat.getBlockSize());
-      waitForCorruptBlocks(1, dfs, parityFile);
-
-      args[1] = parityFile.toUri().getPath();
-      ToolRunner.run(shell, args);
-
-      waitForCorruptBlocks(0, dfs, file1);
-      assertEquals(parityCrc, getCRC(fileSys, parityFile));
-
-    } catch (Exception e) {
-      LOG.info("Test testBlockFix Exception " + e + StringUtils.stringifyException(e));
-      throw e;
-    } finally {
-      myTearDown();
-    }
-    LOG.info("Test testBlockFix completed.");
-  }
-
-  private void waitForCorruptBlocks(
-    int numCorruptBlocks, DistributedFileSystem dfs, Path file)
-    throws Exception {
-    String path = file.toUri().getPath();
-    FileStatus stat = dfs.getFileStatus(file);
-    long start = Time.now();
-    long actual = 0;
-    do {
-      actual = RaidDFSUtil.corruptBlocksInFile(
-          dfs, path, 0, stat.getLen()).size();
-      if (actual == numCorruptBlocks) break;
-      if (Time.now() - start > 120000) break;
-      LOG.info("Waiting for " + numCorruptBlocks + " corrupt blocks in " +
-        path + ", found " + actual);
-      Thread.sleep(1000);
-    } while (true);
-    assertEquals(numCorruptBlocks, actual);
-  }
-
-  private void mySetup(int stripeLength, int timeBeforeHar) throws Exception {
-
-    new File(TEST_DIR).mkdirs(); // Make sure data directory exists
-    conf = new Configuration();
-
-    conf.set("raid.config.file", CONFIG_FILE);
-    conf.setBoolean("raid.config.reload", true);
-    conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL);
-
-    // scan all policies once every 5 second
-    conf.setLong("raid.policy.rescan.interval", 5000);
-
-    // make all deletions not go through Trash
-    conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient");
-
-    // do not use map-reduce cluster for Raiding
-    conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode");
-    conf.set("raid.server.address", "localhost:0");
-    conf.setInt("hdfs.raid.stripeLength", stripeLength);
-    conf.set("hdfs.raid.locations", "/destraid");
-
-    dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build();
-    dfs.waitActive();
-    fileSys = dfs.getFileSystem();
-    namenode = fileSys.getUri().toString();
-
-    FileSystem.setDefaultUri(conf, namenode);
-    hftp = "hftp://localhost.localdomain:" + dfs.getNameNodePort();
-
-    FileSystem.setDefaultUri(conf, namenode);
-
-    FileWriter fileWriter = new FileWriter(CONFIG_FILE);
-    fileWriter.write("<?xml version=\"1.0\"?>\n");
-    String str = "<configuration> " +
-                   "<srcPath prefix=\"/user/dhruba/raidtest\"> " +
-                     "<policy name = \"RaidTest1\"> " +
-                        "<erasureCode>xor</erasureCode> " +
-                        "<destPath> /destraid</destPath> " +
-                        "<property> " +
-                          "<name>targetReplication</name> " +
-                          "<value>1</value> " +
-                          "<description>after RAIDing, decrease the replication factor of a file to this value." +
-                          "</description> " +
-                        "</property> " +
-                        "<property> " +
-                          "<name>metaReplication</name> " +
-                          "<value>1</value> " +
-                          "<description> replication factor of parity file" +
-                          "</description> " +
-                        "</property> " +
-                        "<property> " +
-                          "<name>modTimePeriod</name> " +
-                          "<value>2000</value> " +
-                          "<description> time (milliseconds) after a file is modified to make it " +
-                                         "a candidate for RAIDing " +
-                          "</description> " +
-                        "</property> ";
-    if (timeBeforeHar >= 0) {
-      str +=
-                        "<property> " +
-                          "<name>time_before_har</name> " +
-                          "<value>" + timeBeforeHar + "</value> " +
-                          "<description> amount of time waited before har'ing parity files" +
-                          "</description> " +
-                        "</property> ";
-    }
-
-    str +=
-                     "</policy>" +
-                   "</srcPath>" +
-                 "</configuration>";
-    fileWriter.write(str);
-    fileWriter.close();
-  }
-
-  private void myTearDown() throws Exception {
-    if (cnode != null) { cnode.stop(); cnode.join(); }
-    if (dfs != null) { dfs.shutdown(); }
-  }
-
-  private long getCRC(FileSystem fs, Path p) throws IOException {
-    CRC32 crc = new CRC32();
-    FSDataInputStream stm = fs.open(p);
-    int b;
-    while ((b = stm.read())>=0) {
-      crc.update(b);
-    }
-    stm.close();
-    return crc.getValue();
-  }
-
-  void corruptBlock(ExtendedBlock block) throws IOException {
-    assertTrue("Could not corrupt block",
-        dfs.corruptBlockOnDataNodes(block) > 0);
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestRaidShellFsck.java

@@ -1,724 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http:www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.raid;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.Random;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.RaidDFSUtil;
-import org.apache.hadoop.hdfs.TestRaidDfs;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.After;
-import org.junit.Test;
-
-
-public class TestRaidShellFsck {
-  final static Log LOG =
-    LogFactory.getLog("org.apache.hadoop.raid.TestRaidShellFsck");
-  final static String TEST_DIR = 
-    new File(System.
-             getProperty("test.build.data", "target/test-data")).getAbsolutePath();
-
-  final static String CONFIG_FILE = new File(TEST_DIR, "test-raid.xml").
-    getAbsolutePath();
-  final static long RELOAD_INTERVAL = 1000;
-  final static int NUM_DATANODES = 4;
-  final static int STRIPE_BLOCKS = 3; // number of blocks per stripe
-  final static int FILE_BLOCKS = 6; // number of blocks that file consists of
-  final static short REPL = 1; // replication factor before raiding
-  final static long BLOCK_SIZE = 8192L; // size of block in byte
-  final static String DIR_PATH = "/user/pkling/raidtest";
-  final static Path FILE_PATH0 =
-    new Path("/user/pkling/raidtest/raidfsck.test");
-  final static Path FILE_PATH1 =
-    new Path("/user/pkling/raidtest/raidfsck2.test");
-  final static Path RAID_PATH = new Path("/destraid/user/pkling/raidtest");
-  final static String HAR_NAME = "raidtest_raid.har";
-  final static String RAID_DIR = "/destraid";
-
-  Configuration conf = null;
-  Configuration raidConf = null;
-  Configuration clientConf = null;
-  MiniDFSCluster cluster = null;
-  DistributedFileSystem dfs = null;
-  RaidNode rnode = null;
-  
-  
-  RaidShell shell = null;
-  String[] args = null;
-
-
-  /**
-   * creates a MiniDFS instance with a raided file in it
-   */
-  private void setUp(boolean doHar) throws IOException, ClassNotFoundException {
-
-    final int timeBeforeHar;
-    if (doHar) {
-      timeBeforeHar = 0;
-    } else {
-      timeBeforeHar = -1;
-    }
-
-
-    new File(TEST_DIR).mkdirs(); // Make sure data directory exists
-    conf = new Configuration();
-
-    conf.set("raid.config.file", CONFIG_FILE);
-    conf.setBoolean("raid.config.reload", true);
-    conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL);
-
-    // scan all policies once every 5 second
-    conf.setLong("raid.policy.rescan.interval", 5000);
-
-    // make all deletions not go through Trash
-    conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient");
-
-    // do not use map-reduce cluster for Raiding
-    conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode");
-    // use local block fixer
-    conf.set("raid.blockfix.classname", 
-             "org.apache.hadoop.raid.LocalBlockFixer");
-
-    conf.set("raid.server.address", "localhost:0");
-    conf.setInt("hdfs.raid.stripeLength", STRIPE_BLOCKS);
-    conf.set("hdfs.raid.locations", RAID_DIR);
-
-    conf.setInt("dfs.corruptfilesreturned.max", 500);
-
-    conf.setBoolean("dfs.permissions", false);
-
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES)
-        .build();
-    cluster.waitActive();
-    dfs = (DistributedFileSystem) cluster.getFileSystem();
-    String namenode = dfs.getUri().toString();
-
-    FileSystem.setDefaultUri(conf, namenode);
-
-    FileWriter fileWriter = new FileWriter(CONFIG_FILE);
-    fileWriter.write("<?xml version=\"1.0\"?>\n");
-    String str =
-      "<configuration> " +
-      "  <srcPath prefix=\"" + DIR_PATH + "\"> " +
-      "    <policy name = \"RaidTest1\"> " +
-      "      <erasureCode>xor</erasureCode> " +
-      "      <destPath> " + RAID_DIR + " </destPath> " +
-      "      <property> " +
-      "        <name>targetReplication</name> " +
-      "        <value>1</value> " +
-      "        <description>after RAIDing, decrease the replication " +
-      "factor of a file to this value.</description> " +
-      "      </property> " +
-      "      <property> " +
-      "        <name>metaReplication</name> " +
-      "        <value>1</value> " +
-      "        <description> replication factor of parity file</description> " +
-      "      </property> " +
-      "      <property> " +
-      "        <name>modTimePeriod</name> " +
-      "        <value>2000</value> " +
-      "        <description>time (milliseconds) after a file is modified " + 
-      "to make it a candidate for RAIDing</description> " +
-      "      </property> ";
-   
-    if (timeBeforeHar >= 0) {
-      str +=
-        "      <property> " +
-        "        <name>time_before_har</name> " +
-        "        <value>" + timeBeforeHar + "</value> " +
-        "        <description> amount of time waited before har'ing parity " +
-        "files</description> " +
-        "     </property> ";
-    }
-
-    str +=
-      "    </policy>" +
-      "  </srcPath>" +
-      "</configuration>";
-
-    fileWriter.write(str);
-    fileWriter.close();
-
-    createTestFile(FILE_PATH0);
-    createTestFile(FILE_PATH1);
-
-    Path[] filePaths = { FILE_PATH0, FILE_PATH1 };
-    raidTestFiles(RAID_PATH, filePaths, doHar);
-
-    clientConf = new Configuration(raidConf);
-    clientConf.set("fs.hdfs.impl",
-                   "org.apache.hadoop.hdfs.DistributedRaidFileSystem");
-    clientConf.set("fs.raid.underlyingfs.impl",
-                   "org.apache.hadoop.hdfs.DistributedFileSystem");
-
-    // prepare shell and arguments
-    shell = new RaidShell(clientConf);
-    args = new String[2];
-    args[0] = "-fsck";
-    args[1] = DIR_PATH;
-
-  }
-  
-  /**
-   * Creates test file consisting of random data
-   */
-  private void createTestFile(Path filePath) throws IOException {
-    Random rand = new Random();
-    FSDataOutputStream stm = dfs.create(filePath, true, 
-                                        conf.getInt("io.file.buffer.size",
-                                                    4096), REPL, BLOCK_SIZE);
-
-    final byte[] b = new byte[(int) BLOCK_SIZE];
-    for (int i = 0; i < FILE_BLOCKS; i++) {
-      rand.nextBytes(b);
-      stm.write(b);
-    }
-    stm.close();
-    LOG.info("test file created");
-
-  }
-
-  /**
-   * raids test file
-   */
-  private void raidTestFiles(Path raidPath, Path[] filePaths, boolean doHar)
-    throws IOException, ClassNotFoundException {
-    // create RaidNode
-    raidConf = new Configuration(conf);
-    raidConf.set(RaidNode.RAID_LOCATION_KEY, RAID_DIR);
-    raidConf.setInt("raid.blockfix.interval", 1000);
-    raidConf.setLong("har.block.size", BLOCK_SIZE * 3);
-    // the RaidNode does the raiding inline (instead of submitting to MR node)
-    conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode");
-    rnode = RaidNode.createRaidNode(null, raidConf);
-
-    for (Path filePath: filePaths) {
-      long waitStart = Time.now();
-      boolean raided = false;
-      
-      Path parityFilePath = new Path(RAID_DIR, 
-                                     filePath.toString().substring(1));
-
-      while (!raided) {
-        try {
-          FileStatus[] listPaths = dfs.listStatus(raidPath);
-          if (listPaths != null) {
-            if (doHar) {
-              // case with HAR
-              for (FileStatus f: listPaths) {
-                if (f.getPath().toString().endsWith(".har")) {
-                  // check if the parity file is in the index
-                  final Path indexPath = new Path(f.getPath(), "_index");
-                  final FileStatus indexFileStatus = 
-                    dfs.getFileStatus(indexPath);
-                  final HarIndex harIndex = 
-                    new HarIndex(dfs.open(indexPath), indexFileStatus.getLen());
-                  final HarIndex.IndexEntry indexEntry =
-                    harIndex.findEntryByFileName(parityFilePath.toString());
-                  if (indexEntry != null) {
-                    LOG.info("raid file " + parityFilePath.toString() + 
-                             " found in Har archive: " + 
-                             f.getPath().toString() +
-                             " ts=" + indexEntry.mtime);
-                    raided = true;
-                    break;
-                  }
-                }
-              }
-              
-            } else {
-               // case without HAR
-              for (FileStatus f : listPaths) {
-                Path found = new Path(f.getPath().toUri().getPath());
-                if (parityFilePath.equals(found)) {
-                  LOG.info("raid file found: " + f.getPath().toString());
-                  raided = true;
-                  break;
-                }
-              }
-            }
-          }
-        } catch (FileNotFoundException ignore) {
-        }
-        if (!raided) {
-          if (Time.now() > waitStart + 40000L) {
-            LOG.error("parity file not created after 40s");
-            throw new IOException("parity file not HARed after 40s");
-          } else {
-            try {
-              Thread.sleep(1000);
-            } catch (InterruptedException ignore) {
-            }
-          }
-        }
-      }
-    }
-
-    rnode.stop();
-    rnode.join();
-    rnode = null;
-    LOG.info("test file raided");    
-  }
-
-  /**
-   * sleeps for up to 20s until the number of corrupt files 
-   * in the file system is equal to the number specified
-   */
-  private void waitUntilCorruptFileCount(DistributedFileSystem dfs,
-                                         int corruptFiles)
-    throws IOException {
-    long waitStart = Time.now();
-    while (RaidDFSUtil.getCorruptFiles(dfs).length != corruptFiles) {
-      try {
-        Thread.sleep(1000);
-      } catch (InterruptedException ignore) {
-        
-      }
-
-      if (Time.now() > waitStart + 20000L) {
-        break;
-      }
-    }
-    
-    int corruptFilesFound = RaidDFSUtil.getCorruptFiles(dfs).length;
-    if (corruptFilesFound != corruptFiles) {
-      throw new IOException("expected " + corruptFiles + 
-                            " corrupt files but got " +
-                            corruptFilesFound);
-    }
-  }
-
-  /**
-   * removes a specified block from MiniDFS storage and reports it as corrupt
-   */
-  private void removeAndReportBlock(DistributedFileSystem blockDfs,
-                                    Path filePath,
-                                    LocatedBlock block) 
-    throws IOException {
-    TestRaidDfs.corruptBlock(cluster, filePath, block.getBlock(), NUM_DATANODES, true);
-   
-    // report deleted block to the name node
-    LocatedBlock[] toReport = { block };
-    blockDfs.getClient().getNamenode().reportBadBlocks(toReport);
-
-  }
-
-
-  /**
-   * removes a file block in the specified stripe
-   */
-  private void removeFileBlock(Path filePath, int stripe, int blockInStripe)
-    throws IOException {
-    LocatedBlocks fileBlocks = dfs.getClient().getNamenode().
-      getBlockLocations(filePath.toString(), 0, FILE_BLOCKS * BLOCK_SIZE);
-    if (fileBlocks.locatedBlockCount() != FILE_BLOCKS) {
-      throw new IOException("expected " + FILE_BLOCKS + 
-                            " file blocks but found " + 
-                            fileBlocks.locatedBlockCount());
-    }
-    if (blockInStripe >= STRIPE_BLOCKS) {
-      throw new IOException("blockInStripe is " + blockInStripe +
-                            " but must be smaller than " + STRIPE_BLOCKS);
-    }
-    LocatedBlock block = fileBlocks.get(stripe * STRIPE_BLOCKS + blockInStripe);
-    removeAndReportBlock(dfs, filePath, block);
-    LOG.info("removed file " + filePath.toString() + " block " +
-             stripe * STRIPE_BLOCKS + " in stripe " + stripe);
-  }
-
-  /**
-   * removes a parity block in the specified stripe
-   */
-  private void removeParityBlock(Path filePath, int stripe) throws IOException {
-    // find parity file
-    Path destPath = new Path(RAID_DIR);
-    RaidNode.ParityFilePair ppair = null;
-
-    ppair = RaidNode.getParityFile(destPath, filePath, conf);
-    String parityPathStr = ppair.getPath().toUri().getPath();
-    LOG.info("parity path: " + parityPathStr);
-    FileSystem parityFS = ppair.getFileSystem();
-    if (!(parityFS instanceof DistributedFileSystem)) {
-      throw new IOException("parity file is not on distributed file system");
-    }
-    DistributedFileSystem parityDFS = (DistributedFileSystem) parityFS;
-
-    
-    // now corrupt the block corresponding to the stripe selected
-    FileStatus parityFileStatus =
-      parityDFS.getFileStatus(new Path(parityPathStr));
-    long parityBlockSize = parityFileStatus.getBlockSize();
-    long parityFileLength = parityFileStatus.getLen();
-    long parityFileLengthInBlocks = (parityFileLength / parityBlockSize) + 
-      (((parityFileLength % parityBlockSize) == 0) ? 0L : 1L);
-    if (parityFileLengthInBlocks <= stripe) {
-      throw new IOException("selected stripe " + stripe + 
-                            " but parity file only has " + 
-                            parityFileLengthInBlocks + " blocks");
-    }
-    if (parityBlockSize != BLOCK_SIZE) {
-      throw new IOException("file block size is " + BLOCK_SIZE + 
-                            " but parity file block size is " + 
-                            parityBlockSize);
-    }
-    LocatedBlocks parityFileBlocks = parityDFS.getClient().getNamenode().
-      getBlockLocations(parityPathStr, 0, parityFileLength);
-    if (parityFileBlocks.locatedBlockCount() != parityFileLengthInBlocks) {
-      throw new IOException("expected " + parityFileLengthInBlocks + 
-                            " parity file blocks but got " + 
-                            parityFileBlocks.locatedBlockCount() + 
-                            " blocks");
-    }
-    LocatedBlock parityFileBlock = parityFileBlocks.get(stripe);
-    removeAndReportBlock(parityDFS, new Path(parityPathStr), parityFileBlock);
-    LOG.info("removed parity file block/stripe " + stripe +
-             " for " + filePath.toString());
-
-  }
-
-  /**
-   * removes a block from the har part file
-   */
-  private void removeHarParityBlock(int block) throws IOException {
-    Path harPath = new Path(RAID_PATH, HAR_NAME);
-    FileStatus [] listPaths = dfs.listStatus(harPath);
-    
-    boolean deleted = false;
-    
-    for (FileStatus f: listPaths) {
-      if (f.getPath().getName().startsWith("part-")) {
-        final Path partPath = new Path(f.getPath().toUri().getPath());
-        final LocatedBlocks partBlocks  = dfs.getClient().getNamenode().
-          getBlockLocations(partPath.toString(),
-                            0,
-                            f.getLen());
-        
-        if (partBlocks.locatedBlockCount() <= block) {
-          throw new IOException("invalid har block " + block);
-        }
-
-        final LocatedBlock partBlock = partBlocks.get(block);
-        removeAndReportBlock(dfs, partPath, partBlock);
-        LOG.info("removed block " + block + "/" + 
-                 partBlocks.locatedBlockCount() +
-                 " of file " + partPath.toString() +
-                 " block size " + partBlock.getBlockSize());
-        deleted = true;
-        break;
-      }
-    }
-
-    if (!deleted) {
-      throw new IOException("cannot find part file in " + harPath.toString());
-    }
-  }
-
-  /**
-   * checks fsck with no missing blocks
-   */
-  @Test
-  public void testClean() throws Exception {
-    LOG.info("testClean");
-    setUp(false);
-    int result = ToolRunner.run(shell, args);
-
-    assertTrue("fsck should return 0, but returns " +
-               Integer.toString(result), result == 0);
-  }
-
-
-  /**
-   * checks fsck with missing block in file block but not in parity block
-   */
-  @Test
-  public void testFileBlockMissing() throws Exception {
-    LOG.info("testFileBlockMissing");
-    setUp(false);
-    waitUntilCorruptFileCount(dfs, 0);
-    removeFileBlock(FILE_PATH0, 0, 0);
-    waitUntilCorruptFileCount(dfs, 1);
-
-    int result = ToolRunner.run(shell, args);
-
-    assertTrue("fsck should return 0, but returns " + 
-               Integer.toString(result), result == 0);
-  }
-
-  /**
-   * checks fsck with missing block in parity block but not in file block
-   */
-  @Test
-  public void testParityBlockMissing() throws Exception {
-    LOG.info("testParityBlockMissing");
-    setUp(false);
-    waitUntilCorruptFileCount(dfs, 0);
-    removeParityBlock(FILE_PATH0, 0);
-    waitUntilCorruptFileCount(dfs, 1);
-
-    int result = ToolRunner.run(shell, args);
-
-    assertTrue("fsck should return 0, but returns " +
-               Integer.toString(result), result == 0);
-  }
-
-  /**
-   * checks fsck with missing block in both file block and parity block
-   * in different stripes
-   */
-  @Test
-  public void testFileBlockAndParityBlockMissingInDifferentStripes() 
-    throws Exception {
-    LOG.info("testFileBlockAndParityBlockMissingInDifferentStripes");
-    setUp(false);
-    waitUntilCorruptFileCount(dfs, 0);
-    removeFileBlock(FILE_PATH0, 0, 0);
-    waitUntilCorruptFileCount(dfs, 1);
-    removeParityBlock(FILE_PATH0, 1);
-    waitUntilCorruptFileCount(dfs, 2);
-
-    int result = ToolRunner.run(shell, args);
-
-    assertTrue("fsck should return 0, but returns " + 
-               Integer.toString(result), result == 0);
-  }
-
-  /**
-   * checks fsck with missing block in both file block and parity block
-   * in same stripe
-   */
-  @Test
-  public void testFileBlockAndParityBlockMissingInSameStripe() 
-    throws Exception {
-    LOG.info("testFileBlockAndParityBlockMissingInSameStripe");
-    setUp(false);
-    waitUntilCorruptFileCount(dfs, 0);
-    removeParityBlock(FILE_PATH0, 1);
-    waitUntilCorruptFileCount(dfs, 1);
-    removeFileBlock(FILE_PATH0, 1, 0);
-    waitUntilCorruptFileCount(dfs, 2);
-
-    int result = ToolRunner.run(shell, args);
-
-    assertTrue("fsck should return 1, but returns " + 
-               Integer.toString(result), result == 1);
-  }
-
-  /**
-   * checks fsck with two missing file blocks in same stripe
-   */
-  @Test
-  public void test2FileBlocksMissingInSameStripe() 
-    throws Exception {
-    LOG.info("test2FileBlocksMissingInSameStripe");
-    setUp(false);
-    waitUntilCorruptFileCount(dfs, 0);
-    removeFileBlock(FILE_PATH0, 1, 1);
-    waitUntilCorruptFileCount(dfs, 1);
-    removeFileBlock(FILE_PATH0, 1, 0);
-    waitUntilCorruptFileCount(dfs, 1);
-
-    int result = ToolRunner.run(shell, args);
-
-    assertTrue("fsck should return 1, but returns " + 
-               Integer.toString(result), result == 1);
-  }
-
-  /**
-   * checks fsck with two missing file blocks in different stripes
-   */
-  @Test
-  public void test2FileBlocksMissingInDifferentStripes() 
-    throws Exception {
-    LOG.info("test2FileBlocksMissingInDifferentStripes");
-    setUp(false);
-    waitUntilCorruptFileCount(dfs, 0);
-    removeFileBlock(FILE_PATH0, 1, 1);
-    waitUntilCorruptFileCount(dfs, 1);
-    removeFileBlock(FILE_PATH0, 0, 0);
-    waitUntilCorruptFileCount(dfs, 1);
-
-    int result = ToolRunner.run(shell, args);
-
-    assertTrue("fsck should return 0, but returns " +
-               Integer.toString(result), result == 0);
-  }
-
-  /**
-   * checks fsck with file block missing (HAR)
-   * use 2 files to verify HAR offset logic in RaidShell fsck
-   * both files have one corrupt block, parity blocks are clean
-   *
-   * parity blocks in har (file.stripe):
-   * +-----+-----+-----+  +-----+
-   * | 0.0 | 0.1 | 1.0 |  | 1.1 |
-   * +-----+-----+-----+  +-----+
-   *  0                    1
-   *
-   */
-  @Test
-  public void testFileBlockMissingHar() 
-    throws Exception {
-    LOG.info("testFileBlockMissingHar");
-    setUp(true);
-    waitUntilCorruptFileCount(dfs, 0);
-    removeFileBlock(FILE_PATH0, 1, 1);
-    removeFileBlock(FILE_PATH1, 1, 1);
-    waitUntilCorruptFileCount(dfs, 2);
-
-    int result = ToolRunner.run(shell, args);
-
-    assertTrue("fsck should return 0, but returns " +
-               Integer.toString(result), result == 0);
-  }
-
-  /**
-   * checks fsck with file block missing (HAR)
-   * use 2 files to verify HAR offset logic in RaidShell fsck
-   *
-   * parity blocks in har (file.stripe):
-   * +-----+-----+-----+  +-----+
-   * | 0.0 | 0.1 | 1.0 |  | 1.1 |
-   * +-----+-----+-----+  +-----+
-   *  0                    1
-   *
-   * corrupt file 0, stripe 0 file block 0
-   * corrupt file 0, stripe 1 file block 0
-   * corrupt file 1, stripe 0 file block 0
-   * corrupt file 1, stripe 1 file block 0
-   * corrupt har block 0
-   * both files should be corrupt
-   */
-  @Test
-  public void testFileBlockAndParityBlockMissingHar1() 
-    throws Exception {
-    LOG.info("testFileBlockAndParityBlockMissingHar1");
-    setUp(true);
-    waitUntilCorruptFileCount(dfs, 0);
-    removeFileBlock(FILE_PATH0, 0, 0);
-    removeFileBlock(FILE_PATH0, 1, 0);
-    removeFileBlock(FILE_PATH1, 0, 0);
-    removeFileBlock(FILE_PATH1, 1, 0);
-    removeHarParityBlock(0);
-    waitUntilCorruptFileCount(dfs, 3);
-
-    int result = ToolRunner.run(shell, args);
-
-    assertTrue("fsck should return 2, but returns " +
-               Integer.toString(result), result == 2);
-  }
-
-  /**
-   * checks fsck with file block missing (HAR)
-   * use 2 files to verify HAR offset logic in RaidShell fsck
-   *
-   * parity blocks in har (file.stripe):
-   * +-----+-----+-----+  +-----+
-   * | 0.0 | 0.1 | 1.0 |  | 1.1 |
-   * +-----+-----+-----+  +-----+
-   *  0                    1
-   *
-   * corrupt file 0, stripe 0 file block 0
-   * corrupt file 0, stripe 1 file block 0
-   * corrupt file 1, stripe 0 file block 0
-   * corrupt file 1, stripe 1 file block 0
-   * corrupt har block 1
-   * only file 2 should be corrupt
-   */
-  @Test
-  public void testFileBlockAndParityBlockMissingHar2() 
-    throws Exception {
-    LOG.info("testFileBlockAndParityBlockMissingHar2");
-    setUp(true);
-    waitUntilCorruptFileCount(dfs, 0);
-    removeFileBlock(FILE_PATH0, 0, 0);
-    removeFileBlock(FILE_PATH0, 1, 0);
-    removeFileBlock(FILE_PATH1, 0, 0);
-    removeFileBlock(FILE_PATH1, 1, 0);
-    removeHarParityBlock(1);
-    waitUntilCorruptFileCount(dfs, 3);
-
-    int result = ToolRunner.run(shell, args);
-
-    assertTrue("fsck should return 1, but returns " +
-               Integer.toString(result), result == 1);
-  }
-
-  /**
-   * checks that fsck does not report corrupt file that is not in
-   * the specified path
-   */
-  @Test
-  public void testPathFilter() 
-    throws Exception {
-    LOG.info("testPathFilter");
-    setUp(false);
-    waitUntilCorruptFileCount(dfs, 0);
-    removeParityBlock(FILE_PATH0, 1);
-    waitUntilCorruptFileCount(dfs, 1);
-    removeFileBlock(FILE_PATH0, 1, 0);
-    waitUntilCorruptFileCount(dfs, 2);
-
-    String[] otherArgs = new String[2];
-    otherArgs[0] = "-fsck";
-    otherArgs[1] = "/user/pkling/other";
-    int result = ToolRunner.run(shell, otherArgs);
-
-    assertTrue("fsck should return 0, but returns " + 
-               Integer.toString(result), result == 0);
-  }
-
-
-  @After
-  public void tearDown() throws Exception {
-    if (rnode != null) {
-      rnode.stop();
-      rnode.join();
-      rnode = null;
-    }
-
-    if (cluster != null) {
-      cluster.shutdown();
-      cluster = null;
-    }
-    
-    dfs = null;
-
-    LOG.info("Test cluster shut down");
-  }
-  
-
-}
-

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestReedSolomonDecoder.java

@@ -1,135 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.RaidDFSUtil;
-import org.apache.hadoop.hdfs.TestRaidDfs;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.mapred.Reporter;
-import org.junit.Test;
-
-
-public class TestReedSolomonDecoder {
-  final static Log LOG = LogFactory.getLog(
-                            "org.apache.hadoop.raid.TestReedSolomonDecoder");
-  final static String TEST_DIR = new File(System.getProperty("test.build.data",
-        "target/test-data")).getAbsolutePath();
-  final static int NUM_DATANODES = 3;
-
-  Configuration conf;
-  MiniDFSCluster dfs = null;
-  FileSystem fileSys = null;
-
-  @Test
-  public void testDecoder() throws Exception {
-    mySetup();
-    int stripeSize = 10;
-    int paritySize = 4;
-    long blockSize = 8192;
-    Path file1 = new Path("/user/raidtest/file1");
-    Path recoveredFile1 = new Path("/user/raidtest/file1.recovered");
-    Path parityFile1 = new Path("/rsraid/user/raidtest/file1");
-    long crc1 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1,
-                                                          1, 25, blockSize);
-    FileStatus file1Stat = fileSys.getFileStatus(file1);
-
-    conf.setInt("raid.rsdecoder.bufsize", 512);
-    conf.setInt("raid.rsencoder.bufsize", 512);
-
-    try {
-      // First encode the file.
-      ReedSolomonEncoder encoder = new ReedSolomonEncoder(
-        conf, stripeSize, paritySize);
-      short parityRepl = 1;
-      encoder.encodeFile(fileSys, file1, fileSys, parityFile1, parityRepl,
-        Reporter.NULL);
-
-      // Ensure there are no corrupt files yet.
-      DistributedFileSystem dfs = (DistributedFileSystem)fileSys;
-      String[] corruptFiles = RaidDFSUtil.getCorruptFiles(dfs);
-      assertEquals(corruptFiles.length, 0);
-
-      // Now corrupt the file.
-      long corruptOffset = blockSize * 5;
-      FileStatus srcStat = fileSys.getFileStatus(file1);
-      LocatedBlocks locations = RaidDFSUtil.getBlockLocations(dfs,
-          file1.toUri().getPath(), 0, srcStat.getLen());
-      corruptBlock(locations.get(5).getBlock());
-      corruptBlock(locations.get(6).getBlock());
-      TestBlockFixer.reportCorruptBlocks(dfs, file1, new int[]{5, 6},
-          srcStat.getBlockSize());
-
-      // Ensure file is corrupted.
-      corruptFiles = RaidDFSUtil.getCorruptFiles(dfs);
-      assertEquals(corruptFiles.length, 1);
-      assertEquals(corruptFiles[0], file1.toString());
-
-      // Fix the file.
-      ReedSolomonDecoder decoder = new ReedSolomonDecoder(
-        conf, stripeSize, paritySize);
-      decoder.decodeFile(fileSys, file1, fileSys, parityFile1,
-                corruptOffset, recoveredFile1);
-      assertTrue(TestRaidDfs.validateFile(
-                    fileSys, recoveredFile1, file1Stat.getLen(), crc1));
-    } finally {
-      myTearDown();
-    }
-  }
-
-  void corruptBlock(ExtendedBlock block) throws IOException {
-    assertTrue("Could not corrupt block",
-        dfs.corruptBlockOnDataNodes(block) > 0);
-  }
-
-  private void mySetup() throws Exception {
-
-    new File(TEST_DIR).mkdirs(); // Make sure data directory exists
-    conf = new Configuration();
-
-    // make all deletions not go through Trash
-    conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient");
-
-    conf.setBoolean("dfs.permissions", false);
-
-    dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build();
-    dfs.waitActive();
-    fileSys = dfs.getFileSystem();
-    String namenode = fileSys.getUri().toString();
-    FileSystem.setDefaultUri(conf, namenode);
-  }
-
-  private void myTearDown() throws Exception {
-    if (dfs != null) { dfs.shutdown(); }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-raid/src/test/java/org/apache/hadoop/raid/TestReedSolomonEncoder.java

@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.raid;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.TestRaidDfs;
-import org.apache.hadoop.mapred.Reporter;
-import org.junit.Test;
-
-
-public class TestReedSolomonEncoder {
-  final static Log LOG = LogFactory.getLog(
-                            "org.apache.hadoop.raid.TestReedSolomonEncoder");
-  final static String TEST_DIR = new File(System.getProperty("test.build.data",
-        "target/test-data")).getAbsolutePath();
-  final static int NUM_DATANODES = 3;
-
-  Configuration conf;
-  String namenode = null;
-  MiniDFSCluster dfs = null;
-  FileSystem fileSys = null;
-
-  @Test
-  public void testEncoder() throws Exception {
-    mySetup();
-    int stripeSize = 10;
-    int paritySize = 4;
-    long blockSize = 8192;
-    Path file1 = new Path("/user/raidtest/file1");
-    Path parityFile1 = new Path("/rsraid/user/raidtest/file1");
-    long crc1 = TestRaidDfs.createTestFilePartialLastBlock(fileSys, file1,
-                                                          1, 25, blockSize);
-    try {
-      ReedSolomonEncoder encoder = new ReedSolomonEncoder(
-        conf, stripeSize, paritySize);
-      short parityRepl = 1;
-      encoder.encodeFile(fileSys, file1, fileSys, parityFile1, parityRepl,
-        Reporter.NULL);
-
-      FileStatus parityStat = fileSys.getFileStatus(parityFile1);
-      assertEquals(4*8192*3, parityStat.getLen());
-
-    } finally {
-      myTearDown();
-    }
-  }
-
-  private void mySetup() throws Exception {
-
-    new File(TEST_DIR).mkdirs(); // Make sure data directory exists
-    conf = new Configuration();
-
-    // make all deletions not go through Trash
-    conf.set("fs.shell.delete.classname", "org.apache.hadoop.hdfs.DFSClient");
-
-    dfs = new MiniDFSCluster(conf, NUM_DATANODES, true, null);
-    dfs.waitActive();
-    fileSys = dfs.getFileSystem();
-    namenode = fileSys.getUri().toString();
-
-    FileSystem.setDefaultUri(conf, namenode);
-
-  }
-
-  private void myTearDown() throws Exception {
-    if (dfs != null) { dfs.shutdown(); }
-  }
-}

hadoop-hdfs-project/pom.xml

@@ -34,7 +34,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <module>hadoop-hdfs</module>
     <module>hadoop-hdfs-httpfs</module>
     <module>hadoop-hdfs/src/contrib/bkjournal</module>
-    <module>hadoop-hdfs-raid</module>
   </modules>
 
   <build>

hadoop-mapreduce-project/build.xml

@@ -1711,10 +1711,6 @@
                 output="${build.dir.eclipse-contrib-classes}/gridmix/main" />
         <source path="${contrib.dir}/gridmix/src/test"
                 output="${build.dir.eclipse-contrib-classes}/gridmix/test" />
-        <source path="${contrib.dir}/raid/src/java"
-                output="${build.dir.eclipse-contrib-classes}/raid/main" />
-        <source path="${contrib.dir}/raid/src/test"
-                output="${build.dir.eclipse-contrib-classes}/raid/test" />
         <source path="${contrib.dir}/vaidya/src/java"
                 output="${build.dir.eclipse-contrib-classes}/vaidya/main" />
         <source path="${contrib.dir}/vertica/src/java"

hadoop-mapreduce-project/src/contrib/build.xml

@@ -62,7 +62,6 @@
       <fileset dir="." includes="streaming/build.xml"/> 
       <fileset dir="." includes="gridmix/build.xml"/>
       <fileset dir="." includes="vertica/build.xml"/>
-      <fileset dir="." includes="raid/build.xml"/>
     </subant>
     <available file="${build.contrib.dir}/testsfailed" property="testsfailed"/>
     <fail if="testsfailed">Tests failed!</fail>

hadoop-mapreduce-project/src/contrib/raid/README

@@ -1,201 +0,0 @@
-# Copyright 2008 The Apache Software Foundation Licensed under the
-# Apache License, Version 2.0 (the "License"); you may not use this
-# file except in compliance with the License.  You may obtain a copy
-# of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
-# required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.  See the License for the specific language governing
-# permissions and limitations under the License.
-
-This package implements a Distributed Raid File System. It is used alongwith
-an instance of the Hadoop Distributed File System (HDFS). It can be used to
-provide better protection against data corruption. It can also be used to
-reduce the total storage requirements of HDFS. 
-
-Distributed Raid File System consists of two main software components. The first component 
-is the RaidNode, a daemon that creates parity files from specified HDFS files. 
-The second component "raidfs" is a software that is layered over a HDFS client and it 
-intercepts all calls that an application makes to the HDFS client. If HDFS encounters
-corrupted data while reading a file, the raidfs client detects it; it uses the
-relevant parity blocks to recover the corrupted data (if possible) and returns
-the data to the application. The application is completely transparent to the
-fact that parity data was used to satisfy it's read request.
-
-The primary use of this feature is to save disk space for HDFS files. 
-HDFS typically stores data in triplicate.
-The Distributed Raid File System can be configured in such a way that a set of
-data blocks of a file are combined together to form one or more parity blocks.
-This allows one to reduce the replication factor of a HDFS file from 3 to 2
-while keeping the failure probabilty relatively same as before. This typically
-results in saving 25% to 30% of storage space in a HDFS cluster. 
-
---------------------------------------------------------------------------------
-
-BUILDING:
-
-In HADOOP_PREFIX, run ant package to build Hadoop and its contrib packages.
-
---------------------------------------------------------------------------------
-
-INSTALLING and CONFIGURING:
-
-The entire code is packaged in the form of a single jar file hadoop-*-raid.jar.
-To use HDFS Raid, you need to put the above mentioned jar file on
-the CLASSPATH. The easiest way is to copy the hadoop-*-raid.jar
-from HADOOP_PREFIX/build/contrib/raid to HADOOP_PREFIX/lib. Alternatively
-you can modify HADOOP_CLASSPATH to include this jar, in conf/hadoop-env.sh.
-
-There is a single configuration file named raid.xml that describes the HDFS 
-path(s) that you want to raid. A sample of this file can be found in 
-sc/contrib/raid/conf/raid.xml. Please edit the entries in this file to list the 
-path(s) that you want to raid. Then, edit the hdfs-site.xml file for
-your installation to include a reference to this raid.xml. You can add the
-following to your hdfs-site.xml
-        <property>
-          <name>raid.config.file</name>
-          <value>/mnt/hdfs/DFS/conf/raid.xml</value>
-          <description>This is needed by the RaidNode </description>
-        </property>
-
-Please add an entry to your hdfs-site.xml to enable hdfs clients to use the
-parity bits to recover corrupted data.
-
-       <property>
-         <name>fs.hdfs.impl</name>
-         <value>org.apache.hadoop.dfs.DistributedRaidFileSystem</value>
-         <description>The FileSystem for hdfs: uris.</description>
-       </property>
-
-
---------------------------------------------------------------------------------
-
-OPTIONAL CONFIGIURATION:
-
-The following properties can be set in hdfs-site.xml to further tune you configuration:
-
-    Specifies the location where parity files are located. 
-        <property>
-          <name>hdfs.raid.locations</name>
-          <value>hdfs://newdfs.data:8000/raid</value>
-          <description>The location for parity files. If this is
-          is not defined, then defaults to /raid. 
-          </descrition>
-        </property>
-
-    Specify the parity stripe length 
-        <property>
-          <name>hdfs.raid.stripeLength</name>
-          <value>10</value>
-          <description>The number of blocks in a file to be combined into 
-          a single raid parity block. The default value is 5. The lower
-          the number the greater is the disk space you will save when you
-          enable raid.
-          </description>
-        </property>
-
-    Specify the size of HAR part-files
-        <property>
-          <name>raid.har.partfile.size</name>
-          <value>4294967296</value>
-          <description>The size of HAR part files that store raid parity
-          files. The default is 4GB. The higher the number the fewer the
-          number of files used to store the HAR archive.
-          </description>
-        </property>
-
-    Specify which implementation of RaidNode to use.
-        <property>
-          <name>raid.classname</name>
-          <value>org.apache.hadoop.raid.DistRaidNode</value>
-          <description>Specify which implementation of RaidNode to use
-	  (class name).
-          </description>
-        </property>
-
-
-    Specify the periodicy at which the RaidNode re-calculates (if necessary)
-    the parity blocks
-        <property>
-          <name>raid.policy.rescan.interval</name>
-          <value>5000</value>
-          <description>Specify the periodicity in milliseconds after which
-          all source paths are rescanned and parity blocks recomputed if 
-          necessary. By default, this value is 1 hour.
-          </description>
-        </property>
-
-    By default, the DistributedRaidFileSystem assumes that the underlying file
-    system is the DistributedFileSystem. If you want to layer the DistributedRaidFileSystem
-    over some other file system, then define a property named fs.raid.underlyingfs.impl
-    that specifies the name of the underlying class. For example, if you want to layer
-    The DistributedRaidFileSystem over an instance of the NewFileSystem, then
-        <property>
-          <name>fs.raid.underlyingfs.impl</name>
-          <value>org.apche.hadoop.new.NewFileSystem</value>
-          <description>Specify the filesystem that is layered immediately below the
-          DistributedRaidFileSystem. By default, this value is DistributedFileSystem.
-          </description>
-
-
---------------------------------------------------------------------------------
-
-ADMINISTRATION:
-
-The Distributed Raid File System  provides support for administration at runtime without
-any downtime to cluster services.  It is possible to add/delete new paths to be raided without
-interrupting any load on the cluster. If you change raid.xml, its contents will be
-reload within seconds and the new contents will take effect immediately.
-
-Designate one machine in your cluster to run the RaidNode software. You can run this daemon
-on any machine irrespective of whether that machine is running any other hadoop daemon or not. 
-You can start the RaidNode by running the following on the selected machine:
-nohup $HADOOP_PREFIX/bin/hadoop org.apache.hadoop.raid.RaidNode >> /xxx/logs/hadoop-root-raidnode-hadoop.xxx.com.log &
-
-Optionally, we provide two scripts to start and stop the RaidNode. Copy the scripts
-start-raidnode.sh and stop-raidnode.sh to the directory $HADOOP_PREFIX/bin in the machine
-you would like to deploy the daemon. You can start or stop the RaidNode by directly 
-callying the scripts from that machine. If you want to deploy the RaidNode remotely,
-copy start-raidnode-remote.sh and stop-raidnode-remote.sh to $HADOOP_PREFIX/bin at 
-the machine from which you want to trigger the remote deployment and create a text
-file $HADOOP_PREFIX/conf/raidnode at the same machine containing the name of the server
-where the RaidNode should run. These scripts run ssh to the specified machine and 
-invoke start/stop-raidnode.sh there. As an example, you might want to change
-start-mapred.sh in the JobTracker machine so that it automatically calls 
-start-raidnode-remote.sh (and do the equivalent thing for stop-mapred.sh and 
-stop-raidnode-remote.sh).
-
-To validate the integrity of a file system, run RaidFSCK as follows:
-$HADOOP_PREFIX/bin/hadoop org.apache.hadoop.raid.RaidShell -fsck [path]
-
-This will print a list of corrupt files (i.e., files which have lost too many
-blocks and can no longer be fixed by Raid).
-
-
---------------------------------------------------------------------------------
-
-IMPLEMENTATION:
-
-The RaidNode periodically scans all the specified paths in the configuration
-file. For each path, it recursively scans all files that have more than 2 blocks
-and that has not been modified during the last few hours (default is 24 hours). 
-It picks the specified number of blocks (as specified by the stripe size),
-from the file, generates a parity block by combining them and
-stores the results as another HDFS file in the specified destination 
-directory. There is a one-to-one mapping between a HDFS
-file and its parity file. The RaidNode also periodically finds parity files
-that are orphaned and deletes them.
-
-The Distributed Raid FileSystem is layered over a DistributedFileSystem
-instance intercepts all calls that go into HDFS. HDFS throws a ChecksumException
-or a BlocMissingException when a file read encounters bad data. The layered
-Distributed Raid FileSystem catches these exceptions, locates the corresponding
-parity file, extract the original data from the parity files and feeds the
-extracted data back to the application in a completely transparent way.
-
-The layered Distributed Raid FileSystem does not fix the data-loss that it
-encounters while serving data. It merely make the application transparently
-use the parity blocks to re-create the original data. A command line tool 
-"fsckraid" is currently under development that will fix the corrupted files 
-by extracting the data from the associated parity files. An adminstrator 
-can run "fsckraid" manually as and when needed.

hadoop-mapreduce-project/src/contrib/raid/build.xml

@@ -1,64 +0,0 @@
-<?xml version="1.0"?>
-
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!--
-Before you can run these subtargets directly, you need
-to call at top-level: ant deploy-contrib compile-core-test
--->
-<project name="raid" default="jar">
-
-  <import file="../build-contrib.xml"/>
-
-  <!-- the unit test classpath -->
-  <path id="contrib.classpath.raid">
-    <pathelement location="${hadoop.root}/src/contrib/raid/lib"/>
-    <path refid="contrib-classpath"/>
-  </path>
-
-  <target name="test" depends="compile,compile-test,test-junit" description="Automated Test Framework" if="test.available"/>
-
-  <target name="test-junit" depends="compile,compile-test" if="test.available">
-    <junit maxmemory="512m" showoutput="${test.output}" fork="yes" printsummary="yes" errorProperty="tests.failed" 
-           haltonfailure="no" failureProperty="tests.failed" timeout="${test.timeout}">
-
-        <classpath refid="test.classpath"/>
-        <sysproperty key="test.build.data" value="${build.test}/data"/>
-              <sysproperty key="build.test" value="${build.test}"/>
-              <sysproperty key="user.dir" value="${build.test}/data"/>
-              <sysproperty key="fs.default.name" value="${fs.default.name}"/>
-              <sysproperty key="hadoop.test.localoutputfile" value="${hadoop.test.localoutputfile}"/>
-              <sysproperty key="hadoop.log.dir" value="${hadoop.log.dir}"/>
-        <sysproperty key="test.src.dir" value="${test.src.dir}"/>
-        <formatter type="${test.junit.output.format}" />
-        <batchtest todir="${build.test}" unless="testcase">
-           <fileset dir="${src.test}">
-             <include name="**/Test*.java"/>
-           </fileset>
-        </batchtest>
-        <batchtest todir="${build.test}" if="testcase">
-            <fileset dir="${src.test}">
-                <include name="**/${testcase}.java"/>
-            </fileset>
-         </batchtest>
-    </junit>
-    <fail if="tests.failed">Tests failed!</fail>
- </target>
-
-</project>
-

hadoop-mapreduce-project/src/contrib/raid/ivy.xml

@@ -1,145 +0,0 @@
-<?xml version="1.0" ?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<ivy-module version="1.0" xmlns:m="http://ant.apache.org/ivy/maven">
-  <info organisation="org.apache.hadoop" module="${ant.project.name}">
-    <license name="Apache 2.0"/>
-    <description>Rumen</description>
-  </info>
-  <configurations defaultconfmapping="default">
-    <!--these match the Maven configurations-->
-    <conf name="default" extends="master,runtime"/>
-    <conf name="master" description="contains the artifact but no dependencies"/>
-    <conf name="runtime" description="runtime but not the artifact" />
-
-    <conf name="common" visibility="private" extends="runtime"
-      description="artifacts needed to compile/test the application"/>
-    <conf name="test" visibility="private" extends="runtime"/>
-  </configurations>
-
-  <publications>
-    <!--get the artifact from our module name-->
-    <artifact conf="master"/>
-  </publications>
-  <dependencies>
-   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
-    <dependency org="org.apache.hadoop"
-      name="hadoop-common"
-      rev="${hadoop-common.version}"
-      conf="common->default"/>
-    <dependency org="org.apache.hadoop"
-      name="hadoop-common"
-      rev="${hadoop-common.version}"
-      conf="test->default">
-      <artifact name="hadoop-common" type="tests" ext="jar" m:classifier="tests"/>
-    </dependency>
-    <dependency org="org.apache.hadoop"
-      name="hadoop-hdfs"
-      rev="${hadoop-hdfs.version}"
-      conf="common->default"/>
-    <dependency org="org.apache.hadoop"
-      name="hadoop-hdfs"
-      rev="${hadoop-hdfs.version}"
-      conf="test->default">
-      <artifact name="hadoop-hdfs" type="tests" ext="jar" m:classifier="tests"/>
-    </dependency>
- <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core" 
-               rev="${yarn.version}" conf="common->default"/>
-   <dependency org="org.apache.hadoop" name="hadoop-yarn-common"
-               rev="${yarn.version}" conf="common->default"/>
-   <dependency org="org.apache.hadoop" name="hadoop-archives" 
-               rev="${hadoop-common.version}" conf="common->default"/> 
-
-    <dependency org="commons-logging"
-      name="commons-logging"
-      rev="${commons-logging.version}"
-      conf="common->default"/>
-    <dependency org="log4j"
-      name="log4j"
-      rev="${log4j.version}"
-      conf="common->master"/>
-    <dependency org="junit"
-      name="junit"
-      rev="${junit.version}"
-      conf="common->default"/>
-
-    <!-- necessary for Mini*Clusters -->
-    <dependency org="commons-httpclient"
-      name="commons-httpclient"
-      rev="${commons-httpclient.version}"
-      conf="common->master"/>
-    <dependency org="commons-codec"
-      name="commons-codec"
-      rev="${commons-codec.version}"
-      conf="common->default"/>
-    <dependency org="commons-net"
-      name="commons-net"
-      rev="${commons-net.version}"
-      conf="common->default"/>
-    <dependency org="org.mortbay.jetty"
-      name="jetty"
-      rev="${jetty.version}"
-      conf="common->master"/>
-    <dependency org="org.mortbay.jetty"
-      name="jetty-util"
-      rev="${jetty-util.version}"
-      conf="common->master"/>
-    <dependency org="org.mortbay.jetty"
-      name="jsp-api-2.1"
-      rev="${jetty.version}"
-      conf="common->master"/>
-    <dependency org="org.mortbay.jetty"
-      name="jsp-2.1"
-      rev="${jetty.version}"
-      conf="common->master"/>
-    <dependency org="org.mortbay.jetty"
-      name="servlet-api-2.5"
-      rev="${servlet-api-2.5.version}"
-      conf="common->master"/>
-    <dependency org="commons-cli"
-      name="commons-cli"
-      rev="${commons-cli.version}"
-      conf="common->default"/>
-    <dependency org="org.apache.avro"
-      name="avro"
-      rev="${avro.version}"
-      conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
-    <dependency org="org.codehaus.jackson"
-      name="jackson-mapper-asl"
-      rev="${jackson.version}"
-      conf="common->default"/>
-    <dependency org="org.codehaus.jackson"
-      name="jackson-core-asl"
-      rev="${jackson.version}"
-      conf="common->default"/>
-    <dependency org="com.thoughtworks.paranamer"
-      name="paranamer"
-      rev="${paranamer.version}"
-      conf="common->default"/>
-
-   <!-- Exclusions for transitive dependencies pulled in by log4j -->
-   <exclude org="com.sun.jdmk"/>
-   <exclude org="com.sun.jmx"/>
-   <exclude org="javax.jms"/> 
-   <exclude org="javax.mail"/> 
-
-  </dependencies>
-</ivy-module>

hadoop-mapreduce-project/src/contrib/raid/ivy/libraries.properties

@@ -1,18 +0,0 @@
-#   Licensed to the Apache Software Foundation (ASF) under one or more
-#   contributor license agreements.  See the NOTICE file distributed with
-#   this work for additional information regarding copyright ownership.
-#   The ASF licenses this file to You under the Apache License, Version 2.0
-#   (the "License"); you may not use this file except in compliance with
-#   the License.  You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#This properties file lists the versions of the various artifacts used by hadoop.
-#It drives ivy and the generation of a maven POM
-#These are the versions of our dependencies (in alphabetical order)

hadoop-project/pom.xml

@@ -257,11 +257,6 @@
         <artifactId>hadoop-client</artifactId>
         <version>${project.version}</version>
       </dependency>
-      <dependency>
-        <groupId>org.apache.hadoop</groupId>
-        <artifactId>hadoop-hdfs-raid</artifactId>
-        <version>${project.version}</version>
-      </dependency>
 
       <dependency>
         <groupId>org.apache.hadoop</groupId>