HDFS-15909. Make fnmatch cross platform (#2792)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt

@@ -16,6 +16,7 @@
 # limitations under the License.
 #
 
-add_library(fs_obj OBJECT filesystem.cc filesystem_sync.cc filehandle.cc bad_datanode_tracker.cc namenode_operations.cc)
+add_library(fs_obj OBJECT $<TARGET_OBJECTS:x_platform_obj> filesystem.cc filesystem_sync.cc filehandle.cc bad_datanode_tracker.cc namenode_operations.cc)
+target_include_directories(fs_obj PRIVATE ../lib)
 add_dependencies(fs_obj proto)
-add_library(fs $<TARGET_OBJECTS:fs_obj>)
+add_library(fs $<TARGET_OBJECTS:fs_obj> $<TARGET_OBJECTS:x_platform_obj>)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc

@@ -25,12 +25,12 @@
 #include <limits>
 #include <future>
 #include <tuple>
-#include <iostream>
 #include <pwd.h>
-#include <fnmatch.h>
 
 #include <boost/asio/ip/tcp.hpp>
 
+#include "x-platform/syscall.h"
+
 #define FMT_THIS_ADDR "this=" << (void*)this
 
 namespace hdfs {
@@ -722,8 +722,8 @@ void FileSystemImpl::FindShim(const Status &stat, const std::vector<StatInfo> &
       for (StatInfo const& si : stat_infos) {
         //If we are at the last depth and it matches both path and name, we need to output it.
         if (operational_state->depth == shared_state->dirs.size() - 2
-            && !fnmatch(shared_state->dirs[operational_state->depth + 1].c_str(), si.path.c_str(), 0)
-            && !fnmatch(shared_state->name.c_str(), si.path.c_str(), 0)) {
+            && XPlatform::Syscall::FnMatch(shared_state->dirs[operational_state->depth + 1], si.path)
+            && XPlatform::Syscall::FnMatch(shared_state->name, si.path)) {
           outputs.push_back(si);
         }
         //Skip if not directory
@@ -731,7 +731,7 @@ void FileSystemImpl::FindShim(const Status &stat, const std::vector<StatInfo> &
           continue;
         }
         //Checking for a match with the path at the current depth
-        if(!fnmatch(shared_state->dirs[operational_state->depth + 1].c_str(), si.path.c_str(), 0)){
+        if(XPlatform::Syscall::FnMatch(shared_state->dirs[operational_state->depth + 1], si.path)) {
           //Launch a new requests for every matched directory
           shared_state->outstanding_requests++;
           auto callback = [this, si, operational_state, shared_state](const Status &stat, const std::vector<StatInfo> & stat_infos, bool has_more) {
@@ -755,7 +755,7 @@ void FileSystemImpl::FindShim(const Status &stat, const std::vector<StatInfo> &
           nn_.GetListing(si.full_path, callback);
         }
         //All names that match the specified name are saved to outputs
-        if(!fnmatch(shared_state->name.c_str(), si.path.c_str(), 0)){
+        if(XPlatform::Syscall::FnMatch(shared_state->name, si.path)) {
           outputs.push_back(si);
         }
       }

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h

@@ -48,6 +48,17 @@ class Syscall {
    */
   static int WriteToStdout(const char* message);
 
+  /**
+   * Checks whether the {@link str} argument matches the {@link pattern}
+   * argument, which is a shell wildcard pattern.
+   *
+   * @param pattern The wildcard pattern to use.
+   * @param str The string to match.
+   * @returns A boolean indicating whether the given {@link str}
+   * matches {@link pattern}.
+   */
+  static bool FnMatch(const std::string& pattern, const std::string& str);
+
  private:
   static bool WriteToStdoutImpl(const char* message);
 };

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc

@@ -16,6 +16,7 @@
  * limitations under the License.
  */
 
+#include <fnmatch.h>
 #include <unistd.h>
 
 #include <cstring>
@@ -30,6 +31,11 @@ int XPlatform::Syscall::WriteToStdout(const char* message) {
   return WriteToStdoutImpl(message) ? 1 : 0;
 }
 
+bool XPlatform::Syscall::FnMatch(const std::string& pattern,
+                                 const std::string& str) {
+  return fnmatch(pattern.c_str(), str.c_str(), 0) == 0;
+}
+
 bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
   const auto message_len = strlen(message);
   const auto result = write(1, message, message_len);

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc

@@ -16,10 +16,13 @@
  * limitations under the License.
  */
 
+#include <Shlwapi.h>
 #include <Windows.h>
 
 #include "syscall.h"
 
+#pragma comment(lib, "Shlwapi.lib")
+
 bool XPlatform::Syscall::WriteToStdout(const std::string& message) {
   return WriteToStdoutImpl(message.c_str());
 }
@@ -28,6 +31,12 @@ int XPlatform::Syscall::WriteToStdout(const char* message) {
   return WriteToStdoutImpl(message) ? 1 : 0;
 }
 
+bool XPlatform::Syscall::FnMatch(const std::string& pattern,
+                                 const std::string& str) {
+  return PathMatchSpecA(static_cast<LPCSTR>(str.c_str()),
+                        static_cast<LPCSTR>(pattern.c_str())) == TRUE;
+}
+
 bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
   auto* const stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE);
   if (stdout_handle == INVALID_HANDLE_VALUE || stdout_handle == nullptr) {

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt

@@ -18,9 +18,16 @@
 
 if(WIN32)
     add_executable(x_platform_utils_test $<TARGET_OBJECTS:x_platform_obj> utils_common_test.cc utils_test_main.cc utils_win_test.cc)
+    add_executable(x_platform_syscall_test $<TARGET_OBJECTS:x_platform_obj> syscall_common_test.cc utils_test_main.cc syscall_win_test.cc)
 else(WIN32)
     add_executable(x_platform_utils_test $<TARGET_OBJECTS:x_platform_obj> utils_common_test.cc utils_test_main.cc utils_nix_test.cc)
+    add_executable(x_platform_syscall_test $<TARGET_OBJECTS:x_platform_obj> syscall_common_test.cc utils_test_main.cc syscall_nix_test.cc)
 endif(WIN32)
+
 target_include_directories(x_platform_utils_test PRIVATE ${LIBHDFSPP_LIB_DIR})
 target_link_libraries(x_platform_utils_test gmock_main)
 add_test(x_platform_utils_test x_platform_utils_test)
+
+target_include_directories(x_platform_syscall_test PRIVATE ${LIBHDFSPP_LIB_DIR})
+target_link_libraries(x_platform_syscall_test gmock_main)
+add_test(x_platform_syscall_test x_platform_syscall_test)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc

@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "x-platform/syscall.h"
+
+TEST(XPlatformSyscall, FnMatchBasicAsterisk) {
+  const std::string pattern("a*.doc");
+  const std::string str("abcd.doc");
+  EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
+}
+
+TEST(XPlatformSyscall, FnMatchBasicQuestionMark) {
+  const std::string pattern("a?.doc");
+  const std::string str("ab.doc");
+  EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
+}
+
+TEST(XPlatformSyscall, FnMatchNegativeAsterisk) {
+  const std::string pattern("a*.doc");
+  const std::string str("bcd.doc");
+  EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
+}
+
+TEST(XPlatformSyscall, FnMatchNegativeQuestionMark) {
+  const std::string pattern("a?.doc");
+  const std::string str("abc.doc");
+  EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_nix_test.cc

@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "x-platform/syscall.h"
+
+TEST(XPlatformSyscall, FnMatchBasicPath) {
+  const std::string pattern("*.doc");
+  const std::string str("some/path/abcd.doc");
+  EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
+}
+
+TEST(XPlatformSyscall, FnMatchNegativePath) {
+  const std::string pattern("x*.doc");
+  const std::string str("y/abcd.doc");
+  EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_win_test.cc

@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "x-platform/syscall.h"
+
+TEST(XPlatformSyscall, FnMatchBasicPath) {
+  const std::string pattern("*.doc");
+  const std::string str(R"(some\path\abcd.doc)");
+  EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
+}
+
+TEST(XPlatformSyscall, FnMatchNegativePath) {
+  const std::string pattern("x*.doc");
+  const std::string str(R"(y\abcd.doc)");
+  EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
+}