HADOOP-17724. Add Dockerfile for Debian 10 (#3038)

dev-support/docker/Dockerfile_debian_10

@@ -0,0 +1,107 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for installing the necessary dependencies for building Hadoop.
+# See BUILDING.txt.
+
+FROM debian:10
+
+WORKDIR /root
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+#####
+# Disable suggests/recommends
+#####
+RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
+RUN echo APT::Install-Suggests "0"\; >>  /etc/apt/apt.conf.d/10disableextras
+
+#####
+# For installing the latest packages
+#####
+RUN echo 'deb http://deb.debian.org/debian testing main' >> /etc/apt/sources.list
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_TERSE true
+
+######
+# Platform package dependency resolver
+######
+COPY pkg-resolver pkg-resolver
+RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
+    && chmod a+r pkg-resolver/*.json
+
+######
+# Install packages from apt
+######
+# hadolint ignore=DL3008,SC2046
+RUN apt-get -q update \
+    && apt-get -q install -y --no-install-recommends python3 \
+    && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:10) \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN locale-gen en_US.UTF-8
+ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
+ENV PYTHONIOENCODING=utf-8
+
+######
+# Set env vars required to build Hadoop
+######
+ENV MAVEN_HOME /usr
+# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
+
+#######
+# Set env vars for SpotBugs 4.2.2
+#######
+ENV SPOTBUGS_HOME /opt/spotbugs
+
+#######
+# Set env vars for Google Protobuf 3.7.1
+#######
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PATH "${PATH}:/opt/protobuf/bin"
+
+###
+# Avoid out of memory errors in builds
+###
+ENV MAVEN_OPTS -Xms256m -Xmx3072m
+
+# Skip gpg verification when downloading Yetus via yetus-wrapper
+ENV HADOOP_SKIP_YETUS_VERIFICATION true
+
+####
+# Install packages
+####
+RUN pkg-resolver/install-common-pkgs.sh
+RUN pkg-resolver/install-spotbugs.sh debian:10
+RUN pkg-resolver/install-boost.sh debian:10
+RUN pkg-resolver/install-protobuf.sh debian:10
+RUN pkg-resolver/install-hadolint.sh debian:10
+RUN pkg-resolver/install-intel-isa-l.sh debian:10
+
+###
+# Everything past this point is either not needed for testing or breaks Yetus.
+# So tell Yetus not to read the rest of the file:
+# YETUS CUT HERE
+###
+
+# Add a welcome message and environment checks.
+COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
+RUN chmod 755 /root/hadoop_env_checks.sh
+# hadolint ignore=SC2016
+RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc

dev-support/docker/pkg-resolver/packages.json

@@ -1,30 +1,39 @@
 {
   "ant": {
+    "debian:10": "ant",
     "ubuntu:focal": "ant",
     "ubuntu:focal::arch64": "ant",
     "centos:7": "ant",
     "centos:8": "ant"
   },
   "apt-utils": {
+    "debian:10": "apt-utils",
     "ubuntu:focal": "apt-utils",
     "ubuntu:focal::arch64": "apt-utils"
   },
   "automake": {
+    "debian:10": "automake",
     "ubuntu:focal": "automake",
     "ubuntu:focal::arch64": "automake",
     "centos:7": "automake",
     "centos:8": "automake"
   },
   "bats": {
+    "debian:10": "bats",
     "ubuntu:focal": "bats",
     "ubuntu:focal::arch64": "bats"
   },
   "build-essential": {
+    "debian:10": "build-essential",
     "ubuntu:focal": "build-essential",
     "ubuntu:focal::arch64": "build-essential",
     "centos:7": "build-essential"
   },
   "bzip2": {
+    "debian:10": [
+      "bzip2",
+      "libbz2-dev"
+    ],
     "ubuntu:focal": [
       "bzip2",
       "libbz2-dev"
@@ -43,16 +52,22 @@
     ]
   },
   "clang": {
+    "debian:10": "clang",
     "ubuntu:focal": "clang",
     "ubuntu:focal::arch64": "clang",
     "centos:7": "clang",
     "centos:8": "clang"
   },
   "cmake": {
+    "debian:10": "cmake",
     "ubuntu:focal": "cmake",
     "ubuntu:focal::arch64": "cmake"
   },
   "curl": {
+    "debian:10": [
+      "curl",
+      "libcurl4-openssl-dev"
+    ],
     "ubuntu:focal": [
       "curl",
       "libcurl4-openssl-dev"
@@ -71,6 +86,7 @@
     ]
   },
   "doxygen": {
+    "debian:10": "doxygen",
     "ubuntu:focal": "doxygen",
     "ubuntu:focal::arch64": "doxygen",
     "centos:7": "doxygen"
@@ -79,6 +95,10 @@
     "centos:8": "dnf"
   },
   "fuse": {
+    "debian:10": [
+      "fuse",
+      "libfuse-dev"
+    ],
     "ubuntu:focal": [
       "fuse",
       "libfuse-dev"
@@ -99,6 +119,10 @@
     ]
   },
   "gcc": {
+    "debian:10": [
+      "gcc",
+      "g++"
+    ],
     "ubuntu:focal": [
       "gcc",
       "g++"
@@ -113,36 +137,46 @@
     ]
   },
   "git": {
+    "debian:10": "git",
     "ubuntu:focal": "git",
     "ubuntu:focal::arch64": "git",
     "centos:7": "git",
     "centos:8": "git"
   },
   "gnupg-agent": {
+    "debian:10": "gnupg-agent",
     "ubuntu:focal": "gnupg-agent",
     "ubuntu:focal::arch64": "gnupg-agent"
   },
   "hugo": {
+    "debian:10": "hugo",
     "ubuntu:focal": "hugo",
     "ubuntu:focal::arch64": "hugo"
   },
   "libbcprov-java": {
+    "debian:10": "libbcprov-java",
     "ubuntu:focal": "libbcprov-java",
     "ubuntu:focal::arch64": "libbcprov-java"
   },
   "libtool": {
+    "debian:10": "libtool",
     "ubuntu:focal": "libtool",
     "ubuntu:focal::arch64": "libtool",
     "centos:7": "libtool",
     "centos:8": "libtool"
   },
   "openssl": {
+    "debian:10": "libssl-dev",
     "ubuntu:focal": "libssl-dev",
     "ubuntu:focal::arch64": "libssl-dev",
     "centos:7": "openssl-devel",
     "centos:8": "openssl-devel"
   },
   "protocol-buffers": {
+    "debian:10": [
+      "libprotobuf-dev",
+      "libprotoc-dev"
+    ],
     "ubuntu:focal": [
       "libprotobuf-dev",
       "libprotoc-dev"
@@ -153,17 +187,23 @@
     ]
   },
   "sasl": {
+    "debian:10": "libsasl2-dev",
     "ubuntu:focal": "libsasl2-dev",
     "ubuntu:focal::arch64": "libsasl2-dev",
     "centos:7": "cyrus-sasl-devel",
     "centos:8": "cyrus-sasl-devel"
   },
   "snappy": {
+    "debian:10": "libsnappy-dev",
     "ubuntu:focal": "libsnappy-dev",
     "ubuntu:focal::arch64": "libsnappy-dev",
     "centos:7": "snappy-devel"
   },
   "zlib": {
+    "debian:10": [
+      "libzstd-dev",
+      "zlib1g-dev"
+    ],
     "ubuntu:focal": [
       "libzstd-dev",
       "zlib1g-dev"
@@ -182,6 +222,7 @@
     ]
   },
   "locales": {
+    "debian:10": "locales",
     "ubuntu:focal": "locales",
     "ubuntu:focal::arch64": "locales"
   },
@@ -193,16 +234,23 @@
     "centos:7": "libpmem-devel"
   },
   "make": {
+    "debian:10": "make",
     "ubuntu:focal": "make",
     "ubuntu:focal::arch64": "make",
     "centos:7": "make",
     "centos:8": "make"
   },
   "maven": {
+    "debian:10": "maven",
     "ubuntu:focal": "maven",
     "ubuntu:focal::arch64": "maven"
   },
   "javascript": {
+    "debian:10": [
+      "nodejs",
+      "node-yarn",
+      "npm"
+    ],
     "ubuntu:focal": [
       "nodejs",
       "node-yarn",
@@ -216,6 +264,7 @@
     ]
   },
   "java": {
+    "debian:10": "openjdk-11-jdk",
     "ubuntu:focal": [
       "openjdk-8-jdk",
       "openjdk-11-jdk"
@@ -226,17 +275,26 @@
     ]
   },
   "pinentry-curses": {
+    "debian:10": "pinentry-curses",
     "ubuntu:focal": "pinentry-curses",
     "ubuntu:focal::arch64": "pinentry-curses",
     "centos:7": "pinentry-curses",
     "centos:8": "pinentry-curses"
   },
   "pkg-config": {
+    "debian:10": "pkg-config",
     "ubuntu:focal": "pkg-config",
     "ubuntu:focal::arch64": "pkg-config",
     "centos:8": "pkg-config"
   },
   "python": {
+    "debian:10": [
+      "python3",
+      "python3-pip",
+      "python3-pkg-resources",
+      "python3-setuptools",
+      "python3-wheel"
+    ],
     "ubuntu:focal": [
       "python3",
       "python3-pip",
@@ -266,12 +324,14 @@
     ]
   },
   "rsync": {
+    "debian:10": "rsync",
     "ubuntu:focal": "rsync",
     "ubuntu:focal::arch64": "rsync",
     "centos:7": "rsync",
     "centos:8": "rsync"
   },
   "shellcheck": {
+    "debian:10": "shellcheck",
     "ubuntu:focal": "shellcheck",
     "ubuntu:focal::arch64": "shellcheck"
   },
@@ -280,22 +340,26 @@
     "centos:8": "perl-Digest-SHA"
   },
   "software-properties-common": {
+    "debian:10": "software-properties-common",
     "ubuntu:focal": "software-properties-common",
     "ubuntu:focal::arch64": "software-properties-common"
   },
   "sudo": {
+    "debian:10": "sudo",
     "ubuntu:focal": "sudo",
     "ubuntu:focal::arch64": "sudo",
     "centos:7": "sudo",
     "centos:8": "sudo"
   },
   "valgrind": {
+    "debian:10": "valgrind",
     "ubuntu:focal": "valgrind",
     "ubuntu:focal::arch64": "valgrind",
     "centos:7": "valgrind",
     "centos:8": "valgrind"
   },
   "yasm": {
+    "debian:10": "yasm",
     "ubuntu:focal": "yasm",
     "ubuntu:focal::arch64": "yasm"
   }

dev-support/docker/pkg-resolver/platforms.json

@@ -2,5 +2,6 @@
   "ubuntu:focal",
   "ubuntu:focal::arch64",
   "centos:7",
-  "centos:8"
+  "centos:8",
+  "debian:10"
 ]