YARN-9254. Add support for storing application catalog data to HDFS. Contributed by Eric Yang

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-docker/src/main/scripts/entrypoint.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -x
 
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -32,25 +32,49 @@ template_generator() {
 
 export JAVA_HOME=/usr/lib/jvm/jre
 export HADOOP_CONF_DIR=/etc/hadoop/conf
-/opt/apache/solr/bin/solr start -p 8983 -force
-/opt/apache/solr/bin/solr create_core -c appcatalog -force
-/opt/apache/solr/bin/post -c appcatalog /tmp/samples.xml
-if [ -d /etc/hadoop/conf ]; then
-  sed -i.bak 's/shared.loader=.*$/shared.loader=\/etc\/hadoop\/conf/g' /etc/tomcat/catalina.properties
+
+SOLR_OPTS=()
+
+if [ "${SOLR_STORAGE_TYPE}" == "hdfs" ]; then
+  SOLR_OPTS+=("-Dsolr.directoryFactory=HdfsDirectoryFactory")
+  SOLR_OPTS+=("-Dsolr.lock.type=hdfs")
+  if [ -e "$HADOOP_CONF_DIR" ]; then
+    SOLR_OPTS+=("-Dsolr.hdfs.confdir=${HADOOP_CONF_DIR}")
+  fi
 fi
+
+if [ "${SOLR_DATA_DIR}" != "" ]; then
+  SOLR_OPTS+=("-Dsolr.data.dir=$SOLR_DATA_DIR")
+ fi
+
 if [ -e "$KEYTAB" ]; then
+  SOLR_OPTS+=("-Dsolr.hdfs.security.kerberos.enabled=true")
+  SOLR_OPTS+=("-Dsolr.hdfs.security.kerberos.keytabfile=${KEYTAB}")
+  SOLR_OPTS+=("-Dsolr.hdfs.security.kerberos.principal=${PRINCIPAL}")
   export JAVA_OPTS="$JAVA_OPTS -Djava.security.auth.login.config=/etc/tomcat/jaas.config -Djava.security.krb5.conf=/etc/krb5.conf -Djavax.security.auth.useSubjectCredsOnly=false"
   template_generator /etc/tomcat/jaas.config.template /etc/tomcat/jaas.config
 fi
+
+export SOLR_OPTS
+
+/opt/apache/solr/bin/solr start "${SOLR_OPTS[@]}" -p 8983 -force
+/opt/apache/solr/bin/solr create_core -c appcatalog -force
+/opt/apache/solr/bin/post -c appcatalog /tmp/samples.xml
+if [ -d /etc/hadoop/conf ]; then
+  sed -i.bak 's/shared.loader=.*$/shared.loader=\/etc\/hadoop\/conf/g' /etc/tomcat/catalina.properties
+fi
+
 if [ -e "$SPNEGO_KEYTAB" ]; then
   sed -i.bak 's/authentication.type=.*$/authentication.type=kerberos/g' /etc/tomcat/catalina.properties
   sed -i.bak 's/simple.anonymous.allowed=.*$/simple.anonymous.allowed=false/g' /etc/tomcat/catalina.properties
-  if [ -z "$SPNEGO_PRINCIPAL" ]; then
-    echo "kerberos.principal=HTTP/$HOSTNAME" >> /etc/tomcat/catalina.properties
-  else
-    echo "kerberos.principal=$SPNEGO_PRINCIPAL" >> /etc/tomcat/catalina.properties
-  fi
-  echo "kerberos.keytab=$SPNEGO_KEYTAB" >> /etc/tomcat/catalina.properties
-  echo "hostname=$HOSTNAME" >> /etc/tomcat/catalina.properties
+  {
+    if [ -z "$SPNEGO_PRINCIPAL" ]; then
+      echo "kerberos.principal=HTTP/$HOSTNAME"
+    else
+      echo "kerberos.principal=$SPNEGO_PRINCIPAL"
+    fi
+    echo "kerberos.keytab=$SPNEGO_KEYTAB"
+    echo "hostname=$HOSTNAME"
+  } >> /etc/tomcat/catalina.properties
 fi
 /usr/libexec/tomcat/server start

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md

@@ -188,6 +188,60 @@ For secure cluster, Kerberos settings for application catalog can be configured
 | KEYTAB | /etc/security/keytabs/yarn.service.ketab | Path to keytab file, used by YARN service application master. |
 | PRINCIPAL | yarn/_HOST@EXAMPLE.COM | Service principal used by YARN service application master. |
 
+Application Catalog environment options:
+
+| Environment variables | Description |
+| KEYTAB | Service user keytab file for accessing HDFS. |
+| PRINCIPAL | Service user Kerboers principal. |
+| SOLR_DATA_DIR | Location to store Solr data. |
+| SOLR_STORAGE_TYPE | Storage type for Solr data, supported type are: hdfs, local |
+| SPNEGO_KEYTAB | Location of the keytab file used for authenticating HTTP endpoint. |
+| SPNEGO_PRINCIPAL | The Kerberos principal to be used for HTTP endpoint.  The principal MUST start with 'HTTP'/ as per Kerberos HTTP SPNEGO specification. |
+
+Secure application catalog Yarnfile example:
+```
+{
+  "name": "catalog",
+  "kerberos_principal" : {
+    "principal_name" : "catalog/_HOST@EXAMPLE.COM",
+    "keytab" : "file:///etc/security/keytabs/catalog.service.keytab"
+  },
+  "version": "1",
+  "components" :
+  [
+    {
+      "name": "appcatalog",
+      "number_of_containers": 1,
+      "artifact": {
+        "id": "apache/hadoop-yarn-applications-catalog-docker:3.3.0-SNAPSHOT",
+        "type": "DOCKER"
+      },
+      "resource": {
+        "cpus": 1,
+        "memory": "256"
+      },
+      "configuration": {
+        "env": {
+          "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE":"true",
+          "YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS":"/etc/hadoop/conf:/etc/hadoop/conf:ro,/etc/krb5.conf:/etc/krb5.conf:ro,/etc/security/keytabs/catalog.service.keytab:/etc/security/keytabs/hbase.service.keytab:ro,/etc/security/keytabs/spnego.service.keytab:/etc/security/keytabs/spnego.service.keytab:ro",
+          "SPNEGO_KEYTAB":"/etc/security/keytabs/spnego.service.keytab",
+          "SPNEGO_PRINCIPAL":"HTTP/host-3.example.com@EXAMPLE.COM",
+          "KEYTAB":"/etc/security/keytabs/catalog.service.keytab",
+          "PRINCIPAL":"catalog/host3.example.com@EXAMPLE.COM",
+          "SOLR_DATA_DIR":"hdfs://host-1.example.com:9000/tmp/solr",
+          "SOLR_UPDATE_LOG":"hdfs://host-1.example.com:9000/tmp/solr",
+          "SOLR_STORAGE_TYPE":"hdfs"
+        },
+        "properties": {
+          "docker.network": "host"
+        }
+      }
+    }
+  ]
+}
+
+```
+
 ## Docker image ENTRYPOINT support
 
 Docker images may have built with ENTRYPOINT to enable start up of docker image without any parameters.