AMBARI-19727. Extend security.json of Infra Solr as a configuration property in Ambari (oleewere)

ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py

@@ -143,14 +143,14 @@ def create_znode(zookeeper_quorum, solr_znode, java64_home, retry = 5 , interval
   create_znode_cmd = format('{solr_cli_prefix} --create-znode --retry {retry} --interval {interval}')
   Execute(create_znode_cmd)
 
-def setup_kerberos_plugin(zookeeper_quorum, solr_znode, java64_home, secure=False, jaas_file = None):
+def setup_kerberos_plugin(zookeeper_quorum, solr_znode, java64_home, secure=False, security_json_location = None, jaas_file = None):
   """
   Set Kerberos plugin on the Solr znode in security.json, if secure is False, then clear the security.json
   """
   solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True)
   setup_kerberos_plugin_cmd = format('{solr_cli_prefix} --setup-kerberos-plugin')
-  if secure and jaas_file is not None:
-    setup_kerberos_plugin_cmd+=format(' --jaas-file {jaas_file} --secure')
+  if secure and jaas_file is not None and security_json_location is not None:
+    setup_kerberos_plugin_cmd+=format(' --jaas-file {jaas_file} --secure --security-json-location {security_json_location}')
   Execute(setup_kerberos_plugin_cmd)
 
 def set_cluster_prop(zookeeper_quorum, solr_znode, prop_name, prop_value, java64_home, jaas_file = None):

ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudCLI.java

@@ -50,6 +50,7 @@ public class AmbariSolrCloudCLI {
   private static final String CHECK_ZNODE = "check-znode";
   private static final String SECURE_ZNODE_COMMAND = "secure-znode";
   private static final String SECURE_SOLR_ZNODE_COMMAND = "secure-solr-znode";
+  private static final String SECURITY_JSON_LOCATION = "security-json-location";
   private static final String CMD_LINE_SYNTAX =
     "\n./solrCloudCli.sh --create-collection -z host1:2181,host2:2181/ambari-solr -c collection -cs conf_set"
       + "\n./solrCloudCli.sh --upload-config -z host1:2181,host2:2181/ambari-solr -d /tmp/myconfig_dir -cs config_set"
@@ -61,7 +62,7 @@ public class AmbariSolrCloudCLI {
       + "\n./solrCloudCli.sh --cluster-prop -z host1:2181,host2:2181/ambari-solr -cpn urlScheme -cpn http"
       + "\n./solrCloudCli.sh --secure-znode -z host1:2181,host2:2181 -zn /ambari-solr -su logsearch,atlas,ranger --jaas-file /etc/myconf/jaas_file"
       + "\n./solrCloudCli.sh --secure-solr-znode -z host1:2181,host2:2181 -zn /ambari-solr -su logsearch,atlas,ranger --jaas-file /etc/myconf/jaas_file"
-      + "\n./solrCloudCli.sh --setup-kerberos-plugin -z host1:2181,host2:2181 -zn /ambari-solr\n";
+      + "\n./solrCloudCli.sh --setup-kerberos-plugin -z host1:2181,host2:2181 -zn /ambari-solr --security-json-location /etc/infra-solr/conf/security.json\n";
 
   public static void main(String[] args) {
     Options options = new Options();
@@ -306,6 +307,13 @@ public class AmbariSolrCloudCLI {
       .argName("atlas,ranger,logsearch-solr")
       .build();
 
+    final Option securityJsonLocationOption = Option.builder("sjl")
+      .longOpt(SECURITY_JSON_LOCATION)
+      .desc("Local security.json path")
+      .numberOfArgs(1)
+      .argName("security.json location")
+      .build();
+
     final Option secureOption = Option.builder("sec")
       .longOpt("secure")
       .desc("Flag for enable/disable kerberos (with --setup-kerberos or --setup-kerberos-plugin)")
@@ -349,6 +357,7 @@ public class AmbariSolrCloudCLI {
     options.addOption(saslUsersOption);
     options.addOption(checkZnodeOption);
     options.addOption(setupKerberosPluginOption);
+    options.addOption(securityJsonLocationOption);
 
     AmbariSolrCloudClient solrCloudClient = null;
 
@@ -427,6 +436,7 @@ public class AmbariSolrCloudCLI {
       String znode = cli.hasOption("zn") ? cli.getOptionValue("zn") : null;
       boolean isSecure = cli.hasOption("sec");
       String saslUsers = cli.hasOption("su") ? cli.getOptionValue("su") : "";
+      String securityJsonLocation = cli.hasOption("sjl") ? cli.getOptionValue("sjl") : "";
 
       AmbariSolrCloudClientBuilder clientBuilder = new AmbariSolrCloudClientBuilder()
         .withZkConnectString(zkConnectString)
@@ -450,6 +460,7 @@ public class AmbariSolrCloudCLI {
         .withTrustStoreType(trustStoreType)
         .withClusterPropName(clusterPropName)
         .withClusterPropValue(clusterPropValue)
+        .withSecurityJsonLocation(securityJsonLocation)
         .withZnode(znode)
         .withSecure(isSecure)
         .withSaslUsers(saslUsers);

ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java

@@ -68,6 +68,7 @@ public class AmbariSolrCloudClient {
   private final String saslUsers;
   private final String propName;
   private final String propValue;
+  private final String securityJsonLocation;
   private final boolean secure;
 
   public AmbariSolrCloudClient(AmbariSolrCloudClientBuilder builder) {
@@ -90,6 +91,7 @@ public class AmbariSolrCloudClient {
     this.saslUsers = builder.saslUsers;
     this.propName = builder.propName;
     this.propValue = builder.propValue;
+    this.securityJsonLocation = builder.securityJsonLocation;
     this.secure = builder.secure;
   }
 
@@ -325,4 +327,8 @@ public class AmbariSolrCloudClient {
   public boolean isSecure() {
     return secure;
   }
+
+  public String getSecurityJsonLocation() {
+    return securityJsonLocation;
+  }
 }

ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClientBuilder.java

@@ -51,6 +51,7 @@ public class AmbariSolrCloudClientBuilder {
   String saslUsers;
   String propName;
   String propValue;
+  String securityJsonLocation;
   boolean secure;
 
   public AmbariSolrCloudClient build() {
@@ -195,6 +196,11 @@ public class AmbariSolrCloudClientBuilder {
     return this;
   }
 
+  public AmbariSolrCloudClientBuilder withSecurityJsonLocation(String securityJson) {
+    this.securityJsonLocation = securityJson;
+    return this;
+  }
+
   public AmbariSolrCloudClientBuilder withSecure(boolean isSecure) {
     this.secure = isSecure;
     return this;

ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/EnableKerberosPluginSolrZkCommand.java

@@ -19,16 +19,19 @@
 package org.apache.ambari.logsearch.solr.commands;
 
 import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.solr.common.cloud.SolrZooKeeper;
 import org.apache.zookeeper.CreateMode;
 
+import java.io.File;
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 
 public class EnableKerberosPluginSolrZkCommand extends AbstractZookeeperRetryCommand<String> {
 
   private static final String SECURITY_JSON = "/security.json";
-  private static final String SECURE_CONTENT = "{\"authentication\":{\"class\": \"org.apache.solr.security.KerberosPlugin\"}}";
   private static final String UNSECURE_CONTENT = "{}";
 
   public EnableKerberosPluginSolrZkCommand(int maxRetries, int interval) {
@@ -39,12 +42,13 @@ public class EnableKerberosPluginSolrZkCommand extends AbstractZookeeperRetryCom
   protected String executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception {
     String result = "";
     String filePath = client.getZnode() + SECURITY_JSON;
-    String fileContent = getFileContent(zkClient, filePath);
+    String fileContent = getFileContentFromZnode(zkClient, filePath);
+    String securityContent = getFileContent(client.getSecurityJsonLocation());
     if (client.isSecure()) {
-      if (!fileContent.equals(SECURE_CONTENT)) {
-        putFileContent(zkClient, filePath, SECURE_CONTENT);
+      if (!fileContent.equals(securityContent)) {
+        putFileContent(zkClient, filePath, securityContent);
       }
-      result = SECURE_CONTENT;
+      result = securityContent;
     } else {
       if (!fileContent.equals(UNSECURE_CONTENT)) {
         putFileContent(zkClient, filePath, UNSECURE_CONTENT);
@@ -62,7 +66,7 @@ public class EnableKerberosPluginSolrZkCommand extends AbstractZookeeperRetryCom
     }
   }
 
-  private String getFileContent(SolrZkClient zkClient, String fileName) throws Exception {
+  private String getFileContentFromZnode(SolrZkClient zkClient, String fileName) throws Exception {
     String result;
     if (zkClient.exists(fileName, true)) {
       byte[] data = zkClient.getData(fileName, null, null, true);
@@ -72,4 +76,13 @@ public class EnableKerberosPluginSolrZkCommand extends AbstractZookeeperRetryCom
     }
     return result;
   }
+
+  private String getFileContent(String fileLocation) throws IOException {
+    File securityJson = new File(fileLocation);
+    if (StringUtils.isNotEmpty(fileLocation) && securityJson.exists()) {
+      return FileUtils.readFileToString(securityJson);
+    } else {
+      return UNSECURE_CONTENT;
+    }
+  }
 }

ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/configuration/infra-solr-security-json.xml

@@ -0,0 +1,36 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<!-- This is a special config file for properties used to monitor status of the service -->
+<configuration supports_adding_forbidden="true">
+  <property>
+    <name>content</name>
+    <display-name>infra-solr security.json template</display-name>
+    <description>This is the jinja template for security.json file on the solr znode (only used if the cluster is secure)</description>
+    <value/>
+    <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+    <value-attributes>
+      <property-file-name>infra-solr-security.json.j2</property-file-name>
+      <property-file-type>text</property-file-type>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+</configuration>

ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/metainfo.xml

@@ -64,6 +64,7 @@
             <config-type>infra-solr-env</config-type>
             <config-type>infra-solr-xml</config-type>
             <config-type>infra-solr-log4j</config-type>
+            <config-type>infra-solr-security-json</config-type>
           </configuration-dependencies>
         </component>
 

ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/params.py

@@ -120,6 +120,8 @@ if security_enabled:
   infra_solr_kerberos_name_rules = config['configurations']['infra-solr-env']['infra_solr_kerberos_name_rules'].replace('$', '\$')
   infra_solr_sasl_user = get_name_from_principal(infra_solr_kerberos_principal)
 
+infra_solr_security_json_content = config['configurations']['infra-solr-security-json']['content']
+
 #Solr log4j
 infra_log_maxfilesize = default('configurations/infra-solr-log4j/infra_log_maxfilesize',10)
 infra_log_maxbackupindex = default('configurations/infra-solr-log4j/infra_log_maxbackupindex',9)

ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/setup_infra_solr.py

@@ -72,6 +72,15 @@ def setup_infra_solr(name = None):
          group=params.user_group
          )
 
+    security_json_file_location = format("{infra_solr_conf}/security.json")
+
+    File(security_json_file_location,
+         content=InlineTemplate(params.infra_solr_security_json_content),
+         owner=params.infra_solr_user,
+         group=params.user_group,
+         mode=0644
+         )
+
     jaas_file = params.infra_solr_jaas_file if params.security_enabled else None
     url_scheme = 'https' if params.infra_solr_ssl_enabled else 'http'
 
@@ -96,7 +105,8 @@ def setup_infra_solr(name = None):
       solr_znode=params.infra_solr_znode,
       jaas_file=jaas_file,
       java64_home=params.java64_home,
-      secure=params.security_enabled
+      secure=params.security_enabled,
+      security_json_location=security_json_file_location
     )
 
     if params.security_enabled:

ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/properties/infra-solr-security.json.j2

@@ -0,0 +1,22 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{
+  "authentication": {
+    "class": "org.apache.solr.security.KerberosPlugin"
+  }
+}

ambari-server/src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py

@@ -95,6 +95,13 @@ class TestInfraSolr(RMFTestCase):
                                 content = InlineTemplate(self.getConfig()['configurations']['infra-solr-log4j']['content'])
       )
 
+      self.assertResourceCalled('File', '/etc/ambari-infra-solr/conf/security.json',
+                                owner = 'solr',
+                                group='hadoop',
+                                content = InlineTemplate(self.getConfig()['configurations']['infra-solr-security-json']['content']),
+                                mode = 0644
+                                )
+
       self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --create-znode --retry 30 --interval 5')
       self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --cluster-prop --property-name urlScheme --property-value http')
       self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --setup-kerberos-plugin')

ambari-server/src/test/python/stacks/2.4/configs/default.json

@@ -239,6 +239,9 @@
         "logsearch_admin_password" : "admin",
         "content": "admin json content"
       },
+      "infra-solr-security-json" : {
+        "content" : "{\"authentication\": \"org.apache.solr.security.KerberosPlugin\"}"
+      },
       "infra-solr-client-log4j" : {
         "infra_solr_client_log_dir" : "/var/log/ambari-infra-solr-client",
         "content" : "content"