AMBARI-16141. Add configuration dependencies for Logsearch components (oleewere)

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml

@@ -71,6 +71,20 @@
     <display-name>Logsfeeder max memory</display-name>
   </property>
 
+  <property>
+    <name>logfeeder_solr_service_logs_enable</name>
+    <value>true</value>
+    <display-name>Service Logs enabled</display-name>
+    <description>Enable service logs to Solr</description>
+  </property>
+
+  <property>
+    <name>logfeeder_solr_audit_logs_enable</name>
+    <value>true</value>
+    <display-name>Audit Logs enabled</display-name>
+    <description>Enable audit logs to Solr</description>
+  </property>
+
   <property>
     <name>content</name>
     <description>This is the jinja template for logsearch-env.sh file</description>

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-site.xml

@@ -1,87 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_adding_forbidden="true">
-
-  <property>
-    <name>logfeeder.solr.service.logs.enable</name>
-    <value>true</value>
-    <display-name>Service Logs enabled</display-name>
-    <description>Enable service logs to Solr</description>
-  </property>
-
-  <property>
-    <name>logfeeder.solr.audit.logs.enable</name>
-    <value>true</value>
-    <display-name>Audit Logs enabled</display-name>
-    <description>Enable audit logs to Solr</description>
-  </property>
-
-  <property>
-    <name>logfeeder.kafka.service.logs.enable</name>
-    <value>false</value>
-    <display-name>Kafka Service Logs enabled</display-name>
-    <description>Enable service logs to Kafka</description>
-  </property>
-
-  <property>
-    <name>logfeeder.kafka.topic.service.logs</name>
-    <value>service_logs</value>
-    <display-name>Kafka Service Logs topic</display-name>
-    <description>Kafka topic for service logs</description>
-  </property>
-
-  <property>
-    <name>logfeeder.kafka.audit.logs.enable</name>
-    <value>false</value>
-    <display-name>Kafka Audit Logs enabled</display-name>
-    <description>Enable audit logs to Kafka</description>
-  </property>
-
-  <property>
-    <name>logfeeder.kafka.topic.audit.logs</name>
-    <value>audit_logs</value>
-    <display-name>Kafka Audit Logs topic</display-name>
-    <description>Kafka topic for audit logs</description>
-  </property>
-
-  <property>
-    <name>logfeeder.kafka.broker.list</name>
-    <value>NONE</value>
-    <display-name>Kafka broker list</display-name>
-    <description>List of brokers. e.g. broker1_host:port,broker2_host:port</description>
-  </property>
-
-  <property>
-    <name>logfeeder.kafka.security.protocol</name>
-    <value>NONE</value>
-    <display-name>Kafka security protocol</display-name>
-    <description>If security is enabled, then set this value. E.g. SASL_PLAINTEXT</description>
-  </property>
-
-  <property>
-    <name>logfeeder.kafka.kerberos.service.name</name>
-    <value>NONE</value>
-    <display-name>Kafka Kerberos service name</display-name>
-    <description>If security is enabled, then set this value. E.g. kafka</description>
-  </property>
-
-</configuration>

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-admin-json.xml

@@ -42,11 +42,11 @@
 
   <property>
     <name>content</name>
-    <description>This is the jinja template for user_pass.json file</description>
+    <description>This is the jinja template for logsearch-admin.json file</description>
     <value></value>
     <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
     <value-attributes>
-      <property-file-name>user_pass.json.j2</property-file-name>
+      <property-file-name>logsearch-admin.json.j2</property-file-name>
       <property-file-type>text</property-file-type>
     </value-attributes>
   </property>

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml

@@ -99,15 +99,15 @@
   <property>
     <name>logsearch_solr_minmem</name>
     <value>512m</value>
-    <display-name>Logsearch minimum heap size</display-name>
+    <display-name>Logsearch Solr minimum heap size</display-name>
     <description>Solr minimum heap size e.g.512m</description>
   </property>
 
   <property>
     <name>logsearch_solr_maxmem</name>
     <value>512m</value>
+    <display-name>Logsearch Solr maximum heap size</display-name>
     <description>Solr maximum heap size e.g. 512m</description>
-    <display-name>Logsearch maximum heap size</display-name>
   </property>
 
   <!-- logsearch-solr-env.sh -->

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml

@@ -55,6 +55,12 @@
               </auto-deploy>
             </dependency>
           </dependencies>
+          <configuration-dependencies>
+            <config-type>logsearch-properties</config-type>
+            <config-type>logsearch-env</config-type>
+            <config-type>logsearch-log4j</config-type>
+            <config-type>logsearch-admin-json</config-type>
+          </configuration-dependencies>
         </component>
 
         <component>
@@ -76,6 +82,11 @@
               </auto-deploy>
             </dependency>
           </dependencies>
+          <configuration-dependencies>
+            <config-type>logsearch-solr-env</config-type>
+            <config-type>logsearch-solr-xml</config-type>
+            <config-type>logsearch-solr-log4j</config-type>
+          </configuration-dependencies>
         </component>
 
         <component>
@@ -95,6 +106,11 @@
               <primary>true</primary>
             </log>
           </logs>
+          <configuration-dependencies>
+            <config-type>logfeeder-properties</config-type>
+            <config-type>logfeeder-env</config-type>
+            <config-type>logfeeder-log4j</config-type>
+          </configuration-dependencies>
         </component>
 
         <component>
@@ -166,11 +182,6 @@
         <service>ZOOKEEPER</service>
       </requiredServices>
 
-      <configuration-dependencies>
-        <config-type>logsearch-properties</config-type>
-        <config-type>solr-xml</config-type>
-        <config-type>logfeeder-site</config-type>
-      </configuration-dependencies>
       <restartRequiredAfterChange>false</restartRequiredAfterChange>
 
       <quickLinksConfigurations>

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_common.py

@@ -38,7 +38,8 @@ def kill_process(pid_file, user, log_dir):
 
   hard_kill_cmd = format("{sudo} kill -9 {pid}")
   Execute(hard_kill_cmd,
-          not_if=format("! ({process_id_exists_command}) || ( sleep {wait_time} && ! ({process_id_exists_command}) )"))
+          not_if=format("! ({process_id_exists_command}) || ( sleep {wait_time} && ! ({process_id_exists_command}) )"),
+          ignore_fialures = True)
   try:
     Execute(format("! ({process_id_exists_command})"),
             tries=20,

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py

@@ -234,25 +234,14 @@ logsearch_admin_content = config['configurations']['logsearch-admin-json']['cont
 
 logfeeder_dir = "/usr/lib/ambari-logsearch-logfeeder"
 
-# logfeeder-site configs
-solr_service_logs_enable = default('/configurations/logfeeder-site/logfeeder.solr.service.logs.enable', True)
-solr_audit_logs_enable = default('/configurations/logfeeder-site/logfeeder.solr.audit.logs_enable', True)
-
-kafka_broker_list = default('/configurations/logfeeder-site/logfeeder.kafka.broker.list', '')
-kafka_security_protocol = default('/configurations/logfeeder-site/logfeeder.kafka.security.protocol', '')
-kafka_kerberos_service_name = default('/configurations/logfeeder-site/logfeeder.kafka.kerberos.service.name', '')
-
-kafka_service_logs_enable = default('/configurations/logfeeder-site/logfeeder.kafka.service.logs.enable', False)
-kafka_audit_logs_enable = default('/configurations/logfeeder-site/logfeeder.kafka.audit.logs.enable', False)
-kafka_topic_service_logs = default('/configurations/logfeeder-site/logfeeder.kafka.topic.service.logs', 'service_logs')
-kafka_topic_audit_logs = default('/configurations/logfeeder-site/logfeeder.kafka.topic.audit.logs', 'audit_logs')
-
 # logfeeder-env configs
 logfeeder_user = config['configurations']['logfeeder-env']['logfeeder_user']
 logfeeder_group = config['configurations']['logfeeder-env']['logfeeder_group']
 logfeeder_log_dir = config['configurations']['logfeeder-env']['logfeeder_log_dir']
 logfeeder_log = logfeeder_log_dir + '/logfeeder.out'
 logfeeder_max_mem = config['configurations']['logfeeder-env']['logfeeder_max_mem']
+solr_service_logs_enable = default('/configurations/logfeeder-env/logfeeder_solr_service_logs_enable', True)
+solr_audit_logs_enable = default('/configurations/logfeeder-env/logfeeder_solr_audit_logs_enable', True)
 logfeeder_env_content = config['configurations']['logfeeder-env']['content']
 logfeeder_log4j_content = config['configurations']['logfeeder-log4j']['content']
 

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py

@@ -76,11 +76,11 @@ def setup_logsearch():
 
   random_num = random.random()
 
-  upload_conf_set('hadoop_logs', random_num)
+  upload_conf_set(format('{logsearch_solr_collection_service_logs}'), random_num)
 
   upload_conf_set('history', random_num)
 
-  upload_conf_set('audit_logs', random_num)
+  upload_conf_set(format('{logsearch_solr_collection_audit_logs}'), random_num)
 
   Execute(("chmod", "-R", "ugo+r", format("{logsearch_server_conf}/solr_configsets")),
           sudo=True

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2

@@ -54,42 +54,6 @@
 
       }
 
-    },
-    {
-      "is_enabled":"{{kafka_service_logs_enable}}",
-      "destination":"kafka",
-      "broker_list":"{{kafka_broker_list}}",
-      "topic":"{{kafka_topic_service_logs}}",
-      "kafka.security.protocol":"{{kafka_security_protocol}}",
-      "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}",
-      "conditions":{
-        "fields":{
-          "rowtype":[
-            "service"
-          ]
-
-        }
-
-      }
-
-    },
-    {
-      "is_enabled":"{{kafka_audit_logs_enable}}",
-      "destination":"kafka",
-      "broker_list":"{{kafka_broker_list}}",
-      "topic":"{{kafka_topic_audit_logs}}",
-      "kafka.security.protocol":"{{kafka_security_protocol}}",
-      "kafka.sasl.kerberos.service.name":"{{kafka_kerberos_service_name}}",
-      "conditions":{
-        "fields":{
-          "rowtype":[
-            "audit"
-          ]
-
-        }
-
-      }
-
     }
 
   ]

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/solr-log4j.properties.j2

@@ -16,7 +16,7 @@
 #  Logging level
 solr.log={{logsearch_solr_log_dir}}
 #log4j.rootLogger=INFO, file, CONSOLE
-log4j.rootLogger=INFO, file
+log4j.rootLogger=WARN, file
 
 log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
 

ambari-web/app/data/HDP2/site_properties.js

@@ -1927,83 +1927,33 @@ var hdp2properties = [
     "index": 5
   },
   {
-    "name": "content",
+    "name": "logfeeder_solr_service_logs_enable",
     "serviceName": "LOGSEARCH",
     "filename": "logfeeder-env.xml",
     "category": "Advanced logfeeder-env",
     "index": 6
   },
-  /*logfeeder-log4j*/
-  {
-    "name": "content",
-    "serviceName": "LOGSEARCH",
-    "filename": "logfeeder-log4j.xml",
-    "category": "Advanced logfeeder-log4j",
-    "index": 1
-  },
-  /*logfeeder-site*/
-  {
-    "name": "logfeeder.solr.service.logs.enable",
-    "serviceName": "LOGSEARCH",
-    "filename": "logfeeder-site.xml",
-    "category": "Advanced logfeeder-site",
-    "index": 1
-  },
-  {
-    "name": "logfeeder.solr.audit.logs.enable",
-    "serviceName": "LOGSEARCH",
-    "filename": "logfeeder-site.xml",
-    "category": "Advanced logfeeder-site",
-    "index": 2
-  },
-  {
-    "name": "logfeeder.kafka.broker.list",
-    "serviceName": "LOGSEARCH",
-    "filename": "logfeeder-site.xml",
-    "category": "Advanced logfeeder-site",
-    "index": 3
-  },
-  {
-    "name": "logfeeder.kafka.security.protocol",
-    "serviceName": "LOGSEARCH",
-    "filename": "logfeeder-site.xml",
-    "category": "Advanced logfeeder-site",
-    "index": 4
-  },
-  {
-    "name": "logfeeder.kafka.kerberos.service.name",
-    "serviceName": "LOGSEARCH",
-    "filename": "logfeeder-site.xml",
-    "category": "Advanced logfeeder-site",
-    "index": 5
-  },
-  {
-    "name": "logfeeder.kafka.service.logs.enable",
-    "serviceName": "LOGSEARCH",
-    "filename": "logfeeder-site.xml",
-    "category": "Advanced logfeeder-site",
-    "index": 6
-  },
   {
-    "name": "logfeeder.kafka.topic.service.logs",
+    "name": "logfeeder_solr_audit_logs_enable",
     "serviceName": "LOGSEARCH",
-    "filename": "logfeeder-site.xml",
-    "category": "Advanced logfeeder-site",
+    "filename": "logfeeder-env.xml",
+    "category": "Advanced logfeeder-env",
     "index": 7
   },
   {
-    "name": "logfeeder.kafka.audit.logs.enable",
+    "name": "content",
     "serviceName": "LOGSEARCH",
-    "filename": "logfeeder-site.xml",
-    "category": "Advanced logfeeder-site",
+    "filename": "logfeeder-env.xml",
+    "category": "Advanced logfeeder-env",
     "index": 8
   },
+  /*logfeeder-log4j*/
   {
-    "name": "logfeeder.kafka.topic.audit.logs",
+    "name": "content",
     "serviceName": "LOGSEARCH",
-    "filename": "logfeeder-site.xml",
-    "category": "Advanced logfeeder-site",
-    "index": 9
+    "filename": "logfeeder-log4j.xml",
+    "category": "Advanced logfeeder-log4j",
+    "index": 1
   },
   /*logsearch-admin-json*/
   {
@@ -2156,42 +2106,42 @@ var hdp2properties = [
     "serviceName": "LOGSEARCH",
     "filename": "logsearch-properties.xml",
     "category": "Advanced logsearch-properties",
-    "index": 4
+    "index": 3
   },
   {
     "name": "logsearch.service.logs.fields",
     "serviceName": "LOGSEARCH",
     "filename": "logsearch-properties.xml",
     "category": "Advanced logsearch-properties",
-    "index": 5
+    "index": 4
   },
   {
     "name": "logsearch.service.logs.split.interval.mins",
     "serviceName": "LOGSEARCH",
     "filename": "logsearch-properties.xml",
     "category": "Advanced logsearch-properties",
-    "index": 6
+    "index": 5
   },
   {
     "name": "logsearch.solr.collection.audit.logs",
     "serviceName": "LOGSEARCH",
     "filename": "logsearch-properties.xml",
     "category": "Advanced logsearch-properties",
-    "index": 7
+    "index": 6
   },
   {
     "name": "logsearch.audit.logs.split.interval.mins",
     "serviceName": "LOGSEARCH",
     "filename": "logsearch-properties.xml",
     "category": "Advanced logsearch-properties",
-    "index": 8
+    "index": 7
   },
   {
     "name": "logsearch.logfeeder.include.default.level",
     "serviceName": "LOGSEARCH",
     "filename": "logsearch-properties.xml",
     "category": "Advanced logsearch-properties",
-    "index": 9
+    "index": 8
   },
   /*logsearch-solr-env*/
   {