AMBARI-17492. Logsearch shows all the services not only those which are available in the stack (Miklos Gergely via oleewere)

ambari-logsearch/ambari-logsearch-assembly/pom.xml

@@ -139,6 +139,7 @@
                             <exclude>classes/log4j.xml</exclude>
                             <exclude>classes/logsearch.properties</exclude>
                             <exclude>classes/user_pass.json</exclude>
+                            <exclude>classes/HadoopServiceConfig.json</exclude>
                             <exclude>solr_configsets/**</exclude>
                           </excludes>
                         </source>
@@ -391,7 +392,7 @@
                         <group>root</group>
                       </mapper>
                       <excludes>
-                        classes/log4j.xml,classes/logsearch.properties,classes/user_pass.json,solr_configsets/**
+                        classes/log4j.xml,classes/logsearch.properties,classes/user_pass.json,classes/HadoopServiceConfig.json,solr_configsets/**
                       </excludes>
                     </data>
                     <data>

ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java

@@ -111,6 +111,9 @@ public class ExecutionCommand extends AgentCommand {
   @SerializedName("localComponents")
   private Set<String> localComponents = new HashSet<String>();
 
+  @SerializedName("availableServices")
+  private Set<String> availableServices = new HashSet<String>();
+
   public String getCommandId() {
     return commandId;
   }
@@ -261,6 +264,14 @@ public class ExecutionCommand extends AgentCommand {
     this.localComponents = localComponents;
   }
 
+  public Set<String> getAvailableServices() {
+    return availableServices;
+  }
+
+  public void setAvailableServices(Set<String> availableServices) {
+    this.availableServices = availableServices;
+  }
+
   public Map<String, Map<String, Map<String, String>>> getConfigurationAttributes() {
     return configurationAttributes;
   }

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java

@@ -367,6 +367,8 @@ public class AmbariCustomCommandExecutionHelper {
         execCmd.setForceRefreshConfigTagsBeforeExecution(parseAndValidateComponentsMapping(actionExecutionContext.getParameters().get(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)));
       }
 
+      execCmd.setAvailableServices(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion()).keySet());
+
       Map<String, String> hostLevelParams = new TreeMap<String, String>();
 
       hostLevelParams.put(CUSTOM_COMMAND, commandName);

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java

@@ -2307,6 +2307,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
     }
 
     execCmd.setRoleParams(roleParams);
+    
+    execCmd.setAvailableServices(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion()).keySet());
 
     if ((execCmd != null) && (execCmd.getConfigurationTags().containsKey("cluster-env"))) {
       LOG.info("AmbariManagementControllerImpl.createHostAction: created ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, with cluster-env tags {}",

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py

@@ -70,6 +70,7 @@ zookeeper_hosts_list.sort()
 # get comma separated list of zookeeper hosts from clusterHostInfo
 zookeeper_hosts = ",".join(zookeeper_hosts_list)
 cluster_name = str(config['clusterName'])
+availableServices = config['availableServices']
 
 # for now just pick first collector
 if 'metrics_collector_hosts' in config['clusterHostInfo']:

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py

@@ -56,6 +56,12 @@ def setup_logsearch():
        group=params.user_group
        )
 
+  File(format("{logsearch_server_conf}/HadoopServiceConfig.json"),
+       content=Template("HadoopServiceConfig.json.j2"),
+       owner=params.logsearch_user,
+       group=params.user_group
+       )
+
   File(format("{logsearch_server_conf}/log4j.xml"),
        content=InlineTemplate(params.logsearch_app_log4j_content),
        owner=params.logsearch_user,

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2

@@ -0,0 +1,518 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements.  See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership.  The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License.  You may obtain a copy of the License at
+ #
+ #   http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+  "service": {
+{% if "ACCUMULO" in availableServices %}
+    "accumulo": {
+      "label": "Accumulo",
+      "components": [
+        {
+          "name": "accumulo_gc"
+        },
+        {
+          "name": "accumulo_master"
+        },
+        {
+          "name": "accumulo_monitor"
+        },
+        {
+          "name": "accumulo_tracer"
+        },
+        {
+          "name": "accumulo_tserver"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+{% if "ATLAS" in availableServices %}
+    "atlas": {
+      "label": "Atlas",
+      "components": [
+        {
+          "name": "atlas_app"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+    "ambari": {
+      "label": "Ambari",
+      "components": [
+        {
+          "name": "ambari_agent"
+        },
+        {
+          "name": "ambari_server"
+        },
+        {
+          "name": "ambari_alerts"
+        },
+        {
+          "name": "ambari_audit"
+        },
+        {
+          "name": "ambari_config_changes"
+        },
+        {
+          "name": "ambari_eclipselink"
+        },
+        {
+          "name": "ambari_server_check_database"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% if "AMBARI_METRICS" in availableServices %}
+    "ams": {
+      "label": "AMS",
+      "components": [
+        {
+          "name": "ams_hbase_master"
+        },
+        {
+          "name": "ams_hbase_regionserver"
+        },
+        {
+          "name": "ams_collector"
+        },
+        {
+          "name": "ams_monitor"
+        },
+        {
+          "name": "ams_grafana"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+{% if "FALCON" in availableServices %}
+    "falcon": {
+      "label": "Falcon",
+      "components": [
+        {
+          "name": "falcon_app"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+{% if "FLUME" in availableServices %}
+    "flume": {
+      "label": "Flume",
+      "components": [
+        {
+          "name": "flume_handler"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+{% if "HBASE" in availableServices %}
+    "hbase": {
+      "label": "HBase",
+      "components": [
+        {
+          "name": "hbase_master"
+        },
+        {
+          "name": "hbase_regionserver"
+        },
+        {
+          "name": "hbase_phoenix_server"
+        }
+      ],
+      "dependencies": [
+        {
+          "service": "hdfs",
+          "components": [
+            "hdfs_namenode"
+          ]
+        }
+      ]
+    },
+{% endif %}
+{% if "HDFS" in availableServices %}
+    "hdfs": {
+      "label": "HDFS",
+      "components": [
+        {
+          "name": "hdfs_datanode"
+        },
+        {
+          "name": "hdfs_namenode"
+        },
+        {
+          "name": "hdfs_journalnode"
+        },
+        {
+          "name": "hdfs_secondarynamenode"
+        },
+        {
+          "name": "hdfs_zkfc"
+        },
+        {
+          "name": "hdfs_nfs3"
+        },
+        {
+          "name": "hdfs_audit",
+          "rowtype": "audit"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+{% if "HIVE" in availableServices %}
+    "hive": {
+      "label": "Hive",
+      "components": [
+        {
+          "name": "hive_hiveserver2"
+        },
+        {
+          "name": "hive_metastore"
+        },
+        {
+          "name": "webhcat_server"
+        }
+      ],
+      "dependencies": [
+        {
+          "service": "hdfs",
+          "components": [
+            "hdfs_namenode"
+          ]
+        }
+      ]
+    },
+{% endif %}
+{% if "KAFKA" in availableServices %}
+    "kafka": {
+      "label": "Kafka",
+      "components": [
+        {
+          "name": "kafka_controller"
+        },
+        {
+          "name": "kafka_request"
+        },
+        {
+          "name": "kafka_logcleaner"
+        },
+        {
+          "name": "kafka_server"
+        },
+        {
+          "name": "kafka_statechange"
+        }
+      ],
+      "dependencies": [
+        {
+          "service": "zookeeper",
+          "components": [
+            "zookeeper"
+          ]
+        }
+      ]
+    },
+{% endif %}
+{% if "KNOX" in availableServices %}
+    "knox": {
+      "label": "Knox",
+      "components": [
+        {
+          "name": "knox_gateway"
+        },
+        {
+          "name": "knox_cli"
+        },
+        {
+          "name": "knox_ldap"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+{% if "NIFI" in availableServices %}
+    "nifi": {
+      "label": "NiFi",
+      "components": [
+        {
+          "name": "nifi_app"
+        },
+        {
+          "name": "nifi_bootstrap"
+        },
+        {
+          "name": "nifi_setup"
+        },
+        {
+          "name": "nifi_user"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+{% if "MAPREDUCE2" in availableServices %}
+    "mapred": {
+      "label": "MapReduce",
+      "components": [
+        {
+          "name": "mapred_historyserver"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+    "logsearch": {
+      "label": "Logsearch",
+      "components": [
+        {
+          "name": "logsearch_app"
+        },
+        {
+          "name": "logsearch_feeder"
+        },
+        {
+          "name": "logsearch_perf"
+        },
+        {
+          "name": "logsearch_solr"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% if "RANGER" in availableServices %}
+    "ranger": {
+      "label": "Ranger",
+      "components": [
+        {
+          "name": "ranger_admin"
+        },
+        {
+          "name": "ranger_dbpatch"
+        },
+        {
+          "name": "ranger_kms"
+        },
+        {
+          "name": "ranger_usersync"
+        }
+      ],
+      "dependencies": [
+        {
+          "service": "hdfs",
+          "required": "optional",
+          "components": [
+            "hdfs_namenode"
+          ]
+        },
+        {
+          "service": "hbase",
+          "required": "optional",
+          "components": [
+            "hbase_master",
+            "hbase_regionserver"
+          ]
+        },
+        {
+          "service": "hive",
+          "required": "optional",
+          "components": [
+            "hive_hiveserver2"
+          ]
+        },
+        {
+          "service": "kafka",
+          "required": "optional",
+          "components": [
+            "kafka_ranger"
+          ]
+        },
+        {
+          "service": "knox",
+          "required": "optional",
+          "components": [
+            "knox_gateway"
+          ]
+        },
+        {
+          "service": "storm",
+          "required": "optional",
+          "components": [
+            "storm_supervisor"
+          ]
+        },
+        {
+          "service": "yarn",
+          "required": "optional",
+          "components": [
+            "yarn_resourcemanager"
+          ]
+        }
+      ]
+    },
+{% endif %}
+{% if "OOZIE" in availableServices %}
+    "oozie": {
+      "label": "Oozie",
+      "components": [
+        {
+          "name": "oozie_app"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+{% if "SMARTSENSE" in availableServices %}
+    "hst": {
+      "label": "SmartSense",
+      "components": [
+        {
+          "name": "hst_server"
+        },
+        {
+          "name": "hst_agent"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+{% if "SPARK" in availableServices %}
+    "spark": {
+      "label": "Spark",
+      "components": [
+        {
+          "name": "spark_jobhistory_server"
+        },
+        {
+          "name": "spark_thriftserver"
+        },
+        {
+          "name": "livy_server"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+{% if "SPARK2" in availableServices %}
+    "spark2": {
+      "label": "Spark 2",
+      "components": [
+        {
+          "name": "spark2_jobhistory_server"
+        },
+        {
+          "name": "spark2_thriftserver"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+{% if "STORM" in availableServices %}
+    "storm": {
+      "label": "Storm",
+      "components": [
+        {
+          "name": "storm_drpc"
+        },
+        {
+          "name": "storm_logviewer"
+        },
+        {
+          "name": "storm_nimbus"
+        },
+        {
+          "name": "storm_supervisor"
+        },
+        {
+          "name": "storm_ui"
+        },
+        {
+          "name": "storm_worker"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+{% if "YARN" in availableServices %}
+    "yarn": {
+      "label": "YARN",
+      "components": [
+        {
+          "name": "yarn_nodemanager"
+        },
+        {
+          "name": "yarn_resourcemanager"
+        },
+        {
+          "name": "yarn_timelineserver"
+        },
+        {
+          "name": "yarn_historyserver"
+        },
+        {
+          "name": "yarn_jobsummary"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+{% if "ZEPPELIN" in availableServices %}
+    "zeppelin": {
+      "label": "Zeppelin",
+      "components": [
+        {
+          "name": "zeppelin"
+        }
+      ],
+      "dependencies": [
+      ]
+    },
+{% endif %}
+{% if "ZOOKEEPER" in availableServices %}
+    "zookeeper": {
+      "label": "ZooKeeper",
+      "components": [
+        {
+          "name": "zookeeper"
+        }
+      ],
+      "dependencies": [
+      ]
+    }
+{% endif %}
+  }
+}

ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py

@@ -78,6 +78,11 @@ class TestLogSearch(RMFTestCase):
                               group='hadoop',
                               content = Template('logsearch.properties.j2')
     )
+    self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/HadoopServiceConfig.json',
+                              owner = 'logsearch',
+                              group='hadoop',
+                              content = Template('HadoopServiceConfig.json.j2')
+    )
     self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/log4j.xml',
                               owner = 'logsearch',
                               group='hadoop',

ambari-server/src/test/python/stacks/2.4/configs/default.json

@@ -32,6 +32,10 @@
     },
     "taskId": 152,
     "public_hostname": "c6401.ambari.apache.org",
+    "availableServices": [
+      "ACCUMULO",
+      "ATLAS"
+    ],
     "configurations": {
         "slider-client": {
             "slider.yarn.queue": "default"