AMBARI-18841 : Grafana fails to start (avijayan)

ambari-metrics/ambari-metrics-grafana/conf/unix/ambari-metrics-grafana

@@ -50,6 +50,7 @@ GRAFANA_HOME=${AMS_GRAFANA_HOME_DIR}
 WORK_DIR=$GRAFANA_HOME
 DATA_DIR=${AMS_GRAFANA_DATA_DIR}
 LOG_DIR=${AMS_GRAFANA_LOG_DIR}
+LOG_FILE=$LOG_DIR/grafana.log
 CONF_FILE=$CONF_DIR/ams-grafana.ini
 MAX_OPEN_FILES=10000
 PID_FILE=${AMS_GRAFANA_PID_DIR}/$NAME.pid
@@ -101,6 +102,22 @@ case "$1" in
       exit 0
     fi
 
+    echo "$(date) Checking for previously running Grafana with out of sync PID..." >> $LOG_FILE
+    if [ -n "`ps ax | grep -w ambari-metrics-grafana/bin/grafana-server`" ]; then
+      PID=`ps aux | grep -w ambari-metrics-grafana/bin/grafana-server | head -n 1 | awk '{print $2}'`
+      echo "$(date) Grafana Server already running with PID: ${PID}. Killing that instance"  >> $LOG_FILE
+      kill "${PID}" >/dev/null 2>&1
+      sleep "${STOP_TIMEOUT}"
+
+      if kill -0 "${PID}" > /dev/null 2>&1; then
+        echo "$(date) WARNING: Grafana Server did not stop gracefully. Trying to kill with kill -9" >> $LOG_FILE
+        kill -9 "${PID}" >/dev/null 2>&1
+      fi
+
+    else
+      echo "$(date) No out of sync Grafana server process found" >> $LOG_FILE
+    fi
+
     # Prepare environment
     # mkdir -p "$LOG_DIR" "$DATA_DIR" && chown "$GRAFANA_USER":"$GRAFANA_GROUP" "$LOG_DIR" "$DATA_DIR"
     # touch "$PID_FILE" && chown "$GRAFANA_USER":"$GRAFANA_GROUP" "$PID_FILE"

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py

@@ -23,6 +23,8 @@ from resource_management.libraries.functions import format
 from status import check_service_status
 from ams import ams
 from metrics_grafana_util import create_ams_datasource, create_ams_dashboards
+from resource_management.core.logger import Logger
+from resource_management.core import sudo
 
 class AmsGrafana(Script):
   def install(self, env):
@@ -46,6 +48,12 @@ class AmsGrafana(Script):
             user=params.ams_user,
             not_if = params.grafana_process_exists_cmd,
             )
+    pidfile = format("{ams_grafana_pid_dir}/grafana-server.pid")
+    if not sudo.path_exists(pidfile):
+      Logger.warn("Pid file doesn't exist after starting of the component.")
+    else:
+      Logger.info("Grafana Server has started with pid: {0}".format(sudo.read_file(pidfile).strip()))
+
     # Create datasource
     create_ams_datasource()
     # Create pre-built dashboards