AMBARI-9034. Add ability to change log and pid dirs for AMS daemons. (swagle)

ambari-metrics/ambari-metrics-host-monitoring/conf/unix/ambari-metrics-monitor

@@ -102,6 +102,19 @@ while [[ -z "${_ams_configs_done}" ]]; do
   esac
 done
 
+#execute ams-env.sh
+if [[ -f "${MONITOR_CONF_DIR}/ams-env.sh" ]]; then
+  . "${MONITOR_CONF_DIR}/ams-env.sh"
+else
+  echo "ERROR: Cannot execute ${MONITOR_CONF_DIR}/ams-env.sh." 2>&1
+  exit 1
+fi
+
+# Set log directory path
+if [[ -n "${AMS_MONITOR_LOG_DIR}" ]]; then
+  OUTFILE=${AMS_MONITOR_LOG_DIR}/ambari-metrics-monitor.out
+fi
+
 case "$1" in
 
   start)

ambari-metrics/ambari-metrics-timelineservice/conf/unix/ambari-metrics-collector

@@ -31,6 +31,8 @@ HBASE_CONF_DIR=/etc/ams-hbase/conf
 
 METRIC_COLLECTOR=ambari-metrics-collector
 
+AMS_LOG_DIR=/var/log/ambari-metric-collector
+
 STOP_TIMEOUT=5
 
 function hbase_daemon
@@ -200,7 +202,7 @@ case "$1" in
         rm -f "${PIDFILE}" >/dev/null 2>&1
     fi
 
-    nohup "${JAVA}" "-cp" "/usr/lib/ambari-metrics-collector/*:${COLLECTOR_CONF_DIR}" "-Djava.net.preferIPv4Stack=true" "-Dproc_${DAEMON_NAME}" "${CLASS}" "$@" > $OUTFILE 2>&1 &
+    nohup "${JAVA}" "-cp" "/usr/lib/ambari-metrics-collector/*:${COLLECTOR_CONF_DIR}" "-Djava.net.preferIPv4Stack=true" "-Dams.log.dir=${AMS_COLLECTOR_LOG_DIR}" "-Dproc_${DAEMON_NAME}" "${CLASS}" "$@" > $OUTFILE 2>&1 &
     PID=$!
     write_pidfile "${PIDFILE}"
     sleep 2

ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-env.xml

@@ -25,6 +25,26 @@
     <property-type>USER</property-type>
     <description>AMS User Name.</description>
   </property>
+  <property>
+    <name>ams_collector_log_dir</name>
+    <value>/var/log/ambari-metric-collector</value>
+    <description>Collector log directory.</description>
+  </property>
+  <property>
+    <name>ams_collector_pid_dir</name>
+    <value>/var/run/ambari-metric-collector</value>
+    <description>Collector pid directory.</description>
+  </property>
+  <property>
+    <name>ams_monitor_pid_dir</name>
+    <value>/var/run/ambari-metric-monitor</value>
+    <description>Monitor pid directory.</description>
+  </property>
+  <property>
+    <name>ams_monitor_log_dir</name>
+    <value>/var/log/ambari-metric-monitor</value>
+    <description>Monitor log directory.</description>
+  </property>
 
   <property>
     <name>content</name>
@@ -34,7 +54,11 @@
       # The java implementation to use. Java 1.6 required.
       export JAVA_HOME={{java64_home}}
 
-      #TODO
+      # Collector Log directory for log4j
+      export AMS_COLLECTOR_LOG_DIR={{ams_collector_log_dir}}
+
+      # Monitor Log directory for outfile
+      export AMS_MONITOR_LOG_DIR={{ams_monitor_log_dir}}
     </value>
   </property>
 

ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml

@@ -23,12 +23,12 @@
 <configuration>
   <property>
     <name>hbase_log_dir</name>
-    <value>/var/log/ams-hbase/</value>
+    <value>/var/log/ambari-metrics-collector</value>
     <description>Log Directories for HBase.</description>
   </property>
   <property>
     <name>hbase_pid_dir</name>
-    <value>/var/run/ams-hbase/</value>
+    <value>/var/run/ambari-metrics-collector/</value>
     <description>Pid Directory for HBase.</description>
   </property>
   <property>

ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-log4j.xml

@@ -43,12 +43,15 @@
       #
 
       # Define some default values that can be overridden by system properties
+      ams.log.dir=.
+      ams.log.file=ambari-metrics-collector.log
+
       # Root logger option
       log4j.rootLogger=INFO,file
 
       # Direct log messages to a log file
       log4j.appender.file=org.apache.log4j.RollingFileAppender
-      log4j.appender.file.File=/var/log/ambari-metrics-collector/ambari-metrics-collector.log
+      log4j.appender.file.File=${ams.log.dir}/${ams.log.file}
       log4j.appender.file.MaxFileSize=80MB
       log4j.appender.file.MaxBackupIndex=60
       log4j.appender.file.layout=org.apache.log4j.PatternLayout

ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py

@@ -89,6 +89,11 @@ def ams(name=None):
       template_tag=None
     )
 
+    File(format("{ams_monitor_conf_dir}/ams-env.sh"),
+         owner=params.ams_user,
+         content=InlineTemplate(params.ams_env_sh_template)
+    )
+
     # TODO
     pass
 

ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/params.py

@@ -40,6 +40,8 @@ if metric_collector_port and metric_collector_port.find(':') != -1:
   metric_collector_port = metric_collector_port.split(':')[1]
 pass
 
+ams_collector_log_dir = config['configurations']['ams-env']['ams_collector_log_dir']
+ams_monitor_log_dir = config['configurations']['ams-env']['ams_monitor_log_dir']
 
 ams_monitor_conf_dir = "/etc/ambari-metrics-monitor/conf/"
 ams_monitor_dir = "/usr/lib/python2.6/site-packages/resource_monitoring"

ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/status_params.py

@@ -25,6 +25,6 @@ config = Script.get_config()
 hbase_pid_dir = config['configurations']['ams-hbase-env']['hbase_pid_dir']
 hbase_user = config['configurations']['ams-hbase-env']['hbase_user']
 ams_user = config['configurations']['ams-env']['ams_user']
+ams_collector_pid_dir = config['configurations']['ams-env']['ams_collector_log_dir']
+ams_monitor_pid_dir = config['configurations']['ams-env']['ams_monitor_log_dir']
 
-ams_monitor_pid_dir = "/var/run/ambari-metrics-monitor"
-ams_collector_pid_dir = "/var/run/ambari-metrics-collector"

ambari-web/app/data/HDP2/site_properties.js

@@ -4397,200 +4397,266 @@ module.exports =
     {
       "id": "site property",
       "name": "timeline.metrics.service.operation.mode",
-      "displayName": "timeline.metrics.service.operation.mode",
+      "displayName": "Metrics Service operation mode",
       "description":  "\n      Service Operation modes:\n      1) embedded: Metrics stored on local FS, HBase in Standalone mode\n      2) distributed: HBase daemons writing to HDFS\n      3) external: External HBase storage backend\n    ",
       "defaultValue": "embedded",
       "displayType": "string",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "General"
+      "category": "General",
+      "index" : 1
+    },
+    {
+      "id": "site property",
+      "name": "ams_collector_log_dir",
+      "displayName": "Metrics Collector log dir",
+      "description":  "\n      Log location for collector logs.\n    ",
+      "defaultValue": "embedded",
+      "displayType": "string",
+      "serviceName": "AMS",
+      "filename": "ams-env.xml",
+      "category": "General",
+      "index" : 2
+    },
+    {
+      "id": "site property",
+      "name": "ams_collector_pid_dir",
+      "displayName": "Metrics Collector pid dir",
+      "description":  "\n      pid location for collector.\n    ",
+      "defaultValue": "embedded",
+      "displayType": "string",
+      "serviceName": "AMS",
+      "filename": "ams-env.xml",
+      "category": "General",
+      "index" : 3
+    },
+    {
+      "id": "site property",
+      "name": "ams_monitor_log_dir",
+      "displayName": "Metrics Monitor log dir",
+      "description":  "\n      Log location for monitor logs.\n    ",
+      "defaultValue": "embedded",
+      "displayType": "string",
+      "serviceName": "AMS",
+      "filename": "ams-env.xml",
+      "category": "General",
+      "index" : 4
+    },
+    {
+      "id": "site property",
+      "name": "ams_monitor_pid_dir",
+      "displayName": "Metrics Monitor pid dir",
+      "description":  "\n      pid location for monitor.\n    ",
+      "defaultValue": "embedded",
+      "displayType": "string",
+      "serviceName": "AMS",
+      "filename": "ams-env.xml",
+      "category": "General",
+      "index" : 5
     },
     {
       "id": "site property",
       "name" : "timeline.metrics.aggregator.checkpoint.dir",
-      "displayName": "timeline.metrics.aggregator.checkpoint.dir",
-      "description":  "\n      Directory to store aggregator checkpoints. Change to a permanent\n      location so that checkpoint ar not lost.\n    ",
+      "displayName": "Aggregator checkpoint directory",
+      "description":  "\n      Directory to store aggregator checkpoints.\n    ",
       "defaultValue": "/tmp",
       "displayType": "directory",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 17
     },
     {
       "id": "site property",
       "name": "timeline.metrics.cluster.aggregator.hourly.checkpointCutOffMultiplier",
-      "displayName": "timeline.metrics.cluster.aggregator.hourly.checkpointCutOffMultiplier",
+      "displayName": "Hourly cluster aggregator checkpoint cutOff multiplier",
       "description":  "\n      Multiplier value * interval = Max allowed checkpoint lag. Effectively\n      if aggregator checkpoint is greater than max allowed checkpoint delay,\n      the checkpoint will be discarded by the aggregator.\n    ",
       "defaultValue": "2",
       "displayType": "int",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 16
     },
     {
       "id": "site property",
       "name": "timeline.metrics.cluster.aggregator.hourly.disabled",
-      "displayName": "timeline.metrics.cluster.aggregator.hourly.disabled",
+      "displayName": "Disable Hourly cluster aggregator",
       "description":  "\n      Disable cluster based hourly aggregations.\n    ",
       "defaultValue": "false",
       "displayType": "string",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 14
     },
     {
       "id": "site property",
       "name": "timeline.metrics.cluster.aggregator.hourly.interval",
-      "displayName": "timeline.metrics.cluster.aggregator.hourly.interval",
+      "displayName": "Hourly cluster aggregator Interval",
       "description":  "\n      Time in seconds to sleep for the hourly resolution cluster wide\n      aggregator. Default is 1 hour.\n    ",
       "defaultValue": "3600",
       "displayType": "int",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 15
     },
     {
       "id": "site property",
       "name": "timeline.metrics.cluster.aggregator.minute.checkpointCutOffMultiplier",
-      "displayName": "timeline.metrics.cluster.aggregator.minute.checkpointCutOffMultiplier",
+      "displayName": "Minute cluster aggregator checkpoint cutOff multiplier",
       "description":  "\n      Multiplier value * interval = Max allowed checkpoint lag. Effectively\n      if aggregator checkpoint is greater than max allowed checkpoint delay,\n      the checkpoint will be discarded by the aggregator.\n    ",
       "defaultValue": "2",
       "displayType": "int",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 13
     },
     {
       "id": "site property",
       "name": "timeline.metrics.cluster.aggregator.minute.disabled",
-      "displayName": "timeline.metrics.cluster.aggregator.minute.disabled",
+      "displayName": "Disable minute cluster aggregator",
       "description":  "\n      Disable cluster based minute aggregations.\n    ",
       "defaultValue": "false",
       "displayType": "string",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 10
     },
     {
       "id": "site property",
       "name": "timeline.metrics.cluster.aggregator.minute.interval",
-      "displayName": "timeline.metrics.cluster.aggregator.minute.interval",
+      "displayName": "Minute cluster aggregator interval",
       "description":  "\n      Time in seconds to sleep for the minute resolution cluster wide\n      aggregator. Default resolution is 2 minutes.\n    ",
       "defaultValue": "120",
       "displayType": "int",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 11
     },
     {
       "id": "site property",
       "name": "timeline.metrics.cluster.aggregator.minute.timeslice.interval",
-      "displayName": "timeline.metrics.cluster.aggregator.minute.timeslice.interval",
+      "displayName": "Minute cluster aggregator timeslice interval",
       "description":  "\n      Lowest resolution of desired data for cluster level minute aggregates.\n    ",
       "defaultValue": "15",
       "displayType": "int",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 12
     },
     {
       "id": "site property",
       "name": "timeline.metrics.host.aggregator.hourly.checkpointCutOffMultiplier",
-      "displayName": "timeline.metrics.host.aggregator.hourly.checkpointCutOffMultiplier",
+      "displayName": "Hourly host aggregator checkpoint cutOff multiplier",
       "description":  "\n      Multiplier value * interval = Max allowed checkpoint lag. Effectively\n      if aggregator checkpoint is greater than max allowed checkpoint delay,\n      the checkpoint will be discarded by the aggregator.\n    ",
       "defaultValue": "2",
       "displayType": "int",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 9
     },
     {
       "id": "site property",
       "name": "timeline.metrics.host.aggregator.hourly.disabled",
-      "displayName": "timeline.metrics.host.aggregator.hourly.disabled",
+      "displayName": "Disable Hourly host aggregator",
       "description":  "\n      Disable host based hourly aggregations.\n    ",
       "defaultValue": "false",
       "displayType": "string",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 7
     },
     {
       "id": "site property",
       "name": "timeline.metrics.host.aggregator.hourly.interval",
-      "displayName": "timeline.metrics.host.aggregator.hourly.interval",
+      "displayName": "Hourly host aggregator interval",
       "description":  "\n      Time in seconds to sleep for the hourly resolution host based\n      aggregator. Default resolution is 1 hour.\n    ",
       "defaultValue": "3600",
       "displayType": "int",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 8
     },
     {
       "id": "site property",
       "name": "timeline.metrics.host.aggregator.minute.checkpointCutOffMultiplier",
-      "displayName": "timeline.metrics.host.aggregator.minute.checkpointCutOffMultiplier",
+      "displayName": "Minute host aggregator checkpoint cutOff multiplier",
       "description":  "\n      Multiplier value * interval = Max allowed checkpoint lag. Effectively\n      if aggregator checkpoint is greater than max allowed checkpoint delay,\n      the checkpoint will be discarded by the aggregator.\n    ",
       "defaultValue": "2",
       "displayType": "int",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 6
     },
     {
       "id": "site property",
       "name": "timeline.metrics.host.aggregator.minute.disabled",
-      "displayName": "timeline.metrics.host.aggregator.minute.disabled",
+      "displayName": "Disable Minute host aggregator",
       "description":  "\n      Disable host based minute aggregations.\n    ",
       "defaultValue": "false",
       "displayType": "string",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 4
     },
     {
       "id": "site property",
       "name": "timeline.metrics.host.aggregator.minute.interval",
-      "displayName": "timeline.metrics.host.aggregator.minute.interval",
+      "displayName": "Minute host aggregator interval",
       "description":  "\n      Time in seconds to sleep for the minute resolution host based\n      aggregator. Default resolution is 5 minutes.\n    ",
       "defaultValue": "300",
       "displayType": "int",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 5
     },
     {
       "id": "site property",
       "name": "timeline.metrics.service.checkpointDelay",
-      "displayName": "timeline.metrics.service.checkpointDelay",
+      "displayName": "Metrics service checkpoint delay",
       "description":  "\n      Time in seconds to sleep on the first run or when the checkpoint is\n      too old.\n    ",
       "defaultValue": "120",
       "displayType": "int",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 1
     },
     {
       "id": "site property",
       "name": "timeline.metrics.service.default.result.limit",
-      "displayName": "timeline.metrics.service.default.result.limit",
+      "displayName": "Metrics service default result limit",
       "description":  "\n      Max result limit on number of rows returned. Calculated as follows:\n      4 aggregate metrics/min * 60 * 24: Retrieve aggregate data for 1 day.\n    ",
       "defaultValue": "5760",
       "displayType": "int",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 2
     },
     {
       "id": "site property",
       "name": "timeline.metrics.service.resultset.fetchSize",
-      "displayName": "timeline.metrics.service.resultset.fetchSize",
+      "displayName": "Metrics service resultset fetchSize",
       "description":  "\n      JDBC resultset prefect size for aggregator queries.\n    ",
       "defaultValue": "2000",
       "displayType": "int",
       "serviceName": "AMS",
       "filename": "ams-site.xml",
-      "category": "MetricCollector"
+      "category": "MetricCollector",
+      "index" : 3
     }
   ]
 };

ambari-web/app/models/stack_service.js

@@ -334,8 +334,8 @@ App.StackService.configCategories = function () {
       break;
     case 'AMS':
       serviceConfigCategories.pushObjects([
-        App.ServiceConfigCategory.create({ name: 'MetricCollector', displayName: 'Metric Collector'}),
-        App.ServiceConfigCategory.create({ name: 'General', displayName: 'General'})
+        App.ServiceConfigCategory.create({ name: 'General', displayName: 'General'}),
+        App.ServiceConfigCategory.create({ name: 'MetricCollector', displayName: 'Metric Collector'})
       ]);
       break;
     case 'PIG':