HDDS-1742. Merge ozone-perf and ozonetrace example clusters

Closes #1057

hadoop-ozone/dist/src/main/compose/ozoneperf/README.md

@@ -15,7 +15,8 @@
 # Compose files for local performance tests
 
 This directory contains docker-compose definition for an ozone cluster where
-all the metrics are saved to a prometheus instance.
+all the metrics are saved to a prometheus instance, and profiling and Jaeger
+tracing is turned on and set up.
 
 Prometheus follows a pull based approach where the metrics are published
  on a HTTP endpoint.
@@ -36,8 +37,8 @@ Note: The freon test will be started after 30 seconds.
 
 You can check the ozone web ui:
 
-OzoneManager: https://localhost:9874
-SCM: https://localhost:9876
+OzoneManager: http://localhost:9874
+SCM: http://localhost:9876
 
 You can check the ozone metrics from the prometheus web ui.
 
@@ -50,3 +51,6 @@ http://localhost:3000
 Default dashboards available are:
 Ozone - Object Metrics
 Ozone - RPC Metrics
+
+You can access the Jaeger UI at:
+http://localhost:16686

hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml

@@ -47,6 +47,12 @@ services:
       environment:
           ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
       command: ["ozone","scm"]
+   jaeger:
+     image: jaegertracing/all-in-one:latest
+     environment:
+       COLLECTOR_ZIPKIN_HTTP_PORT: 9411
+     ports:
+       - 16686:16686
    prometheus:
      image: prom/prometheus
      volumes:
@@ -72,3 +78,12 @@ services:
       command: ["-config","/etc/grafana/grafana.ini"]
       ports:
          - 3000:3000
+   s3g:
+     image: apache/ozone-runner:${HADOOP_RUNNER_VERSION}
+     volumes:
+       - ../..:/opt/hadoop
+     ports:
+       - 9878:9878
+     env_file:
+       - ./docker-config
+     command: ["ozone","s3g"]

hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config

@@ -25,10 +25,17 @@ OZONE-SITE.XML_ozone.scm.client.address=scm
 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
 OZONE-SITE.XML_ozone.replication=1
 OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true
+OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
+
+ASYNC_PROFILER_HOME=/opt/profiler
 
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
 
+JAEGER_SAMPLER_PARAM=1
+JAEGER_SAMPLER_TYPE=const
+JAEGER_AGENT_HOST=jaeger
+
 LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
 LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
@@ -37,3 +44,7 @@ LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
 LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
 LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
 LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.grpc.client.GrpcClientProtocolClient=WARN
+LOG4J.PROPERTIES_log4j.logger.http.requests.s3gateway=INFO,s3gatewayrequestlog
+LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
+LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.Filename=/tmp/jetty-s3gateway-yyyy_mm_dd.log
+LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.RetainDays=3

hadoop-ozone/dist/src/main/compose/ozonetrace/.env

@@ -1,19 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-HDDS_VERSION=${hdds.version}
-HADOOP_VERSION=3
-HADOOP_RUNNER_VERSION=${docker.ozone-runner.version}

hadoop-ozone/dist/src/main/compose/ozonetrace/docker-compose.yaml

@@ -1,64 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
-   jaeger:
-      image: jaegertracing/all-in-one:latest
-      environment:
-          COLLECTOR_ZIPKIN_HTTP_PORT: 9411
-      ports:
-         - 16686:16686
-   datanode:
-      image: apache/ozone-runner:${HADOOP_RUNNER_VERSION}
-      volumes:
-        - ../..:/opt/hadoop
-      ports:
-        - 9864
-      command: ["ozone","datanode"]
-      env_file:
-        - ./docker-config
-   om:
-      image: apache/ozone-runner:${HADOOP_RUNNER_VERSION}
-      volumes:
-         - ../..:/opt/hadoop
-      ports:
-         - 9874:9874
-      environment:
-         ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
-      env_file:
-          - ./docker-config
-      command: ["ozone","om"]
-   scm:
-      image: apache/ozone-runner:${HADOOP_RUNNER_VERSION}
-      volumes:
-         - ../..:/opt/hadoop
-      ports:
-         - 9876:9876
-      env_file:
-          - ./docker-config
-      environment:
-          ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
-      command: ["ozone","scm"]
-   s3g:
-      image: apache/ozone-runner:${HADOOP_RUNNER_VERSION}
-      volumes:
-         - ../..:/opt/hadoop
-      ports:
-         - 9878:9878
-      env_file:
-          - ./docker-config
-      command: ["ozone","s3g"]

hadoop-ozone/dist/src/main/compose/ozonetrace/docker-config

@@ -1,84 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=true
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=1
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-
-JAEGER_SAMPLER_PARAM=1
-JAEGER_SAMPLER_TYPE=const
-JAEGER_AGENT_HOST=jaeger
-
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
-LOG4J.PROPERTIES_log4j.logger.http.requests.s3gateway=INFO,s3gatewayrequestlog
-LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.Filename=/tmp/jetty-s3gateway-yyyy_mm_dd.log
-LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.RetainDays=3
-
-#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
-#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
-LOG4J2.PROPERTIES_monitorInterval=30
-LOG4J2.PROPERTIES_filter=read,write
-LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.read.marker=READ
-LOG4J2.PROPERTIES_filter.read.onMatch=DENY
-LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.write.marker=WRITE
-LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_appenders=console, rolling
-LOG4J2.PROPERTIES_appender.console.type=Console
-LOG4J2.PROPERTIES_appender.console.name=STDOUT
-LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.fileName=${sys:hadoop.log.dir}/om-audit-${hostName}.log
-LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
-LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
-LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
-LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
-LOG4J2.PROPERTIES_loggers=audit
-LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
-LOG4J2.PROPERTIES_logger.audit.name=OMAudit
-LOG4J2.PROPERTIES_logger.audit.level=INFO
-LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
-LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
-LOG4J2.PROPERTIES_rootLogger.level=INFO
-LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
-LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT