HDFS-12983. Block Storage: provide docker-compose file for cblock clusters. Contributed by Elek,Marton.

dev-support/compose/cblock/.env

@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+VERSION=3.1.0-SNAPSHOT

dev-support/compose/cblock/README.md

@@ -0,0 +1,42 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+
+# CBlock dozone configuration
+
+This directory contains example cluster definition for CBlock/jscsi servers.
+
+## How to use
+
+1. First of all Start the servers with `docker-compose up -d`
+
+2. Wait until the servers are up and running (check http://localhost:9876 and wait until you have a healthy node)
+
+3. Create a volume: `docker-compose exec cblock hdfs cblock -c bilbo volume2 1GB 4`
+
+4. Mount the iscsi volume (from host machine):
+
+```
+sudo iscsiadm -m node -o new -T bilbo:volume2 -p 127.0.0.1
+sudo iscsiadm -m node -T bilbo:volume2 --login
+```
+
+5. Check the device name from `dmesg` or `lsblk` (eg /dev/sdd). Errors in dmesg could be ignored: jscsi doesn't implement all the jscsi commands.
+
+6. Format the device (`mkfs.ext4 /dev/sdd`). (Yes, format the while device, not just a partition).
+
+7. Mount it (`mount /dev/sdd /mnt/target`).

dev-support/compose/cblock/docker-compose.yaml

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+   namenode:
+      image: elek/hadoop-runner:latest
+      hostname: namenode
+      volumes:
+         - ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
+      ports:
+         - 9870:9870
+      environment:
+          ENSURE_NAMENODE_DIR: /data/namenode
+      env_file:
+         - ./docker-config
+      command: ["/opt/hadoop/bin/hdfs","namenode"]
+   datanode:
+      image: elek/hadoop-runner:latest
+      volumes:
+        - ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
+      ports:
+        - 9864
+      command: ["/opt/hadoop/bin/hdfs","datanode"]
+   jscsi:
+      image: elek/hadoop-runner:latest
+      ports:
+        - 3260:3260
+      volumes:
+         - ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
+      env_file:
+          - ./docker-config
+      command: ["/opt/hadoop/bin/hdfs","jscsi"]
+   cblock:
+      image: elek/hadoop-runner:latest
+      volumes:
+         - ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
+      env_file:
+          - ./docker-config
+      command: ["/opt/hadoop/bin/hdfs","cblockserver"]
+   scm:
+      image: elek/hadoop-runner:latest
+      volumes:
+         - ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
+      ports:
+         - 9876:9876
+      env_file:
+          - ./docker-config
+      command: ["/opt/hadoop/bin/hdfs","scm"]
+      environment:
+          ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION

dev-support/compose/cblock/docker-config

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
+OZONE-SITE.XML_ozone.ksm.address=ksm
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+
+OZONE-SITE.XML_dfs.cblock.jscsi.cblock.server.address=cblock
+OZONE-SITE.XML_dfs.cblock.scm.ipaddress=scm
+OZONE-SITE.XML_dfs.cblock.service.leveldb.path=/tmp
+
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
+HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+LOG4J.PROPERTIES_log4j.rootLogger=info,stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n