AMBARI-5797. Agent function get_port_from_url is broken (aonishuk)

ambari-agent/src/main/python/resource_management/libraries/functions/get_port_from_url.py

@@ -22,10 +22,19 @@ Ambari Agent
 
 from resource_management import *
 from resource_management.libraries.functions.is_empty import *
-from urlparse import urlparse
+from resource_management.core.exceptions import Fail
+import re
 
 def get_port_from_url(address):
+  """
+  Return port from URL. If address is UnknownConfiguration,
+  UnknownConfiguration will be returned. If no port was found, Fail will be
+  raised.
+  """
   if not is_empty(address):
-    return urlparse(address).port
+    port = re.findall(":([\d]{1,5})(?=/|$)", address)
+    if port:
+      return port[0]
+    raise Fail("No port in URL:{0}".format(address))
   else:
     return address

ambari-agent/src/test/python/resource_management/TestLibraryFunctions.py

@@ -0,0 +1,33 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from resource_management.libraries.functions import get_port_from_url
+from resource_management.core.exceptions import Fail
+class TestLibraryFunctions(TestCase):
+
+  def test_get_port_from_url(self):
+    self.assertEqual("8080",get_port_from_url("protocol://host:8080"))
+    self.assertEqual("8080",get_port_from_url("protocol://host:8080/"))
+    self.assertEqual("8080",get_port_from_url("host:8080"))
+    self.assertEqual("8080",get_port_from_url("host:8080/"))
+    self.assertEqual("8080",get_port_from_url("host:8080/dots_in_url8888:"))
+    self.assertEqual("8080",get_port_from_url("protocol://host:8080/dots_in_url8888:"))
+    self.assertEqual("8080",get_port_from_url("127.0.0.1:8080"))
+    self.assertRaises(Fail, get_port_from_url, "http://host/no_port")
+    self.assertRaises(Fail, get_port_from_url, "127.0.0.1:808080")

ambari-server/src/test/python/stacks/1.3.2/configs/default.hbasedecom.json

@@ -233,11 +233,11 @@
             "dfs.datanode.du.reserved": "1073741824", 
             "dfs.webhdfs.enabled": "true", 
             "dfs.namenode.handler.count": "100", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:50075",
             "dfs.datanode.socket.write.timeout": "0", 
             "ipc.server.read.threadpool.size": "5", 
             "dfs.balance.bandwidthPerSec": "6250000", 
-            "dfs.datanode.address": "0.0.0.0:${ambari.dfs.datanode.port}", 
+            "dfs.datanode.address": "0.0.0.0:50075",
             "dfs.blockreport.initialDelay": "120", 
             "dfs.datanode.failed.volumes.tolerated": "0", 
             "dfs.permissions.supergroup": "hdfs", 

ambari-server/src/test/python/stacks/1.3.2/configs/default.json

@@ -235,7 +235,7 @@
             "dfs.datanode.du.reserved": "1073741824", 
             "dfs.webhdfs.enabled": "true", 
             "dfs.namenode.handler.count": "100", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:50075",
             "dfs.datanode.socket.write.timeout": "0", 
             "ipc.server.read.threadpool.size": "5", 
             "dfs.balance.bandwidthPerSec": "6250000", 

ambari-server/src/test/python/stacks/1.3.2/configs/default.non_gmetad_host.json

@@ -233,7 +233,7 @@
             "dfs.datanode.du.reserved": "1073741824", 
             "dfs.webhdfs.enabled": "true", 
             "dfs.namenode.handler.count": "100", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:50075",
             "dfs.datanode.socket.write.timeout": "0", 
             "ipc.server.read.threadpool.size": "5", 
             "dfs.balance.bandwidthPerSec": "6250000", 

ambari-server/src/test/python/stacks/1.3.2/configs/secured.json

@@ -331,7 +331,7 @@
             "dfs.access.time.precision": "0", 
             "dfs.secondary.namenode.kerberos.internal.spnego.principal": "${dfs.web.authentication.kerberos.principal}", 
             "dfs.https.address": "c6401.ambari.apache.org:50470", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:1022",
             "dfs.data.dir": "/hadoop/hdfs/data", 
             "dfs.secondary.https.port": "50490", 
             "dfs.permissions": "true", 

ambari-server/src/test/python/stacks/1.3.2/configs/secured_no_jce_name.json

@@ -329,7 +329,7 @@
             "dfs.access.time.precision": "0", 
             "dfs.secondary.namenode.kerberos.internal.spnego.principal": "${dfs.web.authentication.kerberos.principal}", 
             "dfs.https.address": "c6401.ambari.apache.org:50470", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:1022",
             "dfs.data.dir": "/hadoop/hdfs/data", 
             "dfs.secondary.https.port": "50490", 
             "dfs.permissions": "true", 

ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json

@@ -331,7 +331,7 @@
             "dfs.namenode.handler.count": "100", 
             "dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary", 
             "fs.permissions.umask-mode": "022", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:50075",
             "dfs.datanode.ipc.address": "0.0.0.0:8010", 
             "dfs.datanode.data.dir": "/hadoop/hdfs/data", 
             "dfs.namenode.http-address": "c6401.ambari.apache.org:50070", 

ambari-server/src/test/python/stacks/2.0.6/configs/default.json

@@ -334,7 +334,7 @@
             "dfs.namenode.handler.count": "100", 
             "dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary", 
             "fs.permissions.umask-mode": "022", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:50075",
             "dfs.datanode.ipc.address": "0.0.0.0:8010", 
             "dfs.datanode.data.dir": "/hadoop/hdfs/data", 
             "dfs.namenode.http-address": "c6401.ambari.apache.org:50070", 

ambari-server/src/test/python/stacks/2.0.6/configs/default.non_gmetad_host.json

@@ -332,7 +332,7 @@
             "dfs.namenode.handler.count": "100", 
             "dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary", 
             "fs.permissions.umask-mode": "022", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:50075",
             "dfs.datanode.ipc.address": "0.0.0.0:8010", 
             "dfs.datanode.data.dir": "/hadoop/hdfs/data", 
             "dfs.namenode.http-address": "c6401.ambari.apache.org:50070", 

ambari-server/src/test/python/stacks/2.0.6/configs/flume_target.json

@@ -333,7 +333,7 @@
             "dfs.namenode.handler.count": "100", 
             "dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary", 
             "fs.permissions.umask-mode": "022", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:50075",
             "dfs.datanode.ipc.address": "0.0.0.0:8010", 
             "dfs.datanode.data.dir": "/hadoop/hdfs/data", 
             "dfs.namenode.http-address": "c6401.ambari.apache.org:50070", 

ambari-server/src/test/python/stacks/2.0.6/configs/ha_default.json

@@ -178,7 +178,7 @@
             "dfs.namenode.write.stale.datanode.ratio": "1.0f", 
             "dfs.namenode.secondary.http-address": "c6402.ambari.apache.org:50090", 
             "dfs.ha.fencing.methods": "shell(/bin/true)", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:50075",
             "dfs.datanode.du.reserved": "1073741824", 
             "dfs.client.read.shortcircuit.streams.cache.size": "4096", 
             "dfs.ha.namenodes.ns1": "nn1,nn2", 

ambari-server/src/test/python/stacks/2.0.6/configs/ha_secured.json

@@ -248,7 +248,7 @@
             "dfs.datanode.kerberos.principal": "dn/_HOST@EXAMPLE.COM", 
             "dfs.ha.fencing.methods": "shell(/bin/true)", 
             "dfs.journalnode.keytab.file": "/etc/security/keytabs/jn.service.keytab", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:1022",
             "dfs.datanode.du.reserved": "1073741824", 
             "dfs.client.read.shortcircuit.streams.cache.size": "4096", 
             "dfs.namenode.rpc-address.ns1.nn2": "c6402.ambari.apache.org:8020", 

ambari-server/src/test/python/stacks/2.0.6/configs/secured.json

@@ -446,7 +446,7 @@
             "dfs.namenode.secondary.http-address": "c6402.ambari.apache.org:50090", 
             "dfs.client.read.shortcircuit": "true", 
             "dfs.journalnode.keytab.file": "/etc/security/keytabs/jn.service.keytab", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:1022",
             "dfs.datanode.du.reserved": "1073741824", 
             "dfs.client.read.shortcircuit.streams.cache.size": "4096", 
             "dfs.secondary.namenode.keytab.file": "/etc/security/keytabs/nn.service.keytab", 

ambari-server/src/test/python/stacks/2.0.6/configs/secured_no_jce_name.json

@@ -444,7 +444,7 @@
             "dfs.namenode.secondary.http-address": "c6402.ambari.apache.org:50090", 
             "dfs.client.read.shortcircuit": "true", 
             "dfs.journalnode.keytab.file": "/etc/security/keytabs/jn.service.keytab", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:1022",
             "dfs.datanode.du.reserved": "1073741824", 
             "dfs.client.read.shortcircuit.streams.cache.size": "4096", 
             "dfs.secondary.namenode.keytab.file": "/etc/security/keytabs/nn.service.keytab", 

ambari-server/src/test/python/stacks/2.1/configs/default.json

@@ -372,7 +372,7 @@
             "dfs.namenode.handler.count": "100", 
             "dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary", 
             "fs.permissions.umask-mode": "022", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:50075",
             "dfs.datanode.ipc.address": "0.0.0.0:8010", 
             "dfs.datanode.data.dir": "/hadoop/hdfs/data", 
             "dfs.namenode.http-address": "c6401.ambari.apache.org:50070", 

ambari-server/src/test/python/stacks/2.1/configs/secured.json

@@ -494,7 +494,7 @@
             "dfs.namenode.secondary.http-address": "c6402.ambari.apache.org:50090", 
             "dfs.client.read.shortcircuit": "true", 
             "dfs.journalnode.keytab.file": "/etc/security/keytabs/jn.service.keytab", 
-            "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}", 
+            "dfs.datanode.http.address": "0.0.0.0:1022",
             "dfs.datanode.du.reserved": "1073741824", 
             "dfs.client.read.shortcircuit.streams.cache.size": "4096", 
             "dfs.secondary.namenode.keytab.file": "/etc/security/keytabs/nn.service.keytab",