AMBARI-22674. ServiceAdvisor for OneFS (amagyar)

ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py

@@ -22,7 +22,6 @@ import httplib
 import imp
 import time
 import urllib
-from alerts.base_alert import BaseAlert
 from alerts.metric_alert import MetricAlert
 import ambari_simplejson as json
 import logging
@@ -30,6 +29,7 @@ import re
 import uuid
 
 from resource_management.libraries.functions.get_port_from_url import get_port_from_url
+from ambari_commons import inet_utils
 
 logger = logging.getLogger()
 
@@ -68,7 +68,7 @@ class AmsAlert(MetricAlert):
       logger.debug("[Alert][{0}] Calculated metric URI to be {1} (ssl={2})".format(
         self.get_name(), alert_uri.uri, str(alert_uri.is_ssl_enabled)))
 
-    host = BaseAlert.get_host_from_url(alert_uri.uri)
+    host = inet_utils.get_host_from_url(alert_uri.uri)
     if host is None:
       host = self.host_name
 

ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py

@@ -510,49 +510,4 @@ class BaseAlert(object):
     :param state: the state of the alert in uppercase (such as OK, WARNING, etc)
     :return:  the parameterized text
     '''
-    return '{0}'
-
-  """
-  See RFC3986, Appendix B
-  Tested on the following cases:
-    "192.168.54.1"
-    "192.168.54.2:7661
-    "hdfs://192.168.54.3/foo/bar"
-    "ftp://192.168.54.4:7842/foo/bar"
-
-    Returns None if only a port is passed in
-  """
-  @staticmethod
-  def get_host_from_url(uri):
-    if uri is None:
-      return None
-    
-    # if not a string, return None
-    if not isinstance(uri, basestring):
-      return None    
-        
-    # RFC3986, Appendix B
-    parts = re.findall('^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?', uri)
-
-    # index of parts
-    # scheme    = 1
-    # authority = 3
-    # path      = 4
-    # query     = 6
-    # fragment  = 8
-
-    host_and_port = uri
-    if 0 == len(parts[0][1]):
-      host_and_port = parts[0][4]
-    elif 0 == len(parts[0][2]):
-      host_and_port = parts[0][1]
-    elif parts[0][2].startswith("//"):
-      host_and_port = parts[0][3]
-
-    if -1 == host_and_port.find(':'):
-      if host_and_port.isdigit():
-        return None    
-      
-      return host_and_port
-    else:
-      return host_and_port.split(':')[0]
+    return '{0}'

ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py

@@ -30,6 +30,7 @@ from alerts.base_alert import BaseAlert
 from ambari_commons.urllib_handlers import RefreshHeaderProcessor
 from resource_management.libraries.functions.get_port_from_url import get_port_from_url
 from resource_management.libraries.functions.curl_krb_request import curl_krb_request
+from ambari_commons import inet_utils
 from ambari_agent import Constants
 
 logger = logging.getLogger()
@@ -80,7 +81,7 @@ class MetricAlert(BaseAlert):
     logger.debug("[Alert][{0}] Calculated metric URI to be {1} (ssl={2})".format(
         self.get_name(), alert_uri.uri, str(alert_uri.is_ssl_enabled)))
 
-    host = BaseAlert.get_host_from_url(alert_uri.uri)
+    host = inet_utils.get_host_from_url(alert_uri.uri)
     if host is None:
       host = self.host_name
 

ambari-agent/src/main/python/ambari_agent/alerts/port_alert.py

@@ -24,7 +24,7 @@ import time
 from alerts.base_alert import BaseAlert
 from resource_management.libraries.functions.get_port_from_url import get_port_from_url
 from ambari_commons import OSCheck
-from ambari_commons.inet_utils import resolve_address
+from ambari_commons.inet_utils import resolve_address, get_host_from_url
 logger = logging.getLogger()
 
 # default timeouts
@@ -111,7 +111,7 @@ class PortAlert(BaseAlert):
           break
 
 
-    host = BaseAlert.get_host_from_url(uri_value)
+    host = get_host_from_url(uri_value)
     if host is None or host == "localhost" or host == "0.0.0.0":
       host = self.host_name
       host_not_specified = True

ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py

@@ -21,9 +21,7 @@ limitations under the License.
 import logging
 import time
 import urllib2
-import ssl
 
-from functools import wraps
 from urllib2 import HTTPError
 
 from tempfile import gettempdir
@@ -33,7 +31,7 @@ from resource_management.libraries.functions.get_port_from_url import get_port_f
 from resource_management.libraries.functions.get_path_from_url import get_path_from_url
 from resource_management.libraries.functions.curl_krb_request import curl_krb_request
 from ambari_commons import OSCheck
-from ambari_commons.inet_utils import resolve_address, ensure_ssl_using_protocol
+from ambari_commons.inet_utils import resolve_address, ensure_ssl_using_protocol, get_host_from_url
 from ambari_agent import Constants
 from ambari_agent.AmbariConfig import AmbariConfig
 
@@ -135,7 +133,7 @@ class WebAlert(BaseAlert):
       uri_path = get_path_from_url(string_uri)
 
     # start building the URL manually
-    host = BaseAlert.get_host_from_url(alert_uri.uri)
+    host = get_host_from_url(alert_uri.uri)
     if host is None:
       host = self.host_name
 

ambari-common/src/main/python/ambari_commons/inet_utils.py

@@ -23,6 +23,7 @@ import time
 import sys
 import urllib2
 import socket
+import re
 from ambari_commons import OSCheck
 from functools import wraps
 
@@ -266,3 +267,48 @@ def ensure_ssl_using_protocol(protocol="PROTOCOL_TLSv1", ca_certs=None):
         return context
       _create_default_https_context_patched._ambari_patched = True
       ssl._create_default_https_context = _create_default_https_context_patched
+
+"""
+See RFC3986, Appendix B
+Tested on the following cases:
+  "192.168.54.1"
+  "192.168.54.2:7661
+  "hdfs://192.168.54.3/foo/bar"
+  "ftp://192.168.54.4:7842/foo/bar"
+
+  Returns None if only a port is passed in
+"""
+def get_host_from_url(uri):
+  if uri is None:
+    return None
+
+  # if not a string, return None
+  if not isinstance(uri, basestring):
+    return None
+
+    # RFC3986, Appendix B
+  parts = re.findall('^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?', uri)
+
+  # index of parts
+  # scheme    = 1
+  # authority = 3
+  # path      = 4
+  # query     = 6
+  # fragment  = 8
+
+  host_and_port = uri
+  if 0 == len(parts[0][1]):
+    host_and_port = parts[0][4]
+  elif 0 == len(parts[0][2]):
+    host_and_port = parts[0][1]
+  elif parts[0][2].startswith("//"):
+    host_and_port = parts[0][3]
+
+  if -1 == host_and_port.find(':'):
+    if host_and_port.isdigit():
+      return None
+
+    return host_and_port
+  else:
+    return host_and_port.split(':')[0]
+

contrib/management-packs/isilon-onefs-mpack/pom.xml

@@ -27,6 +27,7 @@
     <minAmbariVersion>3.0.0.0</minAmbariVersion>
     <maxAmbariVersion></maxAmbariVersion>
     <nifiversion>1.0.0</nifiversion>
+    <custom.tests>false</custom.tests>
   </properties>
   <parent>
     <groupId>org.apache.ambari.contrib.mpacks</groupId>
@@ -87,6 +88,28 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <version>1.2.1</version>
+        <executions>
+          <execution>
+            <configuration>
+              <executable>python</executable>
+              <workingDirectory>src/test/python</workingDirectory>
+              <arguments>
+                <argument>unitTests.py</argument>
+                <argument>${custom.tests}</argument>
+              </arguments>
+            </configuration>
+            <id>python-test</id>
+            <phase>test</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <artifactId>maven-assembly-plugin</artifactId>
         <configuration>

contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/core-site.xml

@@ -27,6 +27,12 @@
   literal string "local" or a host:port for HDFS.</description>
     <final>true</final>
     <on-ambari-upgrade add="false"/>
+    <depends-on>
+      <property>
+          <type>onefs</type>
+          <name>onefs_host</name>
+      </property>
+    </depends-on>
   </property>
   <property>
     <name>hadoop.security.authentication</name>
@@ -91,12 +97,4 @@ DEFAULT
     </value-attributes>
     <on-ambari-upgrade add="true"/>
   </property>
-  <property>
-    <name>net.topology.script.file.name</name>
-    <value>/etc/hadoop/conf/topology_script.py</value>
-    <description>
-      Location of topology script used by Hadoop to determine the rack location of nodes.
-    </description>
-    <on-ambari-upgrade add="true"/>
-  </property>
 </configuration>

contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hdfs-site.xml

@@ -30,27 +30,31 @@
     </value-attributes>
     <on-ambari-upgrade add="true"/>
   </property>
-  <property>
-    <name>dfs.datanode.http.address</name>
-    <value>0.0.0.0:8082</value>
-    <description>
-      The datanode http server address and port.
-    </description>
-    <on-ambari-upgrade add="true"/>
-  </property>
   <property>
     <name>dfs.namenode.http-address</name>
     <value>localhost:8082</value>
     <description>The name of the default file system.  Either the
       literal string "local" or a host:port for HDFS.</description>
     <final>true</final>
-    <on-ambari-upgrade add="false"/>
+    <on-ambari-upgrade add="true"/>
+    <depends-on>
+      <property>
+        <type>onefs</type>
+        <name>onefs_host</name>
+      </property>
+    </depends-on>
   </property>
   <property>
     <name>dfs.namenode.https-address</name>
     <value>localhost:8080</value>
     <description>The https address where namenode binds</description>
-    <on-ambari-upgrade add="false"/>
+    <on-ambari-upgrade add="true"/>
+    <depends-on>
+      <property>
+        <type>onefs</type>
+        <name>onefs_host</name>
+      </property>
+    </depends-on>
   </property>
   <property>
     <name>dfs.client-write-packet-size</name>

contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/onefs.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <property>
+    <name>onefs_host</name>
+    <value/>
+    <description>SmartConnect Zone Address of OneFS</description>
+    <display-name>OneFS Host</display-name>
+    <on-ambari-upgrade add="true"/>
+  </property>
+</configuration>

contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml

@@ -39,6 +39,11 @@
             <timeout>1200</timeout>
           </commandScript>
           <configFiles>
+            <configFile>
+              <type>xml</type>
+              <fileName>onefs.xml</fileName>
+              <dictionaryName>onefs</dictionaryName>
+            </configFile>
             <configFile>
               <type>xml</type>
               <fileName>hdfs-site.xml</fileName>
@@ -80,9 +85,17 @@
         <timeout>300</timeout>
       </commandScript>
 
+      <themes>
+        <theme>
+          <fileName>theme.json</fileName>
+          <default>true</default>
+        </theme>
+      </themes>
+
       <configuration-dependencies>
         <config-type>core-site</config-type>
         <config-type>hdfs-site</config-type>
+        <config-type>onefs</config-type>
         <config-type>hadoop-env</config-type>
       </configuration-dependencies>
       <restartRequiredAfterRackChange>true</restartRequiredAfterRackChange>

contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py

@@ -17,15 +17,14 @@ limitations under the License.
 
 """
 
-from resource_management.libraries.functions.version import format_stack_version, compare_versions
-from resource_management import *
-import os
-import itertools
-import re
+from resource_management import Script
 from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions.default import default
 from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
 from resource_management.libraries.resources.hdfs_resource import HdfsResource
 from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import format
 
 config = Script.get_config()
 
@@ -45,6 +44,9 @@ hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
 hdfs_site = config['configurations']['hdfs-site']
 default_fs = config['configurations']['core-site']['fs.defaultFS']
 
+java64_home = config['hostLevelParams']['java_home']
+java_exec = format("{java64_home}/bin/java")
+
 import functools
 #create partial functions with common arguments for every HdfsResource call
 #to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code

contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/service_advisor.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import os
+from ambari_commons import inet_utils
+import imp
+import traceback
+
+def error(message): return {"level": "ERROR", "message": message}
+
+class Uri:
+  @classmethod
+  def default_fs(self, configs):
+    return self.from_config(configs, 'core-site', 'fs.defaultFS')
+
+  @classmethod
+  def http_namenode(self, configs):
+    return self.from_config(configs, 'hdfs-site', 'dfs.namenode.http-address')
+
+  @classmethod
+  def https_namenode(self, configs):
+    return self.from_config(configs, 'hdfs-site', 'dfs.namenode.https-address')
+
+  @classmethod
+  def onefs(self, configs):
+    return self.from_config(configs, 'onefs', 'onefs_host')
+
+  @staticmethod
+  def from_config(configs, config_type, property_name):
+    return Uri(configs['configurations'][config_type]['properties'][property_name])
+
+  def __init__(self, address):
+    self.address = address
+
+  def has_host(self, uri):
+    return uri.hostname() == self.hostname()
+
+  def hostname(self):
+    return inet_utils.get_host_from_url(self.address)
+
+  def fix_host(self, uri):
+    if not uri.hostname() or not self.hostname():
+      return self.address
+    return self.address.replace(self.hostname(), uri.hostname())
+
+  def __str__(self):
+    return self.address
+
+class CoreSite:
+  def __init__(self, configs):
+    self.configs = configs
+
+  def validate(self):
+    invalid_configs = []
+    onefs_host = Uri.onefs(self.configs)
+    if not Uri.default_fs(self.configs).has_host(onefs_host):
+      invalid_configs.append({
+        'config-name': 'fs.defaultFS',
+        'item': error('Hostname should match OneFS host: {0}'.format(onefs_host))
+      })
+    return invalid_configs
+
+class HdfsSite:
+  def __init__(self, configs):
+    self.configs = configs
+
+  def validate(self):
+    invalid_configs = []
+    onefs_host = Uri.onefs(self.configs)
+    if not Uri.http_namenode(self.configs).has_host(onefs_host):
+      invalid_configs.append({
+        'config-name': 'dfs.namenode.http-address',
+        'item': error('Hostname should match OneFS host: {0}'.format(onefs_host))
+      })
+    if not Uri.https_namenode(self.configs).has_host(onefs_host):
+      invalid_configs.append({
+        'config-name': 'dfs.namenode.https-address',
+        'item': error('Hostname should match OneFS host: {0}'.format(onefs_host))
+      })
+    return invalid_configs
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+  with open(PARENT_FILE, 'rb') as fp:
+    service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+  traceback.print_exc()
+  print "Failed to load parent"
+else:
+  class HDP26ONEFSServiceAdvisor(service_advisor.ServiceAdvisor):
+    def getServiceConfigurationRecommendations(self, configs, clusterData, services, hosts):
+      putCoreSiteProperty = self.putProperty(configs, "core-site", services)
+      putHdfsSiteProperty = self.putProperty(configs, "hdfs-site", services)
+      onefs_host = Uri.onefs(services)
+      putCoreSiteProperty("fs.defaultFS", Uri.default_fs(services).fix_host(onefs_host))
+      putHdfsSiteProperty("dfs.namenode.http-address", Uri.http_namenode(services).fix_host(onefs_host))
+      putHdfsSiteProperty("dfs.namenode.https-address", Uri.https_namenode(services).fix_host(onefs_host))
+
+    def getServiceConfigurationsValidationItems(self, configs, recommendedDefaults, services, hosts):
+      validation_errors = []
+      validation_errors.extend(self.toConfigurationValidationProblems(CoreSite(services).validate(), 'core-site'))
+      validation_errors.extend(self.toConfigurationValidationProblems(HdfsSite(services).validate(), 'hdfs-site'))
+      return validation_errors
+

contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/themes/theme.json

@@ -0,0 +1,60 @@
+{
+  "name": "default",
+  "description": "Default theme for OneFS service",
+  "configuration": {
+    "layouts": [
+      {
+        "name": "default",
+        "tabs": [
+          {
+            "name": "settings",
+            "display-name": "Settings",
+            "layout": {
+              "tab-columns": "1",
+              "tab-rows": "1",
+              "sections": [
+                {
+                  "name": "section-onefs",
+                  "display-name": "OneFS",
+                  "row-index": "0",
+                  "column-index": "0",
+                  "row-span": "1",
+                  "column-span": "1",
+                  "section-columns": "1",
+                  "section-rows": "1",
+                  "subsections": [
+                    {
+                      "name": "subsection-onefs-col1",
+                      "row-index": "0",
+                      "column-index": "0",
+                      "row-span": "1",
+                      "column-span": "1"
+                    }
+                  ]
+                }
+              ]
+            }
+          }
+        ]
+      }
+    ],
+    "placement": {
+      "configuration-layout": "default",
+      "configs": [
+        {
+          "config": "onefs/onefs_host",
+          "subsection-name": "subsection-onefs-col1"
+        }
+      ]
+    },
+    "widgets": [
+      {
+        "config": "onefs/onefs_host",
+        "widget": {
+          "type": "text-field"
+        }
+      }
+    ]
+  }
+}
+

contrib/management-packs/isilon-onefs-mpack/src/test/python/TestServiceAdvisor.py

@@ -0,0 +1,67 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from unittest import TestCase
+from service_advisor import Uri, CoreSite, HdfsSite
+
+configs = {
+  'configurations': {
+    'core-site': {
+      'properties': {
+        'fs.defaultFS': 'hdfs://localhost:8020'
+      }
+    },
+    'hdfs-site': {
+      'properties': {
+        'dfs.namenode.http-address'  : 'scisilon.fqdn:8082',
+        'dfs.namenode.https-address' : 'scisilon.fqdn:8080',
+      }
+    },
+    'onefs': {
+      'properties': {
+        'onefs_host': 'scisilon.fqdn'
+      }
+    }
+  }
+}
+
+class TestUri(TestCase):
+  def test_fix_host(self):
+    onefs_host = Uri.onefs(configs)
+    default_fs = Uri.default_fs(configs)
+    fixed = default_fs.fix_host(onefs_host)
+    self.assertEquals('hdfs://scisilon.fqdn:8020', fixed)
+
+  def test_skip_replacing_to_empty_host(self):
+    default_fs = Uri.default_fs(configs)
+    self.assertEquals('hdfs://localhost:8020', default_fs.fix_host(Uri("")))
+
+  def test_skip_fixing_invalid_host(self):
+    default_fs = Uri("hdfs://:8080")
+    self.assertEquals('hdfs://:8080', default_fs.fix_host(Uri("host")))
+
+  def test_core_site_validation_error_on_host_mismatch(self):
+    core_site = CoreSite(configs)
+    erros = core_site.validate()
+    self.assertEquals(len(erros), 1)
+    self.assertEquals(erros[0]['config-name'], 'fs.defaultFS')
+
+  def test_hdfs_site_no_validation_error(self):
+    hdfs_site = HdfsSite(configs)
+    erros = hdfs_site.validate()
+    self.assertEquals(len(erros), 0)

contrib/management-packs/isilon-onefs-mpack/src/test/python/unitTests.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import unittest
+import os
+import sys
+from random import shuffle
+import fnmatch
+
+#excluded directories with non-test staff from stack and service scanning,
+#also we can add service or stack to skip here
+STACK_EXCLUDE = ["utils"]
+SERVICE_EXCLUDE = ["configs"]
+
+TEST_MASK = '[Tt]est*.py'
+CUSTOM_TEST_MASK = '_[Tt]est*.py'
+def get_parent_path(base, directory_name):
+  """
+  Returns absolute path for directory_name, if directory_name present in base.
+  For example, base=/home/user/test2, directory_name=user - will return /home/user
+  """
+  done = False
+  while not done:
+    base = os.path.dirname(base)
+    if base == "/":
+      return None
+    if os.path.split(base)[-1] == directory_name:
+      done = True
+    else:
+      done = False
+  return base
+
+def get_test_files(path, mask = None, recursive=True):
+  """
+  Returns test files for path recursively
+  """
+  current = []
+  directory_items = os.listdir(path)
+  directory_items.sort()
+
+  for item in directory_items:
+    add_to_pythonpath = False
+    item_path = os.path.join(path, item)
+    if os.path.isfile(item_path):
+      if fnmatch.fnmatch(item, mask):
+        add_to_pythonpath = True
+        current.append(item)
+    elif os.path.isdir(item_path):
+      if recursive:
+        current.extend(get_test_files(item_path, mask = mask))
+    if add_to_pythonpath:
+      sys.path.append(path)
+  return current
+
+
+def main():
+  custom_tests = False
+  if len(sys.argv) > 1:
+    if sys.argv[1] == "true":
+      custom_tests = True
+  pwd = os.path.abspath(os.path.dirname(__file__))
+
+  project_folder = get_parent_path(pwd,'isilon-onefs-mpack')
+  sys.path.append(project_folder + "/src/main/resources/addon-services/ONEFS/1.0.0")
+  sys.path.append(project_folder + "/src/test/python")
+  sys.path.append(project_folder + "/../../../ambari-agent/src/main/python")
+  sys.path.append(project_folder + "/../../../ambari-common/src/main/python")
+
+  has_failures = False
+  test_runs = 0
+  test_failures = []
+  test_errors = []
+  sys.stderr.write("Running tests\n")
+  if custom_tests:
+    test_mask = CUSTOM_TEST_MASK
+  else:
+    test_mask = TEST_MASK
+
+  tests = get_test_files(pwd, mask=test_mask, recursive=True)
+  shuffle(tests)
+  modules = [os.path.basename(s)[:-3] for s in tests]
+  suites = [unittest.defaultTestLoader.loadTestsFromName(name) for name in
+            modules]
+  testSuite = unittest.TestSuite(suites)
+  textRunner = unittest.TextTestRunner(verbosity=2).run(testSuite)
+  test_runs += textRunner.testsRun
+  test_errors.extend([(str(item[0]),str(item[1]),"ERROR") for item in textRunner.errors])
+  test_failures.extend([(str(item[0]),str(item[1]),"FAIL") for item in textRunner.failures])
+  tests_status = textRunner.wasSuccessful() and not has_failures
+
+  if not tests_status:
+    sys.stderr.write("----------------------------------------------------------------------\n")
+    sys.stderr.write("Failed tests:\n")
+  for failed_tests in [test_errors,test_failures]:
+    for err in failed_tests:
+      sys.stderr.write("{0}: {1}\n".format(err[2],err[0]))
+      sys.stderr.write("----------------------------------------------------------------------\n")
+      sys.stderr.write("{0}\n".format(err[1]))
+  sys.stderr.write("----------------------------------------------------------------------\n")
+  sys.stderr.write("Total run:{0}\n".format(test_runs))
+  sys.stderr.write("Total errors:{0}\n".format(len(test_errors)))
+  sys.stderr.write("Total failures:{0}\n".format(len(test_failures)))
+
+  if tests_status:
+    sys.stderr.write("OK\n")
+    exit_code = 0
+  else:
+    sys.stderr.write("ERROR\n")
+    exit_code = 1
+  return exit_code
+
+
+if __name__ == "__main__":
+  sys.exit(main())
+