AMBARI-15431 - Atlas Integration : Rename Atlas Configurations

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml

@@ -60,6 +60,16 @@
     <value>./server/webapp</value>
     <description>Atlas expanded WAR directory.</description>
   </property>
+  <property>
+    <name>metadata_conf_file</name>
+    <value>application.properties</value>
+    <description>Atlas configuration file</description>
+    <value-attributes>
+      <read-only>true</read-only>
+      <overridable>false</overridable>
+      <visible>false</visible>
+    </value-attributes>
+  </property>
 
   <!-- metadata-env.sh -->
   <property>

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py

@@ -70,7 +70,7 @@ def metadata():
          content = StaticFile(format('{metadata_home}/server/webapp/atlas.war'))
     )
 
-    PropertiesFile(format('{conf_dir}/application.properties'),
+    PropertiesFile(format('{conf_dir}/{conf_file}'),
          properties = params.application_properties,
          mode=0644,
          owner=params.metadata_user,

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py

@@ -96,7 +96,7 @@ class MetadataServer(Script):
     atlas_expectations.update(atlas_site_expectations)
 
     security_params = get_params_from_filesystem(status_params.conf_dir,
-                                                 {'application.properties': FILE_TYPE_PROPERTIES})
+                                                 {status_params.conf_file: FILE_TYPE_PROPERTIES})
     result_issues = validate_security_config_properties(security_params, atlas_expectations)
     if not result_issues:  # If all validations passed successfully
       try:

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py

@@ -58,6 +58,7 @@ metadata_stop_script = format("{metadata_bin}/atlas_stop.py")
 # metadata local directory structure
 log_dir = config['configurations']['atlas-env']['metadata_log_dir']
 conf_dir = status_params.conf_dir # "/etc/metadata/conf"
+conf_file = status_params.conf_file
 
 # service locations
 hadoop_conf_dir = os.path.join(os.environ["HADOOP_HOME"], "conf") if 'HADOOP_HOME' in os.environ else '/etc/hadoop/conf'

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py

@@ -25,6 +25,7 @@ from resource_management.libraries.functions.default import default
 
 config = Script.get_config()
 
+conf_file = config['configurations']['atlas-env']['metadata_conf_file']
 conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf'
 pid_dir = config['configurations']['atlas-env']['metadata_pid_dir']
 pid_file = format("{pid_dir}/atlas.pid")

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py

@@ -108,9 +108,11 @@ def falcon(type, action = None, upgrade_type=None):
         cd_access = "a")
 
     if params.has_atlas:
-      Link(params.falcon_conf_dir + "/application.properties",
-           to = params.atlas_conf_dir + "/application.properties"
-           )
+      atlas_falcon_hook_dir = os.path.join(params.atlas_home_dir, "hook", "falcon")
+      if os.path.exists(atlas_falcon_hook_dir):
+        Link(os.path.join(params.falcon_conf_dir, params.atlas_conf_file),
+          to = os.path.join(params.atlas_conf_dir, params.atlas_conf_file)
+          )
 
   if type == 'server':
     if action == 'config':
@@ -183,13 +185,14 @@ def falcon(type, action = None, upgrade_type=None):
         environment=environment_dictionary)
 
       if params.has_atlas:
-        atlas_falcon_hook_dir = params.atlas_home_dir + "/hook/falcon"
-        src_files = os.listdir(atlas_falcon_hook_dir)
-        for file_name in src_files:
-          atlas_falcon_hook_file_name = os.path.join(atlas_falcon_hook_dir, file_name)
-          falcon_lib_file_name = os.path.join(params.falcon_webinf_lib, file_name)
-          if (os.path.isfile(atlas_falcon_hook_file_name)):
-            Link(falcon_lib_file_name, to = atlas_falcon_hook_file_name)
+        atlas_falcon_hook_dir = os.path.join(params.atlas_home_dir, "hook", "falcon")
+        if os.path.exists(atlas_falcon_hook_dir):
+          src_files = os.listdir(atlas_falcon_hook_dir)
+          for file_name in src_files:
+            atlas_falcon_hook_file_name = os.path.join(atlas_falcon_hook_dir, file_name)
+            falcon_lib_file_name = os.path.join(params.falcon_webinf_lib, file_name)
+            if (os.path.isfile(atlas_falcon_hook_file_name)):
+              Link(falcon_lib_file_name, to = atlas_falcon_hook_file_name)
 
     if action == 'stop':
       Execute(format('{falcon_home}/bin/falcon-stop'),

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py

@@ -114,6 +114,7 @@ atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', [])
 has_atlas = len(atlas_hosts) > 0
 
 if has_atlas:
+  atlas_conf_file = config['configurations']['atlas-env']['metadata_conf_file']
   atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf'
   atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else '/usr/hdp/current/atlas-server'
 

ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py

@@ -114,6 +114,7 @@ atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', [])
 has_atlas = len(atlas_hosts) > 0
 
 if has_atlas:
+  atlas_conf_file = config['configurations']['atlas-env']['metadata_conf_file']
   atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else '/usr/hdp/current/atlas-server'
   atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf'
   job_data_publish_class = 'org.apache.atlas.sqoop.hook.SqoopHook'

ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py

@@ -60,17 +60,18 @@ def sqoop(type=None):
             )
 
   if params.has_atlas:
-    Link(params.sqoop_conf_dir + "/application.properties",
-         to = params.atlas_conf_dir + "/application.properties"
-         )
+    atlas_sqoop_hook_dir = os.path.join(params.atlas_home_dir, "hook", "sqoop")
+    if os.path.exists(atlas_sqoop_hook_dir):
+      Link(os.path.join(params.sqoop_conf_dir, params.atlas_conf_file),
+           to = os.path.join(params.atlas_conf_dir, params.atlas_conf_file)
+           )
 
-    atlas_sqoop_hook_dir = params.atlas_home_dir + "/hook/sqoop"
-    src_files = os.listdir(atlas_sqoop_hook_dir)
-    for file_name in src_files:
-      atlas_sqoop_hook_file_name = os.path.join(atlas_sqoop_hook_dir, file_name)
-      sqoop_lib_file_name = os.path.join(params.sqoop_lib, file_name)
-      if (os.path.isfile(atlas_sqoop_hook_file_name)):
-        Link(sqoop_lib_file_name, to = atlas_sqoop_hook_file_name)
+      src_files = os.listdir(atlas_sqoop_hook_dir)
+      for file_name in src_files:
+        atlas_sqoop_hook_file_name = os.path.join(atlas_sqoop_hook_dir, file_name)
+        sqoop_lib_file_name = os.path.join(params.sqoop_lib, file_name)
+        if (os.path.isfile(atlas_sqoop_hook_file_name)):
+          Link(sqoop_lib_file_name, to = atlas_sqoop_hook_file_name)
 
   File(format("{sqoop_conf_dir}/sqoop-env.sh"),
     owner=params.sqoop_user,

ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/atlas-env.xml

@@ -0,0 +1,114 @@
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+  <property require-input="false">
+    <name>metadata_log_dir</name>
+    <value>/var/log/atlas</value>
+    <description>Atlas log directory.</description>
+  </property>
+  <property require-input="false">
+    <name>metadata_pid_dir</name>
+    <value>/var/run/atlas</value>
+    <description>Atlas pid-file directory.</description>
+  </property>
+  <property>
+    <name>metadata_user</name>
+    <display-name>Metadata User</display-name>
+    <value>atlas</value>
+    <property-type>USER</property-type>
+    <description>Metadata User Name.</description>
+    <value-attributes>
+      <type>user</type>
+      <overridable>false</overridable>
+    </value-attributes>
+  </property>
+  <property>
+    <name>metadata_opts</name>
+    <value>-Xmx1024m -Dlog4j.configuration=atlas-log4j.xml</value>
+    <description>Metadata Server command line options.</description>
+  </property>
+  <property>
+    <name>metadata_classpath</name>
+    <value> </value>
+    <description>Metadata Server additional classpath.</description>
+  </property>
+  <property require-input="false">
+    <name>metadata_data_dir</name>
+    <value>/var/lib/atlas/data</value>
+    <description>Atlas data directory.</description>
+  </property>
+  <property require-input="false">
+    <name>metadata_expanded_war_dir</name>
+    <value>./server/webapp</value>
+    <description>Atlas expanded WAR directory.</description>
+  </property>
+  <property>
+    <name>metadata_conf_file</name>
+    <value>atlas-application.properties</value>
+    <description>Atlas configuration file</description>
+    <value-attributes>
+      <read-only>true</read-only>
+      <overridable>false</overridable>
+      <visible>false</visible>
+    </value-attributes>
+  </property>
+
+  <!-- metadata-env.sh -->
+  <property>
+    <name>content</name>
+    <description>This is the jinja template for metadata-env.sh file</description>
+    <value>
+      # The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path
+      export JAVA_HOME={{java64_home}}
+
+      # any additional java opts you want to set. This will apply to both client and server operations
+      {% if security_enabled %}
+      export ATLAS_OPTS="{{metadata_opts}} -Djava.security.auth.login.config={{atlas_jaas_file}}"
+      {% else %}
+      export ATLAS_OPTS="{{metadata_opts}}"
+      {% endif %}
+
+      # metadata configuration directory
+      export ATLAS_CONF={{conf_dir}}
+
+      # Where log files are stored. Defatult is logs directory under the base install location
+      export ATLAS_LOG_DIR={{log_dir}}
+
+      # additional classpath entries
+      export ATLASCPPATH={{metadata_classpath}}
+
+      # data dir
+      export ATLAS_DATA_DIR={{data_dir}}
+
+      # pid dir
+      export ATLAS_PID_DIR={{pid_dir}}
+
+      # hbase conf dir
+      export HBASE_CONF_DIR=/etc/hbase/conf
+
+      # Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.
+      export ATLAS_EXPANDED_WEBAPP_DIR={{expanded_war_dir}}
+    </value>
+    <value-attributes>
+      <type>content</type>
+    </value-attributes>
+  </property>
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/metainfo.xml

@@ -21,6 +21,25 @@
     <service>
       <name>ATLAS</name>
       <version>0.5.0.2.6</version>
+      <components>
+        <component>
+          <name>ATLAS_SERVER</name>
+          <configFile>
+            <type>properties</type>
+            <fileName>atlas-application.properties</fileName>
+            <dictionaryName>application-properties</dictionaryName>
+          </configFile>
+        </component>
+
+        <component>
+          <name>ATLAS_CLIENT</name>
+          <configFile>
+            <type>properties</type>
+            <fileName>atlas-application.properties</fileName>
+            <dictionaryName>application-properties</dictionaryName>
+          </configFile>
+        </component>
+      </components>
     </service>
   </services>
 </metainfo>

ambari-server/src/test/python/stacks/2.3/configs/default.json

@@ -209,7 +209,8 @@
         "metadata_pid_dir": "/var/run/atlas",
         "metadata_log_dir": "/var/log/atlas",
         "metadata_data_dir": "/var/lib/atlas/data",
-        "metadata_expanded_war_dir": "/var/lib/atlas/server/webapp"
+        "metadata_expanded_war_dir": "/var/lib/atlas/server/webapp",
+        "metadata_conf_file": "application.properties"
       },
       "ranger-hbase-plugin-properties": {
             "ranger-hbase-plugin-enabled":"yes"

ambari-server/src/test/python/stacks/2.3/configs/secure.json

@@ -190,7 +190,8 @@
       "metadata_pid_dir": "/var/run/atlas",
       "metadata_log_dir": "/var/log/atlas",
       "metadata_data_dir": "/var/lib/atlas/data",
-      "metadata_expanded_war_dir": "/var/lib/atlas/server/webapp"
+      "metadata_expanded_war_dir": "/var/lib/atlas/server/webapp",
+      "metadata_conf_file": "application.properties"
     },
     "ranger-hbase-plugin-properties": {
       "ranger-hbase-plugin-enabled":"yes"

ambari-server/src/test/python/stacks/2.6/ATLAS/test_atlas_server.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from stacks.utils.RMFTestCase import *
+
+from only_for_platform import not_for_platform, PLATFORM_WINDOWS
+
+@not_for_platform(PLATFORM_WINDOWS)
+class TestAtlasServer(RMFTestCase):
+  COMMON_SERVICES_PACKAGE_DIR = "ATLAS/0.1.0.2.3/package"
+  STACK_VERSION = "2.6"
+
+  def configureResourcesCalled(self):
+    self.assertResourceCalled('Directory', '/var/run/atlas',
+                              owner='atlas',
+                              group='hadoop',
+                              create_parents = True,
+                              cd_access='a',
+                              mode=0755
+                              )
+    self.assertResourceCalled('Directory', '/etc/atlas/conf',
+                              owner='atlas',
+                              group='hadoop',
+                              create_parents = True,
+                              cd_access='a',
+                              mode=0755
+                              )
+    self.assertResourceCalled('Directory', '/var/log/atlas',
+                              owner='atlas',
+                              group='hadoop',
+                              create_parents = True,
+                              cd_access='a',
+                              mode=0755
+                              )
+    self.assertResourceCalled('Directory', '/var/lib/atlas/data',
+                              owner='atlas',
+                              group='hadoop',
+                              create_parents = True,
+                              cd_access='a',
+                              mode=0644
+                              )
+    self.assertResourceCalled('Directory', '/var/lib/atlas/server/webapp',
+                              owner='atlas',
+                              group='hadoop',
+                              create_parents = True,
+                              cd_access='a',
+                              mode=0644
+                              )
+    self.assertResourceCalled('File', '/var/lib/atlas/server/webapp/atlas.war',
+                              content = StaticFile('/usr/hdp/current/atlas-server/server/webapp/atlas.war'),
+                              )
+    appprops =  dict(self.getConfig()['configurations'][
+                       'application-properties'])
+    appprops['atlas.http.authentication.kerberos.name.rules'] = ' \\ \n'.join(appprops['atlas.http.authentication.kerberos.name.rules'].splitlines())
+    appprops['atlas.server.bind.address'] = 'c6401.ambari.apache.org'
+
+    self.assertResourceCalled('PropertiesFile',
+                              '/etc/atlas/conf/atlas-application.properties',
+                              properties=appprops,
+                              owner='atlas',
+                              group='hadoop',
+                              mode=0644,
+                              )
+    self.assertResourceCalled('File', '/etc/atlas/conf/atlas-env.sh',
+                              content=InlineTemplate(
+                                  self.getConfig()['configurations'][
+                                    'atlas-env']['content']),
+                              owner='atlas',
+                              group='hadoop',
+                              mode=0755,
+                              )
+    self.assertResourceCalled('File', '/etc/atlas/conf/atlas-log4j.xml',
+                              content=StaticFile('atlas-log4j.xml'),
+                              owner='atlas',
+                              group='hadoop',
+                              mode=0644,
+                              )
+
+  def test_configure_default(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py",
+                       classname = "MetadataServer",
+                       command = "configure",
+                       config_file="default.json",
+                       stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+                       )
+
+    self.configureResourcesCalled()
+    self.assertNoMoreResources()
+