AMBARI-10853. Initial service definition for Atlas (Jon Maron via smohanty)

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3.0.0/alerts.json

@@ -0,0 +1,60 @@
+{
+  "ATLAS": {
+    "service": [],
+    "ATLAS_SERVER": [
+      {
+        "name": "metadata_server_process",
+        "label": "Atlas Metadata Server Process",
+        "description": "This host-level alert is triggered if the individual Metadata server process cannot be established to be up and listening on the network.",
+        "interval": 1,
+        "scope": "ANY",
+        "source": {
+          "type": "PORT",
+          "uri": "{{metadata-env/metadata_port}}",
+          "default_port": 21000,
+          "reporting": {
+            "ok": {
+              "text": "TCP OK - {0:.3f}s response on port {1}"
+            },
+            "warning": {
+              "text": "TCP OK - {0:.3f}s response on port {1}",
+              "value": 1.5
+            },
+            "critical": {
+              "text": "Connection failed: {0} to {1}:{2}",
+              "value": 5.0
+            }
+          }
+        }
+      },
+      {
+        "name": "metadata_server_webui",
+        "label": "Metadata Server Web UI",
+        "description": "This host-level alert is triggered if the Metadata Server Web UI is unreachable.",
+        "interval": 1,
+        "scope": "ANY",
+        "enabled": true,
+        "source": {
+          "type": "WEB",
+          "uri": {
+            "http": "{{metadata-env/metadata_port}}",
+            "default_port": 21000,
+            "kerberos_keytab": "{{metadata-runtime.properties/*.metadata.http.authentication.keytab}}",
+            "kerberos_principal": "{{metadata-runtime.properties/*.metadata.http.authentication.principal}}"
+          },
+          "reporting": {
+            "ok": {
+              "text": "HTTP {0} response in {2:.3f}s"
+            },
+            "warning":{
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
+            },
+            "critical": {
+              "text": "Connection failed to {1} ({3})"
+            }
+          }
+        }
+      }
+    ]
+  }
+}

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3.0.0/configuration/application-properties.xml

@@ -0,0 +1,190 @@
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false">
+  <property>
+    <name>graph_storage_backend</name>
+    <value>berkeleyje</value>
+    <description></description>
+  </property>
+  <property>
+    <name>graph_storage_directory</name>
+    <value>/var/lib/atlas/data/berkeley</value>
+    <description></description>
+  </property>
+  <property>
+    <name>graph_index_search_backend</name>
+    <value>elasticsearch</value>
+    <description></description>
+  </property>
+  <property>
+    <name>graph_index_search_directory</name>
+    <value>/var/lib/atlas/data/es</value>
+    <description></description>
+  </property>
+  <property>
+    <name>graph_index_search_elasticsearch_client_only</name>
+    <value>false</value>
+    <description></description>
+  </property>
+  <property>
+    <name>graph_index_search_elasticsearch_local_mode</name>
+    <value>true</value>
+    <description></description>
+  </property>
+  <property>
+    <name>lineage_hive_table_type_name</name>
+    <value>Table</value>
+    <description></description>
+  </property>
+  <property>
+    <name>lineage_hive_column_type_name</name>
+    <value>Column</value>
+    <description></description>
+  </property>
+  <property>
+    <name>lineage_hive_table_column_name</name>
+    <value>columns</value>
+    <description></description>
+  </property>
+  <property>
+    <name>lineage_hive_process_type_name</name>
+    <value>LoadProcess</value>
+    <description></description>
+  </property>
+  <property>
+    <name>lineage_hive_process_inputs_name</name>
+    <value>inputTables</value>
+    <description></description>
+  </property>
+  <property>
+    <name>lineage_hive_process_outputs_name</name>
+    <value>outputTables</value>
+    <description></description>
+  </property>
+  <property>
+    <name>enableTLS</name>
+    <value>false</value>
+    <description></description>
+  </property>
+  <property>
+    <name>authentication_method</name>
+    <value>simple</value>
+    <description></description>
+  </property>
+  <property>
+    <name>authentication_principal</name>
+    <value>atlas</value>
+    <description></description>
+  </property>
+  <property>
+    <name>authentication_keytab</name>
+    <value>/etc/security/keytabs/atlas.service.keytab</value>
+    <description></description>
+  </property>
+  <property>
+    <name>http_authentication_enabled</name>
+    <value>false</value>
+    <description></description>
+  </property>
+  <property>
+    <name>http_authentication_type</name>
+    <value>simple</value>
+    <description></description>
+  </property>
+  <property>
+    <name>http_authentication_kerberos_principal</name>
+    <value>HTTP/_HOST@EXAMPLE.COM</value>
+    <description></description>
+  </property>
+  <property>
+    <name>http_authentication_kerberos_keytab</name>
+    <value>/etc/security/keytabs/spnego.service.keytab</value>
+    <description></description>
+  </property>
+  <property>
+    <name>http_authentication_kerberos_name_rules</name>
+    <value>RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//
+      DEFAULT</value>
+    <description></description>
+  </property>
+
+  <!-- application.properties -->
+  <property>
+    <name>content</name>
+    <description>This is the jinja template for application.properties file</description>
+    <value>
+  #
+  # Licensed to the Apache Software Foundation (ASF) under one
+  # or more contributor license agreements.  See the NOTICE file
+  # distributed with this work for additional information
+  # regarding copyright ownership.  The ASF licenses this file
+  # to you under the Apache License, Version 2.0 (the
+  # "License"); you may not use this file except in compliance
+  # with the License.  You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+  #
+
+  #########  Graph Database Configs  #########
+  # Graph Storage
+  metadata.graph.storage.backend={{graph_storage_backend}}
+  metadata.graph.storage.directory={{graph_storage_directory}}
+
+  # Graph Search Index
+  metadata.graph.index.search.backend={{graph_index_search_backend}}
+  metadata.graph.index.search.directory={{graph_index_search_directory}}
+  metadata.graph.index.search.elasticsearch.client-only={{graph_index_search_elasticsearch_client_only}}
+  metadata.graph.index.search.elasticsearch.local-mode={{graph_index_search_elasticsearch_local_mode}}
+
+  #########  Hive Lineage Configs  #########
+  metadata.lineage.hive.table.type.name={{lineage_hive_table_type_name}}
+  metadata.lineage.hive.column.type.name={{lineage_hive_column_type_name}}
+  metadata.lineage.hive.table.column.name={{lineage_hive_table_column_name}}
+  metadata.lineage.hive.process.type.name={{lineage_hive_process_type_name}}
+  metadata.lineage.hive.process.inputs.name={{lineage_hive_process_inputs_name}}
+  metadata.lineage.hive.process.outputs.name={{lineage_hive_process_outputs_name}}
+  #########  Security Properties  #########
+
+  # SSL config
+  metadata.enableTLS={{enableTLS}}
+
+  # Service Authentication
+  metadata.authentication.method={{authentication_method}}
+  metadata.authentication.principal={{authentication_principal}}
+  metadata.authentication.keytab={{authentication_keytab}}
+
+  # SPNEGO
+  metadata.http.authentication.enabled={{http_authentication_enabled}}
+  metadata.http.authentication.type={{http_authentication_type}}
+  metadata.http.authentication.kerberos.principal={{http_authentication_kerberos_principal}}
+  metadata.http.authentication.kerberos.keytab={{http_authentication_kerberos_keytab}}
+  metadata.http.authentication.kerberos.name.rules={{http_authentication_kerberos_name_rules}}
+
+  #########  Security Properties  #########
+    </value>
+  </property>
+</configuration>

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3.0.0/configuration/metadata-env.xml

@@ -0,0 +1,94 @@
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <property require-input="false">
+    <name>metadata_port</name>
+    <value>21000</value>
+    <description>Listening port of the Atlas server.</description>
+  </property>
+  <property require-input="false">
+    <name>metadata_log_dir</name>
+    <value>/var/log/atlas</value>
+    <description>Atlas log directory.</description>
+  </property>
+  <property require-input="false">
+    <name>metadata_pid_dir</name>
+    <value>/var/run/atlas</value>
+    <description>Atlas pid-file directory.</description>
+  </property>
+  <property>
+    <name>metadata_user</name>
+    <value>atlas</value>
+    <property-type>USER</property-type>
+    <description>Metadata User Name.</description>
+  </property>
+  <property>
+    <name>metadata_opts</name>
+    <value> </value>
+    <description>Metadata Server command line options.</description>
+  </property>
+  <property>
+    <name>metadata_classpath</name>
+    <value> </value>
+    <description>Metadata Server additional classpath.</description>
+  </property>
+  <property require-input="false">
+    <name>metadata_data_dir</name>
+    <value>/var/lib/atlas/data</value>
+    <description>Atlas data directory.</description>
+  </property>
+  <property require-input="false">
+    <name>metadata_expanded_war_dir</name>
+    <value>./server/webapp</value>
+    <description>Atlas expanded WAR directory.</description>
+  </property>
+
+  <!-- metadata-env.sh -->
+  <property>
+    <name>content</name>
+    <description>This is the jinja template for metadata-env.sh file</description>
+    <value>
+# The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path
+export JAVA_HOME={{java64_home}}
+
+# any additional java opts you want to set. This will apply to both client and server operations
+export METADATA_OPTS={{metadata_opts}}
+
+# metadata configuration directory 
+export METADATA_CONF={{conf_dir}}
+
+# Where log files are stored. Defatult is logs directory under the base install location
+export METADATA_LOG_DIR={{log_dir}}
+
+# additional classpath entries
+export METADATACPPATH={{metadata_classpath}}
+
+# data dir
+export METADATA_DATA_DIR={{data_dir}}
+
+# pid dir
+export METADATA_PID_DIR={{pid_dir}}
+
+# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.
+export METADATA_EXPANDED_WEBAPP_DIR={{expanded_war_dir}}
+    </value>
+  </property>  
+</configuration>

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3.0.0/kerberos.json

@@ -0,0 +1,59 @@
+{
+  "services": [
+    {
+      "name": "ATLAS",
+      "identities": [
+        {
+          "name": "/spnego"
+        }
+      ],
+      "configurations": [
+        {
+          "metadata-runtime.properties": {
+            "*.metadata.authentication.type": "kerberos",
+            "*.metadata.http.authentication.type": "kerberos",
+            "*.dfs.namenode.kerberos.principal": "nn/_HOST@${realm}"
+          }
+        }
+      ],
+      "components": [
+        {
+          "name": "ATLAS_SERVER",
+          "identities": [
+            {
+              "name": "metadata_server",
+              "principal": {
+                "value": "atlas/_HOST@${realm}",
+                "type" : "service",
+                "configuration": "application-properties/*.metadata.authentication.principal",
+                "local_username" : "${metadata-env/metadata_user}"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/atlas.service.keytab",
+                "owner": {
+                  "name": "${metadata-env/metadata_user}",
+                  "access": "r"
+                },
+                "group": {
+                  "name": "${cluster-env/user_group}",
+                  "access": ""
+                },
+                "configuration": "application-properties/*.metadata.authentication.keytab"
+              }
+            },
+            {
+              "name": "/spnego",
+              "principal": {
+                "value": "HTTP/_HOST@${realm}",
+                "configuration": "application-properties/*.metadata.http.authentication.principal"
+              },
+              "keytab": {
+                "configuration": "application-properties/*.metadata.http.authentication.keytab"
+              }
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3.0.0/metainfo.xml

@@ -0,0 +1,79 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>ATLAS</name>
+      <displayName>Atlas</displayName>
+      <comment>Atlas Metadata and Governance platform</comment>
+      <version>0.1.0.2.3.0.0</version>
+      <components>
+        <component>
+          <name>ATLAS_SERVER</name>
+          <displayName>Atlas Metadata Server</displayName>
+          <category>MASTER</category>
+          <cardinality>1</cardinality>
+          <versionAdvertised>false</versionAdvertised>
+          <dependencies>
+          </dependencies>
+          <commandScript>
+            <script>scripts/metadata_server.py</script>
+            <scriptType>PYTHON</scriptType>
+            <timeout>1200</timeout>
+          </commandScript>
+          <configFile>
+            <type>properties</type>
+            <fileName>application.properties</fileName>
+            <dictionaryName>application-properties</dictionaryName>
+          </configFile>
+          <configFile>
+            <type>env</type>
+            <fileName>metadata_env.sh</fileName>
+            <dictionaryName>metadata-env</dictionaryName>
+          </configFile>
+        </component>
+      </components>
+
+      <osSpecifics>
+        <osSpecific>
+          <osFamily>any</osFamily>
+          <packages>
+            <package>
+              <name>atlas</name>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
+
+      <commandScript>
+        <script>scripts/service_check.py</script>
+        <scriptType>PYTHON</scriptType>
+        <timeout>300</timeout>
+      </commandScript>
+      
+
+      <configuration-dependencies>
+        <config-type>application-properties</config-type>
+        <config-type>metadata-env</config-type>
+      </configuration-dependencies>
+
+
+    </service>
+  </services>
+</metainfo>

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3.0.0/package/files/log4j.xml

@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+    <appender name="console" class="org.apache.log4j.ConsoleAppender">
+        <param name="Target" value="System.out"/>
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
+        </layout>
+    </appender>
+
+    <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+        <param name="File" value="${metadata.log.dir}/application.log"/>
+        <param name="Append" value="true"/>
+        <param name="Threshold" value="debug"/>
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
+        </layout>
+    </appender>
+
+    <appender name="AUDIT" class="org.apache.log4j.DailyRollingFileAppender">
+        <param name="File" value="${metadata.log.dir}/audit.log"/>
+        <param name="Append" value="true"/>
+        <param name="Threshold" value="debug"/>
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d %x %m%n"/>
+        </layout>
+    </appender>
+
+    <logger name="org.apache.hadoop.metadata" additivity="false">
+        <level value="debug"/>
+        <appender-ref ref="console"/>
+        <appender-ref ref="FILE"/>
+    </logger>
+
+
+    <logger name="com.thinkaurelius.titan" additivity="false">
+        <level value="info"/>
+        <appender-ref ref="FILE"/>
+    </logger>
+
+    <logger name="org.elasticsearch" additivity="false">
+        <level value="info"/>
+        <appender-ref ref="FILE"/>
+    </logger>
+
+    <logger name="org.apache.lucene" additivity="false">
+        <level value="info"/>
+        <appender-ref ref="FILE"/>
+    </logger>
+
+    <logger name="com.google" additivity="false">
+        <level value="info"/>
+        <appender-ref ref="FILE"/>
+    </logger>
+
+    <logger name="AUDIT">
+        <level value="info"/>
+        <appender-ref ref="AUDIT"/>
+    </logger>
+
+    <root>
+        <priority value="info"/>
+        <appender-ref ref="FILE"/>
+    </root>
+
+</log4j:configuration>

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3.0.0/package/scripts/metadata.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+import shutil
+from resource_management import Directory, Fail, Logger, File, \
+    InlineTemplate, StaticFile
+from resource_management.libraries.functions import format
+
+
+def metadata():
+    import params
+
+    Directory([params.pid_dir],
+              mode=0755,
+              cd_access='a',
+              owner=params.metadata_user,
+              group=params.user_group,
+              recursive=True
+    )
+
+    Directory(params.conf_dir,
+              mode=0755,
+              cd_access='a',
+              owner=params.metadata_user,
+              group=params.user_group,
+              recursive=True
+    )
+
+    Directory(params.log_dir,
+              mode=0755,
+              cd_access='a',
+              owner=params.metadata_user,
+              group=params.user_group,
+              recursive=True
+    )
+
+    Directory(params.data_dir,
+              mode=0644,
+              cd_access='a',
+              owner=params.metadata_user,
+              group=params.user_group,
+              recursive=True
+    )
+
+    Directory(params.expanded_war_dir,
+              mode=0644,
+              cd_access='a',
+              owner=params.metadata_user,
+              group=params.user_group,
+              recursive=True
+    )
+
+    metadata_war_file = format('{params.metadata_home}/server/webapp/metadata.war')
+    if not os.path.isfile(metadata_war_file):
+        raise Fail("Unable to copy {0} because it does not exist".format(metadata_war_file))
+
+    Logger.info("Copying {0} to {1}".format(metadata_war_file, params.expanded_war_dir))
+    shutil.copy2(metadata_war_file, params.expanded_war_dir)
+
+    File(format('{conf_dir}/application.properties'),
+         content=InlineTemplate(params.application_properties_content),
+         mode=0644,
+         owner=params.metadata_user,
+         group=params.user_group
+    )
+
+    File(format("{conf_dir}/metadata-env.sh"),
+         owner=params.metadata_user,
+         group=params.user_group,
+         mode=0755,
+         content=InlineTemplate(params.metadata_env_content)
+    )
+
+    File(format("{conf_dir}/log4j.xml"),
+         mode=0644,
+         owner=params.metadata_user,
+         group=params.user_group,
+         content=StaticFile('log4j.xml')
+    )

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3.0.0/package/scripts/metadata_server.py

@@ -0,0 +1,69 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from metadata import metadata
+from resource_management import Execute, check_process_status, Script
+from resource_management.libraries.functions import format
+
+class MetadataServer(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "atlas-server"}
+
+  def install(self, env):
+    self.install_packages(env)
+    self.configure(env)
+
+  def configure(self, env):
+    import params
+    env.set_params(params)
+    metadata()
+
+  # def pre_rolling_restart(self, env):
+  #   import params
+  #   env.set_params(params)
+  #   upgrade.prestart(env, "metadata-server")
+  #
+  def start(self, env, rolling_restart=False):
+    import params
+    env.set_params(params)
+    daemon_cmd = format('source {params.conf_dir}/metadata-env.sh ; {params.metadata_start_script} --port {params.metadata_port}')
+    no_op_test = format('ls {params.pid_file} >/dev/null 2>&1 && ps -p `cat {params.pid_file}` >/dev/null 2>&1')
+    Execute(daemon_cmd,
+            user=params.metadata_user,
+            not_if=no_op_test
+    )
+
+  def stop(self, env, rolling_restart=False):
+    import params
+    env.set_params(params)
+    daemon_cmd = format('source {params.conf_dir}/metadata-env.sh; {params.metadata_stop_script}')
+    Execute(daemon_cmd,
+            user=params.metadata_user,
+    )
+    Execute (format("rm -f {params.pid_file}"))
+
+
+  def status(self, env):
+    import status_params
+    env.set_params(status_params)
+    check_process_status(status_params.pid_file)
+
+if __name__ == "__main__":
+  MetadataServer().execute()

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3.0.0/package/scripts/params.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+import sys
+from resource_management import format_hdp_stack_version, Script
+from resource_management.libraries.functions import format
+
+import status_params
+
+# server configurations
+config = Script.get_config()
+
+# security enabled
+security_enabled = status_params.security_enabled
+
+# hdp version
+stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
+
+metadata_home = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else '/usr/hdp/current/atlas-server'
+metadata_bin = format("{metadata_home}/bin")
+
+python_binary = os.environ['PYTHON_EXE'] if 'PYTHON_EXE' in os.environ else sys.executable
+metadata_start_script = format("{metadata_bin}/metadata_start.py")
+metadata_stop_script = format("{metadata_bin}/metadata_stop.py")
+
+# metadata local directory structure
+log_dir = config['configurations']['metadata-env']['metadata_log_dir']
+conf_dir = status_params.conf_dir # "/etc/metadata/conf"
+
+# service locations
+hadoop_conf_dir = os.path.join(os.environ["HADOOP_HOME"], "conf") if 'HADOOP_HOME' in os.environ else '/etc/hadoop/conf'
+
+# user and status
+metadata_user = status_params.metadata_user
+user_group = config['configurations']['cluster-env']['user_group']
+pid_dir = status_params.pid_dir
+pid_file = format("{pid_dir}/metadata.pid")
+
+# metadata env
+java64_home = config['hostLevelParams']['java_home']
+env_sh_template = config['configurations']['metadata-env']['content']
+
+# credential provider
+credential_provider = format( "jceks://file@{conf_dir}/atlas-site.jceks")
+
+# command line args
+metadata_port = config['configurations']['metadata-env']['metadata_port']
+metadata_host = config['hostname']
+
+# application properties
+application_properties = config['configurations']['application-properties']
+
+for key, value in application_properties.iteritems():
+    globals()[key] = value
+
+metadata_env_content = config['configurations']['metadata-env']['content']
+application_properties_content = config['configurations']['application-properties']['content']
+
+metadata_opts = config['configurations']['metadata-env']['metadata_opts']
+metadata_classpath = config['configurations']['metadata-env']['metadata_classpath']
+data_dir = config['configurations']['metadata-env']['metadata_data_dir']
+expanded_war_dir = os.environ['METADATA_EXPANDED_WEBAPP_DIR'] if 'METADATA_EXPANDED_WEBAPP_DIR' in os.environ else '/var/lib/atlas/server/webapp'

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3.0.0/package/scripts/properties_config.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core import source, File
+from resource_management.libraries.functions import format
+
+
+def properties_inline_template(configurations):
+  return source.InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}={{ value }}
+{% endfor %}''', configurations_dict=configurations)
+
+def properties_config(filename, configurations = None, conf_dir = None,
+                      mode = None, owner = None, group = None):
+    config_content = properties_inline_template(configurations)
+    File (format("{conf_dir}/{filename}"), content = config_content, owner = owner,
+          group = group, mode = mode)

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3.0.0/package/scripts/service_check.py

@@ -0,0 +1,57 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import httplib
+import socket
+import time
+from resource_management import Script, Logger, ComponentIsNotRunning, Fail
+from resource_management.libraries.functions import format
+
+
+class AtlasServiceCheck(Script):
+  ATLAS_CONNECT_TRIES = 5
+  ATLAS_CONNECT_TIMEOUT = 10
+
+  def service_check(self, env):
+    import params
+
+    env.set_params(params)
+
+    for i in xrange(0, self.ATLAS_CONNECT_TRIES):
+      try:
+        conn = httplib.HTTPConnection(params.metadata_host,
+                                      int(params.metadata_port))
+        conn.request("GET", format("http://{params.metadata_host}:{params.metadata_port}/"))
+      except (httplib.HTTPException, socket.error) as ex:
+        if i < self.ATLAS_CONNECT_TRIES - 1:
+          time.sleep(self.ATLAS_CONNECT_TIMEOUT)
+          Logger.info("Connection failed. Next retry in %s seconds."
+                      % (self.ATLAS_CONNECT_TIMEOUT))
+          continue
+        else:
+          raise Fail("Service check has failed.")
+
+    resp = conn.getresponse()
+    if resp.status == 200 :
+      Logger.info('Atlas server up and running')
+    else:
+      Logger.debug('Atlas server not running')
+      raise ComponentIsNotRunning()
+
+if __name__ == "__main__":
+  AtlasServiceCheck().execute()

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3.0.0/package/scripts/status_params.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+from resource_management import Script
+from resource_management.libraries.functions import  get_kinit_path, format
+
+
+config = Script.get_config()
+
+conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf'
+pid_dir = config['configurations']['metadata-env']['metadata_pid_dir']
+pid_file = format("{pid_dir}/metadata.pid")
+metadata_user = config['configurations']['metadata-env']['metadata_user']
+
+# Security related/required params
+hostname = config['hostname']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+kinit_path_local = get_kinit_path()
+tmp_dir = Script.get_tmp_dir()

ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json

@@ -6,6 +6,7 @@
     "MAHOUT_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
     "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START"],
     "RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"],
-    "PHOENIX_QUERY_SERVER-START": ["HBASE_MASTER-START"]
+    "PHOENIX_QUERY_SERVER-START": ["HBASE_MASTER-START"],
+    "ATLAS_SERVICE_CHECK-SERVICE_CHECK": ["ATLAS_SERVER-START"]
   }
 }

ambari-server/src/main/resources/stacks/HDP/2.3/services/ATLAS/metainfo.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>ATLAS</name>
+      <extends>common-services/ATLAS/0.1.0.2.3.0.0</extends>
+      <version>0.1.0.2.3.0.0</version>
+      <osSpecifics>
+        <osSpecific>
+          <osFamily>redhat7,redhat6,suse11</osFamily>
+          <packages>
+            <package>
+              <name>atlas_2_3_*</name>
+            </package>
+          </packages>
+        </osSpecific>
+        <osSpecific>
+          <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
+          <packages>
+            <package>
+              <name>atlas-2-3-.*</name>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
+    </service>
+  </services>
+</metainfo>

ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from mock.mock import MagicMock, call, patch
+from stacks.utils.RMFTestCase import *
+import json
+import sys
+
+
+class TestMetadataServer(RMFTestCase):
+  COMMON_SERVICES_PACKAGE_DIR = "ATLAS/0.1.0.2.3.0.0/package"
+  STACK_VERSION = "2.3"
+
+  @patch("shutil.copy2", new = MagicMock())
+  @patch("os.path.isfile")
+  def test_configure_default(self, isfile_mock):
+    isfile_mock.return_value = True
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py",
+                       classname = "MetadataServer",
+                       command = "configure",
+                       config_file="default.json",
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+
+    self.assertResourceCalled('Directory', '/var/run/atlas',
+                              owner = 'atlas',
+                              group = 'hadoop',
+                              recursive = True,
+                              cd_access = 'a',
+                              mode=0755
+    )
+    self.assertResourceCalled('Directory', '/etc/atlas/conf',
+                              owner = 'atlas',
+                              group = 'hadoop',
+                              recursive = True,
+                              cd_access = 'a',
+                              mode=0755
+    )
+    self.assertResourceCalled('Directory', '/var/log/atlas',
+                              owner = 'atlas',
+                              group = 'hadoop',
+                              recursive = True,
+                              cd_access = 'a',
+                              mode=0755
+    )
+    self.assertResourceCalled('Directory', '/var/lib/atlas/data',
+                              owner = 'atlas',
+                              group = 'hadoop',
+                              recursive = True,
+                              cd_access = 'a',
+                              mode=0644
+    )
+    self.assertResourceCalled('Directory', '/var/lib/atlas/server/webapp',
+                              owner = 'atlas',
+                              group = 'hadoop',
+                              recursive = True,
+                              cd_access = 'a',
+                              mode=0644
+    )
+    self.assertResourceCalled('File', '/etc/atlas/conf/application.properties',
+                              content = InlineTemplate(self.getConfig()['configurations']['application-properties']['content']),
+                              owner = 'atlas',
+                              group = 'hadoop',
+                              mode = 0644,
+                              )
+    self.assertResourceCalled('File', '/etc/atlas/conf/metadata-env.sh',
+                              content = InlineTemplate(self.getConfig()['configurations']['metadata-env']['content']),
+                              owner = 'atlas',
+                              group = 'hadoop',
+                              mode = 0755,
+                              )
+    self.assertResourceCalled('File', '/etc/atlas/conf/log4j.xml',
+                              content = StaticFile('log4j.xml'),
+                              owner = 'atlas',
+                              group = 'hadoop',
+                              mode = 0644,
+                              )
+    self.assertNoMoreResources()
+
+  def test_start_default(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py",
+                       classname = "MetadataServer",
+                       command = "start",
+                       config_file="default.json",
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    self.assertResourceCalled('Execute', 'source /etc/atlas/conf/metadata-env.sh ; /usr/hdp/current/atlas-server/bin/metadata_start.py --port 21000',
+                              not_if = 'ls /var/run/atlas/metadata.pid >/dev/null 2>&1 && ps -p `cat /var/run/atlas/metadata.pid` >/dev/null 2>&1',
+                              user = 'atlas',
+    )
+
+  def test_stop_default(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py",
+                       classname = "MetadataServer",
+                       command = "stop",
+                       config_file="default.json",
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    self.assertResourceCalled('Execute', 'source /etc/atlas/conf/metadata-env.sh; /usr/hdp/current/atlas-server/bin/metadata_stop.py',
+                              user = 'atlas',
+    )
+    self.assertResourceCalled('Execute', 'rm -f /var/run/atlas/metadata.pid',
+    )

ambari-server/src/test/python/stacks/2.3/configs/default.json

@@ -160,6 +160,39 @@
         "log.index.interval.bytes": "4096",
         "log.retention.hours": "168"
       },
+      "application-properties": {
+        "graph_storage_backend": "berkeleyje",
+        "graph_storage_directory": "data/berkley",
+        "graph_index_search_backend": "elasticsearch",
+        "graph_index_search_directory": "data/es",
+        "graph_index_search_elasticsearch_client_only": false,
+        "graph_index_search_elasticsearch_local_mode": true,
+        "lineage_hive_table_type_name": "Table",
+        "lineage_hive_column_type_name": "Column",
+        "lineage_hive_table_column_name": "columns",
+        "lineage_hive_process_type_name": "LoadProcess",
+        "lineage_hive_process_inputs_name": "inputTables",
+        "lineage_hive_process_outputs_name": "outputTables",
+        "enableTLS": false,
+        "authentication_method": "simple",
+        "authentication_principal": "atlas",
+        "authentication_keytab": "/etc/security/keytabs/atlas.service.keytab",
+        "http_authentication_enabled": false,
+        "http_authentication_type": "simple",
+        "http_authentication_kerberos_principal": "HTTP/_HOST@EXAMPLE.COM",
+        "http_authentication_kerberos_keytab": "/etc/security/keytabs/spnego.service.keytab",
+        "http_authentication_kerberos_name_rules": "DEFAULT",
+        "content": "  # Graph Storage\nmetadata.graph.storage.backend={{graph_storage_backend}}\nmetadata.graph.storage.directory={{graph_storage_directory}}\nmetadata.graph.index.search.backend={{graph_index_search_backend}}\nmetadata.graph.index.search.directory={{graph_index_search_directory}}\nmetadata.graph.index.search.elasticsearch.client-only={{graph_index_search_elasticsearch_client_only}}\nmetadata.graph.index.search.elasticsearch.local-mode={{graph_index_search_elasticsearch_local_mode}}\nmetadata.lineage.hive.table.type.name={{lineage_hive_table_type_name}}\nmetadata.lineage.hive.column.type.name={{lineage_hive_column_type_name}}\nmetadata.lineage.hive.table.column.name={{lineage_hive_table_column_name}}\nmetadata.lineage.hive.process.type.name={{lineage_hive_process_type_name}}\nmetadata.lineage.hive.process.inputs.name={{lineage_hive_process_inputs_name}}\nmetadata.lineage.hive.process.outputs.name={{lineage_hive_process_outputs_name}}\nmetadata.enableTLS={{enableTLS}}\nmetadata.authentication.method={{authentication_method}}\nmetadata.authentication.principal={{authentication_principal}}\nmetadata.authentication.keytab={{authentication_keytab}}\nmetadata.http.authentication.enabled={{http_authentication_enabled}}\nmetadata.http.authentication.type={{http_authentication_type}}\nmetadata.http.authentication.kerberos.principal={{http_authentication_kerberos_principal}}\nmetadata.http.authentication.kerberos.keytab={{http_authentication_kerberos_keytab}}\nmetadata.http.authentication.kerberos.name.rules={{http_authentication_kerberos_name_rules}}"
+      },
+      "metadata-env": {
+        "content": "# The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path\nexport JAVA_HOME={{java64_home}}\n# any additional java opts you want to set. This will apply to both client and server operations\nexport METADATA_OPTS={{metadata_opts}}\n# metadata configuration directory\nexport METADATA_CONF={{conf_dir}}\n# Where log files are stored. Defatult is logs directory under the base install location\nexport METADATA_LOG_DIR={{log_dir}}\n# additional classpath entries\nexport METADATACPPATH={{metadata_classpath}}\n# data dir\nexport METADATA_DATA_DIR={{data_dir}}\n# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.\nexport METADATA_EXPANDED_WEBAPP_DIR={{expanded_war_dir}}",
+        "metadata_user": "atlas",
+        "metadata_port": 21000,
+        "metadata_pid_dir": "/var/run/atlas",
+        "metadata_log_dir": "/var/log/atlas",
+        "metadata_data_dir": "/var/lib/atlas/data",
+        "metadata_expanded_war_dir": "/var/lib/atlas/server/webapp"
+      },
       "ranger-hbase-plugin-properties": {
             "ranger-hbase-plugin-enabled":"yes"
       },
@@ -194,7 +227,8 @@
       "users-ldif": {},
       "kafka-env": {},
       "kafka-log4j": {},
-      "kafka-broker": {}
+      "kafka-broker": {},
+      "metadata-env": {}
     },
     "configurationTags": {
         "slider-client": {
@@ -232,6 +266,9 @@
       },
       "kafka-broker": {
         "tag": "version1"
+      },
+      "metadata-env": {
+        "tag": "version1"
       }
     },
     "commandId": "7-1",