AMBARI-18874 : Provide SSL related configurations for Ranger-Tagsync (Vishal Suvagia via mugdha)

ambari-common/src/main/python/resource_management/libraries/functions/constants.py

@@ -104,3 +104,4 @@ class StackFeature:
   RANGER_HIVE_PLUGIN_JDBC_URL = "ranger_hive_plugin_jdbc_url"
   ZKFC_VERSION_ADVERTISED = "zkfc_version_advertised"
   PHOENIX_CORE_HDFS_SITE_REQUIRED = "phoenix_core_hdfs_site_required"
+  RANGER_TAGSYNC_SSL_XML_SUPPORT="ranger_tagsync_ssl_xml_support"

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py

@@ -71,6 +71,7 @@ stack_supports_infra_client = check_stack_feature(StackFeature.RANGER_INSTALL_IN
 stack_supports_pid = check_stack_feature(StackFeature.RANGER_PID_SUPPORT, version_for_stack_feature_checks)
 stack_supports_ranger_admin_password_change = check_stack_feature(StackFeature.RANGER_ADMIN_PASSWD_CHANGE, version_for_stack_feature_checks)
 stack_supports_ranger_setup_db_on_start = check_stack_feature(StackFeature.RANGER_SETUP_DB_ON_START, version_for_stack_feature_checks)
+stack_supports_ranger_tagsync_ssl_xml_support = check_stack_feature(StackFeature.RANGER_TAGSYNC_SSL_XML_SUPPORT, version_for_stack_feature_checks)
 
 downgrade_from_version = default("/commandParams/downgrade_from_version", None)
 upgrade_direction = default("/commandParams/upgrade_direction", None)
@@ -81,6 +82,14 @@ ranger_tagsync_home  = format('{stack_root}/current/ranger-tagsync')
 ranger_tagsync_conf = format('{stack_root}/current/ranger-tagsync/conf')
 tagsync_bin = '/usr/bin/ranger-tagsync'
 tagsync_services_file = format('{stack_root}/current/ranger-tagsync/ranger-tagsync-services.sh')
+security_store_path = '/etc/security/serverKeys'
+tagsync_etc_path = '/etc/ranger/tagsync/'
+ranger_tagsync_credential_file= os.path.join(tagsync_etc_path,'rangercred.jceks')
+atlas_tagsync_credential_file= os.path.join(tagsync_etc_path,'atlascred.jceks')
+ranger_tagsync_keystore_password = config['configurations']['ranger-tagsync-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']
+ranger_tagsync_truststore_password = config['configurations']['ranger-tagsync-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']
+atlas_tagsync_keystore_password = config['configurations']['atlas-tagsync-ssl']['xasecure.policymgr.clientssl.keystore.password']
+atlas_tagsync_truststore_password = config['configurations']['atlas-tagsync-ssl']['xasecure.policymgr.clientssl.truststore.password']
 
 if upgrade_direction == Direction.DOWNGRADE and version and not check_stack_feature(StackFeature.CONFIG_VERSIONING, version):
   stack_supports_rolling_upgrade = True
@@ -250,6 +259,7 @@ has_ranger_tagsync = len(ranger_tagsync_hosts) > 0
 
 tagsync_log_dir = default("/configurations/ranger-tagsync-site/ranger.tagsync.logdir", "/var/log/ranger/tagsync")
 tagsync_jceks_path = config["configurations"]["ranger-tagsync-site"]["ranger.tagsync.keystore.filename"]
+atlas_tagsync_jceks_path = config["configurations"]["ranger-tagsync-site"]["ranger.tagsync.source.atlasrest.keystore.filename"]
 tagsync_application_properties = dict(config["configurations"]["tagsync-application-properties"]) if has_ranger_tagsync else None
 tagsync_pid_file = format('{ranger_pid_dir}/tagsync.pid')
 tagsync_cred_lib = os.path.join(ranger_tagsync_home, "lib", "*")

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_tagsync.py

@@ -28,6 +28,7 @@ from resource_management.core.logger import Logger
 from resource_management.core import shell
 from ranger_service import ranger_service
 from setup_ranger_xml import ranger, ranger_credential_helper
+from resource_management.core.exceptions import Fail
 import upgrade
 
 class RangerTagsync(Script):
@@ -43,6 +44,12 @@ class RangerTagsync(Script):
        group = params.unix_group,
        mode = 0640
     )
+    if params.stack_supports_ranger_tagsync_ssl_xml_support:
+      Logger.info("Stack support Atlas user for Tagsync, creating keystore for same.")
+      self.create_atlas_user_keystore(env)
+    else:
+      Logger.info("Stack does not support Atlas user for Tagsync, skipping keystore creation for same.")
+
     self.configure(env)
 
   def configure(self, env, upgrade_type=None):
@@ -92,5 +99,37 @@ class RangerTagsync(Script):
     import params
     return params.unix_user
 
+  def configure_atlas_user_for_tagsync(self, env):
+    Logger.info("Configuring Atlas user for Tagsync service.")
+    import params
+    env.set_params(params)
+
+    upgrade_stack = stack_select._get_upgrade_stack()
+    if upgrade_stack is None:
+      raise Fail('Unable to determine the stack and stack version')
+
+    stack_name = upgrade_stack[0]
+    stack_version = upgrade_stack[1]
+
+    stack_select.select("ranger-tagsync", stack_version)
+    conf_select.select(stack_name, "ranger-tagsync", stack_version)
+    if params.stack_supports_ranger_tagsync_ssl_xml_support:
+      Logger.info("Upgrading Tagsync, stack support Atlas user for Tagsync, creating keystore for same.")
+      self.create_atlas_user_keystore(env)
+    else:
+      Logger.info("Upgrading Tagsync, stack does not support Atlas user for Tagsync, skipping keystore creation for same.")
+
+    Logger.info("Configuring Atlas user for Tagsync service done.")
+
+  def create_atlas_user_keystore(self,env):
+    import params
+    env.set_params(params)
+    ranger_credential_helper(params.tagsync_cred_lib, 'atlas.user.password', 'admin', params.atlas_tagsync_jceks_path)
+    File(params.atlas_tagsync_jceks_path,
+         owner = params.unix_user,
+         group = params.unix_group,
+         mode = 0640
+    )
+
 if __name__ == "__main__":
   RangerTagsync().execute()

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py

@@ -542,6 +542,11 @@ def setup_tagsync(upgrade_type=None):
     owner=params.unix_user,
     group=params.unix_group,
     mode=0644)
+  if params.stack_supports_ranger_tagsync_ssl_xml_support:
+    Logger.info("Stack supports tagsync-ssl configurations, performing the same.")
+    setup_tagsync_ssl_configs()
+  else:
+    Logger.info("Stack doesnt support tagsync-ssl configurations, skipping the same.")
 
   PropertiesFile(format('{ranger_tagsync_conf}/atlas-application.properties'),
     properties = params.tagsync_application_properties,
@@ -648,3 +653,52 @@ def check_znode():
     zookeeper_quorum=params.zookeeper_quorum,
     solr_znode=params.solr_znode,
     java64_home=params.java_home)
+
+
+def setup_tagsync_ssl_configs():
+  import params
+  Directory(params.security_store_path,
+            cd_access="a",
+            create_parents=True)
+
+  Directory(params.tagsync_etc_path,
+            cd_access="a",
+            owner=params.unix_user,
+            group=params.unix_group,
+            mode=0775,
+            create_parents=True)
+
+  XmlConfig("ranger-policymgr-ssl.xml",
+            conf_dir=params.ranger_tagsync_conf,
+            configurations=params.config['configurations']['ranger-tagsync-policymgr-ssl'],
+            configuration_attributes=params.config['configuration_attributes']['ranger-tagsync-policymgr-ssl'],
+            owner=params.unix_user,
+            group=params.unix_group,
+            mode=0644)
+
+  ranger_credential_helper(params.tagsync_cred_lib, 'sslKeyStore', params.ranger_tagsync_keystore_password, params.ranger_tagsync_credential_file)
+  ranger_credential_helper(params.tagsync_cred_lib, 'sslTrustStore', params.ranger_tagsync_truststore_password, params.ranger_tagsync_credential_file)
+
+  File(params.ranger_tagsync_credential_file,
+       owner = params.unix_user,
+       group = params.unix_group,
+       mode = 0640
+       )
+
+  XmlConfig("atlas-tagsync-ssl.xml",
+            conf_dir=params.ranger_tagsync_conf,
+            configurations=params.config['configurations']['atlas-tagsync-ssl'],
+            configuration_attributes=params.config['configuration_attributes']['atlas-tagsync-ssl'],
+            owner=params.unix_user,
+            group=params.unix_group,
+            mode=0644)
+
+  ranger_credential_helper(params.tagsync_cred_lib, 'sslKeyStore', params.atlas_tagsync_keystore_password, params.atlas_tagsync_credential_file)
+  ranger_credential_helper(params.tagsync_cred_lib, 'sslTrustStore', params.atlas_tagsync_truststore_password, params.atlas_tagsync_credential_file)
+
+  File(params.atlas_tagsync_credential_file,
+       owner = params.unix_user,
+       group = params.unix_group,
+       mode = 0640
+       )
+  Logger.info("Configuring tagsync-ssl configurations done successfully.")

ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/atlas-tagsync-ssl.xml

@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <property>
+    <name>xasecure.policymgr.clientssl.keystore</name>
+    <value>/etc/security/serverKeys/atlas-tagsync-keystore.jks</value>
+    <description>Java Keystore files</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>xasecure.policymgr.clientssl.keystore.password</name>
+    <value>myKeyFilePassword</value>
+    <property-type>PASSWORD</property-type>
+    <description>password for keystore</description>
+    <value-attributes>
+      <type>password</type>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>xasecure.policymgr.clientssl.truststore</name>
+    <value>/etc/security/serverKeys/atlas-tagsync-mytruststore.jks</value>
+    <description>java truststore file</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>xasecure.policymgr.clientssl.truststore.password</name>
+    <value>changeit</value>
+    <property-type>PASSWORD</property-type>
+    <description>java truststore password</description>
+    <value-attributes>
+      <type>password</type>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>xasecure.policymgr.clientssl.keystore.credential.file</name>
+    <value>jceks://file{{atlas_tagsync_credential_file}}</value>
+    <description>java keystore credential file</description>
+    <on-ambari-upgrade add="false" />
+  </property>
+
+  <property>
+    <name>xasecure.policymgr.clientssl.truststore.credential.file</name>
+    <value>jceks://file{{atlas_tagsync_credential_file}}</value>
+    <description>java truststore credential file</description>
+    <on-ambari-upgrade add="false" />
+  </property>
+
+</configuration>

ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-tagsync-policymgr-ssl.xml

@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <property>
+    <name>xasecure.policymgr.clientssl.keystore</name>
+    <value>/etc/security/serverKeys/ranger-tagsync-keystore.jks</value>
+    <description>Java Keystore files</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>xasecure.policymgr.clientssl.keystore.password</name>
+    <value>myKeyFilePassword</value>
+    <property-type>PASSWORD</property-type>
+    <description>password for keystore</description>
+    <value-attributes>
+      <type>password</type>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>xasecure.policymgr.clientssl.truststore</name>
+    <value>/etc/security/serverKeys/ranger-tagsync-mytruststore.jks</value>
+    <description>java truststore file</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>xasecure.policymgr.clientssl.truststore.password</name>
+    <value>changeit</value>
+    <property-type>PASSWORD</property-type>
+    <description>java truststore password</description>
+    <value-attributes>
+      <type>password</type>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>xasecure.policymgr.clientssl.keystore.credential.file</name>
+    <value>jceks://file{{ranger_tagsync_credential_file}}</value>
+    <description>java keystore credential file</description>
+    <on-ambari-upgrade add="false" />
+  </property>
+
+  <property>
+    <name>xasecure.policymgr.clientssl.truststore.credential.file</name>
+    <value>jceks://file{{ranger_tagsync_credential_file}}</value>
+    <description>java truststore credential file</description>
+    <on-ambari-upgrade add="false" />
+  </property>
+
+</configuration>

ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-tagsync-site.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+  <property>
+    <name>ranger.tagsync.dest.ranger.ssl.config.filename</name>
+    <value>/etc/ranger/tagsync/conf/ranger-policymgr-ssl.xml</value>
+    <description>Keystore and truststore information used for tagsync, required if tagsync to ranger admin communication is SSL enabled</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>ranger.tagsync.source.atlasrest.ssl.config.filename</name>
+    <value>/etc/ranger/tagsync/conf/atlas-tagsync-ssl.xml</value>
+    <description>Keystore and truststore information used for tagsync, required if tagsync to atlas communication is SSL enabled</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+</configuration>

ambari-server/src/main/resources/common-services/RANGER/0.7.0/metainfo.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>RANGER</name>
+      <displayName>Ranger</displayName>
+      <comment>Comprehensive security for Hadoop</comment>
+      <extends>common-services/RANGER/0.6.0</extends>
+      <version>0.7.0</version>
+    </service>
+  </services>
+</metainfo>

ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json

@@ -323,6 +323,11 @@
       "name": "phoenix_core_hdfs_site_required",
       "description": "HDFS and CORE site required for Phoenix",
       "max_version": "2.5.9.9"
+    },
+    {
+      "name": "ranger_tagsync_ssl_xml_support",
+      "description": "Ranger Tagsync ssl xml support.",
+      "min_version": "2.6.0.0"
     }
   ]
 }

ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml

@@ -784,6 +784,14 @@
       </component>
 
       <component name="RANGER_TAGSYNC">
+
+        <pre-upgrade>
+          <task xsi:type="execute" hosts="all">
+            <script>scripts/ranger_tagsync.py</script>
+            <function>configure_atlas_user_for_tagsync</function>
+          </task>
+        </pre-upgrade>
+
         <upgrade>
           <task xsi:type="restart-task"/>
         </upgrade>

ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml

@@ -518,6 +518,14 @@
       </component>
 
       <component name="RANGER_TAGSYNC">
+
+        <pre-upgrade>
+          <task xsi:type="execute" hosts="all">
+            <script>scripts/ranger_tagsync.py</script>
+            <function>configure_atlas_user_for_tagsync</function>
+          </task>
+        </pre-upgrade>
+
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>

ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER/configuration/ranger-tagsync-site.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+  <property>
+    <name>ranger.tagsync.keystore.filename</name>
+    <value>/usr/hdp/current/ranger-tagsync/conf/rangertagsync.jceks</value>
+    <description>Keystore file</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>ranger.tagsync.source.atlasrest.keystore.filename</name>
+    <value>/usr/hdp/current/ranger-tagsync/conf/atlasuser.jceks</value>
+    <description>Tagsync atlasrest keystore file</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>ranger.tagsync.dest.ranger.ssl.config.filename</name>
+    <value>{{stack_root}}/current/ranger-tagsync/conf/ranger-policymgr-ssl.xml</value>
+    <description>Keystore and truststore information used for tagsync, required if tagsync -&gt; ranger admin communication is SSL enabled</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>ranger.tagsync.source.atlasrest.ssl.config.filename</name>
+    <value>{{stack_root}}/current/ranger-tagsync/conf/atlas-tagsync-ssl.xml</value>
+    <description>Keystore and truststore information used for tagsync, required if tagsync to atlas communication is SSL enabled</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER/configuration/ranger-ugsync-site.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<configuration>
+  <property>
+    <name>ranger.usersync.keystore.file</name>
+    <value>/usr/hdp/current/ranger-usersync/conf/unixauthservice.jks</value>
+    <description>Keystore file used for usersync</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>ranger.usersync.truststore.file</name>
+    <value>/usr/hdp/current/ranger-usersync/conf/mytruststore.jks</value>
+    <description>Truststore used for usersync, required if usersync -&gt; ranger admin communication is SSL enabled</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>ranger.usersync.ldap.bindkeystore</name>
+    <value/>
+    <description>Set same value as ranger.usersync.keystore.file property i.e default value /usr/hdp/current/ranger-usersync/conf/ugsync.jceks</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>ranger.usersync.credstore.filename</name>
+    <value>/usr/hdp/current/ranger-usersync/conf/ugsync.jceks</value>
+    <description>Credential store file name for user sync, specify full path</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>ranger.usersync.policymgr.keystore</name>
+    <value>/usr/hdp/current/ranger-usersync/conf/ugsync.jceks</value>
+    <description/>
+    <on-ambari-upgrade add="false"/>
+  </property>
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER/metainfo.xml

@@ -23,7 +23,12 @@
   <services>
     <service>
       <name>RANGER</name>
+      <extends>common-services/RANGER/0.7.0</extends>
       <version>0.7.0.2.6</version>
+      <credential-store>
+        <supported>true</supported>
+        <enabled>false</enabled>
+      </credential-store>
     </service>
   </services>
 </metainfo>

ambari-server/src/test/python/stacks/2.6/RANGER/test_ranger_tagsync.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import json
+from mock.mock import MagicMock, patch
+from stacks.utils.RMFTestCase import *
+from only_for_platform import not_for_platform, PLATFORM_WINDOWS
+
+@not_for_platform(PLATFORM_WINDOWS)
+class TestRangerTagsync(RMFTestCase):
+  COMMON_SERVICES_PACKAGE_DIR = "RANGER/0.4.0/package"
+  STACK_VERSION = "2.6"
+
+  def test_configure_default(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/ranger_tagsync.py",
+                   classname = "RangerTagsync",
+                   command = "configure",
+                   config_file="ranger-admin-default.json",
+                   stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    self.assert_configure_default()
+    self.assertNoMoreResources()
+
+  def test_start_default(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/ranger_tagsync.py",
+                   classname = "RangerTagsync",
+                   command = "start",
+                   config_file="ranger-admin-default.json",
+                   stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    self.assert_configure_default()
+    self.assertResourceCalled('Execute', '/usr/hdp/current/ranger-tagsync/ranger-tagsync-services.sh start',
+        environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+        not_if = 'ps -ef | grep proc_rangertagsync | grep -v grep',
+        user = 'ranger',
+    )
+    self.assertNoMoreResources()
+    
+  def test_stop_default(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/ranger_tagsync.py",
+                   classname = "RangerTagsync",
+                   command = "stop",
+                   config_file="ranger-admin-default.json",
+                   stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    self.assertResourceCalled('Execute', '/usr/hdp/current/ranger-tagsync/ranger-tagsync-services.sh stop',
+        environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+        user = 'ranger'
+    )
+
+    self.assertResourceCalled('File', '/var/run/ranger/tagsync.pid',
+      action = ['delete']
+    )
+    self.assertNoMoreResources()
+
+  def test_configure_secured(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/ranger_tagsync.py",
+                   classname = "RangerTagsync",
+                   command = "configure",
+                   config_file="ranger-admin-secured.json",
+                   stack_version = self.STACK_VERSION,
+                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+    self.assert_configure_default()
+    self.assertNoMoreResources()
+
+  def assert_configure_default(self):
+    self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-tagsync/conf',
+      owner = 'ranger',
+      group = 'ranger',
+      create_parents = True
+    )
+
+    self.assertResourceCalled('Directory', '/var/run/ranger',
+      mode=0755,
+      owner = 'ranger',
+      group = 'hadoop',
+      cd_access = "a",
+      create_parents=True
+    )
+
+    self.assertResourceCalled('File', '/usr/hdp/current/ranger-tagsync/conf/ranger-tagsync-env-piddir.sh',
+      content = 'export TAGSYNC_PID_DIR_PATH=/var/run/ranger\nexport UNIX_TAGSYNC_USER=ranger',
+      owner = 'ranger',
+      group = 'ranger',
+      mode = 0755
+    )
+
+    self.assertResourceCalled('Directory', '/var/log/ranger/tagsync',
+      owner = 'ranger',
+      group = 'ranger',
+      cd_access = "a",
+      mode=0755,
+      create_parents = True
+    )
+
+    self.assertResourceCalled('File',
+      '/usr/hdp/current/ranger-tagsync/conf/ranger-tagsync-env-logdir.sh',
+      owner = 'ranger',
+      content = 'export RANGER_TAGSYNC_LOG_DIR=/var/log/ranger/tagsync',
+      group = 'ranger',
+      mode=0755
+    )
+
+    self.assertResourceCalled('XmlConfig', 'ranger-tagsync-site.xml',
+      owner = 'ranger',
+      group = 'ranger',
+      conf_dir = '/usr/hdp/current/ranger-tagsync/conf',
+      configurations = self.getConfig()['configurations']['ranger-tagsync-site'],
+      configuration_attributes = self.getConfig()['configuration_attributes']['ranger-tagsync-site'],
+      mode=0644
+    )
+
+    self.assertResourceCalled('Directory', '/etc/security/serverKeys',
+      create_parents = True,
+      cd_access = 'a',
+    )
+
+    self.assertResourceCalled('Directory', '/etc/ranger/tagsync/',
+      owner = 'ranger',
+      group = 'ranger',
+      create_parents = True,
+      mode = 0775,
+      cd_access = 'a',
+    )
+
+    self.assertResourceCalled('XmlConfig', 'ranger-policymgr-ssl.xml',
+      owner = 'ranger',
+      group = 'ranger',
+      conf_dir = '/usr/hdp/current/ranger-tagsync/conf',
+      configurations = self.getConfig()['configurations']['ranger-tagsync-policymgr-ssl'],
+      configuration_attributes = self.getConfig()['configuration_attributes']['ranger-tagsync-policymgr-ssl'],
+      mode = 0644,
+    )
+
+    self.assertResourceCalled('Execute', (u'/usr/jdk64/jdk1.7.0_45/bin/java',
+      '-cp',
+      u'/usr/hdp/current/ranger-tagsync/lib/*',
+      'org.apache.ranger.credentialapi.buildks',
+      'create',
+      'sslKeyStore',
+      '-value',
+      'myKeyFilePassword',
+      '-provider',
+      'jceks://file/etc/ranger/tagsync/rangercred.jceks'),
+      logoutput = True,
+      environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+      sudo = True,
+    )
+
+    self.assertResourceCalled('Execute', (u'/usr/jdk64/jdk1.7.0_45/bin/java',
+      '-cp',
+      u'/usr/hdp/current/ranger-tagsync/lib/*',
+      'org.apache.ranger.credentialapi.buildks',
+      'create',
+      'sslTrustStore',
+      '-value',
+      'changeit',
+      '-provider',
+      'jceks://file/etc/ranger/tagsync/rangercred.jceks'),
+      logoutput = True,
+      environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+      sudo = True,
+    )
+
+    self.assertResourceCalled('File', '/etc/ranger/tagsync/rangercred.jceks',
+      owner = 'ranger',
+      group = 'ranger',
+      mode = 0640,
+    )
+
+    self.assertResourceCalled('XmlConfig', 'atlas-tagsync-ssl.xml',
+      group = 'ranger',
+      conf_dir = '/usr/hdp/current/ranger-tagsync/conf',
+      mode = 0644,
+      configuration_attributes = UnknownConfigurationMock(),
+      owner = 'ranger',
+      configurations = self.getConfig()['configurations']['atlas-tagsync-ssl']
+    )
+
+
+
+    self.assertResourceCalled('Execute', (u'/usr/jdk64/jdk1.7.0_45/bin/java',
+      '-cp',
+      u'/usr/hdp/current/ranger-tagsync/lib/*',
+      'org.apache.ranger.credentialapi.buildks',
+      'create',
+      'sslKeyStore',
+      '-value',
+      'myKeyFilePassword',
+      '-provider',
+      'jceks://file/etc/ranger/tagsync/atlascred.jceks'),
+      logoutput = True,
+      environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+      sudo = True,
+    )
+
+    self.assertResourceCalled('Execute', (u'/usr/jdk64/jdk1.7.0_45/bin/java',
+      '-cp',
+      u'/usr/hdp/current/ranger-tagsync/lib/*',
+      'org.apache.ranger.credentialapi.buildks',
+      'create',
+      'sslTrustStore',
+      '-value',
+      'changeit',
+      '-provider',
+      'jceks://file/etc/ranger/tagsync/atlascred.jceks'),
+      logoutput = True,
+      environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+      sudo = True,
+    )
+
+    self.assertResourceCalled('File', '/etc/ranger/tagsync/atlascred.jceks',
+      owner = 'ranger',
+      group = 'ranger',
+      mode = 0640,
+    )
+
+
+    self.assertResourceCalled('PropertiesFile', '/usr/hdp/current/ranger-tagsync/conf/atlas-application.properties',
+      properties = self.getConfig()['configurations']['tagsync-application-properties'],
+      mode=0755,
+      owner='ranger',
+      group='ranger'
+    )
+
+    self.assertResourceCalled('File', '/usr/hdp/current/ranger-tagsync/conf/log4j.properties',
+      owner = 'ranger',
+      group = 'ranger',
+      content = self.getConfig()['configurations']['tagsync-log4j']['content'],
+      mode = 0644
+    )
+
+    self.assertResourceCalled('File', '/usr/hdp/current/ranger-tagsync/ranger-tagsync-services.sh',
+      mode = 0755,
+    )
+
+    self.assertResourceCalled('Execute', ('ln', '-sf', '/usr/hdp/current/ranger-tagsync/ranger-tagsync-services.sh', '/usr/bin/ranger-tagsync'),
+      not_if='ls /usr/bin/ranger-tagsync',
+      only_if='ls /usr/hdp/current/ranger-tagsync/ranger-tagsync-services.sh',
+      sudo=True
+    )
+
+    self.assertResourceCalled('XmlConfig', 'core-site.xml',
+      owner = 'ranger',
+      group = 'ranger',
+      conf_dir = '/usr/hdp/current/ranger-tagsync/conf',
+      configurations = self.getConfig()['configurations']['core-site'],
+      configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+      mode = 0644
+    )

ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json

@@ -35,9 +35,10 @@
                 "dfs.webhdfs.enabled": "true"
             }
         }, 
-        "ranger-tagsync-site": {}, 
-        "zoo.cfg": {}, 
-        "hadoop-policy": {}, 
+        "ranger-tagsync-site": {},
+        "ranger-tagsync-policymgr-ssl": {}, 
+        "zoo.cfg": {},
+        "hadoop-policy": {},
         "hdfs-log4j": {}, 
         "krb5-conf": {}, 
         "core-site": {
@@ -147,7 +148,10 @@
         }, 
         "tagsync-log4j": {
             "tag": "version1466705299949"
-        }, 
+        },
+        "ranger-tagsync-policymgr-ssl": {
+            "tag": "version1479216811014"
+        },
         "ranger-hdfs-security": {
             "tag": "version1466705299922"
         }, 
@@ -171,26 +175,26 @@
     "hostLevelParams": {
         "agent_stack_retry_on_unavailability": "false", 
         "stack_name": "HDP", 
-        "package_version": "2_5_0_0_*", 
+        "package_version": "2_6_0_0_*",
         "custom_mysql_jdbc_name": "mysql-connector-java.jar",
         "previous_custom_mysql_jdbc_name": "mysql-connector-java-old.jar",
         "host_sys_prepped": "false", 
         "ambari_db_rca_username": "mapred", 
         "current_version": "2.6.0.0-801",
-        "mysql_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar", 
-        "agent_stack_retry_count": "5", 
+        "mysql_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar",
+        "agent_stack_retry_count": "5",
         "stack_version": "2.6",
         "jdk_name": "jdk-8u60-linux-x64.tar.gz", 
         "ambari_db_rca_driver": "org.postgresql.Driver", 
         "java_home": "/usr/jdk64/jdk1.7.0_45",
-        "repository_version_id": "1", 
-        "jdk_location": "http://c6401.ambari.apache.org:8080/resources/", 
+        "repository_version_id": "1",
+        "jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
         "not_managed_hdfs_path_list": "[\"/tmp\"]", 
         "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca", 
         "java_version": "8", 
-        "repo_info": "[{\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.5.0.0-801\",\"osType\":\"redhat6\",\"repoId\":\"HDP-2.5\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0\",\"latestBaseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.5.0.0-801\",\"baseSaved\":true},{\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6\",\"osType\":\"redhat6\",\"repoId\":\"HDP-UTILS-1.1.0.21\",\"repoName\":\"HDP-UTILS\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6\",\"latestBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6\",\"baseSaved\":true}]", 
-        "package_list": "[{\"name\":\"ranger_${stack_version}-admin\",\"condition\":\"\",\"skipUpgrade\":false},{\"name\":\"ranger_${stack_version}-usersync\",\"condition\":\"\",\"skipUpgrade\":false},{\"name\":\"ranger_${stack_version}-tagsync\",\"condition\":\"should_install_ranger_tagsync\",\"skipUpgrade\":false},{\"name\":\"ambari-logsearch-solr-client\",\"condition\":\"should_install_logsearch_solr_client\",\"skipUpgrade\":false}]", 
-        "db_name": "ambari", 
+        "repo_info": "[{\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.6.0.0-801\",\"osType\":\"redhat6\",\"repoId\":\"HDP-2.6\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/updates/2.6.0.0\",\"latestBaseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.6.0.0-801\",\"baseSaved\":true},{\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6\",\"osType\":\"redhat6\",\"repoId\":\"HDP-UTILS-1.1.0.21\",\"repoName\":\"HDP-UTILS\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6\",\"latestBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6\",\"baseSaved\":true}]",
+        "package_list": "[{\"name\":\"ranger_${stack_version}-admin\",\"condition\":\"\",\"skipUpgrade\":false},{\"name\":\"ranger_${stack_version}-usersync\",\"condition\":\"\",\"skipUpgrade\":false},{\"name\":\"ranger_${stack_version}-tagsync\",\"condition\":\"should_install_ranger_tagsync\",\"skipUpgrade\":false},{\"name\":\"ambari-logsearch-solr-client\",\"condition\":\"should_install_logsearch_solr_client\",\"skipUpgrade\":false}]",
+        "db_name": "ambari",
         "group_list": "[\"ranger\",\"hadoop\",\"users\"]", 
         "agentCacheDir": "/var/lib/ambari-agent/cache", 
         "ambari_db_rca_password": "mapred", 
@@ -205,8 +209,8 @@
         "script": "scripts/ranger_admin.py", 
         "hooks_folder": "HDP/2.0.6/hooks", 
         "version": "2.6.0.0-801",
-        "max_duration_for_retries": "0", 
-        "command_retry_enabled": "false", 
+        "max_duration_for_retries": "0",
+        "command_retry_enabled": "false",
         "command_timeout": "600", 
         "script_type": "PYTHON"
     }, 
@@ -270,7 +274,23 @@
             "xasecure.audit.provider.summary.enabled": "false", 
             "xasecure.audit.destination.hdfs.dir": "hdfs://c6401.ambari.apache.org:8020/ranger/audit", 
             "xasecure.audit.is.enabled": "true"
-        }, 
+        },
+        "ranger-tagsync-policymgr-ssl": {
+            "xasecure.policymgr.clientssl.keystore": "/etc/security/serverKeys/ranger-tagsync-keystore.jks", 
+            "xasecure.policymgr.clientssl.truststore.password": "changeit", 
+            "xasecure.policymgr.clientssl.keystore.credential.file": "jceks://file{{ranger_tagsync_credential_file}}",
+            "xasecure.policymgr.clientssl.truststore": "/etc/security/serverKeys/ranger-tagsync-mytruststore.jks", 
+            "xasecure.policymgr.clientssl.truststore.credential.file": "jceks://file{{ranger_tagsync_credential_file}}",
+            "xasecure.policymgr.clientssl.keystore.password": "myKeyFilePassword"
+        },
+        "atlas-tagsync-ssl": {
+            "xasecure.policymgr.clientssl.keystore": "/etc/security/serverKeys/atlas-tagsync-keystore.jks",
+            "xasecure.policymgr.clientssl.truststore.password": "changeit",
+            "xasecure.policymgr.clientssl.keystore.credential.file": "jceks://file{{atlas_tagsync_credential_file}}",
+            "xasecure.policymgr.clientssl.truststore": "/etc/security/serverKeys/atlas-tagsync-mytruststore.jks",
+            "xasecure.policymgr.clientssl.truststore.credential.file": "jceks://file{{atlas_tagsync_credential_file}}",
+            "xasecure.policymgr.clientssl.keystore.password": "myKeyFilePassword"
+        },
         "ssl-client": {
             "ssl.client.truststore.reload.interval": "10000", 
             "ssl.client.keystore.password": "bigdata", 
@@ -579,9 +599,9 @@
             "dfs.namenode.name.dir": "/grid/0/hadoop/hdfs/namenode"
         }, 
         "ranger-tagsync-site": {
-            "ranger.tagsync.dest.ranger.ssl.config.filename": "/usr/hdp/current/ranger-tagsync/conf/mytruststore.jks", 
-            "ranger.tagsync.source.atlasrest.username": "", 
-            "ranger.tagsync.logdir": "/var/log/ranger/tagsync", 
+            "ranger.tagsync.dest.ranger.ssl.config.filename": "{{stack_root}}/current/ranger-tagsync/conf/ranger-policymgr-ssl.xml",
+            "ranger.tagsync.source.atlasrest.username": "",
+            "ranger.tagsync.logdir": "/var/log/ranger/tagsync",
             "ranger.tagsync.source.atlasrest.download.interval.millis": "", 
             "ranger.tagsync.keystore.filename": "/usr/hdp/current/ranger-tagsync/conf/rangertagsync.jceks", 
             "ranger.tagsync.source.file.check.interval.millis": "",