فهرست منبع

AMBARI-20527. make home directory check as optional in Pig view (Nitiraj Rathore via pallavkul)

pallavkul 8 سال پیش
والد
کامیت
b7cd5f9d88

+ 1 - 0
ambari-server/conf/unix/ambari.properties

@@ -69,6 +69,7 @@ views.request.connect.timeout.millis=5000
 views.request.read.timeout.millis=10000
 views.ambari.request.connect.timeout.millis=30000
 views.ambari.request.read.timeout.millis=45000
+views.skip.home-directory-check.file-system.list=wasb,adls,adl
 
 # Scheduler settings
 server.execution.scheduler.isClustered=false

+ 3 - 0
ambari-server/conf/windows/ambari.properties

@@ -106,3 +106,6 @@ views.http.cache-control=no-store
 views.http.pragma=no-cache
 
 mpacks.staging.path=resources\\mpacks
+
+views.skip.home-directory-check.file-system.list=wasb,adls,adl
+

+ 23 - 2
contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/HelpService.java

@@ -21,11 +21,14 @@ package org.apache.ambari.view.pig.services;
 import org.apache.ambari.view.ViewContext;
 import org.apache.ambari.view.ViewResourceHandler;
 import org.apache.ambari.view.pig.persistence.DataStoreStorage;
-import org.apache.ambari.view.pig.persistence.InstanceKeyValueStorage;
 import org.apache.ambari.view.pig.resources.files.FileService;
 import org.apache.ambari.view.pig.resources.jobs.JobResourceManager;
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.ambari.view.pig.utils.ServiceCheck;
+import org.apache.ambari.view.pig.utils.ServiceFormattedException;
+import org.apache.ambari.view.utils.hdfs.HdfsApiException;
 import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
@@ -37,6 +40,9 @@ import java.util.HashMap;
  * Help service
  */
 public class HelpService extends BaseService {
+  private final static Logger LOG =
+    LoggerFactory.getLogger(HelpService.class);
+
   private ViewContext context;
   private ViewResourceHandler handler;
 
@@ -130,6 +136,21 @@ public class HelpService extends BaseService {
     return getOKResponse();
   }
 
+  @GET
+  @Path("/service-check-policy")
+  public Response getServiceCheckList(){
+    ServiceCheck serviceCheck = new ServiceCheck(context);
+    try {
+      ServiceCheck.Policy policy = serviceCheck.getServiceCheckPolicy();
+      JSONObject policyJson = new JSONObject();
+      policyJson.put("serviceCheckPolicy", policy);
+      return Response.ok(policyJson).build();
+    } catch (HdfsApiException e) {
+      LOG.error("Error occurred while generating service check policy : ", e);
+      throw new ServiceFormattedException(e);
+    }
+  }
+
   private Response getOKResponse() {
     JSONObject response = new JSONObject();
     response.put("message", "OK");

+ 25 - 0
contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/Constants.java

@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.utils;
+
+public interface Constants {
+  String VIEW_CONF_KEYVALUES = "view.conf.keyvalues";
+  String DEFAULT_FS = "fs.defaultFS";
+  String AMBARI_SKIP_HOME_DIRECTORY_CHECK_PROTOCOL_LIST = "views.skip.home-directory-check.file-system.list";
+}

+ 132 - 0
contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/ServiceCheck.java

@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.utils;
+
+import com.google.common.base.Optional;
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.commons.hdfs.ViewPropertyHelper;
+import org.apache.ambari.view.utils.hdfs.ConfigurationBuilder;
+import org.apache.ambari.view.utils.hdfs.HdfsApiException;
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.*;
+
+public class ServiceCheck {
+  protected static final Logger LOG = LoggerFactory.getLogger(ServiceCheck.class);
+
+  private final ViewContext viewContext;
+
+  public ServiceCheck(ViewContext viewContext){
+    this.viewContext = viewContext;
+  }
+
+  public static class Policy {
+    private boolean checkHdfs = true;
+    private boolean checkHomeDirectory = true;
+    private boolean checkWebhcat = true;
+    private boolean checkStorage = true;
+
+    public Policy() {
+    }
+
+    public Policy(boolean checkHdfs, boolean checkHomeDirectory, boolean checkWebhcat, boolean checkStorage) {
+      this.checkHdfs = checkHdfs;
+      this.checkHomeDirectory = checkHomeDirectory;
+      this.checkWebhcat = checkWebhcat;
+      this.checkStorage = checkStorage;
+    }
+
+    public boolean isCheckHdfs() {
+      return checkHdfs;
+    }
+
+    public void setCheckHdfs(boolean checkHdfs) {
+      this.checkHdfs = checkHdfs;
+    }
+
+    public boolean isCheckHomeDirectory() {
+      return checkHomeDirectory;
+    }
+
+    public void setCheckHomeDirectory(boolean checkHomeDirectory) {
+      this.checkHomeDirectory = checkHomeDirectory;
+    }
+
+    public boolean isCheckWebhcat() {
+      return checkWebhcat;
+    }
+
+    public void setCheckWebhcat(boolean checkWebhcat) {
+      this.checkWebhcat = checkWebhcat;
+    }
+
+    public boolean isCheckStorage() {
+      return checkStorage;
+    }
+
+    public void setCheckStorage(boolean checkStorage) {
+      this.checkStorage = checkStorage;
+    }
+
+    @Override
+    public String toString() {
+      return "Policy{" +
+        "checkHdfs=" + checkHdfs +
+        ", checkHomeDirectory=" + checkHomeDirectory +
+        ", checkWebhcat=" + checkWebhcat +
+        ", checkStorage=" + checkStorage +
+        '}';
+    }
+  }
+
+  public Policy getServiceCheckPolicy() throws HdfsApiException {
+    Policy policy = new Policy();
+    Optional<Map<String, String>> viewConfigs = ViewPropertyHelper.getViewConfigs(viewContext, Constants.VIEW_CONF_KEYVALUES);
+    ConfigurationBuilder configBuilder;
+    if(viewConfigs.isPresent()) {
+      configBuilder = new ConfigurationBuilder(this.viewContext, viewConfigs.get());
+    }else{
+      configBuilder = new ConfigurationBuilder(this.viewContext);
+    }
+
+    Configuration configurations = configBuilder.buildConfig();
+    String defaultFS = configurations.get(Constants.DEFAULT_FS);
+
+    URI fsUri = null;
+    try {
+      fsUri = new URI(defaultFS);
+      String protocol = fsUri.getScheme();
+      String ambariSkipCheckValues = viewContext.getAmbariProperty(Constants.AMBARI_SKIP_HOME_DIRECTORY_CHECK_PROTOCOL_LIST);
+      List<String> protocolSkipList = (ambariSkipCheckValues == null? new LinkedList<String>() : Arrays.asList(ambariSkipCheckValues.split(",")));
+      if(null != protocol && protocolSkipList.contains(protocol)){
+        policy.setCheckHomeDirectory(false);
+        return policy;
+      }
+    } catch (URISyntaxException e) {
+      LOG.error("Error occurred while parsing the defaultFS URI.", e);
+      return policy;
+    }
+
+    return policy;
+  }
+}

+ 37 - 2
contrib/views/pig/src/main/resources/ui/pig-web/app/controllers/splash.js

@@ -57,9 +57,38 @@ App.SplashController = Ember.ObjectController.extend({
       }
       model.set(name + 'TestDone', true);
       var percent = model.get('percent');
-      model.set('percent', percent + 25);
+      model.set('percent', percent + (100/model.get("numberOfChecks")));
     };
-    var promises = ['storage', 'webhcat', 'hdfs', 'userhome'].map(function(name) {
+
+    let checks = [];
+    if(model.get("serviceCheckPolicy").checkHdfs){
+      checks.push("hdfs");
+    }else{
+      model.set("hdfs" + 'TestDone', true);
+      model.set("hdfs" + 'Test', true);
+    }
+    if(model.get("serviceCheckPolicy").checkStorage){
+      checks.push("storage");
+    }else{
+      model.set("storage" + 'TestDone', true);
+      model.set("storage" + 'Test', true);
+    }
+
+    if(model.get("serviceCheckPolicy").checkWebhcat){
+      checks.push("webhcat");
+    }else{
+      model.set("webhcat" + 'TestDone', true);
+      model.set("webhcat" + 'Test', true);
+    }
+
+    if(model.get("serviceCheckPolicy").checkHomeDirectory){
+      checks.push("userhome");
+    }else{
+      model.set("userhome" + 'TestDone', true);
+      model.set("userhome" + 'Test', true);
+    }
+
+    var promises = checks.map(function(name) {
       return Ember.$.getJSON('/' + url + name + 'Status')
                .then(
                  function(data) {
@@ -73,6 +102,12 @@ App.SplashController = Ember.ObjectController.extend({
 
     return Ember.RSVP.all(promises);
   },
+  fetchServiceCheckPolicy: function(){
+    var url = App.getNamespaceUrl() + '/resources/pig/help/';
+
+    return Ember.$.getJSON('/' + url + "service-check-policy");
+  },
+
   progressBarStyle: function() {
     return 'width: ' + this.get("model").get("percent") + '%;';
   }.property("model.percent"),

+ 28 - 14
contrib/views/pig/src/main/resources/ui/pig-web/app/routes/splash.js

@@ -29,26 +29,40 @@ App.SplashRoute = Em.Route.extend({
       hdfsTestDone: null,
       userhomeTest: null,
       userhomeTestDone: null,
-      percent: 0
+      percent: 0,
+      numberOfChecks: null,
+      serviceCheckPolicy: null,
     });
   },
   renderTemplate: function() {
     this.render('splash');
   },
-  setupController: function(controller, model) {
+  setupController: function (controller, model) {
     controller.set('model', model);
     var self = this;
-    controller.startTests(model).then(function() {
-      if (model.get("storageTest") && model.get("webhcatTest") && model.get("hdfsTest") && model.get("userhomeTest")) {
-        Ember.run.later(this, function() {
-          previousTransition = App.get('previousTransition');
-          if (previousTransition) {
-            previousTransition.retry();
-          } else {
-            self.transitionTo('pig.scripts');
+    controller.fetchServiceCheckPolicy()
+      .then(function(data){
+        let numberOfChecks = 0;
+        let serviceCheckPolicy = data.serviceCheckPolicy;
+        for (let serviceCheck in serviceCheckPolicy) {
+          if (serviceCheckPolicy[serviceCheck] === true) {
+            numberOfChecks++;
           }
-        }, 2000);
-      }
-    });
-  }
+        }
+        model.set("numberOfChecks", numberOfChecks);
+        model.set("serviceCheckPolicy", serviceCheckPolicy);
+        controller.startTests(model).then(function () {
+          if (model.get("storageTest") && model.get("webhcatTest") && model.get("hdfsTest") && model.get("userhomeTest")) {
+            Ember.run.later(this, function () {
+              previousTransition = App.get('previousTransition');
+              if (previousTransition) {
+                previousTransition.retry();
+              } else {
+                self.transitionTo('pig.scripts');
+              }
+            }, 2000);
+          }
+        });
+      });
+  },
 });

+ 12 - 4
contrib/views/pig/src/main/resources/ui/pig-web/app/templates/splash.hbs

@@ -35,7 +35,8 @@
 
   <table class="table">
     <tbody>
-      <tr>
+    {{#if model.serviceCheckPolicy.checkStorage}}
+    <tr>
         <td>
           {{#if storageTestDone}}
             {{#if storageTest}}
@@ -49,8 +50,10 @@
         </td>
         <td>{{t 'splash.storage_test'}}</td>
       </tr>
+    {{/if}}
 
-      <tr>
+    {{#if model.serviceCheckPolicy.checkHdfs}}
+    <tr>
         <td>
           {{#if hdfsTestDone}}
             {{#if hdfsTest}}
@@ -64,8 +67,10 @@
         </td>
         <td>{{t 'splash.hdfs_test'}}</td>
       </tr>
+    {{/if}}
 
-      <tr>
+    {{#if model.serviceCheckPolicy.checkWebhcat}}
+    <tr>
         <td>
           {{#if webhcatTestDone}}
             {{#if webhcatTest}}
@@ -79,8 +84,10 @@
         </td>
         <td>{{t 'splash.webhcat_test'}}</td>
       </tr>
+    {{/if}}
 
-      <tr>
+    {{#if model.serviceCheckPolicy.checkHomeDirectory}}
+    <tr>
         <td>
           {{#if userhomeTestDone}}
             {{#if userhomeTest}}
@@ -94,6 +101,7 @@
         </td>
         <td>{{t 'splash.userhome_test'}}</td>
       </tr>
+    {{/if}}
 
     </tbody>
   </table>