AMBARI-5786 - Views: Pig view parameters adjusting (Roman Rader via tbeerbower)

contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java

@@ -87,4 +87,11 @@ public class PersonalCRUDResourceManager<T extends PersonalResource> extends CRU
     }
     return result;
   }
+
+  protected String getUsername() {
+    String userName = context.getProperties().get("dataworker.username");
+    if (userName == null)
+      userName = context.getUsername();
+    return userName;
+  }
 }

contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java

@@ -64,7 +64,7 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
    */
   public TempletonApi getTempletonApi() {
     if (api == null) {
-      api = new TempletonApi(context.getProperties().get("dataworker.templeton_url"),
+      api = new TempletonApi(context.getProperties().get("dataworker.webhcat.url"),
           getTempletonUser(), getTempletonUser(), context);
     }
     return api;
@@ -135,8 +135,8 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
    */
   private void submitJob(PigJob job) {
     String date = new SimpleDateFormat("dd-MM-yyyy-HH-mm-ss").format(new Date());
-    String statusdir = String.format(context.getProperties().get("dataworker.pigJobsPath") +
-            "/%s/%s_%s", getTempletonUser(),
+    String statusdir = String.format(context.getProperties().get("dataworker.jobs.path") +
+            "/%s/%s_%s", getUsername(),
         job.getTitle().toLowerCase().replaceAll("[^a-zA-Z0-9 ]+", "").replace(" ", "_"),
         date);
 
@@ -295,6 +295,6 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
    * @return username in templeton
    */
   private String getTempletonUser() {
-    return context.getProperties().get("dataworker.templeton_user");
+    return context.getProperties().get("dataworker.webhcat.user");
   }
 }

contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptResourceManager.java

@@ -58,9 +58,9 @@ public class ScriptResourceManager extends PersonalCRUDResourceManager<PigScript
   }
 
   private void createDefaultScriptFile(PigScript object) {
-    String userScriptsPath = context.getProperties().get("dataworker.userScriptsPath");
+    String userScriptsPath = context.getProperties().get("dataworker.scripts.path");
     if (userScriptsPath == null) {
-      String msg = "dataworker.userScriptsPath is not configured!";
+      String msg = "dataworker.scripts.path is not configured!";
       LOG.error(msg);
       throw new WebServiceException(msg);
     }
@@ -72,7 +72,7 @@ public class ScriptResourceManager extends PersonalCRUDResourceManager<PigScript
       String normalizedName = object.getTitle().replaceAll("[^a-zA-Z0-9 ]+", "").replaceAll(" ", "_").toLowerCase();
       String timestamp = new SimpleDateFormat("yyyy-MM-dd_hh-mm").format(new Date());
       newFilePath = String.format(userScriptsPath +
-              "/%s/%s-%s%s.pig", context.getUsername(),
+              "/%s/%s-%s%s.pig", getUsername(),
           normalizedName, timestamp, (checkId == 0)?"":"_"+checkId);
       LOG.debug("Trying to create new file " + newFilePath);
 
@@ -105,4 +105,5 @@ public class ScriptResourceManager extends PersonalCRUDResourceManager<PigScript
     object.setPigScript(newFilePath);
     getPigStorage().store(object);
   }
+
 }

contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/BaseService.java

@@ -69,8 +69,6 @@ public class BaseService {
     if (hdfsApi == null) {
       Thread.currentThread().setContextClassLoader(null);
 
-      String userName = context.getUsername();
-
       String defaultFS = context.getProperties().get("dataworker.defaultFs");
       if (defaultFS == null) {
         String message = "dataworker.defaultFs is not configured!";
@@ -79,7 +77,7 @@ public class BaseService {
       }
 
       try {
-        hdfsApi = new HdfsApi(defaultFS, userName);
+        hdfsApi = new HdfsApi(defaultFS, getHdfsUsername(context));
         LOG.info("HdfsApi connected OK");
       } catch (IOException e) {
         String message = "HdfsApi IO error: " + e.getMessage();
@@ -94,6 +92,13 @@ public class BaseService {
     return hdfsApi;
   }
 
+  public static String getHdfsUsername(ViewContext context) {
+    String userName = context.getProperties().get("dataworker.hdfs.username");
+    if (userName == null)
+      userName = context.getUsername();
+    return userName;
+  }
+
   protected HdfsApi getHdfsApi()  {
     return getHdfsApi(context);
   }

contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java

@@ -52,7 +52,7 @@ public class TempletonApi {
 
   /**
    * TempletonApi constructor
-   * @param api dataworker.templeton_url
+   * @param api dataworker.webhcat.url
    * @param username templeton username
    * @param doAs doAs argument
    * @param context context with URLStreamProvider

contrib/views/pig/src/main/resources/view.xml

@@ -27,24 +27,24 @@
     </parameter>
 
     <parameter>
-        <name>dataworker.hdfs_user</name>
+        <name>dataworker.hdfs.username</name>
         <description>HDFS user.name and doAs</description>
         <required>false</required>
     </parameter>
 
     <parameter>
-        <name>dataworker.use_current_user</name>
-        <description>If true, get user from Ambari auth</description>
-        <required>true</required>
+        <name>dataworker.username</name>
+        <description>The username (defaults to ViewContext username)</description>
+        <required>false</required>
     </parameter>
 
     <parameter>
-        <name>dataworker.userScriptsPath</name>
+        <name>dataworker.scripts.path</name>
         <description>Directory to store scripts</description>
         <required>true</required>
     </parameter>
     <parameter>
-        <name>dataworker.pigJobsPath</name>
+        <name>dataworker.jobs.path</name>
         <description>Directory to store jobs (for templeton status dir)</description>
         <required>true</required>
     </parameter>
@@ -58,13 +58,13 @@
     </parameter>
 
     <parameter>
-        <name>dataworker.templeton_url</name>
+        <name>dataworker.webhcat.url</name>
         <description>Templeton url</description>
         <required>true</required>
     </parameter>
 
     <parameter>
-        <name>dataworker.templeton_user</name>
+        <name>dataworker.webhcat.user</name>
         <description>Templeton user.name and doAs</description>
         <required>true</required>
     </parameter>

contrib/views/pig/src/test/java/org/apache/ambari/view/pig/BasePigTest.java

@@ -37,6 +37,7 @@ public abstract class BasePigTest {
   protected ViewContext context;
   protected static File pigStorageFile;
   protected static File baseDir;
+  protected Map<String, String> properties;
 
   protected static String DATA_DIRECTORY = "./target/PigTest";
 
@@ -52,17 +53,17 @@ public abstract class BasePigTest {
     handler = createNiceMock(ViewResourceHandler.class);
     context = createNiceMock(ViewContext.class);
 
-    Map<String, String> properties = new HashMap<String, String>();
+    properties = new HashMap<String, String>();
     baseDir = new File(DATA_DIRECTORY)
         .getAbsoluteFile();
     pigStorageFile = new File("./target/BasePigTest/storage.dat")
         .getAbsoluteFile();
 
     properties.put("dataworker.storagePath", pigStorageFile.toString());
-    properties.put("dataworker.templeton_url", "localhost:50111/templeton/v1");
-    properties.put("dataworker.templeton_user", "admin");
-    properties.put("dataworker.userScriptsPath", "/tmp/.pigscripts");
-    properties.put("dataworker.pigJobsPath", "/tmp/.pigjobs");
+    properties.put("dataworker.webhcat.url", "localhost:50111/templeton/v1");
+    properties.put("dataworker.webhcat.user", "admin");
+    properties.put("dataworker.scripts.path", "/tmp/.pigscripts");
+    properties.put("dataworker.jobs.path", "/tmp/.pigjobs");
 
     setupProperties(properties, baseDir);
 

contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/JobTest.java

@@ -106,13 +106,38 @@ public class JobTest extends BasePigTest {
     Assert.assertTrue(obj.containsKey("job"));
     Assert.assertNotNull(((PigJob) obj.get("job")).getId());
     Assert.assertFalse(((PigJob) obj.get("job")).getId().isEmpty());
-    Assert.assertTrue(((PigJob) obj.get("job")).getStatusDir().startsWith("/tmp/.pigjobs/admin/test"));
+    Assert.assertTrue(((PigJob) obj.get("job")).getStatusDir().startsWith("/tmp/.pigjobs/ambari-qa/test"));
 
     PigJob job = ((PigJob) obj.get("job"));
     Assert.assertEquals(PigJob.Status.SUBMITTED, job.getStatus());
     Assert.assertTrue(job.isInProgress());
   }
 
+  @Test
+  public void testSubmitJobUsernameProvided() throws Exception {
+    HdfsApi hdfsApi = createNiceMock(HdfsApi.class);
+    expect(hdfsApi.copy(eq("/tmp/script.pig"), startsWith("/tmp/.pigjobs/"))).andReturn(true);
+
+    ByteArrayOutputStream do_stream = new ByteArrayOutputStream();
+
+    FSDataOutputStream stream = new FSDataOutputStream(do_stream);
+    expect(hdfsApi.create(anyString(), eq(true))).andReturn(stream);
+    replay(hdfsApi);
+    JobService.setHdfsApi(hdfsApi);
+
+    TempletonApi api = createNiceMock(TempletonApi.class);
+    jobService.getResourceManager().setTempletonApi(api);
+    TempletonApi.JobData data = api.new JobData();
+    expect(api.runPigQuery((File) anyObject(), anyString(), eq("-useHCatalog"))).andReturn(data);
+    replay(api);
+
+    properties.put("dataworker.username", "luke");
+    Response response = doCreateJob("Test", "/tmp/script.pig", "-useHCatalog");
+    JSONObject obj = (JSONObject)response.getEntity();
+    Assert.assertTrue(obj.containsKey("job"));
+    Assert.assertTrue(((PigJob) obj.get("job")).getStatusDir().startsWith("/tmp/.pigjobs/luke/test"));
+  }
+
   @Test
   public void testSubmitJobNoArguments() throws Exception {
     HdfsApi hdfsApi = createNiceMock(HdfsApi.class);
@@ -140,7 +165,7 @@ public class JobTest extends BasePigTest {
     Assert.assertTrue(obj.containsKey("job"));
     Assert.assertNotNull(((PigJob) obj.get("job")).getId());
     Assert.assertFalse(((PigJob) obj.get("job")).getId().isEmpty());
-    Assert.assertTrue(((PigJob) obj.get("job")).getStatusDir().startsWith("/tmp/.pigjobs/admin/test"));
+    Assert.assertTrue(((PigJob) obj.get("job")).getStatusDir().startsWith("/tmp/.pigjobs/ambari-qa/test"));
 
     PigJob job = ((PigJob) obj.get("job"));
     Assert.assertEquals(PigJob.Status.SUBMITTED, job.getStatus());

contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTest.java

@@ -60,7 +60,7 @@ public class ScriptTest extends HDFSTest {
     @Override
     protected void setupProperties(Map<String, String> properties, File baseDir) throws Exception {
         super.setupProperties(properties, baseDir);
-        properties.put("dataworker.userScriptsPath", "/tmp/.pigscripts");
+        properties.put("dataworker.scripts.path", "/tmp/.pigscripts");
     }
 
     private Response doCreateScript() {

contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestHDFSUnmanaged.java

@@ -27,6 +27,9 @@ import org.apache.ambari.view.pig.persistence.Storage;
 import org.apache.ambari.view.pig.resources.files.FileService;
 import org.apache.ambari.view.pig.resources.scripts.ScriptService;
 import org.apache.ambari.view.pig.persistence.utils.StorageUtil;
+import org.apache.ambari.view.pig.resources.scripts.models.PigScript;
+import org.apache.ambari.view.pig.services.BaseService;
+import org.json.simple.JSONObject;
 import org.junit.*;
 
 import javax.ws.rs.core.Response;
@@ -70,7 +73,7 @@ public class ScriptTestHDFSUnmanaged extends HDFSTest {
         .getAbsoluteFile();
 
     properties.put("dataworker.storagePath", pigStorageFile.toString());
-//        properties.put("dataworker.userScriptsPath", "/tmp/.pigscripts");
+//        properties.put("dataworker.scripts.path", "/tmp/.pigscripts");
     properties.put("dataworker.defaultFs", hdfsURI);
 
     expect(context.getProperties()).andReturn(properties).anyTimes();
@@ -82,6 +85,34 @@ public class ScriptTestHDFSUnmanaged extends HDFSTest {
     doCreateScript("Test", null);
   }
 
+  @Test
+  public void createScriptAutoCreateUsername() throws IOException, InterruptedException {
+    Map<String, String> properties = new HashMap<String, String>();
+    baseDir = new File(DATA_DIRECTORY)
+        .getAbsoluteFile();
+    pigStorageFile = new File("./target/BasePigTest/storage.dat")
+        .getAbsoluteFile();
+
+    properties.put("dataworker.storagePath", pigStorageFile.toString());
+    properties.put("dataworker.scripts.path", "/tmp/.pigscripts");
+    properties.put("dataworker.defaultFs", hdfsURI);
+
+    expect(context.getProperties()).andReturn(properties).anyTimes();
+    expect(context.getUsername()).andReturn("ambari-qa").anyTimes();
+
+    replay(handler, context);
+    scriptService = getService(ScriptService.class, handler, context);
+
+    Response createdScript = doCreateScript("Test", null);
+    String createdScriptPath = ((PigScript) ((JSONObject) createdScript.getEntity()).get("script")).getPigScript();
+    Assert.assertTrue(createdScriptPath.startsWith("/tmp/.pigscripts/ambari-qa/"));
+
+    properties.put("dataworker.username", "luke");
+    Response createdScript2 = doCreateScript("Test", null);
+    String createdScriptPath2 = ((PigScript) ((JSONObject) createdScript2.getEntity()).get("script")).getPigScript();
+    Assert.assertTrue(createdScriptPath2.startsWith("/tmp/.pigscripts/luke/"));
+  }
+
   @Test
   public void createScriptAutoCreateNoStoragePath() throws IOException, InterruptedException {
     Map<String, String> properties = new HashMap<String, String>();
@@ -91,7 +122,7 @@ public class ScriptTestHDFSUnmanaged extends HDFSTest {
         .getAbsoluteFile();
 
 //        properties.put("dataworker.storagePath", pigStorageFile.toString());
-    properties.put("dataworker.userScriptsPath", "/tmp/.pigscripts");
+    properties.put("dataworker.scripts.path", "/tmp/.pigscripts");
     properties.put("dataworker.defaultFs", hdfsURI);
 
     expect(context.getProperties()).andReturn(properties).anyTimes();
@@ -103,6 +134,24 @@ public class ScriptTestHDFSUnmanaged extends HDFSTest {
     Assert.assertEquals(InstanceKeyValueStorage.class.getSimpleName(), storage.getClass().getSimpleName());
   }
 
+  @Test
+  public void hdfsApiNoUsernameProvided() throws IOException, InterruptedException {
+    Map<String, String> properties = new HashMap<String, String>();
+    properties.put("dataworker.defaultFs", hdfsURI);
+
+    expect(context.getProperties()).andReturn(properties).anyTimes();
+    expect(context.getUsername()).andReturn("ambari-qa").anyTimes();
+
+    replay(context);
+
+    // no dataworker.hdfs.username property
+    Assert.assertEquals("ambari-qa", BaseService.getHdfsUsername(context));
+
+    // with dataworker.hdfs.username property
+    properties.put("dataworker.hdfs.username", "luke");
+    Assert.assertEquals("luke", BaseService.getHdfsUsername(context));
+  }
+
   private Response doCreateScript(String title, String path) {
     return ScriptTest.doCreateScript(title, path, scriptService);
   }

contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestUnmanaged.java

@@ -79,7 +79,7 @@ public class ScriptTestUnmanaged extends BasePigTest {
   public void createScriptAutoCreateNoDefaultFS() {
     Map<String, String> properties = new HashMap<String, String>();
     properties.put("dataworker.storagePath", pigStorageFile.toString());
-    properties.put("dataworker.userScriptsPath", "/tmp/.pigscripts");
+    properties.put("dataworker.scripts.path", "/tmp/.pigscripts");
 
     expect(context.getProperties()).andReturn(properties).anyTimes();
     expect(context.getUsername()).andReturn("ambari-qa").anyTimes();