AMBARI-7345 - Views : Exception from ambari-server startup, extract views

ambari-server/src/main/java/org/apache/ambari/server/view/ViewRegistry.java

@@ -28,7 +28,6 @@ import org.apache.ambari.server.api.resources.ViewExternalSubResourceDefinition;
 import org.apache.ambari.server.api.services.ViewExternalSubResourceService;
 import org.apache.ambari.server.api.services.ViewSubResourceService;
 import org.apache.ambari.server.configuration.Configuration;
-import org.apache.ambari.server.controller.ControllerModule;
 import org.apache.ambari.server.controller.spi.Resource;
 import org.apache.ambari.server.orm.dao.MemberDAO;
 import org.apache.ambari.server.orm.dao.PrivilegeDAO;
@@ -101,6 +100,7 @@ public class ViewRegistry {
    * Constants
    */
   private static final String EXTRACTED_ARCHIVES_DIR = "work";
+  private static final String EXTRACT_COMMAND = "extract";
 
   /**
    * Thread pool
@@ -219,15 +219,31 @@ public class ViewRegistry {
    * Registry main method.
    *
    * @param args  the command line arguments
-   *
-   * @throws Exception if the registry command can not be completed
    */
-  public static void main(String[] args) throws Exception {
+  public static void main(String[] args) {
+
+    if (args.length >= 2) {
+      String archivePath = args[1];
+
+      try {
+        Injector injector = Guice.createInjector();
 
-    Injector injector = Guice.createInjector(new ControllerModule());
-    initInstance(injector.getInstance(ViewRegistry.class));
+        ViewExtractor      extractor      = injector.getInstance(ViewExtractor.class);
+        ViewArchiveUtility archiveUtility = injector.getInstance(ViewArchiveUtility.class);
+        Configuration      configuration  = injector.getInstance(Configuration.class);
 
-    singleton.readViewArchives(true, false);
+        if (args[0].equals(EXTRACT_COMMAND)) {
+          if (extractViewArchive(archivePath, extractor, archiveUtility, configuration, true)) {
+            System.exit(0);
+          }
+        }
+      } catch (Exception e) {
+        String msg = "Caught exception extracting view archive " + archivePath + ".";
+        LOG.error(msg, e);
+        System.exit(2);
+      }
+    }
+    System.exit(1);
   }
 
   /**
@@ -1229,6 +1245,43 @@ public class ViewRegistry {
     }
   }
 
+  // extract the view archive for the given path.
+  protected static boolean extractViewArchive(String archivePath,
+                                            ViewExtractor extractor,
+                                            ViewArchiveUtility archiveUtility,
+                                            Configuration configuration,
+                                            boolean systemOnly ) throws Exception {
+
+    File viewDir = configuration.getViewsDir();
+
+    String extractedArchivesPath = viewDir.getAbsolutePath() +
+        File.separator + EXTRACTED_ARCHIVES_DIR;
+
+    if (extractor.ensureExtractedArchiveDirectory(extractedArchivesPath)) {
+
+      File archiveFile = archiveUtility.getFile(archivePath);
+
+      ViewConfig viewConfig = archiveUtility.getViewConfigFromArchive(archiveFile);
+
+      String commonName = viewConfig.getName();
+      String version    = viewConfig.getVersion();
+      String viewName   = ViewEntity.getViewName(commonName, version);
+
+      String extractedArchiveDirPath = extractedArchivesPath + File.separator + viewName;
+      File   extractedArchiveDirFile = archiveUtility.getFile(extractedArchiveDirPath);
+
+      if (!extractedArchiveDirFile.exists()) {
+        ViewEntity viewDefinition = new ViewEntity(viewConfig, configuration, extractedArchiveDirPath);
+
+        if (!systemOnly || viewDefinition.isSystem()) {
+          extractor.extractViewArchive(viewDefinition, archiveFile, extractedArchiveDirFile);
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
   // set the status of the given view.
   private void setViewStatus(ViewEntity viewDefinition, ViewEntity.ViewStatus status, String statusDetail) {
     viewDefinition.setStatus(status);

ambari-server/src/main/python/ambari-server.py

@@ -189,7 +189,7 @@ STACK_UPGRADE_HELPER_CMD = "{0}" + os.sep + "bin" + os.sep + "java -cp {1}" +\
 
 VIEW_EXTRACT_CMD = "{0}" + os.sep + "bin" + os.sep + "java -cp {1}" +\
                           os.pathsep + "{2} " +\
-                          "org.apache.ambari.server.view.ViewRegistry " +\
+                          "org.apache.ambari.server.view.ViewRegistry extract {3} " +\
                           "> " + SERVER_OUT_FILE + " 2>&1"
 
 
@@ -283,6 +283,9 @@ JDBC_HOSTNAME_PROPERTY = "server.jdbc.hostname"
 JDBC_PORT_PROPERTY = "server.jdbc.port"
 JDBC_POSTGRES_SCHEMA_PROPERTY = "server.jdbc.postgres.schema"   # Only for postgres, defaults to same value as DB name
 
+VIEWS_DIR_PROPERTY = "views.dir"
+DEFAULT_VIEWS_DIR = "/var/lib/ambari-server/resources/views"
+
 JDBC_USER_NAME_PROPERTY = "server.jdbc.user.name"
 JDBC_PASSWORD_PROPERTY = "server.jdbc.user.passwd"
 JDBC_PASSWORD_FILENAME = "password.dat"
@@ -1225,6 +1228,7 @@ def prompt_db_properties(args):
 
 # extract the system views
 def extract_views():
+
   jdk_path = find_jdk()
   if jdk_path is None:
     print_error_msg("No JDK found, please run the \"setup\" "
@@ -1232,12 +1236,36 @@ def extract_views():
                     "JDK manually to " + JDK_INSTALL_DIR)
     return 1
 
-  command = VIEW_EXTRACT_CMD.format(jdk_path, get_conf_dir(),
-    get_ambari_classpath())
-  (retcode, stdout, stderr) = run_os_command(command)
-  print_info_msg("Return code from view extraction: " +
-                 str(retcode))
-  return retcode
+  properties = get_ambari_properties()
+  if properties == -1:
+    print_error_msg("Error getting ambari properties")
+    return -1
+
+  if not VIEWS_DIR_PROPERTY in properties.keys():
+    vdir = DEFAULT_VIEWS_DIR
+  else:
+    vdir = properties.get_property(VIEWS_DIR_PROPERTY)
+
+  files = [f for f in os.listdir(vdir) if os.path.isfile(os.path.join(vdir,f))]
+  for f in files:
+
+    command = VIEW_EXTRACT_CMD.format(jdk_path, get_conf_dir(),
+      get_ambari_classpath(), os.path.join(vdir,f))
+
+    retcode, stdout, stderr = run_os_command(command)
+    if retcode == 0:
+      sys.stdout.write(f + "\n")
+    elif retcode == 2:
+      sys.stdout.write("Error extracting " + f + "\n")
+    else:
+      sys.stdout.write(".")
+      sys.stdout.flush()
+
+    print_info_msg("Return code from extraction of view archive " + f + ": " +
+                   str(retcode))
+
+  sys.stdout.write("\n")
+  return 0
 
 # Store set of properties for remote database connection
 def store_remote_properties(args):

ambari-server/src/test/java/org/apache/ambari/server/view/ViewRegistryTest.java

@@ -18,8 +18,10 @@
 
 package org.apache.ambari.server.view;
 
+import static org.easymock.EasyMock.capture;
 import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.eq;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.reset;
@@ -81,6 +83,7 @@ import org.apache.ambari.server.view.events.EventImplTest;
 import org.apache.ambari.view.ViewDefinition;
 import org.apache.ambari.view.events.Event;
 import org.apache.ambari.view.events.Listener;
+import org.easymock.Capture;
 import org.easymock.EasyMock;
 import org.junit.Assert;
 import org.junit.Before;
@@ -926,6 +929,93 @@ public class ViewRegistryTest {
     verify(securityHelper, viewEntity, configuration);
   }
 
+  @Test
+  public void testExtractViewArchive() throws Exception {
+
+    File viewDir = createNiceMock(File.class);
+    File extractedArchiveDir = createNiceMock(File.class);
+    File viewArchive = createNiceMock(File.class);
+    File archiveDir = createNiceMock(File.class);
+    File entryFile  = createNiceMock(File.class);
+    File classesDir = createNiceMock(File.class);
+    File libDir = createNiceMock(File.class);
+    File fileEntry = createNiceMock(File.class);
+
+    JarFile viewJarFile = createNiceMock(JarFile.class);
+    Enumeration<JarEntry> enumeration = createMock(Enumeration.class);
+    JarEntry jarEntry = createNiceMock(JarEntry.class);
+    InputStream is = createMock(InputStream.class);
+    FileOutputStream fos = createMock(FileOutputStream.class);
+    ViewExtractor viewExtractor = createMock(ViewExtractor.class);
+
+    ResourceTypeEntity resourceTypeEntity = new ResourceTypeEntity();
+    resourceTypeEntity.setId(10);
+    resourceTypeEntity.setName("MY_VIEW{1.0.0}");
+
+    ViewEntity viewDefinition = ViewEntityTest.getViewEntity();
+    viewDefinition.setResourceType(resourceTypeEntity);
+
+    Set<ViewInstanceEntity> viewInstanceEntities = ViewInstanceEntityTest.getViewInstanceEntities(viewDefinition);
+    viewDefinition.setInstances(viewInstanceEntities);
+
+    Map<File, ViewConfig> viewConfigs =
+        Collections.singletonMap(viewArchive, viewDefinition.getConfiguration());
+
+    long resourceId = 99L;
+    for (ViewInstanceEntity viewInstanceEntity : viewInstanceEntities) {
+      ResourceEntity resourceEntity = new ResourceEntity();
+      resourceEntity.setId(resourceId);
+      resourceEntity.setResourceType(resourceTypeEntity);
+      viewInstanceEntity.setResource(resourceEntity);
+    }
+
+    Map<String, File> files = new HashMap<String, File>();
+
+    files.put("/var/lib/ambari-server/resources/views/my_view-1.0.0.jar", viewArchive);
+    files.put("/var/lib/ambari-server/resources/views/work", extractedArchiveDir);
+    files.put("/var/lib/ambari-server/resources/views/work/MY_VIEW{1.0.0}", archiveDir);
+    files.put("/var/lib/ambari-server/resources/views/work/MY_VIEW{1.0.0}/view.xml", entryFile);
+    files.put("/var/lib/ambari-server/resources/views/work/MY_VIEW{1.0.0}/WEB-INF/classes", classesDir);
+    files.put("/var/lib/ambari-server/resources/views/work/MY_VIEW{1.0.0}/WEB-INF/lib", libDir);
+
+    Map<File, FileOutputStream> outputStreams = new HashMap<File, FileOutputStream>();
+    outputStreams.put(entryFile, fos);
+
+    Map<File, JarFile> jarFiles = new HashMap<File, JarFile>();
+    jarFiles.put(viewArchive, viewJarFile);
+
+    // set expectations
+    expect(configuration.getViewsDir()).andReturn(viewDir);
+    expect(viewDir.getAbsolutePath()).andReturn("/var/lib/ambari-server/resources/views");
+
+    expect(configuration.getViewExtractionThreadPoolCoreSize()).andReturn(2).anyTimes();
+    expect(configuration.getViewExtractionThreadPoolMaxSize()).andReturn(3).anyTimes();
+    expect(configuration.getViewExtractionThreadPoolTimeout()).andReturn(10000L).anyTimes();
+
+    expect(viewArchive.getAbsolutePath()).andReturn("/var/lib/ambari-server/resources/views/work/MY_VIEW{1.0.0}").anyTimes();
+
+    expect(archiveDir.exists()).andReturn(false);
+    expect(archiveDir.getAbsolutePath()).andReturn(
+        "/var/lib/ambari-server/resources/views/work/MY_VIEW{1.0.0}").anyTimes();
+
+    Capture<ViewEntity> viewEntityCapture = new Capture<ViewEntity>();
+    expect(viewExtractor.ensureExtractedArchiveDirectory("/var/lib/ambari-server/resources/views/work")).andReturn(true);
+    expect(viewExtractor.extractViewArchive(capture(viewEntityCapture), eq(viewArchive), eq(archiveDir))).andReturn(null);
+
+    // replay mocks
+    replay(configuration, viewDir, extractedArchiveDir, viewArchive, archiveDir, entryFile, classesDir,
+        libDir, fileEntry, viewJarFile, enumeration, jarEntry, is, fos, viewExtractor, resourceDAO, viewDAO, viewInstanceDAO);
+
+    TestViewArchiveUtility archiveUtility = new TestViewArchiveUtility(viewConfigs, files, outputStreams, jarFiles);
+
+    Assert.assertTrue(ViewRegistry.extractViewArchive("/var/lib/ambari-server/resources/views/my_view-1.0.0.jar",
+        viewExtractor, archiveUtility, configuration, true));
+
+    // verify mocks
+    verify(configuration, viewDir, extractedArchiveDir, viewArchive, archiveDir, entryFile, classesDir,
+        libDir, fileEntry, viewJarFile, enumeration, jarEntry, is, fos, viewExtractor, resourceDAO, viewDAO, viewInstanceDAO);
+  }
+
   public static class TestViewArchiveUtility extends ViewArchiveUtility {
     private final Map<File, ViewConfig> viewConfigs;
     private final Map<String, File> files;

ambari-server/src/test/java/org/apache/ambari/server/view/configuration/ViewConfigTest.java

@@ -47,6 +47,7 @@ public class ViewConfigTest {
       "    <label>My View!</label>\n" +
       "    <description>Description</description>" +
       "    <version>1.0.0</version>\n" +
+      "    <system>true</system>\n" +
       "    <icon64>/this/is/the/icon/url/icon64.png</icon64>\n" +
       "    <icon>/this/is/the/icon/url/icon.png</icon>\n" +
       "    <masker-class>org.apache.ambari.server.view.DefaultMasker</masker-class>" +