AMBARI-2112. Agent should not cache repo info to avoid situations of incorrect repo. (Dmitry Sen via swagle)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/trunk@1493187 13f79535-47bb-0310-9956-ffa450edef68

ambari-agent/src/test/python/TestPuppetExecutor.py

@@ -20,6 +20,7 @@ limitations under the License.
 
 from unittest import TestCase
 from PuppetExecutor import PuppetExecutor
+from RepoInstaller import RepoInstaller
 from Grep import Grep
 from pprint import pformat
 import socket, threading, tempfile
@@ -67,6 +68,39 @@ class TestPuppetExecutor(TestCase):
     self.assertFalse(puppetInstance.reposInstalled)
     os.unlink(tmpdir + os.sep + 'site-' + str(parsedJson["taskId"]) + '.pp')
 
+  @patch.object(RepoInstaller, 'generate_repo_manifests')
+  @patch.object(PuppetExecutor, 'runPuppetFile')
+  def test_overwrite_repos(self, runPuppetFileMock, generateRepoManifestMock):
+    tmpdir = AmbariConfig().getConfig().get("stack", "installprefix")
+    puppetInstance = PuppetExecutor("/tmp", "/x", "/y", tmpdir, AmbariConfig().getConfig())
+    jsonFile = open('../../main/python/ambari_agent/test.json', 'r')
+    jsonStr = jsonFile.read()
+    parsedJson = json.loads(jsonStr)
+    parsedJson["taskId"] = 77
+    def side_effect(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile):
+      result["exitcode"] = 0
+    runPuppetFileMock.side_effect = side_effect
+
+    #If ambari-agent has been just started and no any commands were executed by
+    # PuppetExecutor.runCommand, then no repo files were updated by
+    # RepoInstaller.generate_repo_manifests
+    self.assertEquals(0, generateRepoManifestMock.call_count)
+    self.assertFalse(puppetInstance.reposInstalled)
+
+    # After executing of the first command, RepoInstaller.generate_repo_manifests
+    # generates a .pp file for updating repo files
+    puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt')
+    self.assertTrue(puppetInstance.reposInstalled)
+    self.assertEquals(1, generateRepoManifestMock.call_count)
+
+    # After executing of the next commands, repo manifest aren't generated again
+    puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt')
+    self.assertTrue(puppetInstance.reposInstalled)
+    self.assertEquals(1, generateRepoManifestMock.call_count)
+    puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt')
+    self.assertTrue(puppetInstance.reposInstalled)
+    self.assertEquals(1, generateRepoManifestMock.call_count)
+
   @patch("os.path.exists")
   def test_configure_environ(self, osPathExistsMock):
     config = AmbariConfig().getConfig()

ambari-server/sbin/ambari-server

@@ -86,6 +86,10 @@ case "$1" in
         echo -e "Upgrading stack of ambari-server"
         $PYTHON /usr/sbin/ambari-server.py $@
         ;;
+  update-metainfo)
+        echo -e "Updating ambari-server meta information"
+        $PYTHON /usr/sbin/ambari-server.py $@
+        ;;
   setup)
         initdb_res=`/sbin/service postgresql initdb 2>&1 > /dev/null`
         initdb_res=`/sbin/service postgresql start 2>&1 > /dev/null`

ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java

@@ -118,6 +118,7 @@ public class AmbariMetaInfo {
    */
   @Inject
   public void init() throws Exception {
+    stacksResult = new ArrayList<StackInfo>();
     readServerVersion();
     getConfigurationInformation(stackRoot);
   }

ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java

@@ -22,10 +22,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
+import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.Response;
@@ -45,7 +42,15 @@ public class StacksService extends BaseService {
   public Response getStacks(@Context HttpHeaders headers, @Context UriInfo ui) {
 
     return handleRequest(headers, null, ui, Request.Type.GET,
-        createStackResource(null));
+      createStackResource(null));
+  }
+
+  @PUT
+  @Produces("text/plain")
+  public Response updateStacks(@Context HttpHeaders headers, @Context UriInfo ui) {
+
+    return handleRequest(headers, null, ui, Request.Type.PUT,
+      createStackResource(null));
   }
 
   @GET

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java

@@ -423,7 +423,14 @@ public interface AmbariManagementController {
    * @throws  AmbariException if the resources cannot be read
    */
   public Set<StackResponse> getStacks(Set<StackRequest> requests) throws AmbariException;
-  
+
+  /**
+   * Update stacks from the files at stackRoot.
+   *
+   * @return a track action response
+   * @throws AmbariException if
+   */
+  public RequestStatusResponse updateStacks() throws AmbariException;
   
   /**
    * Get supported stacks versions.

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java

@@ -20,16 +20,8 @@ package org.apache.ambari.server.controller;
 
 import java.io.File;
 import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.ClusterNotFoundException;
@@ -4227,6 +4219,19 @@ public class AmbariManagementControllerImpl implements
     return response;
   }
 
+  @Override
+  public synchronized RequestStatusResponse updateStacks() throws AmbariException {
+
+    try {
+      ambariMetaInfo.init();
+    } catch (Exception e) {
+      throw new AmbariException(
+        "Ambari metainormation can't be read from the stack root directory");
+    }
+
+    return null;
+  }
+
   @Override
   public Set<RepositoryResponse> getRepositories(Set<RepositoryRequest> requests)
       throws AmbariException {

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackResourceProvider.java

@@ -18,24 +18,15 @@
 
 package org.apache.ambari.server.controller.internal;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.RequestStatusResponse;
 import org.apache.ambari.server.controller.StackRequest;
 import org.apache.ambari.server.controller.StackResponse;
-import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
-import org.apache.ambari.server.controller.spi.NoSuchResourceException;
-import org.apache.ambari.server.controller.spi.Predicate;
-import org.apache.ambari.server.controller.spi.Request;
-import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.spi.*;
 import org.apache.ambari.server.controller.spi.Resource.Type;
-import org.apache.ambari.server.controller.spi.SystemException;
-import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
 import org.apache.ambari.server.controller.utilities.PredicateHelper;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
 
@@ -87,6 +78,25 @@ public class StackResourceProvider extends ReadOnlyResourceProvider {
     return resources;
   }
 
+  @Override
+  public RequestStatus updateResources(Request request, Predicate predicate)
+    throws SystemException, UnsupportedPropertyException,
+    NoSuchResourceException, NoSuchParentResourceException {
+
+    RequestStatusResponse response = modifyResources(
+      new Command<RequestStatusResponse>() {
+
+      @Override
+      public RequestStatusResponse invoke() throws AmbariException {
+        return getManagementController().updateStacks();
+      }
+    });
+
+    notifyUpdate(Type.Stack, request, predicate);
+
+    return getRequestStatus(response);
+  }
+
   private StackRequest getRequest(Map<String, Object> properties) {
     return new StackRequest((String) properties.get(STACK_NAME_PROPERTY_ID));
   }

ambari-server/src/main/python/ambari-server.py

@@ -35,6 +35,7 @@ import fileinput
 import urllib2
 import time
 import getpass
+import socket
 import datetime
 import socket
 import tempfile
@@ -51,6 +52,7 @@ STOP_ACTION = "stop"
 RESET_ACTION = "reset"
 UPGRADE_ACTION = "upgrade"
 UPGRADE_STACK_ACTION = "upgradestack"
+UPDATE_METAINFO_ACTION = "update-metainfo"
 STATUS_ACTION = "status"
 LDAP_SETUP_ACTION = "setupldap"
 RESET_MASTER_KEY_ACTION = "resetmasterkey"
@@ -162,6 +164,7 @@ SETUP_DB_CMD = ['su', '-', 'postgres',
         '--command=psql -f {0} -v username=\'"{1}"\' -v password="\'{2}\'"']
 UPGRADE_STACK_CMD = ['su', 'postgres',
         '--command=psql -f {0} -v stack_name="\'{1}\'"  -v stack_version="\'{2}\'"']
+UPDATE_METAINFO_CMD = 'curl -X PUT "http://{0}:{1}/api/v1/stacks2" -u "{2}":"{3}"'
 PG_ST_CMD = "/sbin/service postgresql status"
 PG_INITDB_CMD = "/sbin/service postgresql initdb"
 PG_START_CMD = "/sbin/service postgresql start"
@@ -184,6 +187,9 @@ JDBC_PASSWORD_PROPERTY = "server.jdbc.user.passwd"
 JDBC_PASSWORD_FILENAME = "password.dat"
 JDBC_RCA_PASSWORD_FILENAME = "rca_password.dat"
 
+CLIENT_API_PORT_PROPERTY = "client.api.port"
+CLIENT_API_PORT = "8080"
+
 PERSISTENCE_TYPE_PROPERTY = "server.persistence.type"
 JDBC_DRIVER_PROPERTY = "server.jdbc.driver"
 JDBC_URL_PROPERTY = "server.jdbc.url"
@@ -415,6 +421,65 @@ def run_os_command(cmd):
   (stdoutdata, stderrdata) = process.communicate()
   return process.returncode, stdoutdata, stderrdata
 
+#
+# Updates metainfo information from stack root. Re-cache information from
+# repoinfo.xml , metainfo.xml files , etc.
+#
+def update_metainfo(args):
+  configure_update_metainfo_args(args)
+
+  hostname = args.hostname
+  port = args.port
+  username = args.username
+  password = args.password
+
+  command = UPDATE_METAINFO_CMD
+  command = command.format(hostname, port, username, password)
+  retcode, outdata, errdata = run_os_command(command)
+
+  if outdata.find("Bad credentials") > 0:
+    print 'Incorrect credential provided. Please try again.'
+
+  if not retcode == 0:
+    print errdata
+  return retcode
+
+def configure_update_metainfo_args(args):
+  conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
+  properties = Properties()
+
+  try:
+    properties.load(open(conf_file))
+  except Exception, e:
+    print 'Could not read ambari config file "%s": %s' % (conf_file, e)
+    return -1
+
+
+  default_username = "admin"
+
+  username_prompt = 'Username [' + default_username + ']: '
+  password_prompt = 'Password: '
+  input_pattern = "^[a-zA-Z_][a-zA-Z0-9_\-]*$"
+
+  hostname = socket.gethostname()
+  port = properties[CLIENT_API_PORT_PROPERTY]
+
+  if not port:
+    port = CLIENT_API_PORT
+
+  input_descr = "Invalid characters in received. Start with _ or alpha "\
+                  "followed by alphanumeric or _ or - characters"
+
+  print 'Full authentication is required to access the Ambari API'
+  username = get_validated_string_input(username_prompt, default_username,
+      input_pattern, input_descr, False)
+  password = get_validated_string_input(password_prompt, "", input_pattern,
+      input_descr, True)
+
+  args.hostname = hostname
+  args.port = port
+  args.username = username
+  args.password = password
 
 #
 # Checks SELinux
@@ -2543,6 +2608,8 @@ def main():
       setup_ldap()
     elif action == RESET_MASTER_KEY_ACTION:
       reset_master_key()
+    elif action == UPDATE_METAINFO_ACTION:
+      update_metainfo(options)
     else:
       parser.error("Invalid action")
   except FatalException as e:

ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java

@@ -18,10 +18,6 @@
 
 package org.apache.ambari.server.controller;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -70,6 +66,7 @@ import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.persist.PersistService;
 
+import static org.junit.Assert.*;
 
 public class AmbariManagementControllerTest {
 
@@ -92,6 +89,7 @@ public class AmbariManagementControllerTest {
   private static final int OS_CNT = 2;
 
   private static final String NON_EXT_VALUE = "XXX";
+  private static final String INCORRECT_BASE_URL = "http://incorrect.url";
 
   private static final String COMPONENT_NAME = "NAMENODE";
   
@@ -6480,5 +6478,25 @@ public class AmbariManagementControllerTest {
         State.INSTALLED.toString());
     controller.updateServices(Collections.singleton(req), Collections.<String, String>emptyMap(), true, false);
   }
+
+  @Test
+  public void testUpdateStacks() throws Exception {
+
+    StackInfo stackInfo = ambariMetaInfo.getStackInfo(STACK_NAME, STACK_VERSION);
+
+    for (RepositoryInfo repositoryInfo: stackInfo.getRepositories()) {
+      assertFalse(INCORRECT_BASE_URL.equals(repositoryInfo.getBaseUrl()));
+      repositoryInfo.setBaseUrl(INCORRECT_BASE_URL);
+      assertTrue(INCORRECT_BASE_URL.equals(repositoryInfo.getBaseUrl()));
+    }
+
+    controller.updateStacks();
+
+    stackInfo = ambariMetaInfo.getStackInfo(STACK_NAME, STACK_VERSION);
+
+    for (RepositoryInfo repositoryInfo: stackInfo.getRepositories()) {
+      assertFalse(INCORRECT_BASE_URL.equals(repositoryInfo.getBaseUrl()));
+    }
+  }
   
 }