AMBARI-9203. Repository Version Management: Base URL validation (Yurii Shylov via ncole)

ambari-server/src/main/java/org/apache/ambari/server/api/resources/RepositoryResourceDefinition.java

@@ -17,10 +17,16 @@
  */
 package org.apache.ambari.server.api.resources;
 
+import java.util.Collection;
+
 import org.apache.ambari.server.controller.spi.Resource;
 
+import com.google.common.collect.Sets;
+
 public class RepositoryResourceDefinition extends BaseStacksResourceDefinition {
 
+  public static String VALIDATE_ONLY_DIRECTIVE = "validate_only";
+
   public RepositoryResourceDefinition() {
     super(Resource.Type.Repository);
   }
@@ -35,4 +41,9 @@ public class RepositoryResourceDefinition extends BaseStacksResourceDefinition {
     return "repository";
   }
 
+  @Override
+  public Collection<String> getCreateDirectives() {
+    return Sets.newHashSet(VALIDATE_ONLY_DIRECTIVE);
+  }
+
 }

ambari-server/src/main/java/org/apache/ambari/server/api/services/RepositoryService.java

@@ -38,6 +38,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import javax.ws.rs.GET;
+import javax.ws.rs.POST;
 import javax.ws.rs.PUT;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
@@ -69,6 +70,36 @@ public class RepositoryService extends BaseService {
     this.parentKeyProperties = parentKeyProperties;
   }
 
+  /**
+   * Creates repository.
+   * Handles: POST /repositories requests.
+   *
+   * @param body    body
+   * @param headers http headers
+   * @param ui      uri info
+   */
+  @POST
+  @Produces("text/plain")
+  public Response createRepository(String body, @Context HttpHeaders headers, @Context UriInfo ui) {
+    return handleRequest(headers, body, ui, Request.Type.POST, createResource(null));
+  }
+
+  /**
+   * Creates repository.
+   * Handles: POST /repositories requests.
+   *
+   *@param body    body
+   * @param headers http headers
+   * @param repoId  repository id
+   * @param ui      uri info
+   */
+  @POST
+  @Path("{repoId}")
+  @Produces("text/plain")
+  public Response createRepository(String body, @Context HttpHeaders headers, @Context UriInfo ui, @PathParam("repoId") String repoId) {
+    return handleRequest(headers, body, ui, Request.Type.POST, createResource(repoId));
+  }
+
   /**
    * Gets all repositories.
    * Handles: GET /repositories requests.

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java

@@ -376,8 +376,16 @@ public interface AmbariManagementController {
    *
    * @throws AmbariException
    */
-  void updateRespositories(Set<RepositoryRequest> requests) throws AmbariException;
+  public void updateRepositories(Set<RepositoryRequest> requests) throws AmbariException;
 
+  /**
+   * Verifies repositories' base urls.
+   *
+   * @param requests the repositories
+   *
+   * @throws AmbariException if verification of any of urls fails
+   */
+  public void verifyRepositories(Set<RepositoryRequest> requests) throws AmbariException;
 
   /**
    * Get repositories by stack name, version.

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java

@@ -3004,7 +3004,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
   }
 
   @Override
-  public void updateRespositories(Set<RepositoryRequest> requests) throws AmbariException {
+  public void updateRepositories(Set<RepositoryRequest> requests) throws AmbariException {
     for (RepositoryRequest rr : requests) {
       if (null == rr.getStackName() || rr.getStackName().isEmpty()) {
         throw new AmbariException("Stack name must be specified.");
@@ -3023,61 +3023,71 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
       }
 
       if (null != rr.getBaseUrl()) {
-        if (!rr.isVerifyBaseUrl()) {
-          if (rr.getRepositoryVersionId() != null) {
-            throw new AmbariException("Can't directly update repositories in repository_version, update the repository_version instead");
-          }
-          ambariMetaInfo.updateRepoBaseURL(rr.getStackName(),
-              rr.getStackVersion(), rr.getOsType(), rr.getRepoId(),
-              rr.getBaseUrl());
-        } else {
-          URLStreamProvider usp = new URLStreamProvider(REPO_URL_CONNECT_TIMEOUT,
-              REPO_URL_READ_TIMEOUT, null, null, null);
+        if (rr.isVerifyBaseUrl()) {
+          verifyRepository(rr);
+        }
+        if (rr.getRepositoryVersionId() != null) {
+          throw new AmbariException("Can't directly update repositories in repository_version, update the repository_version instead");
+        }
+        ambariMetaInfo.updateRepoBaseURL(rr.getStackName(), rr.getStackVersion(), rr.getOsType(), rr.getRepoId(), rr.getBaseUrl());
+      }
+    }
+  }
 
-          RepositoryInfo repositoryInfo = ambariMetaInfo.getRepository(rr.getStackName(), rr.getStackVersion(), rr.getOsType(), rr.getRepoId());
-          String repoName = repositoryInfo.getRepoName();
+  @Override
+  public void verifyRepositories(Set<RepositoryRequest> requests) throws AmbariException {
+    for (RepositoryRequest request: requests) {
+      if (request.getBaseUrl() == null) {
+        throw new AmbariException("Base url is missing for request " + request);
+      }
+      verifyRepository(request);
+    }
+  }
 
-          boolean bFound = true;
-          String errorMessage = null;
+  /**
+   * Verifies single repository, see {{@link #verifyRepositories(Set)}.
+   *
+   * @param request request
+   * @throws AmbariException if verification fails
+   */
+  private void verifyRepository(RepositoryRequest request) throws AmbariException {
+    URLStreamProvider usp = new URLStreamProvider(REPO_URL_CONNECT_TIMEOUT, REPO_URL_READ_TIMEOUT, null, null, null);
 
-          String[] suffixes = configs.getRepoValidationSuffixes(rr.getOsType());
-          for (String suffix : suffixes) {
-            String formatted_suffix = String.format(suffix, repoName);
-            String spec = rr.getBaseUrl();
+    RepositoryInfo repositoryInfo = ambariMetaInfo.getRepository(request.getStackName(), request.getStackVersion(), request.getOsType(), request.getRepoId());
+    String repoName = repositoryInfo.getRepoName();
 
-            if (spec.charAt(spec.length() - 1) != '/' && formatted_suffix.charAt(0) != '/') {
-              spec = rr.getBaseUrl() + "/" + formatted_suffix;
-            } else if (spec.charAt(spec.length() - 1) == '/' && formatted_suffix.charAt(0) == '/') {
-              spec = rr.getBaseUrl() + formatted_suffix.substring(1);
-            } else {
-              spec = rr.getBaseUrl() + formatted_suffix;
-            }
+    String errorMessage = null;
 
-            try {
-              IOUtils.readLines(usp.readFrom(spec));
-            } catch (IOException ioe) {
-              errorMessage = "Could not access base url . " + rr.getBaseUrl() + " . ";
-              if (LOG.isDebugEnabled()) {
-                errorMessage += ioe;
-              } else {
-                errorMessage += ioe.getMessage();
-              }
-              bFound = false;
-              break;
-            }
-          }
+    String[] suffixes = configs.getRepoValidationSuffixes(request.getOsType());
+    for (String suffix : suffixes) {
+      String formatted_suffix = String.format(suffix, repoName);
+      String spec = request.getBaseUrl();
 
-          if (bFound) {
-            ambariMetaInfo.updateRepoBaseURL(rr.getStackName(),
-                rr.getStackVersion(), rr.getOsType(), rr.getRepoId(),
-                rr.getBaseUrl());
-          } else {
-            LOG.error(errorMessage);
-            throw new IllegalArgumentException(errorMessage);
-          }
+      if (spec.charAt(spec.length() - 1) != '/' && formatted_suffix.charAt(0) != '/') {
+        spec = request.getBaseUrl() + "/" + formatted_suffix;
+      } else if (spec.charAt(spec.length() - 1) == '/' && formatted_suffix.charAt(0) == '/') {
+        spec = request.getBaseUrl() + formatted_suffix.substring(1);
+      } else {
+        spec = request.getBaseUrl() + formatted_suffix;
+      }
+
+      try {
+        IOUtils.readLines(usp.readFrom(spec));
+      } catch (IOException ioe) {
+        errorMessage = "Could not access base url . " + request.getBaseUrl() + " . ";
+        if (LOG.isDebugEnabled()) {
+          errorMessage += ioe;
+        } else {
+          errorMessage += ioe.getMessage();
         }
+        break;
       }
     }
+
+    if (errorMessage != null) {
+      LOG.error(errorMessage);
+      throw new IllegalArgumentException(errorMessage);
+    }
   }
 
   @Override

ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryRequest.java

@@ -69,4 +69,13 @@ public class RepositoryRequest extends OperatingSystemRequest {
     verify = verifyUrl;
   }
 
+  @Override
+  public String toString() {
+    return "RepositoryRequest [repoId=" + repoId + ", baseUrl=" + baseUrl
+        + ", verify=" + verify + ", getOsType()=" + getOsType()
+        + ", getRepositoryVersionId()=" + getRepositoryVersionId()
+        + ", getStackVersion()=" + getStackVersion() + ", getStackName()="
+        + getStackName() + "]";
+  }
+
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java

@@ -27,6 +27,7 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.api.resources.RepositoryResourceDefinition;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.RepositoryRequest;
 import org.apache.ambari.server.controller.RepositoryResponse;
@@ -41,6 +42,7 @@ import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException;
 import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
+import org.apache.commons.lang.BooleanUtils;
 
 public class RepositoryResourceProvider extends AbstractControllerResourceProvider {
 
@@ -115,7 +117,7 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid
     modifyResources(new Command<Void>() {
       @Override
       public Void invoke() throws AmbariException {
-        getManagementController().updateRespositories(requests);
+        getManagementController().updateRepositories(requests);
         return null;
       }
     });
@@ -171,10 +173,29 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid
   }
 
   @Override
-  public RequestStatus createResources(Request request) throws SystemException,
-      UnsupportedPropertyException, ResourceAlreadyExistsException,
-      NoSuchParentResourceException {
-    throw new SystemException("Cannot create repositories.", null);
+  public RequestStatus createResources(Request request) throws SystemException, UnsupportedPropertyException, ResourceAlreadyExistsException, NoSuchParentResourceException {
+    final String validateOnlyProperty = request.getRequestInfoProperties().get(RepositoryResourceDefinition.VALIDATE_ONLY_DIRECTIVE);
+    if (BooleanUtils.toBoolean(validateOnlyProperty)) {
+      final Set<RepositoryRequest> requests = new HashSet<RepositoryRequest>();
+      final Iterator<Map<String,Object>> iterator = request.getProperties().iterator();
+      if (iterator.hasNext()) {
+        for (Map<String, Object> propertyMap : request.getProperties()) {
+          requests.add(getRequest(propertyMap));
+        }
+      }
+      createResources(new Command<Void>() {
+
+        @Override
+        public Void invoke() throws AmbariException {
+          getManagementController().verifyRepositories(requests);
+          return null;
+        }
+
+      });
+      return getRequestStatus(null);
+    } else {
+      throw new SystemException("Cannot create repositories.", null);
+    }
   }
 
   @Override

ambari-server/src/test/java/org/apache/ambari/server/api/resources/RepositoryResourceDefinitionTest.java

@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.api.resources;
+
+import java.util.Collection;
+import java.util.Set;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * RepositoryResourceDefinition tests.
+ */
+public class RepositoryResourceDefinitionTest {
+  @Test
+  public void testGetPluralName() throws Exception {
+    final RepositoryResourceDefinition resourceDefinition = new RepositoryResourceDefinition();
+    Assert.assertEquals("repositories", resourceDefinition.getPluralName());
+  }
+
+  @Test
+  public void testGetSingularName() throws Exception {
+    final RepositoryResourceDefinition resourceDefinition = new RepositoryResourceDefinition();
+    Assert.assertEquals("repository", resourceDefinition.getSingularName());
+  }
+
+  @Test
+  public void testGetSubResourceDefinitions() throws Exception {
+    final RepositoryResourceDefinition resourceDefinition = new RepositoryResourceDefinition();
+    final Set<SubResourceDefinition> subResourceDefinitions = resourceDefinition.getSubResourceDefinitions ();
+    Assert.assertEquals(0, subResourceDefinitions.size());
+  }
+
+  @Test
+  public void testGetCreateDirectives() throws Exception {
+    final RepositoryResourceDefinition resourceDefinition = new RepositoryResourceDefinition();
+    final Collection<String> createDirectives = resourceDefinition.getCreateDirectives();
+    Assert.assertEquals(1, createDirectives.size());
+    Assert.assertEquals(RepositoryResourceDefinition.VALIDATE_ONLY_DIRECTIVE, createDirectives.iterator().next());
+  }
+}

ambari-server/src/test/java/org/apache/ambari/server/api/services/RepositoryServiceTest.java

@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.api.services;
+
+import org.apache.ambari.server.api.resources.ResourceInstance;
+import org.apache.ambari.server.api.services.parsers.RequestBodyParser;
+import org.apache.ambari.server.api.services.serializers.ResultSerializer;
+import org.apache.ambari.server.controller.spi.Resource.Type;
+
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.UriInfo;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Unit tests for RepositoryService.
+ */
+public class RepositoryServiceTest extends BaseServiceTest {
+
+  @Override
+  public List<ServiceTestInvocation> getTestInvocations() throws Exception {
+    List<ServiceTestInvocation> listInvocations = new ArrayList<ServiceTestInvocation>();
+
+    RepositoryService service;
+    Method m;
+    Object[] args;
+
+    //createRepository
+    service = new TestRepositoryService();
+    m = service.getClass().getMethod("createRepository", String.class, HttpHeaders.class, UriInfo.class);
+    args = new Object[] {"body", getHttpHeaders(), getUriInfo()};
+    listInvocations.add(new ServiceTestInvocation(Request.Type.POST, service, m, args, "body"));
+
+    //createRepository
+    service = new TestRepositoryService();
+    m = service.getClass().getMethod("createRepository", String.class, HttpHeaders.class, UriInfo.class, String.class);
+    args = new Object[] {"body", getHttpHeaders(), getUriInfo(), "HDP-2.2"};
+    listInvocations.add(new ServiceTestInvocation(Request.Type.POST, service, m, args, "body"));
+
+    //getRepositories
+    service = new TestRepositoryService();
+    m = service.getClass().getMethod("getRepositories", HttpHeaders.class, UriInfo.class);
+    args = new Object[] {getHttpHeaders(), getUriInfo()};
+    listInvocations.add(new ServiceTestInvocation(Request.Type.GET, service, m, args, null));
+
+    //getRepository
+    service = new TestRepositoryService();
+    m = service.getClass().getMethod("getRepository", HttpHeaders.class, UriInfo.class, String.class);
+    args = new Object[] {getHttpHeaders(), getUriInfo(), "HDP-2.2"};
+    listInvocations.add(new ServiceTestInvocation(Request.Type.GET, service, m, args, null));
+
+    //updateRepository
+    service = new TestRepositoryService();
+    m = service.getClass().getMethod("updateRepository", String.class, HttpHeaders.class, UriInfo.class, String.class);
+    args = new Object[] {"body", getHttpHeaders(), getUriInfo(), "HDP-2.2"};
+    listInvocations.add(new ServiceTestInvocation(Request.Type.PUT, service, m, args, "body"));
+
+    return listInvocations;
+  }
+
+  private class TestRepositoryService extends RepositoryService {
+
+    public TestRepositoryService() {
+      super(new HashMap<Type, String>());
+    }
+
+    @Override
+    protected ResourceInstance createResource(Type type, Map<Type, String> mapIds) {
+      return getTestResource();
+    }
+
+    @Override
+    RequestFactory getRequestFactory() {
+      return getTestRequestFactory();
+    }
+
+    @Override
+    protected RequestBodyParser getBodyParser() {
+      return getTestBodyParser();
+    }
+
+    @Override
+    protected ResultSerializer getResultSerializer() {
+      return getTestResultSerializer();
+    }
+  }
+}

ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java

@@ -8139,7 +8139,7 @@ public class AmbariManagementControllerTest {
 
     // test bad url
     try {
-      controller.updateRespositories(requests);
+      controller.updateRepositories(requests);
       Assert.fail("Expected a bad URL to throw an exception");
     } catch (Exception e) {
       assertNotNull(e);
@@ -8147,7 +8147,7 @@ public class AmbariManagementControllerTest {
     }
     // test bad url, but allow to set anyway
     request.setVerifyBaseUrl(false);
-    controller.updateRespositories(requests);
+    controller.updateRepositories(requests);
     Assert.assertEquals(request.getBaseUrl(), repo.getBaseUrl());
 
     requests.clear();
@@ -8156,7 +8156,7 @@ public class AmbariManagementControllerTest {
     requests.add(request);
     // test bad url
     try {
-      controller.updateRespositories(requests);
+      controller.updateRepositories(requests);
     } catch (Exception e) {
       System.err.println(e.getMessage());
       assertTrue(e.getMessage().contains(IOException.class.getName()));
@@ -8168,7 +8168,7 @@ public class AmbariManagementControllerTest {
     requests.add(request);
     // test bad url
     try {
-      controller.updateRespositories(requests);
+      controller.updateRepositories(requests);
     } catch (Exception e) {
       assertTrue(e.getMessage().contains(MalformedURLException.class.getName()));
     }
@@ -8179,7 +8179,7 @@ public class AmbariManagementControllerTest {
     requests.add(request);
     // test bad url
     try {
-      controller.updateRespositories(requests);
+      controller.updateRepositories(requests);
     } catch (Exception e) {
       String exceptionMsg = e.getMessage();
       assertTrue(exceptionMsg.contains(UnknownHostException.class.getName())
@@ -8191,7 +8191,7 @@ public class AmbariManagementControllerTest {
     request = new RepositoryRequest(STACK_NAME, STACK_VERSION, OS_TYPE, REPO_ID);
     request.setBaseUrl(repo.getDefaultBaseUrl());
     requests.add(request);
-    controller.updateRespositories(requests);
+    controller.updateRepositories(requests);
     Assert.assertEquals(repo.getBaseUrl(), repo.getDefaultBaseUrl());
 
     String baseUrl = repo.getDefaultBaseUrl();
@@ -8203,35 +8203,35 @@ public class AmbariManagementControllerTest {
     backingProperties.setProperty(Configuration.REPO_SUFFIX_KEY_UBUNTU, "/repodata/repomd.xml");
     Assert.assertTrue(baseUrl.endsWith("/") && configuration.getRepoValidationSuffixes("ubuntu12")[0].startsWith("/"));
     request.setBaseUrl(baseUrl);
-    controller.updateRespositories(requests);
+    controller.updateRepositories(requests);
     Assert.assertEquals(baseUrl, repo.getBaseUrl());
 
     // variation #2: url with trailing slash, suffix no preceding slash
     backingProperties.setProperty(Configuration.REPO_SUFFIX_KEY_DEFAULT, "repodata/repomd.xml");
     Assert.assertTrue(baseUrl.endsWith("/") && !configuration.getRepoValidationSuffixes("redhat6")[0].startsWith("/"));
     request.setBaseUrl(baseUrl);
-    controller.updateRespositories(requests);
+    controller.updateRepositories(requests);
     Assert.assertEquals(baseUrl, repo.getBaseUrl());
 
     baseUrl = baseUrl.substring(0, baseUrl.length()-1);
     // variation #3: url with no trailing slash, suffix no prededing slash
     Assert.assertTrue(!baseUrl.endsWith("/") && !configuration.getRepoValidationSuffixes("redhat6")[0].startsWith("/"));
     request.setBaseUrl(baseUrl);
-    controller.updateRespositories(requests);
+    controller.updateRepositories(requests);
     Assert.assertEquals(baseUrl, repo.getBaseUrl());
 
     // variation #4: url with no trailing slash, suffix preceding slash
     backingProperties.setProperty(Configuration.REPO_SUFFIX_KEY_DEFAULT, "/repodata/repomd.xml");
     Assert.assertTrue(!baseUrl.endsWith("/") && configuration.getRepoValidationSuffixes("suse11")[0].startsWith("/"));
     request.setBaseUrl(baseUrl);
-    controller.updateRespositories(requests);
+    controller.updateRepositories(requests);
     Assert.assertEquals(baseUrl, repo.getBaseUrl());
 
     // variation #5: multiple suffix tests
     backingProperties.setProperty(Configuration.REPO_SUFFIX_KEY_UBUNTU, "/foo/bar.xml,/repodata/repomd.xml");
     Assert.assertTrue(configuration.getRepoValidationSuffixes("ubuntu12").length > 1);
     request.setBaseUrl(baseUrl);
-    controller.updateRespositories(requests);
+    controller.updateRepositories(requests);
     Assert.assertEquals(baseUrl, repo.getBaseUrl());
 
   }

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java

@@ -128,7 +128,7 @@ public class RepositoryResourceProviderTest {
 
     // set expectations
     expect(managementController.getRepositories(EasyMock.<Set<RepositoryRequest>>anyObject())).andReturn(allResponse).times(1);
-    managementController.updateRespositories(EasyMock.<Set<RepositoryRequest>>anyObject());
+    managementController.updateRepositories(EasyMock.<Set<RepositoryRequest>>anyObject());
 
     // replay
     replay(managementController);