AMBARI-16021. Update LogSearch integration to use configuration to obtain credential. (rnettleton)

ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperFactoryImpl.java

@@ -86,7 +86,7 @@ public class LoggingRequestHelperFactoryImpl implements LoggingRequestHelperFact
         final String logSearchPortNumber =
           logSearchSiteConfig.getProperties().get(LOGSEARCH_UI_PORT_PROPERTY_NAME);
 
-        return new LoggingRequestHelperImpl(logSearchHostName, logSearchPortNumber, ambariManagementController.getCredentialStoreService(), clusterName);
+        return new LoggingRequestHelperImpl(logSearchHostName, logSearchPortNumber, ambariManagementController.getCredentialStoreService(), cluster);
       }
     } catch (AmbariException ambariException) {
       LOG.error("Error occurred while trying to obtain the cluster, cluster name = " + clusterName, ambariException);

ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java

@@ -23,6 +23,8 @@ import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.security.credential.Credential;
 import org.apache.ambari.server.security.credential.PrincipalKeyCredential;
 import org.apache.ambari.server.security.encryption.CredentialStoreService;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Config;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.log4j.Logger;
@@ -53,15 +55,17 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
 
   private static Logger LOG = Logger.getLogger(LoggingRequestHelperImpl.class);
 
-  public static String LOGSEARCH_QUERY_PATH = "/service/dashboard/solr/logs_search";
+  private static final String LOGSEARCH_ADMIN_PROPERTIES_CONFIG_TYPE_NAME = "logsearch-admin-properties";
 
-  public static String LOGSEARCH_GET_LOG_LEVELS_PATH = "/service/dashboard/getLogLevelCounts";
+  private static final String LOGSEARCH_ADMIN_USERNAME_PROPERTY_NAME = "logsearch_admin_username";
 
-  private static String DEFAULT_LOGSEARCH_USER = "admin";
+  private static final String LOGSEARCH_ADMIN_PASSWORD_PROPERTY_NAME = "logsearch_admin_password";
 
-  private static String DEFAULT_LOGSEARCH_PWD = "admin";
+  private static final String LOGSEARCH_QUERY_PATH = "/service/dashboard/solr/logs_search";
 
-  public static final String LOGSEARCH_ADMIN_CREDENTIAL_NAME = "logsearch.admin.credential";
+  private static final String LOGSEARCH_GET_LOG_LEVELS_PATH = "/service/dashboard/getLogLevelCounts";
+
+  private static final String LOGSEARCH_ADMIN_CREDENTIAL_NAME = "logsearch.admin.credential";
 
   private final String hostName;
 
@@ -69,13 +73,20 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
 
   private final CredentialStoreService credentialStoreService;
 
-  private final String clusterName;
+  private final Cluster cluster;
+
+  private final NetworkConnection networkConnection;
+
+  public LoggingRequestHelperImpl(String hostName, String portNumber, CredentialStoreService credentialStoreService, Cluster cluster) {
+    this(hostName, portNumber, credentialStoreService, cluster, new DefaultNetworkConnection());
+  }
 
-  public LoggingRequestHelperImpl(String hostName, String portNumber, CredentialStoreService credentialStoreService, String clusterName) {
+  protected LoggingRequestHelperImpl(String hostName, String portNumber, CredentialStoreService credentialStoreService, Cluster cluster, NetworkConnection networkConnection) {
     this.hostName = hostName;
     this.portNumber = portNumber;
     this.credentialStoreService = credentialStoreService;
-    this.clusterName = clusterName;
+    this.cluster = cluster;
+    this.networkConnection = networkConnection;
   }
 
   public LogQueryResponse sendQueryRequest(Map<String, String> queryParameters) {
@@ -89,7 +100,7 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
 
       setupCredentials(httpURLConnection);
 
-      StringBuffer buffer = readQueryResponseFromServer(httpURLConnection);
+      StringBuffer buffer = networkConnection.readQueryResponseFromServer(httpURLConnection);
 
       // setup a reader for the JSON response
       StringReader stringReader =
@@ -108,21 +119,55 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
   }
 
   private void setupCredentials(HttpURLConnection httpURLConnection) {
-    PrincipalKeyCredential principalKeyCredential =
-      getLogSearchCredentials();
-
-    // determine the credential to use for connecting to LogSearch
-    if (principalKeyCredential != null) {
-      // setup credential stored in credential service
-      LOG.debug("Credential found in CredentialStore, will be used to connect to LogSearch");
-      setupBasicAuthentication(httpURLConnection, createEncodedCredentials(principalKeyCredential));
+    final String logSearchAdminUser =
+      getLogSearchAdminUser();
+    final String logSearchAdminPassword =
+      getLogSearchAdminPassword();
+
+    // first attempt to use the LogSearch admin configuration to
+    // obtain the LogSearch server credential
+    if ((logSearchAdminUser != null) && (logSearchAdminPassword != null)) {
+      LOG.debug("Credential found in config, will be used to connect to LogSearch");
+      networkConnection.setupBasicAuthentication(httpURLConnection, createEncodedCredentials(logSearchAdminUser, logSearchAdminPassword));
     } else {
-      // fall back to hard-coded credential for now
-      LOG.debug("No credential found in CredentialStore, defaulting to fall-back credential for now");
-      setupBasicAuthentication(httpURLConnection, createDefaultEncodedCredentials());
+      // if no credential found in config, attempt to locate the credential using
+      // the Ambari CredentialStoreService
+      PrincipalKeyCredential principalKeyCredential =
+        getLogSearchCredentials();
+
+      // determine the credential to use for connecting to LogSearch
+      if (principalKeyCredential != null) {
+        // setup credential stored in credential service
+        LOG.debug("Credential found in CredentialStore, will be used to connect to LogSearch");
+        networkConnection.setupBasicAuthentication(httpURLConnection, createEncodedCredentials(principalKeyCredential));
+      } else {
+        LOG.debug("No LogSearch credential could be found, this is probably an error in configuration");
+      }
     }
   }
 
+  private String getLogSearchAdminUser() {
+    Config logSearchAdminConfig =
+      cluster.getDesiredConfigByType(LOGSEARCH_ADMIN_PROPERTIES_CONFIG_TYPE_NAME);
+
+    if (logSearchAdminConfig != null) {
+      return logSearchAdminConfig.getProperties().get(LOGSEARCH_ADMIN_USERNAME_PROPERTY_NAME);
+    }
+
+    return null;
+  }
+
+  private String getLogSearchAdminPassword() {
+    Config logSearchAdminConfig =
+      cluster.getDesiredConfigByType(LOGSEARCH_ADMIN_PROPERTIES_CONFIG_TYPE_NAME);
+
+    if (logSearchAdminConfig != null) {
+      return logSearchAdminConfig.getProperties().get(LOGSEARCH_ADMIN_PASSWORD_PROPERTY_NAME);
+    }
+
+    return null;
+  }
+
   public Set<String> sendGetLogFileNamesRequest(String componentName, String hostName) {
     Map<String, String> queryParameters =
       new HashMap<String, String>();
@@ -161,7 +206,7 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
 
       setupCredentials(httpURLConnection);
 
-      StringBuffer buffer = readQueryResponseFromServer(httpURLConnection);
+      StringBuffer buffer = networkConnection.readQueryResponseFromServer(httpURLConnection);
 
       // setup a reader for the JSON response
       StringReader stringReader =
@@ -185,9 +230,6 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
     return mapper.reader(type);
   }
 
-
-
-
   private URI createLogSearchQueryURI(String scheme, Map<String, String> queryParameters) throws URISyntaxException {
     URIBuilder uriBuilder = createBasicURI(scheme);
     uriBuilder.setPath(LOGSEARCH_QUERY_PATH);
@@ -238,36 +280,10 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
     return mapper;
   }
 
-  private StringBuffer readQueryResponseFromServer(HttpURLConnection httpURLConnection) throws IOException {
-    InputStream resultStream = null;
-    try {
-      // read in the response from LogSearch
-      resultStream = httpURLConnection.getInputStream();
-      BufferedReader reader = new BufferedReader(new InputStreamReader(resultStream));
-      LOG.debug("Response code from LogSearch Service is = " + httpURLConnection.getResponseCode());
-
-      String line = reader.readLine();
-      StringBuffer buffer = new StringBuffer();
-      while (line != null) {
-        buffer.append(line);
-        line = reader.readLine();
-      }
-
-      LOG.debug("Sucessfully retrieved response from server, response = " + buffer);
-
-      return buffer;
-    } finally {
-      // make sure to close the stream after request is completed
-      if (resultStream != null) {
-        resultStream.close();
-      }
-    }
-  }
-
   private PrincipalKeyCredential getLogSearchCredentials() {
     try {
       Credential credential =
-        credentialStoreService.getCredential(clusterName, LOGSEARCH_ADMIN_CREDENTIAL_NAME);
+        credentialStoreService.getCredential(cluster.getClusterName(), LOGSEARCH_ADMIN_CREDENTIAL_NAME);
       if ((credential != null)  && (credential instanceof PrincipalKeyCredential)) {
         return (PrincipalKeyCredential)credential;
       }
@@ -281,19 +297,9 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
       LOG.error("Error encountered while trying to obtain LogSearch admin credentials.", ambariException);
     }
 
-
     return null;
   }
 
-  private static void setupBasicAuthentication(HttpURLConnection httpURLConnection, String encodedCredentials) {
-    httpURLConnection.setRequestProperty("Authorization", "Basic " + encodedCredentials);
-  }
-
-  // might need to remove this once the credential integration is in place
-  private static String createDefaultEncodedCredentials() {
-    return createEncodedCredentials(DEFAULT_LOGSEARCH_USER, DEFAULT_LOGSEARCH_PWD);
-  }
-
   private static String createEncodedCredentials(PrincipalKeyCredential principalKeyCredential) {
     return createEncodedCredentials(principalKeyCredential.getPrincipal(), new String(principalKeyCredential.getKey()));
   }
@@ -302,5 +308,58 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
     return Base64.encodeBase64String((userName + ":" + password).getBytes());
   }
 
+  /**
+   * Interface used to abstract out the network access needed to
+   * connect to the LogSearch Server.
+   *
+   * This abstraction is useful for unit testing this class, and simulating
+   * different output and error conditions.
+   */
+  interface NetworkConnection {
+    StringBuffer readQueryResponseFromServer(HttpURLConnection httpURLConnection) throws IOException;
+
+    void setupBasicAuthentication(HttpURLConnection httpURLConnection, String encodedCredentials);
+  }
+
+  /**
+   * The default implementation of NetworkConnection, that reads
+   * the InputStream associated with the HttpURL connection passed in.
+   */
+  private static class DefaultNetworkConnection implements NetworkConnection {
+    @Override
+    public StringBuffer readQueryResponseFromServer(HttpURLConnection httpURLConnection) throws IOException {
+      InputStream resultStream = null;
+      try {
+        // read in the response from LogSearch
+        resultStream = httpURLConnection.getInputStream();
+        BufferedReader reader = new BufferedReader(new InputStreamReader(resultStream));
+        LOG.debug("Response code from LogSearch Service is = " + httpURLConnection.getResponseCode());
+
+        String line = reader.readLine();
+        StringBuffer buffer = new StringBuffer();
+        while (line != null) {
+          buffer.append(line);
+          line = reader.readLine();
+        }
+
+        LOG.debug("Sucessfully retrieved response from server, response = " + buffer);
+
+        return buffer;
+      } finally {
+        // make sure to close the stream after request is completed
+        if (resultStream != null) {
+          resultStream.close();
+        }
+      }
+    }
+
+    @Override
+    public void setupBasicAuthentication(HttpURLConnection httpURLConnection, String encodedCredentials) {
+      // default implementation for this method should just set the Authorization header
+      // required for Basic Authentication to the LogSearch Server
+      httpURLConnection.setRequestProperty("Authorization", "Basic " + encodedCredentials);
+    }
+  }
+
 
 }

ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImplTest.java

@@ -0,0 +1,672 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.controller.logging;
+
+import org.apache.ambari.server.security.credential.PrincipalKeyCredential;
+import org.apache.ambari.server.security.encryption.CredentialStoreService;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Config;
+import org.apache.commons.codec.binary.Base64;
+import org.easymock.Capture;
+import org.easymock.EasyMockSupport;
+import org.junit.Test;
+
+import java.net.HttpURLConnection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+
+public class LoggingRequestHelperImplTest {
+
+  private static final String TEST_JSON_INPUT_TWO_LIST_ENTRIES =
+    "{" +
+      "  \"startIndex\" : 0," +
+      "  \"pageSize\" : 5," +
+      "  \"totalCount\" : 10452," +
+      "  \"resultSize\" : 5," +
+      "  \"queryTimeMS\" : 1458148754113," +
+      "  \"logList\" : [ {" +
+      "    \"cluster\" : \"clusterone\"," +
+      "    \"method\" : \"chooseUnderReplicatedBlocks\"," +
+      "    \"level\" : \"INFO\"," +
+      "    \"event_count\" : 1," +
+      "    \"ip\" : \"192.168.1.1\"," +
+      "    \"type\" : \"hdfs_namenode\"," +
+      "    \"seq_num\" : 10584," +
+      "    \"path\" : \"/var/log/hadoop/hdfs/hadoop-hdfs-namenode-c6401.ambari.apache.org.log\"," +
+      "    \"file\" : \"UnderReplicatedBlocks.java\"," +
+      "    \"line_number\" : 394," +
+      "    \"host\" : \"c6401.ambari.apache.org\"," +
+      "    \"log_message\" : \"chooseUnderReplicatedBlocks selected 2 blocks at priority level 0;  Total=2 Reset bookmarks? false\"," +
+      "    \"logger_name\" : \"BlockStateChange\"," +
+      "    \"id\" : \"9c5562fb-123f-47c8-aaf5-b5e407326c08\"," +
+      "    \"message_md5\" : \"-3892769501348410581\"," +
+      "    \"logtime\" : 1458148749036," +
+      "    \"event_md5\" : \"1458148749036-2417481968206345035\"," +
+      "    \"logfile_line_number\" : 2084," +
+      "    \"_ttl_\" : \"+7DAYS\"," +
+      "    \"_expire_at_\" : 1458753550322," +
+      "    \"_version_\" : 1528979784023932928" +
+      "  }, {" +
+      "    \"cluster\" : \"clusterone\"," +
+      "    \"method\" : \"putMetrics\"," +
+      "    \"level\" : \"WARN\"," +
+      "    \"event_count\" : 1," +
+      "    \"ip\" : \"192.168.1.1\"," +
+      "    \"type\" : \"yarn_resourcemanager\"," +
+      "    \"seq_num\" : 10583," +
+      "    \"path\" : \"/var/log/hadoop-yarn/yarn/yarn-yarn-resourcemanager-c6401.ambari.apache.org.log\"," +
+      "    \"file\" : \"HadoopTimelineMetricsSink.java\"," +
+      "    \"line_number\" : 262," +
+      "    \"host\" : \"c6401.ambari.apache.org\"," +
+      "    \"log_message\" : \"Unable to send metrics to collector by address:http://c6401.ambari.apache.org:6188/ws/v1/timeline/metrics\"," +
+      "    \"logger_name\" : \"timeline.HadoopTimelineMetricsSink\"," +
+      "    \"id\" : \"8361c5a9-5b1c-4f44-bc8f-4c6f07d94228\"," +
+      "    \"message_md5\" : \"5942185045779825717\"," +
+      "    \"logtime\" : 1458148746937," +
+      "    \"event_md5\" : \"14581487469371427138486123628676\"," +
+      "    \"logfile_line_number\" : 549," +
+      "    \"_ttl_\" : \"+7DAYS\"," +
+      "    \"_expire_at_\" : 1458753550322," +
+      "    \"_version_\" : 1528979784022884357" +
+      "  }" +
+      "]" +
+      "}";
+
+  private static final String TEST_JSON_INPUT_LOG_LEVEL_QUERY =
+    "{\"pageSize\":\"0\",\"queryTimeMS\":\"1459970731998\",\"resultSize\":\"6\",\"startIndex\":\"0\",\"totalCount\":\"0\"," +
+      "\"vNameValues\":[{\"name\":\"FATAL\",\"value\":\"0\"},{\"name\":\"ERROR\",\"value\":\"0\"}," +
+      "{\"name\":\"WARN\",\"value\":\"41\"},{\"name\":\"INFO\",\"value\":\"186\"},{\"name\":\"DEBUG\",\"value\":\"0\"}," +
+      "{\"name\":\"TRACE\",\"value\":\"0\"}]}";
+
+
+  private final String EXPECTED_HOST_NAME = "c6401.ambari.apache.org";
+
+  private final String EXPECTED_PORT_NUMBER = "61888";
+
+  private static final String EXPECTED_USER_NAME = "admin-user";
+
+  private static final String EXPECTED_ADMIN_PASSWORD = "admin-pwd";
+
+  private static final String EXPECTED_ENCODED_CREDENTIALS =
+    Base64.encodeBase64String((EXPECTED_USER_NAME + ":" + EXPECTED_ADMIN_PASSWORD).getBytes());
+
+
+
+  @Test
+  public void testLogQueryRequestBasic() throws Exception {
+    EasyMockSupport mockSupport =
+      new EasyMockSupport();
+
+    CredentialStoreService credentialStoreServiceMock =
+      mockSupport.createMock(CredentialStoreService.class);
+
+    Cluster clusterMock =
+      mockSupport.createMock(Cluster.class);
+
+    LoggingRequestHelperImpl.NetworkConnection networkConnectionMock =
+      mockSupport.createMock(LoggingRequestHelperImpl.NetworkConnection.class);
+
+    Config adminPropertiesConfigMock =
+      mockSupport.createMock(Config.class);
+
+    Map<String, String> testConfigProperties =
+      new HashMap<String, String>();
+
+    testConfigProperties.put("logsearch_admin_username", EXPECTED_USER_NAME);
+    testConfigProperties.put("logsearch_admin_password", EXPECTED_ADMIN_PASSWORD);
+    testConfigProperties = Collections.unmodifiableMap(testConfigProperties);
+
+    Capture<HttpURLConnection> captureURLConnection = new Capture<HttpURLConnection>();
+    Capture<HttpURLConnection> captureURLConnectionForAuthentication = new Capture<HttpURLConnection>();
+
+    expect(clusterMock.getDesiredConfigByType("logsearch-admin-properties")).andReturn(adminPropertiesConfigMock).atLeastOnce();
+    expect(adminPropertiesConfigMock.getProperties()).andReturn(testConfigProperties).atLeastOnce();
+    expect(networkConnectionMock.readQueryResponseFromServer(capture(captureURLConnection))).andReturn(new StringBuffer(TEST_JSON_INPUT_TWO_LIST_ENTRIES)).atLeastOnce();
+
+    // expect that basic authentication is setup, with the expected encoded credentials
+    networkConnectionMock.setupBasicAuthentication(capture(captureURLConnectionForAuthentication), eq(EXPECTED_ENCODED_CREDENTIALS));
+
+    mockSupport.replayAll();
+
+    LoggingRequestHelper helper =
+      new LoggingRequestHelperImpl(EXPECTED_HOST_NAME, EXPECTED_PORT_NUMBER, credentialStoreServiceMock, clusterMock, networkConnectionMock);
+
+    // invoke query request
+    LogQueryResponse result =
+      helper.sendQueryRequest(Collections.<String,String>emptyMap());
+
+    // verify that the HttpURLConnection was created with the proper values
+    HttpURLConnection httpURLConnection =
+      captureURLConnection.getValue();
+
+    assertEquals("URLConnection did not have the correct hostname information",
+      EXPECTED_HOST_NAME, httpURLConnection.getURL().getHost());
+    assertEquals("URLConnection did not have the correct port information",
+      EXPECTED_PORT_NUMBER, httpURLConnection.getURL().getPort() + "");
+    assertEquals("URLConnection did not have the expected http protocol scheme",
+      "http", httpURLConnection.getURL().getProtocol());
+    assertEquals("URLConnection did not have the expected method set",
+      "GET", httpURLConnection.getRequestMethod());
+
+    assertSame("HttpUrlConnection instances passed into NetworkConnection mock should have been the same instance",
+      httpURLConnection, captureURLConnectionForAuthentication.getValue());
+
+
+    assertNotNull("Response object should not be null",
+      result);
+
+    // verify that the JSON response returned from the simulated server
+    // is parsed properly, and has the expected values
+    assertEquals("startIndex not parsed properly",
+      "0", result.getStartIndex());
+    assertEquals("pageSize not parsed properly",
+      "5", result.getPageSize());
+    assertEquals("totalCount not parsed properly",
+      "10452", result.getTotalCount());
+    assertEquals("resultSize not parsed properly",
+      "5", result.getResultSize());
+    assertEquals("queryTimeMS not parsed properly",
+      "1458148754113", result.getQueryTimeMS());
+
+    assertEquals("incorrect number of LogLineResult items parsed",
+      2, result.getListOfResults().size());
+
+    List<LogLineResult> listOfLineResults =
+      result.getListOfResults();
+
+    {
+      LogLineResult resultOne = listOfLineResults.get(0);
+      // verify that all fields in this class are parsed as expected
+      assertEquals("Cluster name not parsed properly",
+        "clusterone", resultOne.getClusterName());
+      assertEquals("Method Name not parsed properly",
+        "chooseUnderReplicatedBlocks", resultOne.getLogMethod());
+      assertEquals("Log Level not parsed properly",
+        "INFO", resultOne.getLogLevel());
+      assertEquals("event_count not parsed properly",
+        "1", resultOne.getEventCount());
+      assertEquals("ip address not parsed properly",
+        "192.168.1.1", resultOne.getIpAddress());
+      assertEquals("component type not parsed properly",
+        "hdfs_namenode", resultOne.getComponentType());
+      assertEquals("sequence number not parsed properly",
+        "10584", resultOne.getSequenceNumber());
+      assertEquals("log file path not parsed properly",
+        "/var/log/hadoop/hdfs/hadoop-hdfs-namenode-c6401.ambari.apache.org.log", resultOne.getLogFilePath());
+      assertEquals("log src file name not parsed properly",
+        "UnderReplicatedBlocks.java", resultOne.getSourceFile());
+      assertEquals("log src line number not parsed properly",
+        "394", resultOne.getSourceFileLineNumber());
+      assertEquals("host name not parsed properly",
+        "c6401.ambari.apache.org", resultOne.getHostName());
+      assertEquals("log message not parsed properly",
+        "chooseUnderReplicatedBlocks selected 2 blocks at priority level 0;  Total=2 Reset bookmarks? false", resultOne.getLogMessage());
+      assertEquals("logger name not parsed properly",
+        "BlockStateChange", resultOne.getLoggerName());
+      assertEquals("id not parsed properly",
+        "9c5562fb-123f-47c8-aaf5-b5e407326c08", resultOne.getId());
+      assertEquals("message MD5 not parsed properly",
+        "-3892769501348410581", resultOne.getMessageMD5());
+      assertEquals("log time not parsed properly",
+        "1458148749036", resultOne.getLogTime());
+      assertEquals("event MD5 not parsed properly",
+        "1458148749036-2417481968206345035", resultOne.getEventMD5());
+      assertEquals("logfile line number not parsed properly",
+        "2084", resultOne.getLogFileLineNumber());
+      assertEquals("ttl not parsed properly",
+        "+7DAYS", resultOne.getTtl());
+      assertEquals("expire at not parsed properly",
+        "1458753550322", resultOne.getExpirationTime());
+      assertEquals("version not parsed properly",
+        "1528979784023932928", resultOne.getVersion());
+    }
+
+    {
+      LogLineResult resultTwo = listOfLineResults.get(1);
+      // verify second log line record's data is parsed correctly
+      assertEquals("Cluster name not parsed properly",
+        "clusterone", resultTwo.getClusterName());
+      assertEquals("Method Name not parsed properly",
+        "putMetrics", resultTwo.getLogMethod());
+      assertEquals("Log Level not parsed properly",
+        "WARN", resultTwo.getLogLevel());
+      assertEquals("event_count not parsed properly",
+        "1", resultTwo.getEventCount());
+      assertEquals("ip address not parsed properly",
+        "192.168.1.1", resultTwo.getIpAddress());
+      assertEquals("component type not parsed properly",
+        "yarn_resourcemanager", resultTwo.getComponentType());
+      assertEquals("sequence number not parsed properly",
+        "10583", resultTwo.getSequenceNumber());
+      assertEquals("log file path not parsed properly",
+        "/var/log/hadoop-yarn/yarn/yarn-yarn-resourcemanager-c6401.ambari.apache.org.log", resultTwo.getLogFilePath());
+      assertEquals("log src file name not parsed properly",
+        "HadoopTimelineMetricsSink.java", resultTwo.getSourceFile());
+      assertEquals("log src line number not parsed properly",
+        "262", resultTwo.getSourceFileLineNumber());
+      assertEquals("host name not parsed properly",
+        "c6401.ambari.apache.org", resultTwo.getHostName());
+      assertEquals("log message not parsed properly",
+        "Unable to send metrics to collector by address:http://c6401.ambari.apache.org:6188/ws/v1/timeline/metrics", resultTwo.getLogMessage());
+      assertEquals("logger name not parsed properly",
+        "timeline.HadoopTimelineMetricsSink", resultTwo.getLoggerName());
+      assertEquals("id not parsed properly",
+        "8361c5a9-5b1c-4f44-bc8f-4c6f07d94228", resultTwo.getId());
+      assertEquals("message MD5 not parsed properly",
+        "5942185045779825717", resultTwo.getMessageMD5());
+      assertEquals("log time not parsed properly",
+        "1458148746937", resultTwo.getLogTime());
+      assertEquals("event MD5 not parsed properly",
+        "14581487469371427138486123628676", resultTwo.getEventMD5());
+      assertEquals("logfile line number not parsed properly",
+        "549", resultTwo.getLogFileLineNumber());
+      assertEquals("ttl not parsed properly",
+        "+7DAYS", resultTwo.getTtl());
+      assertEquals("expire at not parsed properly",
+        "1458753550322", resultTwo.getExpirationTime());
+      assertEquals("version not parsed properly",
+        "1528979784022884357", resultTwo.getVersion());
+    }
+
+    mockSupport.verifyAll();
+  }
+
+  @Test
+  public void testLogLevelRequestBasic() throws Exception {
+    EasyMockSupport mockSupport =
+      new EasyMockSupport();
+
+    CredentialStoreService credentialStoreServiceMock =
+      mockSupport.createMock(CredentialStoreService.class);
+
+    Cluster clusterMock =
+      mockSupport.createMock(Cluster.class);
+
+    LoggingRequestHelperImpl.NetworkConnection networkConnectionMock =
+      mockSupport.createMock(LoggingRequestHelperImpl.NetworkConnection.class);
+
+    Config adminPropertiesConfigMock =
+      mockSupport.createMock(Config.class);
+
+    Map<String, String> testConfigProperties =
+      new HashMap<String, String>();
+    testConfigProperties.put("logsearch_admin_username", "admin-user");
+    testConfigProperties.put("logsearch_admin_password", "admin-pwd");
+    testConfigProperties = Collections.unmodifiableMap(testConfigProperties);
+
+    Capture<HttpURLConnection> captureURLConnection = new Capture<HttpURLConnection>();
+    Capture<HttpURLConnection> captureURLConnectionForAuthentication = new Capture<HttpURLConnection>();
+
+    expect(clusterMock.getDesiredConfigByType("logsearch-admin-properties")).andReturn(adminPropertiesConfigMock).atLeastOnce();
+    expect(adminPropertiesConfigMock.getProperties()).andReturn(testConfigProperties).atLeastOnce();
+    expect(networkConnectionMock.readQueryResponseFromServer(capture(captureURLConnection))).andReturn(new StringBuffer(TEST_JSON_INPUT_LOG_LEVEL_QUERY)).atLeastOnce();
+
+    // expect that basic authentication is setup, with the expected encoded credentials
+    networkConnectionMock.setupBasicAuthentication(capture(captureURLConnectionForAuthentication), eq(EXPECTED_ENCODED_CREDENTIALS));
+
+
+    mockSupport.replayAll();
+
+
+    LoggingRequestHelper helper =
+      new LoggingRequestHelperImpl(EXPECTED_HOST_NAME, EXPECTED_PORT_NUMBER, credentialStoreServiceMock, clusterMock, networkConnectionMock);
+
+    // invoke query request
+    LogLevelQueryResponse result =
+      helper.sendLogLevelQueryRequest("hdfs_datanode", EXPECTED_HOST_NAME);
+
+    // verify that the HttpURLConnection was created with the proper values
+    HttpURLConnection httpURLConnection =
+      captureURLConnection.getValue();
+
+    assertEquals("URLConnection did not have the correct hostname information",
+      EXPECTED_HOST_NAME, httpURLConnection.getURL().getHost());
+    assertEquals("URLConnection did not have the correct port information",
+      EXPECTED_PORT_NUMBER, httpURLConnection.getURL().getPort() + "");
+    assertEquals("URLConnection did not have the expected http protocol scheme",
+      "http", httpURLConnection.getURL().getProtocol());
+    assertEquals("URLConnection did not have the expected method set",
+      "GET", httpURLConnection.getRequestMethod());
+
+    assertSame("HttpUrlConnection instances passed into NetworkConnection mock should have been the same instance",
+      httpURLConnection, captureURLConnectionForAuthentication.getValue());
+
+    assertNotNull("Response object should not be null",
+      result);
+
+    // expected values taken from JSON input string declared above
+    assertEquals("startIndex not parsed properly",
+      "0", result.getStartIndex());
+    assertEquals("pageSize not parsed properly",
+      "0", result.getPageSize());
+    assertEquals("totalCount not parsed properly",
+      "0", result.getTotalCount());
+    assertEquals("resultSize not parsed properly",
+      "6", result.getResultSize());
+    assertEquals("queryTimeMS not parsed properly",
+      "1459970731998", result.getQueryTimeMS());
+
+    assertEquals("Incorrect number of log level count items parsed",
+      6, result.getNameValueList().size());
+
+    List<NameValuePair> resultList =
+      result.getNameValueList();
+    assertNameValuePair("FATAL", "0", resultList.get(0));
+    assertNameValuePair("ERROR", "0", resultList.get(1));
+    assertNameValuePair("WARN", "41", resultList.get(2));
+    assertNameValuePair("INFO", "186", resultList.get(3));
+    assertNameValuePair("DEBUG", "0", resultList.get(4));
+    assertNameValuePair("TRACE", "0", resultList.get(5));
+
+    mockSupport.verifyAll();
+  }
+
+  @Test
+  public void testLogFileNameRequestBasic() throws Exception {
+    final String expectedComponentName = "hdfs_namenode";
+
+    EasyMockSupport mockSupport =
+      new EasyMockSupport();
+
+    CredentialStoreService credentialStoreServiceMock =
+      mockSupport.createMock(CredentialStoreService.class);
+
+    Cluster clusterMock =
+      mockSupport.createMock(Cluster.class);
+
+    LoggingRequestHelperImpl.NetworkConnection networkConnectionMock =
+      mockSupport.createMock(LoggingRequestHelperImpl.NetworkConnection.class);
+
+    Config adminPropertiesConfigMock =
+      mockSupport.createMock(Config.class);
+
+    Map<String, String> testConfigProperties =
+      new HashMap<String, String>();
+    testConfigProperties.put("logsearch_admin_username", "admin-user");
+    testConfigProperties.put("logsearch_admin_password", "admin-pwd");
+    testConfigProperties = Collections.unmodifiableMap(testConfigProperties);
+
+    Capture<HttpURLConnection> captureURLConnection = new Capture<HttpURLConnection>();
+    Capture<HttpURLConnection> captureURLConnectionForAuthentication = new Capture<HttpURLConnection>();
+
+    expect(clusterMock.getDesiredConfigByType("logsearch-admin-properties")).andReturn(adminPropertiesConfigMock).atLeastOnce();
+    expect(adminPropertiesConfigMock.getProperties()).andReturn(testConfigProperties).atLeastOnce();
+    expect(networkConnectionMock.readQueryResponseFromServer(capture(captureURLConnection))).andReturn(new StringBuffer(TEST_JSON_INPUT_TWO_LIST_ENTRIES)).atLeastOnce();
+
+    // expect that basic authentication is setup, with the expected encoded credentials
+    networkConnectionMock.setupBasicAuthentication(capture(captureURLConnectionForAuthentication), eq(EXPECTED_ENCODED_CREDENTIALS));
+
+    mockSupport.replayAll();
+
+    LoggingRequestHelper helper =
+      new LoggingRequestHelperImpl(EXPECTED_HOST_NAME, EXPECTED_PORT_NUMBER, credentialStoreServiceMock, clusterMock, networkConnectionMock);
+
+    // invoke query request
+    Set<String> result =
+      helper.sendGetLogFileNamesRequest(expectedComponentName, EXPECTED_HOST_NAME);
+
+    // verify that the HttpURLConnection was created with the propert values
+    HttpURLConnection httpURLConnection =
+      captureURLConnection.getValue();
+
+    assertEquals("URLConnection did not have the correct hostname information",
+      EXPECTED_HOST_NAME, httpURLConnection.getURL().getHost());
+    assertEquals("URLConnection did not have the correct port information",
+      EXPECTED_PORT_NUMBER, httpURLConnection.getURL().getPort() + "");
+    assertEquals("URLConnection did not have the expected http protocol scheme",
+      "http", httpURLConnection.getURL().getProtocol());
+    assertEquals("URLConnection did not have the expected method set",
+      "GET", httpURLConnection.getRequestMethod());
+
+    assertSame("HttpUrlConnection instances passed into NetworkConnection mock should have been the same instance",
+      httpURLConnection, captureURLConnectionForAuthentication.getValue());
+
+    final String resultQuery =
+      httpURLConnection.getURL().getQuery();
+
+    // verify that the query contains the three required parameters
+    assertTrue("host parameter was not included in query",
+      resultQuery.contains("host=c6401.ambari.apache.org"));
+    assertTrue("components_name parameter was not included in the query",
+      resultQuery.contains("components_name=" + expectedComponentName));
+    assertTrue("pageSize parameter was not included in query",
+      resultQuery.contains("pageSize=1"));
+
+    assertNotNull("Response object should not be null",
+      result);
+    assertEquals("Response Set was not of the expected size",
+      1, result.size());
+    assertEquals("Response did not include the expected file name",
+      "/var/log/hadoop/hdfs/hadoop-hdfs-namenode-c6401.ambari.apache.org.log",
+      result.iterator().next());
+  }
+
+  /**
+   * Verifies that if the LogSearch admin configuration for user/password credentials
+   * is not available, the integration layer will attempt to locate the LogSearch credential
+   * in the CredentialStoreService as a fallback mechanism.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testLogQueryRequestBasicCredentialsNotInConfig() throws Exception {
+    final String expectedClusterName = "my-test-cluster";
+
+    EasyMockSupport mockSupport =
+      new EasyMockSupport();
+
+    CredentialStoreService credentialStoreServiceMock =
+      mockSupport.createMock(CredentialStoreService.class);
+
+    Cluster clusterMock =
+      mockSupport.createMock(Cluster.class);
+
+    LoggingRequestHelperImpl.NetworkConnection networkConnectionMock =
+      mockSupport.createMock(LoggingRequestHelperImpl.NetworkConnection.class);
+
+    Config adminPropertiesConfigMock =
+      mockSupport.createMock(Config.class);
+
+    Capture<HttpURLConnection> captureURLConnection = new Capture<HttpURLConnection>();
+    Capture<HttpURLConnection> captureURLConnectionForAuthentication = new Capture<HttpURLConnection>();
+
+    expect(clusterMock.getDesiredConfigByType("logsearch-admin-properties")).andReturn(adminPropertiesConfigMock).atLeastOnce();
+    expect(clusterMock.getClusterName()).andReturn(expectedClusterName).atLeastOnce();
+    expect(adminPropertiesConfigMock.getProperties()).andReturn(Collections.<String, String>emptyMap()).atLeastOnce();
+    expect(networkConnectionMock.readQueryResponseFromServer(capture(captureURLConnection))).andReturn(new StringBuffer(TEST_JSON_INPUT_TWO_LIST_ENTRIES)).atLeastOnce();
+    // the credential store service should be consulted in this case, in order
+    // to attempt to obtain the LogSearch credential from the store
+    expect(credentialStoreServiceMock.getCredential(expectedClusterName, "logsearch.admin.credential")).andReturn(new PrincipalKeyCredential(EXPECTED_USER_NAME, EXPECTED_ADMIN_PASSWORD)).atLeastOnce();
+
+    // expect that basic authentication is setup, with the expected encoded credentials
+    networkConnectionMock.setupBasicAuthentication(capture(captureURLConnectionForAuthentication), eq(EXPECTED_ENCODED_CREDENTIALS));
+
+    mockSupport.replayAll();
+
+    LoggingRequestHelper helper =
+      new LoggingRequestHelperImpl(EXPECTED_HOST_NAME, EXPECTED_PORT_NUMBER, credentialStoreServiceMock, clusterMock, networkConnectionMock);
+
+    // invoke query request
+    LogQueryResponse result =
+      helper.sendQueryRequest(Collections.<String,String>emptyMap());
+
+    // verify that the HttpURLConnection was created with the proper values
+    HttpURLConnection httpURLConnection =
+      captureURLConnection.getValue();
+
+    assertEquals("URLConnection did not have the correct hostname information",
+      EXPECTED_HOST_NAME, httpURLConnection.getURL().getHost());
+    assertEquals("URLConnection did not have the correct port information",
+      EXPECTED_PORT_NUMBER, httpURLConnection.getURL().getPort() + "");
+    assertEquals("URLConnection did not have the expected http protocol scheme",
+      "http", httpURLConnection.getURL().getProtocol());
+    assertEquals("URLConnection did not have the expected method set",
+      "GET", httpURLConnection.getRequestMethod());
+
+    assertSame("HttpUrlConnection instances passed into NetworkConnection mock should have been the same instance",
+      httpURLConnection, captureURLConnectionForAuthentication.getValue());
+
+
+    assertNotNull("Response object should not be null",
+      result);
+
+    // verify that the JSON response returned from the simulated server
+    // is parsed properly, and has the expected values
+    assertEquals("startIndex not parsed properly",
+      "0", result.getStartIndex());
+    assertEquals("pageSize not parsed properly",
+      "5", result.getPageSize());
+    assertEquals("totalCount not parsed properly",
+      "10452", result.getTotalCount());
+    assertEquals("resultSize not parsed properly",
+      "5", result.getResultSize());
+    assertEquals("queryTimeMS not parsed properly",
+      "1458148754113", result.getQueryTimeMS());
+
+    assertEquals("incorrect number of LogLineResult items parsed",
+      2, result.getListOfResults().size());
+
+    List<LogLineResult> listOfLineResults =
+      result.getListOfResults();
+
+    {
+      LogLineResult resultOne = listOfLineResults.get(0);
+      // verify that all fields in this class are parsed as expected
+      assertEquals("Cluster name not parsed properly",
+        "clusterone", resultOne.getClusterName());
+      assertEquals("Method Name not parsed properly",
+        "chooseUnderReplicatedBlocks", resultOne.getLogMethod());
+      assertEquals("Log Level not parsed properly",
+        "INFO", resultOne.getLogLevel());
+      assertEquals("event_count not parsed properly",
+        "1", resultOne.getEventCount());
+      assertEquals("ip address not parsed properly",
+        "192.168.1.1", resultOne.getIpAddress());
+      assertEquals("component type not parsed properly",
+        "hdfs_namenode", resultOne.getComponentType());
+      assertEquals("sequence number not parsed properly",
+        "10584", resultOne.getSequenceNumber());
+      assertEquals("log file path not parsed properly",
+        "/var/log/hadoop/hdfs/hadoop-hdfs-namenode-c6401.ambari.apache.org.log", resultOne.getLogFilePath());
+      assertEquals("log src file name not parsed properly",
+        "UnderReplicatedBlocks.java", resultOne.getSourceFile());
+      assertEquals("log src line number not parsed properly",
+        "394", resultOne.getSourceFileLineNumber());
+      assertEquals("host name not parsed properly",
+        "c6401.ambari.apache.org", resultOne.getHostName());
+      assertEquals("log message not parsed properly",
+        "chooseUnderReplicatedBlocks selected 2 blocks at priority level 0;  Total=2 Reset bookmarks? false", resultOne.getLogMessage());
+      assertEquals("logger name not parsed properly",
+        "BlockStateChange", resultOne.getLoggerName());
+      assertEquals("id not parsed properly",
+        "9c5562fb-123f-47c8-aaf5-b5e407326c08", resultOne.getId());
+      assertEquals("message MD5 not parsed properly",
+        "-3892769501348410581", resultOne.getMessageMD5());
+      assertEquals("log time not parsed properly",
+        "1458148749036", resultOne.getLogTime());
+      assertEquals("event MD5 not parsed properly",
+        "1458148749036-2417481968206345035", resultOne.getEventMD5());
+      assertEquals("logfile line number not parsed properly",
+        "2084", resultOne.getLogFileLineNumber());
+      assertEquals("ttl not parsed properly",
+        "+7DAYS", resultOne.getTtl());
+      assertEquals("expire at not parsed properly",
+        "1458753550322", resultOne.getExpirationTime());
+      assertEquals("version not parsed properly",
+        "1528979784023932928", resultOne.getVersion());
+    }
+
+    {
+      LogLineResult resultTwo = listOfLineResults.get(1);
+      // verify second log line record's data is parsed correctly
+      assertEquals("Cluster name not parsed properly",
+        "clusterone", resultTwo.getClusterName());
+      assertEquals("Method Name not parsed properly",
+        "putMetrics", resultTwo.getLogMethod());
+      assertEquals("Log Level not parsed properly",
+        "WARN", resultTwo.getLogLevel());
+      assertEquals("event_count not parsed properly",
+        "1", resultTwo.getEventCount());
+      assertEquals("ip address not parsed properly",
+        "192.168.1.1", resultTwo.getIpAddress());
+      assertEquals("component type not parsed properly",
+        "yarn_resourcemanager", resultTwo.getComponentType());
+      assertEquals("sequence number not parsed properly",
+        "10583", resultTwo.getSequenceNumber());
+      assertEquals("log file path not parsed properly",
+        "/var/log/hadoop-yarn/yarn/yarn-yarn-resourcemanager-c6401.ambari.apache.org.log", resultTwo.getLogFilePath());
+      assertEquals("log src file name not parsed properly",
+        "HadoopTimelineMetricsSink.java", resultTwo.getSourceFile());
+      assertEquals("log src line number not parsed properly",
+        "262", resultTwo.getSourceFileLineNumber());
+      assertEquals("host name not parsed properly",
+        "c6401.ambari.apache.org", resultTwo.getHostName());
+      assertEquals("log message not parsed properly",
+        "Unable to send metrics to collector by address:http://c6401.ambari.apache.org:6188/ws/v1/timeline/metrics", resultTwo.getLogMessage());
+      assertEquals("logger name not parsed properly",
+        "timeline.HadoopTimelineMetricsSink", resultTwo.getLoggerName());
+      assertEquals("id not parsed properly",
+        "8361c5a9-5b1c-4f44-bc8f-4c6f07d94228", resultTwo.getId());
+      assertEquals("message MD5 not parsed properly",
+        "5942185045779825717", resultTwo.getMessageMD5());
+      assertEquals("log time not parsed properly",
+        "1458148746937", resultTwo.getLogTime());
+      assertEquals("event MD5 not parsed properly",
+        "14581487469371427138486123628676", resultTwo.getEventMD5());
+      assertEquals("logfile line number not parsed properly",
+        "549", resultTwo.getLogFileLineNumber());
+      assertEquals("ttl not parsed properly",
+        "+7DAYS", resultTwo.getTtl());
+      assertEquals("expire at not parsed properly",
+        "1458753550322", resultTwo.getExpirationTime());
+      assertEquals("version not parsed properly",
+        "1528979784022884357", resultTwo.getVersion());
+    }
+
+    mockSupport.verifyAll();
+  }
+
+
+  /**
+   * Convenience method for asserting on the values of NameValuePair instances
+   *
+   * @param expectedName the expected name
+   * @param expectedValue the expected value
+   * @param nameValuePair the NameValuePair instance to test
+   */
+  static void assertNameValuePair(String expectedName, String expectedValue, NameValuePair nameValuePair) {
+    assertEquals("Unexpected name found in this pair",
+      expectedName, nameValuePair.getName());
+    assertEquals("Unexpected value found in this pair",
+      expectedValue, nameValuePair.getValue());
+  }
+
+}