AMBARI-18076. JBehave and docker based Integration Test Framework for Log Search components (oleewere)

ambari-logsearch/README.md

@@ -33,4 +33,13 @@ mvn -Dbuild-rpm clean package
 mvn -Dbuild-deb clean package
 ```
 3. Generated RPM/DPKG files will be found in ambari-logsearch-assembly/target folder
-  
+
+## Running Integration Tests
+
+By default integration tests are not a part of the build process, you need to set ${it.skip} variable to true (docker needed here too)
+
+```bash
+# from ambari-logsearch folder
+mvn clean integration-test -Dit.skip=false
+```
+Also you can run from the IDE, but make sure all of the ambari logsearch modules are built.

ambari-logsearch/ambari-logsearch-it/pom.xml

@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>ambari-logsearch</artifactId>
+    <groupId>org.apache.ambari</groupId>
+    <version>2.0.0.0-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+
+  <artifactId>ambari-logsearch-it</artifactId>
+  <url>http://maven.apache.org</url>
+  <name>Ambari Logsearch Integration Test</name>
+  <packaging>jar</packaging>
+
+  <properties>
+    <it.skip>true</it.skip>
+    <jbehave.version>4.0.5</jbehave.version>
+    <jersey.version>2.23.1</jersey.version>
+    <jackson-jaxrs.version>2.6.4</jackson-jaxrs.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.jbehave</groupId>
+      <artifactId>jbehave-core</artifactId>
+      <version>${jbehave.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>2.5</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.ambari</groupId>
+      <artifactId>ambari-logsearch-solr-client</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.github.docker-java</groupId>
+      <artifactId>docker-java</artifactId>
+      <version>3.0.0</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.jaxrs</groupId>
+      <artifactId>jackson-jaxrs-json-provider</artifactId>
+      <version>${jackson-jaxrs.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.glassfish.jersey.core</groupId>
+      <artifactId>jersey-client</artifactId>
+      <version>${jersey.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>1.7.7</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <version>1.7.10</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <testOutputDirectory>target/classes</testOutputDirectory>
+    <testResources>
+      <testResource>
+        <directory>src/test/java/</directory>
+        <includes>
+          <include>**/*.story</include>
+        </includes>
+      </testResource>
+      <testResource>
+        <directory>src/test/resources</directory>
+      </testResource>
+    </testResources>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-failsafe-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>run-integration-tests</id>
+            <phase>integration-test</phase>
+            <goals>
+              <goal>integration-test</goal>
+            </goals>
+            <configuration>
+              <includes>
+                <include>**/*Stories.java</include>
+                <include>**/*Story.java</include>
+              </includes>
+            </configuration>
+          </execution>
+          <execution>
+            <id>verify-integration-tests</id>
+            <phase>verify</phase>
+            <goals>
+              <goal>verify</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/StoryDataRegistry.java

@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.domain;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.core.DockerClientConfig;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+
+public class StoryDataRegistry {
+  public static final StoryDataRegistry INSTANCE = new StoryDataRegistry();
+
+  private DockerClient dockerClient;
+  private DockerClientConfig dockerClientConfig;
+  private CloudSolrClient cloudSolrClient;
+  private boolean logsearchContainerStarted = false;
+  private String dockerHost;
+  private String ambariFolder;
+  private final int solrPort = 8886;
+  private final int logsearchPort = 61888;
+  private final int zookeeperPort = 9983;
+  private final String serviceLogsCollection = "hadoop_logs";
+  private final String auditLogsCollection = "audit_logs";
+
+  private StoryDataRegistry() {
+  }
+
+  public String getDockerHost() {
+    return dockerHost;
+  }
+
+  public void setDockerHost(String dockerHost) {
+    this.dockerHost = dockerHost;
+  }
+
+  public int getSolrPort() {
+    return solrPort;
+  }
+
+  public int getLogsearchPort() {
+    return logsearchPort;
+  }
+
+  public int getZookeeperPort() {
+    return zookeeperPort;
+  }
+
+  public DockerClient getDockerClient() {
+    return dockerClient;
+  }
+
+  public void setDockerClient(DockerClient dockerClient) {
+    this.dockerClient = dockerClient;
+  }
+
+  public String getServiceLogsCollection() {
+    return serviceLogsCollection;
+  }
+
+  public String getAuditLogsCollection() {
+    return auditLogsCollection;
+  }
+
+  public CloudSolrClient getCloudSolrClient() {
+    return cloudSolrClient;
+  }
+
+  public void setCloudSolrClient(CloudSolrClient cloudSolrClient) {
+    this.cloudSolrClient = cloudSolrClient;
+  }
+
+  public String getAmbariFolder() {
+    return ambariFolder;
+  }
+
+  public void setAmbariFolder(String ambariFolder) {
+    this.ambariFolder = ambariFolder;
+  }
+
+  public DockerClientConfig getDockerClientConfig() {
+    return dockerClientConfig;
+  }
+
+  public void setDockerClientConfig(DockerClientConfig dockerClientConfig) {
+    this.dockerClientConfig = dockerClientConfig;
+  }
+
+  public boolean isLogsearchContainerStarted() {
+    return logsearchContainerStarted;
+  }
+
+  public void setLogsearchContainerStarted(boolean logsearchContainerStarted) {
+    this.logsearchContainerStarted = logsearchContainerStarted;
+  }
+}

ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchDockerSteps.java

@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.steps;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CreateContainerResponse;
+import com.github.dockerjava.api.model.Bind;
+import com.github.dockerjava.api.model.Container;
+import com.github.dockerjava.api.model.ExposedPort;
+import com.github.dockerjava.api.model.Ports;
+import com.github.dockerjava.api.model.Volume;
+import com.github.dockerjava.core.DockerClientBuilder;
+import com.github.dockerjava.core.DockerClientConfig;
+import com.github.dockerjava.core.command.BuildImageResultCallback;
+import com.google.common.base.Preconditions;
+import org.apache.ambari.logsearch.domain.StoryDataRegistry;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocumentList;
+import org.jbehave.core.annotations.AfterStories;
+import org.jbehave.core.annotations.BeforeStories;
+import org.jbehave.core.annotations.Given;
+import org.jbehave.core.annotations.When;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.URL;
+import java.util.List;
+
+public class LogSearchDockerSteps {
+
+  private static final Logger LOG = LoggerFactory.getLogger(LogSearchDockerSteps.class);
+
+  @Given("logsearch docker container")
+  public void setupLogSearchContainer() throws Exception {
+    boolean logsearchStarted = StoryDataRegistry.INSTANCE.isLogsearchContainerStarted();
+    if (!logsearchStarted) {
+      DockerClient dockerClient = StoryDataRegistry.INSTANCE.getDockerClient();
+      LOG.info("Create new docker container for Log Search ..");
+      URL location = LogSearchDockerSteps.class.getProtectionDomain().getCodeSource().getLocation();
+      String ambariFolder = new File(location.toURI()).getParentFile().getParentFile().getParentFile().getParent();
+      StoryDataRegistry.INSTANCE.setAmbariFolder(ambariFolder);
+      String dockerBaseDirectory = ambariFolder + "/ambari-logsearch/docker";
+      String dockerFileLocation = dockerBaseDirectory + "/Dockerfile";
+
+      String imageId = dockerClient.buildImageCmd()
+        .withTag("ambari-logsearch:v1.0")
+        .withBaseDirectory(new File(dockerBaseDirectory))
+        .withDockerfile(new File(dockerFileLocation))
+        .exec(new BuildImageResultCallback())
+        .awaitImageId();
+      LOG.info("Docker image id: {}", imageId);
+
+      removeLogSearchContainerIfExists();
+
+      // volume bindings
+      Volume testLogsVolume = new Volume("/root/test-logs");
+      Volume testConfigVolume = new Volume("/root/test-config");
+      Volume ambariVolume = new Volume("/root/ambari");
+      Volume logfeederClassesVolume = new Volume("/root/ambari/ambari-logsearch/ambari-logsearch-logfeeder/target/package/classes");
+      Volume logsearchClassesVolume = new Volume("/root/ambari/ambari-logsearch/ambari-logsearch-portal/target/package/classes");
+      Volume logsearchWebappVolume = new Volume("/root/ambari/ambari-logsearch/ambari-logsearch-portal/target/package/classes/webapps/app");
+      Bind testLogsBind = new Bind(ambariFolder +"/ambari-logsearch/docker/test-logs", testLogsVolume);
+      Bind testConfigBind = new Bind(ambariFolder +"/ambari-logsearch/docker/test-config", testConfigVolume);
+      Bind ambariRootBind = new Bind(ambariFolder, ambariVolume);
+      Bind logfeederClassesBind = new Bind(ambariFolder + "/ambari-logsearch/ambari-logsearch-logfeeder/target/classes", logfeederClassesVolume);
+      Bind logsearchClassesBind = new Bind(ambariFolder + "/ambari-logsearch/ambari-logsearch-portal/target/classes", logsearchClassesVolume);
+      Bind logsearchWebappBind = new Bind(ambariFolder + "/ambari-logsearch/ambari-logsearch-portal/src/main/webapp", logsearchWebappVolume);
+
+      // port bindings
+      Ports ports = new Ports();
+      ports.bind(new ExposedPort(5005), new Ports.Binding("0.0.0.0", "5005"));
+      ports.bind(new ExposedPort(5006), new Ports.Binding("0.0.0.0", "5006"));
+      ports.bind(new ExposedPort(StoryDataRegistry.INSTANCE.getSolrPort()), new Ports.Binding("0.0.0.0", "8886"));
+      ports.bind(new ExposedPort(StoryDataRegistry.INSTANCE.getLogsearchPort()), new Ports.Binding("0.0.0.0", "61888"));
+      ports.bind(new ExposedPort(StoryDataRegistry.INSTANCE.getZookeeperPort()), new Ports.Binding("0.0.0.0", "9983"));
+
+      LOG.info("Creating docker cointainer...");
+      CreateContainerResponse createResponse = dockerClient.createContainerCmd("ambari-logsearch:v1.0")
+        .withHostName("logsearch.apache.org")
+        .withName("logsearch")
+        .withVolumes(testLogsVolume, testConfigVolume, ambariVolume, logfeederClassesVolume, logsearchClassesVolume, logsearchWebappVolume)
+        .withBinds(testLogsBind, testConfigBind, ambariRootBind, logfeederClassesBind, logsearchClassesBind, logsearchWebappBind)
+        .withExposedPorts(
+          new ExposedPort(StoryDataRegistry.INSTANCE.getLogsearchPort()),
+          new ExposedPort(5005),
+          new ExposedPort(5006),
+          new ExposedPort(StoryDataRegistry.INSTANCE.getSolrPort()),
+          new ExposedPort(StoryDataRegistry.INSTANCE.getZookeeperPort()))
+        .withPortBindings(ports)
+        .exec();
+      LOG.info("Created docker container id: {}", createResponse.getId());
+
+      dockerClient.startContainerCmd(createResponse.getId()).exec();
+      StoryDataRegistry.INSTANCE.setLogsearchContainerStarted(true);
+      String dockerHostFromUri = StoryDataRegistry.INSTANCE.getDockerClientConfig().getDockerHost().getHost();
+      StoryDataRegistry.INSTANCE.setDockerHost(dockerHostFromUri);
+      checkHostAndPortReachable(dockerHostFromUri, StoryDataRegistry.INSTANCE.getLogsearchPort(), "LogSearch");
+      waitUntilSolrHasAnyData();
+
+      LOG.info("Waiting for logfeeder to finish the test log parsings... (10 sec)");
+      Thread.sleep(10000);
+    }
+  }
+
+  @When("logfeeder started (parse logs & send data to solr)")
+  public void logfeederStarted() throws Exception {
+    // TODO: run ps aux to check LogFeeder process with docker exec
+    /**
+    DockerClient dockerClient = StoryDataRegistry.INSTANCE.getDockerClient();
+    ExecCreateCmdResponse execResp = dockerClient
+      .execCreateCmd(containerId)
+      .withAttachStdout(true)
+      .withCmd("ps", "aux").exec();
+    execResp.getId();
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+    ExecStartResultCallback res = dockerClient
+      .execStartCmd(execResp.getId())
+      .withDetach(true)
+      .withTty(true)
+      .exec(new ExecStartResultCallback(outputStream,  outputStream)).awaitCompletion();
+     **/
+  }
+
+  @BeforeStories
+  public void checkDockerApi() {
+    LOG.info("Tries to setup docker client configuration");
+    final String dockerHost = System.getenv("DOCKER_HOST");
+    final String dockerCertPath = System.getenv("DOCKER_CERT_PATH");
+    final String dockerApiVersion = System.getenv("DOCKER_API_VERSION") == null ? "1.20" : System.getenv("DOCKER_API_VERSION");
+
+    Preconditions.checkArgument(dockerHost != null, "Set 'DOCKER_HOST' env variable");
+    Preconditions.checkArgument(dockerCertPath != null, "Set 'DOCKER_CERT_PATH' env variable");
+    LOG.info("DOCKER_HOST: {}", dockerHost);
+    LOG.info("DOCKER_CERT_PATH: {}", dockerCertPath);
+    LOG.info("DOCKER_API_VERSION: {}", dockerApiVersion);
+    DockerClientConfig dockerClientConfig = DockerClientConfig.createDefaultConfigBuilder()
+      .withDockerHost(dockerHost)
+      .withDockerCertPath(dockerCertPath)
+      .withApiVersion(dockerApiVersion)
+      .withDockerTlsVerify(true)
+      .build();
+    StoryDataRegistry.INSTANCE.setDockerClientConfig(dockerClientConfig);
+    DockerClient dockerClient = DockerClientBuilder.getInstance(dockerClientConfig).build();
+    StoryDataRegistry.INSTANCE.setDockerClient(dockerClient);
+    LOG.info("Docker client setup successfully.");
+  }
+
+  @AfterStories
+  public void removeLogSearchContainer() {
+    removeLogSearchContainerIfExists();
+  }
+
+  private void removeLogSearchContainerIfExists() {
+    DockerClient dockerClient = StoryDataRegistry.INSTANCE.getDockerClient();
+    List<Container> containerList = dockerClient
+      .listContainersCmd()
+      .withShowAll(true)
+      .exec();
+
+    boolean isLogSearchContainerExists = false;
+    String containerId = null;
+    for (Container container : containerList) {
+      isLogSearchContainerExists = ArrayUtils.contains(container.getNames(), "/logsearch");
+      if (isLogSearchContainerExists) {
+        containerId = container.getId();
+        break;
+      }
+    }
+
+    if (isLogSearchContainerExists) {
+      LOG.info("Remove logsearch container: {}", containerId);
+      dockerClient.removeContainerCmd(containerId).withForce(true).exec();
+    }
+  }
+
+  private void waitUntilSolrHasAnyData() throws IOException, SolrServerException, InterruptedException {
+    boolean solrHasData = false;
+    CloudSolrClient solrClient = new CloudSolrClient(String.format("%s:%d",
+      StoryDataRegistry.INSTANCE.getDockerHost(),
+      StoryDataRegistry.INSTANCE.getZookeeperPort()));
+    StoryDataRegistry.INSTANCE.setCloudSolrClient(solrClient);
+    SolrQuery solrQuery = new SolrQuery();
+    solrQuery.setQuery("*:*");
+
+    int maxTries = 60;
+    for (int tries = 1; tries < maxTries; tries++) {
+      QueryResponse queryResponse = solrClient.query(StoryDataRegistry.INSTANCE.getServiceLogsCollection(), solrQuery);
+      SolrDocumentList list = queryResponse.getResults();
+      if (list.size() > 0) {
+        solrHasData = true;
+        break;
+      } else {
+        Thread.sleep(2000);
+        LOG.info("Solr has no data yet, retrying...");
+      }
+    }
+    if (!solrHasData) {
+      throw new IllegalStateException(String.format("Solr has no data after %d tries", maxTries));
+    }
+  }
+
+
+  private void checkHostAndPortReachable(String host, int port, String serviceName) throws InterruptedException {
+    boolean reachable = false;
+    int maxTries = 60;
+    for (int tries = 1; tries < maxTries; tries++ ) {
+      try (Socket socket = new Socket()) {
+        socket.connect(new InetSocketAddress(host, port), 1000);
+        reachable = true;
+        break;
+      } catch (IOException e) {
+        Thread.sleep(2000);
+        LOG.info("{} is not reachable yet, retrying..", serviceName);
+      }
+    }
+    if (!reachable) {
+      throw new IllegalStateException(String.format("%s is not reachable after %s tries", serviceName, maxTries));
+    }
+  }
+}

ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/SolrSteps.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.steps;
+
+import org.apache.ambari.logsearch.domain.StoryDataRegistry;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocumentList;
+import org.jbehave.core.annotations.Given;
+import org.jbehave.core.annotations.Then;
+import org.jbehave.core.annotations.When;
+import org.jbehave.core.model.Story;
+import org.junit.Assert;
+
+import java.io.IOException;
+
+public class SolrSteps {
+
+  @Then("the number of $component docs is: $docSize")
+  public void numberOfDocsForComponent(String component, int docSize) throws IOException, SolrServerException {
+    SolrClient solrClient = StoryDataRegistry.INSTANCE.getCloudSolrClient();
+    SolrQuery solrQuery = new SolrQuery();
+    solrQuery.setQuery(String.format("type:%s", component));
+    QueryResponse queryResponse = solrClient.query(StoryDataRegistry.INSTANCE.getServiceLogsCollection(), solrQuery);
+    SolrDocumentList list = queryResponse.getResults();
+    Assert.assertEquals(docSize, list.size());
+  }
+}

ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchStory.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.story;
+
+import org.apache.ambari.logsearch.steps.SolrSteps;
+import org.apache.ambari.logsearch.steps.LogSearchDockerSteps;
+import org.jbehave.core.configuration.Configuration;
+import org.jbehave.core.configuration.MostUsefulConfiguration;
+import org.jbehave.core.io.LoadFromClasspath;
+import org.jbehave.core.io.StoryPathResolver;
+import org.jbehave.core.io.UnderscoredCamelCaseResolver;
+import org.jbehave.core.junit.JUnitStory;
+import org.jbehave.core.reporters.Format;
+import org.jbehave.core.reporters.StoryReporterBuilder;
+import org.jbehave.core.steps.InjectableStepsFactory;
+import org.jbehave.core.steps.InstanceStepsFactory;
+import org.junit.Test;
+
+abstract public class LogSearchStory extends JUnitStory {
+  @Override
+  public Configuration configuration() {
+    StoryPathResolver storyPathResolver = new UnderscoredCamelCaseResolver(".story");
+    return new MostUsefulConfiguration()
+      .useStoryPathResolver(storyPathResolver)
+      .useStoryLoader(new LoadFromClasspath(this.getClass()))
+      .useStoryReporterBuilder(
+        new StoryReporterBuilder().withFailureTrace(true).withDefaultFormats().withFormats(Format.CONSOLE, Format.TXT));
+  }
+
+  @Override
+  public InjectableStepsFactory stepsFactory() {
+    return new InstanceStepsFactory(configuration(),
+      new LogSearchDockerSteps(),
+      new SolrSteps());
+  }
+
+  @Test
+  public void run() throws Throwable {
+    super.run();
+  }
+
+}

ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogfeederParsingStory.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.story;
+
+public class LogfeederParsingStory extends LogSearchStory {
+}

ambari-logsearch/ambari-logsearch-it/src/test/resources/log4j.properties

@@ -0,0 +1,16 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+log4j.rootLogger=INFO, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n

ambari-logsearch/ambari-logsearch-it/src/test/resources/org/apache/ambari/logsearch/story/logfeeder_parsing_story.story

@@ -0,0 +1,18 @@
+Story Service logs are parsed and stored into Solr
+
+Narrative:
+As a user
+I want to start logsearch/logfeeder/solr components in a docker container with test logs
+So that I can parse and store the logs into Solr
+
+Scenario: Logsearch logs are stored into Solr.
+
+Given logsearch docker container
+When logfeeder started (parse logs & send data to solr)
+Then the number of logsearch_app docs is: 1
+
+Scenario: Zookeeper logs are stored into Solr.
+
+Given logsearch docker container
+When logfeeder started (parse logs & send data to solr)
+Then the number of zookeeper docs is: 3

ambari-logsearch/pom.xml

@@ -34,6 +34,7 @@
     <module>ambari-logsearch-portal</module>
     <module>ambari-logsearch-logfeeder</module>
     <module>ambari-logsearch-solr-client</module>
+    <module>ambari-logsearch-it</module>
   </modules>
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -162,6 +163,7 @@
             <exclude>**/*.json</exclude>
             <exclude>**/*.log</exclude>
             <exclude>**/*.txt</exclude>
+            <exclude>**/*.story</exclude>
             <exclude>**/docker/Profile</exclude>
           </excludes>
         </configuration>