AMBARI-8894. Remove unnecessary dependencies from flume and storm sinks. (mpapirkovskyy)

ambari-metrics/ambari-metrics-assembly/pom.xml

@@ -40,7 +40,9 @@
     <deb.python.ver>python (&gt;= 2.6)</deb.python.ver>
     <deb.architecture>amd64</deb.architecture>
     <deb.dependency.list>${deb.python.ver},python-dev,gcc</deb.dependency.list>
-    <sinkJarName>ambari-metrics-hadoop-sink-with-common-${project.version}.jar</sinkJarName>
+    <hadoop.sink.jar>ambari-metrics-hadoop-sink-with-common-${project.version}.jar</hadoop.sink.jar>
+    <storm.sink.jar>ambari-metrics-storm-sink-with-common-${project.version}.jar</storm.sink.jar>
+    <flume.sink.jar>ambari-metrics-flume-sink-with-common-${project.version}.jar</flume.sink.jar>
   </properties>
 
   <build>
@@ -177,9 +179,6 @@
                   <needarch>x86_64</needarch>
                   <autoRequires>false</autoRequires>
                   <requires>
-                    <require>ambari-metrics-hadoop-sink</require>
-                    <require>ambari-metrics-flume-sink</require>
-                    <require>ambari-metrics-storm-sink</require>
                     <require>${python.ver}</require>
                     <require>gcc</require>
                     <require>python-devel</require>
@@ -409,7 +408,7 @@
                   <defaultGroupname>root</defaultGroupname>
 
                   <postinstallScriptlet>
-                    <scriptFile>${hadoop-sink.dir}/target/classes/rpm/postinstall.sh</scriptFile>
+                    <scriptFile>${project.basedir}/src/main/package/rpm/sink/postinstall.sh</scriptFile>
                     <fileEncoding>utf-8</fileEncoding>
                   </postinstallScriptlet>
 
@@ -423,75 +422,27 @@
                       </sources>
 
                     </mapping>
-
-                  </mappings>
-                </configuration>
-
-              </execution>
-
-              <!-- flume-sink -->
-              <execution>
-                <id>ambari-metrics-flume-sink</id>
-                <phase>package</phase>
-                <goals>
-                  <goal>rpm</goal>
-                </goals>
-                <configuration>
-                  <name>ambari-metrics-flume-sink</name>
-                  <copyright>2012, Apache Software Foundation</copyright>
-                  <group>Development</group>
-                  <description>Maven Recipe: RPM Package.</description>
-                  <mappings>
                     <mapping>
                       <directory>/usr/lib/flume/lib</directory>
-                      <filemode>644</filemode>
-                      <username>root</username>
-                      <groupname>root</groupname>
                       <sources>
                         <source>
-                          <location>${flume-sink.dir}/target/ambari-metrics-flume-sink-${project.version}.jar</location>
-                        </source>
-                        <source>
-                          <location>${flume-sink.dir}/target/lib</location>
+                          <location>${flume-sink.dir}/target/ambari-metrics-flume-sink-with-common-${project.version}.jar</location>
                         </source>
                       </sources>
                     </mapping>
-                  </mappings>
-                </configuration>
-              </execution>
-
-              <!-- storm-sink -->
-              <execution>
-                <id>ambari-metrics-storm-sink</id>
-                <phase>package</phase>
-                <goals>
-                  <goal>rpm</goal>
-                </goals>
-
-                <configuration>
-                  <name>ambari-metrics-storm-sink</name>
-                  <copyright>2012, Apache Software Foundation</copyright>
-                  <group>Development</group>
-                  <description>Maven Recipe: RPM Package.</description>
-                  <defaultDirmode>755</defaultDirmode>
-                  <defaultFilemode>644</defaultFilemode>
-                  <defaultUsername>root</defaultUsername>
-                  <defaultGroupname>root</defaultGroupname>
-                  <mappings>
                     <mapping>
                       <directory>/usr/lib/storm/lib</directory>
                       <sources>
                         <source>
-                          <location>${storm-sink.dir}/target/ambari-metrics-storm-sink-${project.version}.jar</location>
-                        </source>
-                        <source>
-                          <location>${storm-sink.dir}/target/lib</location>
+                          <location>${storm-sink.dir}/target/ambari-metrics-storm-sink-with-common-${project.version}.jar</location>
                         </source>
                       </sources>
                     </mapping>
                   </mappings>
                 </configuration>
+
               </execution>
+
             </executions>
           </plugin>
         </plugins>
@@ -730,7 +681,7 @@
                 <!-- flume sink -->
 
                 <data>
-                  <src>${flume-sink.dir}/target/ambari-metrics-flume-sink-${project.version}.jar</src>
+                  <src>${flume-sink.dir}/target/${flume.sink.jar}</src>
                   <type>file</type>
                   <mapper>
                     <type>perm</type>
@@ -739,20 +690,11 @@
                     <prefix>/usr/lib/flume/lib</prefix>
                   </mapper>
                 </data>
-                <data>
-                  <src>${flume-sink.dir}/target/lib</src>
-                  <type>directory</type>
-                  <mapper>
-                    <type>perm</type>
-                    <filemode>644</filemode>
-                    <prefix>/usr/lib/flume/lib</prefix>
-                  </mapper>
-                </data>
 
                 <!-- storm sink -->
 
                 <data>
-                  <src>${storm-sink.dir}/target/ambari-metrics-storm-sink-${project.version}.jar</src>
+                  <src>${storm-sink.dir}/target/${storm.sink.jar}</src>
                   <type>file</type>
                   <mapper>
                     <type>perm</type>
@@ -761,15 +703,6 @@
                     <prefix>/usr/lib/storm/lib</prefix>
                   </mapper>
                 </data>
-                <data>
-                  <src>${storm-sink.dir}/target/lib</src>
-                  <type>directory</type>
-                  <mapper>
-                    <type>perm</type>
-                    <filemode>644</filemode>
-                    <prefix>/usr/lib/storm/lib</prefix>
-                  </mapper>
-                </data>
 
               </dataSet>
             </configuration>

ambari-metrics/ambari-metrics-assembly/src/main/package/deb/control/postinst

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -14,11 +14,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License
 
-LINK_NAME="/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar"
-JAR_NAME="/usr/lib/ambari-metrics-hadoop-sink/${sinkJarName}"
+HADOOP_LINK_NAME="/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar"
+FLUME_LINK_NAME="/usr/lib/flume/lib/ambari-metrics-flume-sink.jar"
+HADOOP_SINK_JAR="/usr/lib/ambari-metrics-hadoop-sink/${hadoop.sink.jar}"
+FLUME_SINK_JAR="/usr/lib/flume/lib/${flume.sink.jar}"
+#link for storm jar not required with current loading
+#STORM_SINK_JAR="/usr/lib/storm/lib/${storm.sink.jar}"
+#STORM_LINK_NAME="/usr/lib/storm/lib/ambari-metrics-storm-sink.jar"
 
-if [ -e "$LINK_NAME" ]; then
-  rm -f $LINK_NAME
-fi
+JARS=(${HADOOP_SINK_JAR} ${FLUME_SINK_JAR})
+LINKS=(${HADOOP_LINK_NAME} ${FLUME_LINK_NAME})
 
-ln -s $JAR_NAME $LINK_NAME
+for index in ${!LINKS[*]}
+do
+  if [ -e "${LINKS[$index]}" ]; then
+    rm -f ${LINKS[$index]}
+  fi
+
+  ln -s ${JARS[$index]} ${LINKS[$index]}
+done

ambari-metrics/ambari-metrics-hadoop-sink/src/main/package/rpm/postinstall.sh → ambari-metrics/ambari-metrics-assembly/src/main/package/rpm/sink/postinstall.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -14,11 +14,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License
 
-LINK_NAME="/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar"
-JAR_NAME="/usr/lib/ambari-metrics-hadoop-sink/${sinkJarName}"
+HADOOP_LINK_NAME="/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar"
+FLUME_LINK_NAME="/usr/lib/flume/lib/ambari-metrics-flume-sink.jar"
+HADOOP_SINK_JAR="/usr/lib/ambari-metrics-hadoop-sink/${hadoop.sink.jar}"
+FLUME_SINK_JAR="/usr/lib/flume/lib/${flume.sink.jar}"
+#link for storm jar not required with current loading
+#STORM_SINK_JAR="/usr/lib/storm/lib/${storm.sink.jar}"
+#STORM_LINK_NAME="/usr/lib/storm/lib/ambari-metrics-storm-sink.jar"
 
-if [ -e "$LINK_NAME" ]; then
-  rm -f $LINK_NAME
-fi
+JARS=(${HADOOP_SINK_JAR} ${FLUME_SINK_JAR})
+LINKS=(${HADOOP_LINK_NAME} ${FLUME_LINK_NAME})
 
-ln -s $JAR_NAME $LINK_NAME
+for index in ${!LINKS[*]}
+do
+  if [ -e "${LINKS[$index]}" ]; then
+    rm -f ${LINKS[$index]}
+  fi
+
+  ln -s ${JARS[$index]} ${LINKS[$index]}
+done

ambari-metrics/ambari-metrics-flume-sink/pom.xml

@@ -30,20 +30,6 @@ limitations under the License.
   <packaging>jar</packaging>
   <build>
     <plugins>
-      <plugin>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/lib</outputDirectory>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <artifactId>maven-assembly-plugin</artifactId>
         <executions>
@@ -114,12 +100,12 @@ limitations under the License.
       <groupId>org.apache.flume</groupId>
       <artifactId>flume-ng-core</artifactId>
       <version>1.5.1</version>
-      <scope>provided</scope>
+      <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.ambari</groupId>
       <artifactId>ambari-metrics-common</artifactId>
-      <version>0.1.0-SNAPSHOT</version>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>junit</groupId>

ambari-metrics/ambari-metrics-storm-sink/pom.xml

@@ -30,20 +30,6 @@ limitations under the License.
   <packaging>jar</packaging>
   <build>
     <plugins>
-      <plugin>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/lib</outputDirectory>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <artifactId>maven-assembly-plugin</artifactId>
         <executions>
@@ -114,12 +100,12 @@ limitations under the License.
       <groupId>org.apache.storm</groupId>
       <artifactId>storm-core</artifactId>
       <version>0.9.3</version>
-      <scope>provided</scope>
+      <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.ambari</groupId>
       <artifactId>ambari-metrics-common</artifactId>
-      <version>0.1.0-SNAPSHOT</version>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>junit</groupId>

ambari-server/src/main/resources/common-services/AMS/0.1.0/metainfo.xml

@@ -73,6 +73,9 @@
             <package>
               <name>ambari-metrics-monitor</name>
             </package>
+            <package>
+              <name>ambari-metrics-hadoop-sink</name>
+            </package>
             <package>
               <name>hadoop_2_2_*</name>
             </package>

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/configuration/flume-env.xml

@@ -69,8 +69,10 @@ export JAVA_HOME={{java_home}}
 # export JAVA_OPTS="-Xms100m -Xmx2000m -Dcom.sun.management.jmxremote"
 
 # Note that the Flume conf directory is always included in the classpath.
-#TODO temporary addition
-export FLUME_CLASSPATH=$FLUME_CLASSPATH:/usr/lib/flume/lib/*
+# Add flume sink to classpath
+if [ -e "/usr/lib/flume/lib/ambari-metrics-flume-sink.jar" ]; then
+  export FLUME_CLASSPATH=$FLUME_CLASSPATH:/usr/lib/flume/lib/ambari-metrics-flume-sink.jar
+fi
 
 export HIVE_HOME={{flume_hive_home}}
 export HCAT_HOME={{flume_hcat_home}}