AMBARI-5720. pig.properties should set pig.location.check.strict to false. (Jonathan Hurley via swagle)

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-properties.xml

@@ -0,0 +1,89 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+  <property>
+    <name>pig-content</name>
+    <description>Describe all the Pig agent configurations</description>
+    <value>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments.
+# see bin/pig -help
+
+# brief logging (no timestamps)
+brief=false
+
+# debug level, INFO is default
+debug=INFO
+
+# verbose print all log messages to screen (default to print only INFO and above to screen)
+verbose=false
+
+# exectype local|mapreduce, mapreduce is default
+exectype=mapreduce
+
+# Enable insertion of information about script into hadoop job conf 
+pig.script.info.enabled=true
+
+# Do not spill temp files smaller than this size (bytes)
+pig.spill.size.threshold=5000000
+
+# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
+# This should help reduce the number of files being spilled.
+pig.spill.gc.activation.size=40000000
+
+# the following two parameters are to help estimate the reducer number
+pig.exec.reducers.bytes.per.reducer=1000000000
+pig.exec.reducers.max=999
+
+# Temporary location to store the intermediate data.
+pig.temp.dir=/tmp/
+
+# Threshold for merging FRJoin fragment files
+pig.files.concatenation.threshold=100
+pig.optimistic.files.concatenation=false;
+
+pig.disable.counter=false
+
+hcat.bin=/usr/bin/hcat
+
+    </value>
+  </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig.properties

@@ -1,52 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments.
-# see bin/pig -help
-
-# brief logging (no timestamps)
-brief=false
-
-#debug level, INFO is default
-debug=INFO
-
-#verbose print all log messages to screen (default to print only INFO and above to screen)
-verbose=false
-
-#exectype local|mapreduce, mapreduce is default
-exectype=mapreduce
-
-#Enable insertion of information about script into hadoop job conf 
-pig.script.info.enabled=true
-
-#Do not spill temp files smaller than this size (bytes)
-pig.spill.size.threshold=5000000
-#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
-#This should help reduce the number of files being spilled.
-pig.spill.gc.activation.size=40000000
-
-#the following two parameters are to help estimate the reducer number
-pig.exec.reducers.bytes.per.reducer=1000000000
-pig.exec.reducers.max=999
-
-#Temporary location to store the intermediate data.
-pig.temp.dir=/tmp/
-
-#Threshold for merging FRJoin fragment files
-pig.files.concatenation.threshold=100
-pig.optimistic.files.concatenation=false;
-
-pig.disable.counter=false

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml

@@ -55,6 +55,7 @@
       <configuration-dependencies>
         <config-type>global</config-type>
         <config-type>pig-log4j</config-type>
+        <config-type>pig-properties</config-type>
       </configuration-dependencies>
 
     </service>

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py

@@ -38,7 +38,16 @@ kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "
 java64_home = config['hostLevelParams']['java_home']
 hadoop_home = "/usr"
 
-#log4j.properties
+# pig.properties - if not in the JSON command, then we need to esnure some 
+# basic properties are set; this is a safety mechanism
+if (('pig-properties' in config['configurations']) and ('pig-content' in config['configurations']['pig-properties'])):
+  pig_properties = config['configurations']['pig-properties']['pig-content']
+else:
+  pig_properties = {}
+  pig_properties['hcat.bin'] = '/usr/bin/hcat'
+  pig_properties['pig.location.check.strict'] = 'false'
+
+# log4j.properties
 if (('pig-log4j' in config['configurations']) and ('content' in config['configurations']['pig-log4j'])):
   log4j_props = config['configurations']['pig-log4j']['content']
 else:

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py

@@ -30,7 +30,14 @@ def pig():
     group = params.user_group
   )
 
-  pig_TemplateConfig( ['pig-env.sh','pig.properties'])
+  pig_TemplateConfig( ['pig-env.sh'])
+
+  # pig_properties is always set to a default even if it's not in the payload
+  PropertiesFile(format("{params.pig_conf_dir}/pig.properties"),    
+    mode = 0644,
+    group=params.user_group,
+    owner=params.hdfs_user,
+    properties=params.pig_properties)    
 
   if (params.log4j_props != None):
     File(format("{params.pig_conf_dir}/log4j.properties"),

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/templates/pig.properties.j2

@@ -1,73 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-#
-
-# Pig configuration file. All values can be overwritten by command line arguments.
-
-# log4jconf log4j configuration file
-# log4jconf=./conf/log4j.properties
-
-# a file that contains pig script
-#file=
-
-# load jarfile, colon separated
-#jar=
-
-#verbose print all log messages to screen (default to print only INFO and above to screen)
-#verbose=true
-
-#exectype local|mapreduce, mapreduce is default
-#exectype=local
-
-#pig.logfile=
-
-#Do not spill temp files smaller than this size (bytes)
-#pig.spill.size.threshold=5000000
-#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
-#This should help reduce the number of files being spilled.
-#pig.spill.gc.activation.size=40000000
-
-#the following two parameters are to help estimate the reducer number
-#pig.exec.reducers.bytes.per.reducer=1000000000
-#pig.exec.reducers.max=999
-
-#Use this option only when your Pig job will otherwise die because of
-#using more counter than hadoop configured limit
-#pig.disable.counter=true
-hcat.bin=/usr/bin/hcat

ambari-server/src/main/resources/stacks/HDP/2.1/services/PIG/configuration/pig-properties.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+  <property>
+    <name>pig-content</name>
+    <value>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments.
+# see bin/pig -help
+
+# brief logging (no timestamps)
+brief=false
+
+# debug level, INFO is default
+debug=INFO
+
+# verbose print all log messages to screen (default to print only INFO and above to screen)
+verbose=false
+
+# exectype local|mapreduce, mapreduce is default
+exectype=mapreduce
+
+# Enable insertion of information about script into hadoop job conf 
+pig.script.info.enabled=true
+
+# Do not spill temp files smaller than this size (bytes)
+pig.spill.size.threshold=5000000
+
+# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
+# This should help reduce the number of files being spilled.
+pig.spill.gc.activation.size=40000000
+
+# the following two parameters are to help estimate the reducer number
+pig.exec.reducers.bytes.per.reducer=1000000000
+pig.exec.reducers.max=999
+
+# Temporary location to store the intermediate data.
+pig.temp.dir=/tmp/
+
+# Threshold for merging FRJoin fragment files
+pig.files.concatenation.threshold=100
+pig.optimistic.files.concatenation=false;
+
+pig.disable.counter=false
+
+# Avoid pig failures when multiple jobs write to the same location
+pig.location.check.strict=false
+
+hcat.bin=/usr/bin/hcat
+
+    </value>
+  </property>
+
+</configuration>

ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py

@@ -31,13 +31,16 @@ class TestPigClient(RMFTestCase):
 
     self.assertResourceCalled('Directory', '/etc/pig/conf',
       owner = 'hdfs',
-      group = 'hadoop',
+      group = 'hadoop'
     )
     self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig-env.sh',
-      owner = 'hdfs',
+      owner = 'hdfs'
     )
-    self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties',
+    self.assertResourceCalled('PropertiesFile', '/etc/pig/conf/pig.properties',
       owner = 'hdfs',
+      group = 'hadoop',
+      mode = 0644,
+      properties = 'pigproperties\nline2'      
     )
     self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties',
       owner = 'hdfs',
@@ -56,13 +59,16 @@ class TestPigClient(RMFTestCase):
     
     self.assertResourceCalled('Directory', '/etc/pig/conf',
       owner = 'hdfs',
-      group = 'hadoop',
+      group = 'hadoop'
     )
     self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig-env.sh',
-      owner = 'hdfs',
+      owner = 'hdfs'
     )
-    self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties',
+    self.assertResourceCalled('PropertiesFile', '/etc/pig/conf/pig.properties',
       owner = 'hdfs',
+      group = 'hadoop',
+      mode = 0644,
+      properties = 'pigproperties\nline2'
     )
     self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties',
       owner = 'hdfs',

ambari-server/src/test/python/stacks/2.0.6/configs/default.json

@@ -512,6 +512,9 @@
         "pig-log4j": {
             "content": "log4jproperties\nline2"
         },
+        "pig-properties": {
+            "pig-content": "pigproperties\nline2"
+        },        
         "oozie-log4j": {
             "content": "log4jproperties\nline2"
         },
@@ -579,7 +582,10 @@
         },
         "pig-log4j": {
             "tag": "version1"
-        }
+        },
+        "pig-properties": {
+            "tag": "version1"
+        }        
     }, 
     "commandId": "7-1", 
     "clusterHostInfo": {

ambari-server/src/test/python/stacks/2.0.6/configs/secured.json

@@ -636,6 +636,9 @@
         "pig-log4j": {
             "content": "log4jproperties\nline2"
         },
+        "pig-properties": {
+            "pig-content": "pigproperties\nline2"
+        },        
         "oozie-log4j": {
             "content": "log4jproperties\nline2"
         }