ambari/contrib/views/pig/blueprint.json

{
    "configurations" : [
        {
            "capacity-scheduler" : {
                "yarn.scheduler.capacity.root.default.maximum-am-resource-percent" :  "0.5",
                "yarn.scheduler.capacity.maximum-am-resource-percent" : "0.5"
            }
        },
        {
            "cluster-env": {
                "cluster_name": "PigView",
                "smokeuser": "ambari-qa",
                "user_group": "hadoop",
                "security_enabled": "false"
            }
        },
        {
            "core-site" : {
                "hadoop.proxyuser.root.hosts" : "*",
                "hadoop.proxyuser.root.groups" : "*",
                "hadoop.proxyuser.hcat.hosts" : "*",
                "hadoop.proxyuser.hcat.groups" : "*"
            }
        },
        {
            "hadoop-env": {
                "dtnode_heapsize" : "250",
                "hadoop_heapsize" : "250",
                "namenode_heapsize" : "250",
                "namenode_opt_newsize": "50",
                "namenode_opt_maxnewsize": "100",
                "content" : "\n# Set Hadoop-specific environment variables here.\n\n# The only required environment variable is JAVA_HOME.  All others are\n# optional.  When running a distributed configuration it is best to\n# set JAVA_HOME in this file, so that it is correctly defined on\n# remote nodes.\n\n# The java implementation to use.  Required.\nexport JAVA_HOME={{java_home}}\nexport HADOOP_HOME_WARN_SUPPRESS=1\n\n# Hadoop home directory\nexport HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop}\n\n# Hadoop Configuration Directory\n#TODO: if env var set that can cause problems\nexport HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-{{hadoop_conf_dir}}}\n\n{# this is different for HDP1 #}\n# Path to jsvc required by secure HDP 2.0 datanode\nexport JSVC_HOME={{jsvc_path}}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\n\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\n\n# Extra Java runtime options.  Empty by default.\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\n\n# Command specific options appended to HADOOP_OPTS when specified\nexport HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}\"\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\n\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\nHADOOP_DATANODE_OPTS=\"-Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}\"\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\n\nexport HADOOP_SECONDARYNAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps ${HADOOP_NAMENODE_INIT_HEAPSIZE} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}\"\n\n# The following applies to multiple commands (fs, dfs, fsck, distcp etc)\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS\"\n\n# Extra ssh options.  Empty by default.\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\n\n# Where log files are stored.  $HADOOP_HOME/logs by default.\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\n\n# History server logs\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\n\n# File naming remote slave hosts.  $HADOOP_HOME/conf/slaves by default.\n# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\n\n# host:path where hadoop code should be rsync'd from.  Unset by default.\n# export HADOOP_MASTER=master:/home/$USER/src/hadoop\n\n# Seconds to sleep between slave commands.  Unset by default.  This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HADOOP_SLAVE_SLEEP=0.1\n\n# The directory where pid files are stored. /tmp by default.\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\n\n# History server pid\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\n\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\n\n# A string representing this instance of hadoop. $USER by default.\nexport HADOOP_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes.  See 'man nice'.\n\n# export HADOOP_NICENESS=10\n\n# Use libraries from standard classpath\nJAVA_JDBC_LIBS=\"\"\n#Add libraries required by mysql connector\nfor jarFile in `ls /usr/share/java/*mysql* 2>/dev/null`\ndo\n  JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n#Add libraries required by oracle connector\nfor jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`\ndo\n  JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n#Add libraries required by nodemanager\nMAPREDUCE_LIBS={{mapreduce_libs_path}}\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}\n\nif [ -d \"/usr/hdp/current/tez-client\" ]; then\n  export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf\nfi\n\n# Setting path to hdfs command line\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n\n#Mostly required for hadoop 2.0\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop/lib/native/Linux-amd64-64\n\n#Hadoop logging options\nexport HADOOP_ROOT_LOGGER={{hadoop_root_logger}}\n    "
            }
        },
        {
            "hive-env": {
                "content" : "\n if [ \"$SERVICE\" = \"cli\" ]; then\n   if [ -z \"$DEBUG\" ]; then\n     export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit\"\n   else\n     export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit\"\n   fi\n fi\n\n# The heap size of the jvm stared by hive shell script can be controlled via:\n\nexport HADOOP_HEAPSIZE=\"{{hive_heapsize}}\"\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS\"\n\n# Larger heap size may be required when running queries over large number of files or partitions.\n# By default hive shell scripts use a heap size of 256 (MB).  Larger heap size would also be\n# appropriate for hive server (hwi etc).\n\n\n# Set HADOOP_HOME to point to a specific hadoop install directory\nHADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hive Configuration Directory can be controlled by:\nexport HIVE_CONF_DIR={{hive_config_dir}}\n\n# Folder containing extra ibraries required for hive compilation/execution can be controlled by:\nif [ \"${HIVE_AUX_JARS_PATH}\" != \"\" ]; then\n  export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}\nelif [ -d \"/usr/lib/hive-webhcat/\" ]; then\n  export HIVE_AUX_JARS_PATH=/usr/lib/hive-webhcat/share/hcatalog/hive-hcatalog-core-*.jar\nelse\n  export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar\nfi\nexport METASTORE_PORT={{hive_metastore_port}}\n    "
            }
        },
        {
            "hbase-site" : {
                "hbase.security.authorization": "true",
                "hbase.rpc.engine": "org.apache.hadoop.hbase.ipc.SecureRpcEngine",
                "hbase_master_heapsize": "250",
                "hbase_regionserver_heapsize": "250",
                "hbase.rpc.protection": "PRIVACY"
            }
        },
        {
            "hdfs-site" : {
                "dfs.block.size" : "34217472",
                "dfs.replication" : "1",
                "dfs.namenode.accesstime.precision" : "3600000",
                "dfs.nfs3.dump.dir" : "/tmp/.hdfs-nfs",
                "dfs.nfs.exports.allowed.hosts" : "* rw",
                "dfs.datanode.max.xcievers" : "1024",
                "dfs.block.access.token.enable" : "false"
            }
        },
        {
            "hive-site" : {
                "javax.jdo.option.ConnectionPassword" : "hive",
                "hive.tez.container.size" : "250",
                "hive.tez.java.opts" : "-server -Xmx200m -Djava.net.preferIPv4Stack=true",
                "hive.heapsize" : "250",
                "hive.users.in.admin.role" : "hive",
                "hive_metastore_user_passwd" : "hive",
                "hive.server2.enable.impersonation": "true",
                "hive.compactor.check.interval": "300s",
                "hive.compactor.initiator.on": "true",
                "hive.compactor.worker.timeout": "86400s",
                "hive.enforce.bucketing": "true",
                "hive.support.concurrency": "true",
                "hive.exec.dynamic.partition.mode": "nonstrict",
                "hive.server2.enable.doAs": "false",
                "hive.txn.manager": "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager",
                "hive.txn.max.open.batch": "1000",
                "hive.txn.timeout": "300",
                "hive.security.authorization.enabled": "false"
            }
        },
        {
            "mapred-env": {
                "jobhistory_heapsize" : "250"
            }
        },
        {
            "mapred-site" : {
                "mapreduce.map.memory.mb" : "250",
                "mapreduce.reduce.memory.mb" : "250",
                "mapreduce.task.io.sort.mb" : "64",
                "yarn.app.mapreduce.am.resource.mb" : "250",
                "yarn.app.mapreduce.am.command-opts" : "-Xmx200m",
                "mapred.job.reduce.memory.mb" : "250",
                "mapred.child.java.opts" : "-Xmx200m",
                "mapred.job.map.memory.mb" : "250",
                "io.sort.mb" : "64",
                "mapreduce.map.java.opts" : "-Xmx200m",
                "mapreduce.reduce.java.opts" : "-Xmx200m"
            }
        },
        {
            "nagios-env":{
                "nagios_contact" : "admin@localhost.localdomain",
                "nagios_web_login" : "nagiosadmin",
                "nagios_web_password" : "admin"
            }
        },
        {
            "tez-site" : {
                "tez.am.java.opts" : "-server -Xmx200m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC",
                "tez.am.resource.memory.mb" : "250",
                "tez.dag.am.resource.memory.mb" : "250",
                "yarn.app.mapreduce.am.command-opts" : "-Xmx200m"
            }
        },
        {
            "webhcat-site" : {
                "webhcat.proxyuser.hcat.hosts" : "*",
                "webhcat.proxyuser.hcat.groups" : "*"
            }
        },
        {
            "yarn-env": {
                "apptimelineserver_heapsize" : "250",
                "resourcemanager_heapsize" : "250",
                "nodemanager_heapsize" : "250",
                "yarn_heapsize" : "250"
            }

        },
        {
            "yarn-site" : {
                "yarn.nodemanager.resource.memory-mb": "2250",
                "yarn.nodemanager.vmem-pmem-ratio" : "10",
                "yarn.scheduler.minimum-allocation-mb" : "250",
                "yarn.scheduler.maximum-allocation-mb" : "2250",
                "yarn.nodemanager.pmem-check-enabled" : "false",
                "yarn.acl.enable" : "false",
                "yarn.resourcemanager.webapp.proxyuser.hcat.groups" : "*",
                "yarn.resourcemanager.webapp.proxyuser.hcat.hosts" : "*"
            }
        }
    ],
    "host_groups" : [
        {
            "name" : "single_host",
            "components" : [
                {"name": "APP_TIMELINE_SERVER"},
                {"name" : "DATANODE"},
                {"name" : "GANGLIA_SERVER"},
                {"name" : "HDFS_CLIENT"},
                {"name" : "HISTORYSERVER"},
                {"name" : "HIVE_CLIENT"},
                {"name" : "HIVE_METASTORE"},
                {"name" : "HIVE_SERVER"},
                {"name" : "MAPREDUCE2_CLIENT"},
                {"name" : "MYSQL_SERVER"},
                {"name" : "NAGIOS_SERVER"},
                {"name" : "NAMENODE"},
                {"name" : "NODEMANAGER"},
                {"name" : "PIG"},
                {"name" : "RESOURCEMANAGER"},
                {"name" : "SECONDARY_NAMENODE"},
                {"name" : "TEZ_CLIENT"},
                {"name" : "WEBHCAT_SERVER"},
                {"name" : "YARN_CLIENT"},
                {"name" : "ZOOKEEPER_CLIENT"},
                {"name" : "ZOOKEEPER_SERVER"}
            ],
            "cardinality" : "1"
        }
    ],
    "Blueprints" : {
        "blueprint_name" : "pig-view",
        "stack_name" : "HDP",
        "stack_version" : "2.2"
    }
}