ambari/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-hdfs.json

{
  "input": [
    {
      "type": "hdfs_audit",
      "rowtype": "audit",
      "is_enabled": "true",
      "add_fields": {
        "logType": "HDFSAudit",
        "enforcer": "hadoop-acl",
        "repoType": "1",
        "repo": "hdfs"
      },
      "path": "/root/test-logs/hdfs-audit/hdfs-audit.log"
    }
  ],
  "filter": [
    {
      "filter":"grok",
      "conditions":{
        "fields":{
          "type":[
            "hdfs_audit"
          ]

        }

      },
      "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
      "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})",
      "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
      "post_map_values":{
        "evtTime":{
          "map_date":{
            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
          }

        }

      }

    },
    {
      "filter":"keyvalue",
      "sort_order":1,
      "conditions":{
        "fields":{
          "type":[
            "hdfs_audit"
          ]

        }

      },
      "source_field":"log_message",
      "value_split":"=",
      "field_split":"\t",
      "post_map_values":{
        "src":{
          "map_fieldname":{
            "new_fieldname":"resource"
          }

        },
        "ip":{
          "map_fieldname":{
            "new_fieldname":"cliIP"
          }

        },
        "allowed":[
          {
            "map_fieldvalue":{
              "pre_value":"true",
              "post_value":"1"
            }

          },
          {
            "map_fieldvalue":{
              "pre_value":"false",
              "post_value":"0"
            }

          },
          {
            "map_fieldname":{
              "new_fieldname":"result"
            }

          }

        ],
        "cmd":{
          "map_fieldname":{
            "new_fieldname":"action"
          }

        },
        "proto":{
          "map_fieldname":{
            "new_fieldname":"cliType"
          }

        },
        "callerContext":{
          "map_fieldname":{
            "new_fieldname":"req_caller_id"
          }

        }

      }

    },
    {
      "filter":"grok",
      "sort_order":2,
      "source_field":"ugi",
      "remove_source_field":"false",
      "conditions":{
        "fields":{
          "type":[
            "hdfs_audit"
          ]

        }

      },
      "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}",
      "post_map_values":{
        "user":{
          "map_fieldname":{
            "new_fieldname":"reqUser"
          }

        },
        "x_user":{
          "map_fieldname":{
            "new_fieldname":"reqUser"
          }

        },
        "p_user":{
          "map_fieldname":{
            "new_fieldname":"reqUser"
          }

        },
        "k_user":{
          "map_fieldname":{
            "new_fieldname":"proxyUsers"
          }

        },
        "p_authType":{
          "map_fieldname":{
            "new_fieldname":"authType"
          }

        },
        "k_authType":{
          "map_fieldname":{
            "new_fieldname":"proxyAuthType"
          }

        }

      }

    }
  ]
}