This website uses Cookies. Click Accept to agree to our website's cookie use as described in our Privacy Policy. Click Preferences to customize your cookie settings.
Accept
Reject
Preferences
Log in to ask a question
Announcements
New SecOps Webinar May 14th! Learn about Gemini's generative AI within Google SecOps
Log in to ask a question

Creating Custom Parser

Posted on 11-11-2024 12:07 PM
ayush_gupta_onb
Bronze 1
Bronze 1
Reply posted on --/--/---- --:-- AM

@Hieveryone ,
I am trying to create a custom parser for Mambu logs but got stuck with an error "generic::unknown: pipeline.ParseLogEntry failed: LOG_PARSING_CBN_ERROR: "generic::invalid_argument: failed to convert raw output to events: failed to convert raw message 0: field \"idm\": index 0: recursive rawDataToProto failed: field \"read_only_udm\": index 0: recursive rawDataToProto failed: field \"user\": no descriptor found"

Sample Logs: 

 

[
  {
    "occurred_at": "2024-11-07T05:51:11.132Z",
    "response_code": 302,
    "resource": "login",
    "event_source": "UI",
    "client_ip": "10.30.40.60",
    "request_method": "POST",
    "request_payload": "{}",
    "resource_fragment": "/saml/login",
    "request_uri": "/saml/login",
    "user_agent": "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
    "response_payload": "",
    "username": "john.doe@sampledomain.com"
  },
  {
    "occurred_at": "2024-11-07T05:51:03.373Z",
    "response_code": 302,
    "resource": "login",
    "event_source": "UI",
    "client_ip": "10.30.40.60",
    "request_method": "POST",
    "request_payload": "{}",
    "resource_fragment": "/saml/login",
    "request_uri": "/saml/login",
    "user_agent": "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
    "response_payload": "",
    "username": "john.doe@sampledomain.com"
  },
  {
    "occurred_at": "2024-11-07T05:50:39.913Z",
    "response_code": 302,
    "resource": "login",
    "event_source": "UI",
    "client_ip": "10.30.40.60",
    "request_method": "POST",
    "request_payload": "{\"RelayState\":[\"https://service.sample.com/saml/login\"]}",
    "resource_fragment": "/saml/login",
    "request_uri": "/saml/login",
    "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
    "response_payload": "",
    "username": "jane.doe@sampledomain.com"
  },
  {
    "occurred_at": "2024-11-07T05:50:33.135Z",
    "response_code": 401,
    "resource": "login",
    "event_source": "UI",
    "client_ip": "10.30.40.60",
    "request_method": "POST",
    "request_payload": "{\"username\":[\"john.doe\"],\"loginType\":[\"onlineLogin\"]}",
    "resource_fragment": "/servlet/login",
    "request_uri": "/servlet/login",
    "user_agent": "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
    "response_payload": "FAIL",
    "username": ""
  },
  {
    "occurred_at": "2024-11-07T05:50:06.470Z",
    "response_code": 429,
    "resource": "login",
    "event_source": "UI",
    "client_ip": "10.30.40.60",
    "request_method": "POST",
    "request_payload": "{\"username\":[\"john.doe\"],\"loginType\":[\"onlineLogin\"]}",
    "resource_fragment": "/servlet/login",
    "request_uri": "/servlet/login",
    "user_agent": "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
    "response_payload": "CAPTCHA_REQUIRED",
    "username": ""
  },
  {
    "occurred_at": "2024-11-07T05:48:51.489Z",
    "response_code": 401,
    "resource": "login",
    "event_source": "UI",
    "client_ip": "10.30.40.60",
    "request_method": "POST",
    "request_payload": "{\"username\":[\"john.doe\"],\"loginType\":[\"onlineLogin\"]}",
    "resource_fragment": "/servlet/login",
    "request_uri": "/servlet/login",
    "user_agent": "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
    "response_payload": "FAIL",
    "username": ""
  },
  {
    "occurred_at": "2024-11-07T05:48:34.643Z",
    "response_code": 429,
    "resource": "login",
    "event_source": "UI",
    "client_ip": "10.30.40.60",
    "request_method": "POST",
    "request_payload": "{\"username\":[\"john.doe\"],\"loginType\":[\"onlineLogin\"]}",
    "resource_fragment": "/servlet/login",
    "request_uri": "/servlet/login",
    "user_agent": "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
    "response_payload": "CAPTCHA_REQUIRED",
    "username": ""
  },
  {
    "occurred_at": "2024-11-07T05:48:01.810Z",
    "response_code": 302,
    "resource": "login",
    "event_source": "UI",
    "client_ip": "10.30.40.60",
    "request_method": "POST",
    "request_payload": "{}",
    "resource_fragment": "/saml/login",
    "request_uri": "/saml/login",
    "user_agent": "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
    "response_payload": "",
    "username": "john.doe@sampledomain.com"
  }
]

 

 

 

Parser Code: 

 

 

filter {
    # Wrap the message in JSON format
    mutate {
        replace => {
            "message" => "{\"records\":%{message}}"
        }
    }

    # Parse the JSON message
    json {
        source         => "message"
        array_function => "split_columns"
        on_error       => "not_json"
    }

    # Drop malformed messages
    if [not_json] {
        drop {
            tag => "TAG_MALFORMED_MESSAGE"
        }
    }

    # Process valid JSON messages
    if ![not_json] {
        for index, data in records {
            # Set metadata fields
            mutate {
                replace => {
                    "event.idm.read_only_udm.metadata.vendor_name" => "Mambu"
                    "event.idm.read_only_udm.metadata.product_name" => "Mambu"
                    "event.idm.read_only_udm.metadata.event_type" => "GENERIC_EVENT"
                }
            }

            # Map event source
            mutate {
                replace => {
                    "event.idm.read_only_udm.metadata.event_source" => "%{data.event_source}"
                }
                on_error => "invalid_event_source"
            }

            # Set principal IP address
            mutate {
                replace => {
                    "event.idm.read_only_udm.principal.ip" => "%{data.client_ip}"
                }
                on_error => "invalid_principal_ip"
            }

            # Map HTTP method and user agent
            mutate {
                replace => {
                    "event.idm.read_only_udm.network.http.method" => "%{data.request_method}"
                    "event.idm.read_only_udm.network.http.user_agent" => "%{data.user_agent}"
                    "event.idm.read_only_udm.network.http.request_payload" => "%{data.request_payload}"
                }
                on_error => "invalid_http_fields"
            }

            # Convert response code to string and handle errors
            mutate {
                convert => {
                    "data.response_code" => "string"
                }
                on_error => "invalid_response_code_conversion"
            }
            if ![invalid_response_code_conversion] {
                mutate {
                    replace => {
                        "event.idm.read_only_udm.network.http.response_code" => "%{data.response_code}"
                    }
                    on_error => "invalid_response_code"
                }
            }

            # Map target URL and resource
            mutate {
                replace => {
                    "event.idm.read_only_udm.target.url" => "%{data.request_uri}"
                    "event.idm.read_only_udm.target.resource" => "%{data.resource}"
                }
                on_error => "invalid_target_fields"
            }

            # Handle resource fragment label
            mutate {
                replace => {
                    "resource_fragment_label.key" => "resource_fragment"
                    "resource_fragment_label.value" => "%{data.resource_fragment}"
                }
                on_error => "invalid_resource_fragment_value"
            }

            if ![invalid_resource_fragment_value] {
                mutate {
                    merge => {
                        "event.idm.read_only_udm.principal.resource.attribute.labels" => "resource_fragment_label"
                    }
                    on_error => "resource_fragment_label_merge_failed"
                }
            }

            # Map user ID
            mutate {
                replace => {
                    "event.idm.read_only_udm.user.user_id" => "%{data.username}"
                }
                on_error => "invalid_user_id"
            }

            # Output the event
            statedump {}
            mutate {
                merge => {
                    "@output" => "event"
                }
                on_error => "output_merge_failed"
            }
        }
    }
}

 

 

 

0 1 108
Topic Labels
0 Likes
1 REPLY 1

Posted on --/--/---- --:-- AM
dnehoda
Staff
Reply posted on --/--/---- --:-- AM

This has been answered above.  

0 Likes
Top Solution Authors
View all