Help Understanding Parser Logic

[GSCoNist-]

Hey everyone,

I've started working on a custom parser for NetSuite, since it's a major product my company currently uses. Initially, I was having some success extracting the necessary data from my logs. However, when I switched to a different raw log, I began running into issues.

I initialized all the variables I could identify from the raw logs, assuming that if a field didn’t exist, the parser would simply skip it and continue processing the rest. Unfortunately, that doesn’t seem to be happening.

Right now, I’m just dumping these fields into the additional_fields section. I know that’s not best practice, but since I’m the only one using the SIEM at the moment and enrichment for this log type isn’t a high priority, it’s working for now.

Here is my current parser logic (I removed some to make this post shorter):

filter {
  # This parser is just working for the incident we are currently having with GSM - it will need to be changed to include other fields
  mutate {
    replace => {
      "udm_event.idm.read_only_udm.metadata.vendor_name" => "Net Suite"
      "udm_event.idm.read_only_udm.metadata.event_type" => "GENERIC_EVENT"

      "title" => ""
      "type" => ""
      "user" => ""
      "scripttype" => ""
      "detail.clientReferenceInformation.code" => ""
      "detail.clientReferenceInformation.partner.solutionId" => ""
      #"detail.processingInformation.authorizationOptions.initiator.credentialStoredOnFile" => ""
      #"detail.processingInformation.authorizationOptions.ignoreAvsResult" => ""
      #"detail.processingInformation.authorizationOptions.ignoreCvResult" => ""
      "detail.processingInformation.commerceIndicator" => ""
      "detail.paymentInformation.card.number" => ""
      "detail.paymentInformation.card.expirationMonth" => ""
      "detail.paymentInformation.card.expirationYear" => ""
      "detail.paymentInformation.card.securityCode" => ""
      "detail.paymentInformation.card.type" => ""
      "detail.orderInformation.amountDetails.totalAmount" => ""
      "detail.orderInformation.amountDetails.currency" => ""
      "detail.orderInformation.amountDetails.discountAmount" => ""
      "detail.orderInformation.amountDetails.taxAmount" => ""
      "detail.orderInformation.billTo.firstName" => ""
      "detail.orderInformation.billTo.lastName" => ""
      "detail.orderInformation.billTo.address1" => ""
      "detail.orderInformation.billTo.locality" => ""
      "detail.orderInformation.billTo.administrativeArea" => ""
      "detail.orderInformation.billTo.postalCode" => ""
      "detail.orderInformation.billTo.country" => ""
      "detail.orderInformation.billTo.email" => ""
      "detail.orderInformation.billTo.phoneNumber" => ""
      "detail.orderInformation.shipTo.firstName" => ""
      "detail.orderInformation.shipTo.lastName" => ""
      "detail.orderInformation.shipTo.address1" => ""
      "detail.orderInformation.shipTo.locality" => ""
      "detail.orderInformation.shipTo.administrativeArea" => ""
      "detail.orderInformation.shipTo.postalCode" => ""
      "detail.orderInformation.shipTo.country" => ""
      "detail.paymentInsightsInformation.responseInsights.category" => ""
    }
  }

  # Parse JSON
  json {
    source => "message"
    on_error => "not_json"
  }

  statedump{label=TheStart}

  # Handle the title field
  if [title] != "" {
    mutate {
      replace => {
        "title_field.key" => "Title"
        "title_field.value.string_value" => "%{title}"
      }
    }
    mutate {
      merge => {
        "udm_event.idm.read_only_udm.additional.fields" => "title_field"
      }
    }
  }

  # Handle the type field
  if [type] != "" {
    mutate {
      replace => {
        "type_field.key" => "Type"
        "type_field.value.string_value" => "%{type}"
      }
    }
    mutate {
      merge => {
        "udm_event.idm.read_only_udm.additional.fields" => "type_field"
      }
    }
  }

  # Handle the user field
  if [user] != "" {
    mutate {
      replace => {
        "user_field.key" => "User"
        "user_field.value.string_value" => "%{user}"
      }
    }
    mutate {
      merge => {
        "udm_event.idm.read_only_udm.additional.fields" => "user_field"
      }
    }
  }

  # Handle the scripttype field
  if [scripttype] != "" {
    mutate {
      replace => {
        "scripttype_field.key" => "ScriptType"
        "scripttype_field.value.string_value" => "%{scripttype}"
      }
    }
    mutate {
      merge => {
        "udm_event.idm.read_only_udm.additional.fields" => "scripttype_field"
      }
    }
  }

  # Handle client reference information code
  if [detail][clientReferenceInformation][code] != "" {
    mutate {
        replace => {
            "clientrefcode_field.key" => "ClientReferenceCode"
            "clientrefcode_field.value.string_value" => "%{detail.clientReferenceInformation.code}"
        }
    }
    mutate {
        merge => {
            "udm_event.idm.read_only_udm.additional.fields" => "clientrefcode_field"
        }
    }
  }

  # Handle client solution ID
  if [detail][clientReferenceInformation][partner][solutionId] != "" {
    mutate {
        replace => {
            "solutionid_field.key" => "ClientSolutionId"
            "solutionid_field.value.string_value" => "%{detail.clientReferenceInformation.partner.solutionId}"
        }
    }
    mutate {
        merge => {
            "udm_event.idm.read_only_udm.additional.fields" => "solutionid_field"
        }
    }
  }

  statedump{label=TheEnd}

  mutate {
    merge => {
      "@output" => "udm_event"
    }
  }

This is the error that I am running into currently:
generic::unknown: pipeline.ParseLogEntry failed: LOG_PARSING_CBN_ERROR: "generic::invalid_argument: pipeline failed: filter conditional (2) failed: failed to evaluate expression: generic::invalid_argument: \"detail.clientReferenceInformation\" not found in state data"

And here is the raw log from Netsuite that I am attempting to parse:
{
"title": " Basic AUTHORIZATION Response Code",
"type": "Audit",
"date": "6/17/2025",
"time": "11:00 pm",
"user": "889990397 USERNAME",
"scripttype": "Payment Processing Plug-in",
"detail": 201,
"name.script": "Cybersource for NetSuite"
}

[Ben_T]

Hi GCSoNist,

Some additional conditional checks might help with this issue, and might look something like:

if [detail] and [detail][clientReferenceInformation] and [detail][clientReferenceInformation][code] != "" { mutate { replace => { "clientrefcode_field.key" => "ClientReferenceCode" "clientrefcode_field.value.string_value" => "%{detail.clientReferenceInformation.code}" } } mutate { merge => { "udm_event.idm.read_only_udm.additional.fields" => "clientrefcode_field" } } }

Hope this helps.

[GSCoNist-]

Thanks for the reply @Ben_T, I added in those additional checks but im still getting the same error:
generic::unknown: pipeline.ParseLogEntry failed: LOG_PARSING_CBN_ERROR: "generic::invalid_argument: pipeline failed: filter conditional (15) failed: failed to evaluate expression: generic::invalid_argument: \"detail.clientReferenceInformaton\" not found in state data"

I'm not sure why the fields arent being skipped since I initialize them at the beginning to be empty. I thought since they are empty it would just skip over them if the field is not in the raw log, is that not the case?

[GSCoNist-]

Just to add some additional context I just tried this raw log:
{
"title": "ERROR_SSP_LIBRARIES_EXT",
"type": "Error",
"date": "6/17/2025",
"time": "7:56 pm",
"user": "889990431 USERNAME",
"scripttype": "SSP Application",
"detail": {
"message": "Newsletter.ServiceController.Site missing ReCaptcha.Configuration.Model"
},
"name.script": "SuiteCommerce Advanced - Dev 2020.1"
}

And got no errors - I think it might be something with how I am using "detail" in the initialization but I am not 100% for sure.
Here is the UDM I got:

metadata.event_timestamp: "2025-06-18T13:24:43Z"
metadata.event_type: "GENERIC_EVENT"
metadata.vendor_name: "Net Suite"
metadata.log_type: "NET_SUITE"
additional.fields["ScriptType"]: "SSP Application"
additional.fields["Title"]: "ERROR_SSP_LIBRARIES_EXT"
additional.fields["Type"]: "Error"
additional.fields["User"]: "889990431 USER"

[Ben_T]

I think the root of the issue is that you're pre-creating empty placeholder fields for nested data at the start of your parser:
#Remove this "detail.clientReferenceInformation.code" => ""
#Remove this "detail.clientReferenceInformation.partner.solutionId" => ""

When a log comes in where detail is a simple number (like 201) instead of a complex object, your JSON parser correctly overwrites detail to be 201. However, your parser still thinks there might be a detail.clientReferenceInformation field because you created it initially. When it tries to check [detail][clientReferenceInformation], it finds that detail is now a number, not an object, so it can't look inside which is causing the error.

If you let your json parser create the 'detail' structure as it comes in your log then I think the condition checks you added previously will skip processing those parts when 'detail' isn't an object of in the cases when those nested fields are not present. Here's some sample code that may help:

filter {
mutate {
replace => {
"udm_event.idm.read_only_udm.metadata.vendor_name" => "Net Suite"
"udm_event.idm.read_only_udm.metadata.event_type" => "GENERIC_EVENT"

"title" => ""
"type" => ""
"user" => ""
"scripttype" => ""
#Remove this "detail.clientReferenceInformation.code" => ""
#Remove this "detail.clientReferenceInformation.partner.solutionId" => ""
# ... remove all other "detail.*" initializations ...
}
}

# Parse JSON - this will correctly create 'detail' as 201 or as an object
json {
source => "message"
on_error => "not_json"
}

statedump{label=TheStart}

# Placeholder for your existing title, type, user, scripttype handling

# Keep suggested conditional checks here
# Handle client reference information code
if [detail] and [detail][clientReferenceInformation] and [detail][clientReferenceInformation][code] != "" {
mutate {
replace => {
"clientrefcode_field.key" => "ClientReferenceCode"
"clientrefcode_field.value.string_value" => "%{detail.clientReferenceInformation.code}"
}
}
mutate {
merge => {
"udm_event.idm.read_only_udm.additional.fields" => "clientrefcode_field"
}
}
}

# Handle client solution ID
if [detail] and [detail][clientReferenceInformation] and [detail][clientReferenceInformation][partner] and [detail][clientReferenceInformation][partner][solutionId] != "" {
mutate {
replace => {
"solutionid_field.key" => "ClientSolutionId"
"solutionid_field.value.string_value" => "%{detail.clientReferenceInformation.partner.solutionId}"
}
}
mutate {
merge => {
"udm_event.idm.read_only_udm.additional.fields" => "solutionid_field"
}
}
}

# ... (continue with similar checks for all other detail fields) ...

statedump{label=TheEnd}

mutate {
merge => {
"@output" => "udm_event"
}
}
}

[GSCoNist-]

Hey @Ben_T, I removed the fields from the initialization and added the additional conditional checks and i'm still getting the generic error:
 generic::unknown: pipeline.ParseLogEntry failed: LOG_PARSING_CBN_ERROR: "generic::invalid_argument: pipeline failed: filter conditional (15) failed: failed to evaluate expression: generic::invalid_argument: \"detail.clientReferenceInformation\" not found in state data"

It doesnt seem to like my nested JSON and it doesnt look like it is creating the detail structure.

[GSCoNist-]

Hey @Ben_T any other ideas that I could try? I've removed all the initialized fields and i'm still getting the same generic error message about fields not be in the state data.

[Ben_T]

Looking at this further, additional conditional checks might be required for cases where dtail is not an object(integer).  I would try removing any lines that initialize detail fields that are nested like: detail.clientReferenceInformation.code. You could then add conditional checks to verify the existence of each parent level.
Something like: if [detail] and [detail][firstLevel] and [detail][firstLevel][secondLevel].

Then validate, using both log types where detail is an integer and another where it's an object.

Hope this helps.

[GSCoNist-]

Hey @Ben_T I found what seems to be a hacky way around what was happening with nested JSON. I initialized the field names and then used a mutate replace on the initialized field with the value from the log. Idk if that is proper way but I also added some error handling just in case.

filter {
  mutate {
    replace => {
      "udm_event.idm.read_only_udm.metadata.vendor_name" => "Net Suite"
      "udm_event.idm.read_only_udm.metadata.event_type" => "GENERIC_EVENT"

      "title" => ""
      "type" => ""
      "user" => ""
      "scripttype" => ""
      "name.script" => ""

      "detail_message" => ""
      
      
      "client_ref_code" => ""
      "client_ref_solutionid" => ""
      #"action_list" => ""
      "card_number" => ""
      "commerce_indicator" => ""
    }
  }

  # Parse JSON
  json {
    source => "message"
    array_function => "split_columns"
    on_error => "_not_json"
  }

  statedump{label=TheStart}

  if [_not_json] {
    drop { tag => "TAG_MALFORMED_MESSAGE"}
  }

  # Title field
  if [title] != "" {
    mutate {
        replace => {
            "title_field.key" => "title"
            "title_field.value.string_value" => "%{title}"
        }
    }
    mutate {
        merge => {
            "udm_event.idm.read_only_udm.additional.fields" => "title_field"
        }
    }
  }

  # Type field
  if [type] != "" {
    mutate {
        replace => {
            "type_field.key" => "type"
            "type_field.value.string_value" => "%{type}"
        }
    }
    mutate {
        merge => {
            "udm_event.idm.read_only_udm.additional.fields" => "type_field"
        }
    }
  }

  # User field
  if [user] != "" {
    mutate {
        replace => {
            "user_field.key" => "user"
            "user_field.value.string_value" => "%{user}"
        }
    }
    mutate {
        merge => {
            "udm_event.idm.read_only_udm.additional.fields" => "user_field"
        }
    }
  }

  # Script type field
  if [scripttype] != "" {
    mutate {
        replace => {
            "scripttype_field.key" => "scriptType"
            "scripttype_field.value.string_value" => "%{scripttype}"
        }
    }
    mutate {
        merge => {
            "udm_event.idm.read_only_udm.additional.fields" => "scripttype_field"
        }
    }
  }

  # Script name field
  if [name][script] != "" {
    mutate {
        replace => {
            "scriptname_field.key" => "scriptName"
            "scriptname_field.value.string_value" => "%{name.script}"
        }
    }
    mutate {
        merge => {
            "udm_event.idm.read_only_udm.additional.fields" => "scriptname_field"
        }
    }
  }

  # Replace detail_message
  mutate {
    replace => {
        "detail_message" => "%{detail.message}"
    }
    on_error => "_no_detail_message"
  }
  # Logic for detail_message field
  if ![_no_detail_message] and [detail_message] != "" {
    mutate {
        replace => {
            "detailmessage_field.key" => "detailMessage"
            "detailmessage_field.value.string_value" => "%{detail.message}"
        }
    }
    mutate {
        merge => {
            "udm_event.idm.read_only_udm.additional.fields" => "detailmessage_field"
        }
    }
  }

  # Replace client_ref_code
  mutate {
    replace => {
        "client_ref_code" => "%{detail.clientReferenceInformation.code}"
    }
    on_error => "_no_clientref_path"
  }
  # Logic for client_ref_code field
  if ![_no_clientref_path] and [client_ref_code] != "" {
    mutate {
        replace => {
            "clientref_field.key" => "clientReferenceCode"
            "clientref_field.value.string_value" => "%{detail.clientReferenceInformation.code}"
        }
    }
    mutate {
        merge => {
            "udm_event.idm.read_only_udm.additional.fields" => "clientref_field"
        }
    }
  }

  # Replace client_ref_solutionid
  mutate {
    replace => {
        "client_ref_solutionid" => "%{detail.clientReferenceInformation.partner.solutionId}"
    }
    on_error => "_no_clientref_solutionid"
  }
  # Logic for client_ref_solutionid
  if ![_no_clientref_solutionid] and [client_ref_solutionid] != "" {
    mutate {
        replace => {
            "solutionid_field.key" => "clientSolutionId"
            "solutionid_field.value.string_value" => "%{detail.clientReferenceInformation.partner.solutionId}"
        }
    }
    mutate {
        merge => {
            "udm_event.idm.read_only_udm.additional.fields" => "solutionid_field"
        }
    }
  }



  # Replace commerce_indicator
  mutate {
    replace => {
        "commerce_indicator" => "%{detail.processingInformation.commerceIndicator}"
    }
    on_error => "_no_commerce_indicator"
  }
  # Logic for commerce_indicator
  if ![_no_commerce_indicator] and [commerce_indicator] != "" {
    mutate {
        replace => {
            "commerceindicator_field.key" => "commerceIndicator"
            "commerceindicator_field.value.string_value" => "%{detail.processingInformation.commerceIndicator}"
        }
    }
    mutate {
        merge => {
            "udm_event.idm.read_only_udm.additional.fields" => "commerceindicator_field"
        }
    }
  }

  statedump{label=TheEnd}

  mutate {
    merge => {
      "@output" => "udm_event"
    }
  }
}

 I still have a lot to add but so far its working pretty nice! 

[Ben_T]

Excellent, glad to see the progress!