The Google Cloud Security Community will be in read-only mode July 16 - July 22 as we migrate platforms. Read more and check out our FAQ.

[Roni]

Hello,

How do i parse the values from the raw log which is viewed in a JSON format into a code snippet whether the value is a top level or nested? 

In the example - 

{

  "textPayload": "  customer: 12345,",

  "insertId": "avhvhjmk",

  "resource": {

    "type": "k8s_container",

    "labels": {

      "cluster_name": "test-gke",

      "project_id": "project-gke",

      "location": "me-west1",

      "container_name": "test-container",

      "namespace_name": "default",

      "pod_name": ”test-pod"

    }

  },

  "timestamp": "2025-01-07T13:29:46.060650557Z",

  "severity": "INFO",

  "labels": {

    "k8s-pod/app": "test-app",

    "k8s-pod/pod-template-hash": "513gvhcd",

    "compute.googleapis.com/resource_name": "test-gke-pool"

  },

  "logName": "projects/project-gke/logs/stdout",

  "receiveTimestamp": "2025-01-07T13:29:46.193718123Z"

}

I’d like to convert the “location” field into the udm.target.location.name value, “severity” field into the udm.principal.security_result.severity_details value, and the “k8s-pod/app” field into the udm.principal.application value.

How can I do this?

Thank you!

[mikewilusz]

Hi,

I'd recommend taking a look at a recent blog I did related to parsing JSON logs to UDM. It should contain all the example to do what you're looking for: https://medium.com/@cloudymike/parsing-netflow-data-in-google-secops-2f1b0f58ea49

-mike

[sudeep_singh]

Hi @Roni ,

Please use the below mentioned snippet it will successfully parse the fields which you have mentioned. Using "gsub" function i have converted the embedded Json into KV format.

filter {

  mutate {

    replace => {

       "event.idm.read_only_udm.metadata.event_type" => "GENERIC_EVENT"

       "severity" => ""

       "location" => ""

       "app" => ""

    }

  }

  mutate {

    gsub => ["message", "\\{", ""]

}

mutate {

    gsub => ["message", "\\}", ""]

}

mutate {

    gsub => ["message", '"',""]

}

mutate {

    gsub => ["message",", ",""]

}

mutate {

    gsub => ["message","\\\n",""]

}

mutate {

    replace => {

        "msg2" => "%{message}"

    }

}

    kv {

    source => "message"

    field_split => ","

    value_split => ":"

    on_error => "not_kv"

  }

 if [severity] != "" {

        mutate {

          replace => {

            "security_result.severity" => "INFORMATIONAL"

            "security_result.severity_details" => "Informational message only"

          }

        }

      }

  if [location] != "" {

    mutate {

      replace => {

        "event.idm.read_only_udm.principal.location.country_or_region" => "%{location}"

      }

    }

  }

   if [labels] =~ "k8s-pod/app" {

    grok {

    match => {

        "labels" => [

        'k8s-pod/app: %{GREEDYDATA:app}%{GREEDYDATA}'

        ]

    }

    overwrite => ["app"]

    on_error => "not_grok3"

    }

    }

    if [app] != "" {  

    mutate {

        replace => {

            "event.idm.read_only_udm.principal.application" => "%{app}"

        }

    }

  }

mutate {

        merge => {

            "event.idm.read_only_udm.security_result" => "security_result"

            }

          on_error => "no_security_result"  

            }

statedump {label => "1"}

mutate {

    merge => {

      "@output" => "event"

    }

  }

}

[Roni]

I tried this code but I got an error -

Error: generic::unimplemented: failed to create augmentor pipeline: failed to extract filters: failed to create filter from agent 17: filter factory failed for "statedump": line 81, col 9: no filter implemented for "statedump" label

Thanks

[sudeep_singh]

Hi @Roni ,

Statedump is just for debugging, while validating the parser please comment it out. The parser will be working as expected.