Hi ,
I have a dataflow streaming job running with a datastream to big query template , now my source data base has only 4 columns but after the dataflow job my final table has so many columns like _metadata_timestamp, _metadata_read_timestamp etc , I have tried to remove these columns with the help of udf parameter but now i am not getting any data in the staging or final table. this is my udf
function process(inJson) {
const obj = JSON.parse(inJson);
// Concatenate two fields: column1 and column2
obj.Name = obj.First_name +" "+ obj.Last_Name;
// Optionally, you can remove the original fields if needed
delete obj.First_name;
delete obj.Last_Name;
delete obj._metadata_timestamp;
delete obj._metadata_read_timestamp;
delete obj._metadata_source_timestamp;
delete obj._metadata_read_method;
delete obj._metadata_source_type;
delete obj._metadata_deleted;
delete obj._metadata_change_type;
delete obj._metadata_uuid;
delete obj._metadata_log_file;
delete obj._metadata_log_position;
return JSON.stringify(obj);
}
how can i remove those columns in the dataflow job itself?