Hi,
I am loading data into a big query table as follows
async loadDataToTable(project_id, dataset_id, table_id, uris, source_format, schema) {
const result = new CustomResponse().result;
try {
const options = {
sourceUris: uris,
sourceFormat: source_format,
schema: schema,
location: this.bigquery_location,
ignoreUnknownValues: true,
destinationTable: {
datasetId: dataset_id,
projectId: project_id,
tableId: table_id,
},
};
const [job] = await this.connection.createJob({ configuration: { load: options } });
logInfo(
`Job ${job.id} started ${job.metadata.status.state} for dataset ${dataset_id} for table ${table_id}`,
this.pipe_id,
Events.MONGODB_QUERY,
);
const [metadata] = await job.promise();
// Wait for job to complete
const status = metadata.status;
if (status.state === "DONE") {
logInfo(
`Job ${job.id} done for dataset ${dataset_id} for table ${table_id}`,
this.pipe_id,
Events.MONGODB_QUERY,
);
result.data = {
rows_processed:
metadata.statistics && metadata.statistics.load && metadata.statistics.load.outputRows
? Number(metadata.statistics.load.outputRows)
: 0,
};
} else {
logInfo(
`Job current status: ${status.state} for dataset ${dataset_id} for table ${table_id}`,
this.pipe_id,
Events.MONGODB_QUERY,
);
if (status.errors) {
result.error_occurred = true;
result.data = { rows_processed: 0 };
result.error_info = {
error: JSON.stringify(status.errors),
error_source: "BigqueryService.loadDataToTable",
error_description: `Error occurred while doing load job at BigQuery table ${table_id} in dataset ${dataset_id} for pipe ID: ${this.pipe_id}`,
};
return result;
}
}
} catch (e) {
result.error_occurred = true;
result.data = { rows_processed: 0 };
result.error_info = {
error: e.errors ? JSON.stringify(e.errors) : `${e.name}: ${e.message}`,
error_stack: e.stack,
error_source: "BigqueryService.loadDataToTable",
error_description: `Error occurred while loading data into BigQuery table ${table_id} in dataset ${dataset_id} for pipe ID: ${this.pipe_id}`,
};
}
return result;
}the problem I am facing is, suppose I have 10 URIs to upload, and like 2 or 3 URIs are having the problem due to which the whole load job is failing, I want to find out which URIs are causing the issue,
I have tried to get the metadata of the job but that does not contain the URIs
I would really appreciate it if someone can tell me the way to do it
LinkedIn
Twitter
