Vertex AI Endpoint won't deploy - unknown error

[raviravi6]

Every time I try to deploy any model and endpoint on Vertex AI - standard Llama 3.1 model or a Huggingface embedding model, I get an error saying the endpoint can't be created because of an unknown error. This is happening for every model I deploy through the model garden GUI

This is what my logs say each time:
{
"protoPayload": {
"@type": "type.googleapis.com/google.cloud.audit.AuditLog",
"status": {
"code": 13,
"message": "java.util.NoSuchElementException: No value present\n\tat java.base/java.util.Optional.get(Unknown Source)\n\tat com.google.cloud.ai.platform.boq.shared.api.endpoint.CreateEndpointApiHandler.lambda$startCreateEndpointPipeline$31(CreateEndpointApiHandler.java:938)\n\tat com.google.intelligence.cloud.automl.boq.publics.pipeline.fluentapi.TaskFactory.createFromTaskMetadataGroup(TaskFactory.java:107)\n\tat com.google.intelligence.cloud.automl.boq.publics.pipeline.fluentapi.TaskFactory.createFromTaskMetadataGroup(TaskFactory.java:95)\n\tat com.google.intelligence.cloud.automl.boq.publics.pipeline.fluentapi.TaskFactory.createFromTaskMetadataGroup(TaskFactory.java:95)\n\tat com.google.intelligence.cloud.automl.boq.publics.pipeline.fluentapi.TaskFactory.createFromTaskMetadataGroup(TaskFactory.java:95)\n\tat com.google.intelligence.cloud.automl.boq.publics.pipeline.fluentapi.TaskFactory.createFromTaskMetadataGroup(TaskFactory.java:95)\n\tat com.google.intelligence.cloud.automl.boq.publics.pipeline.fluentapi.TaskFactory.create(TaskFactory.java:50)\n\tat com.google.intelligence.cloud.automl.boq.publics.pipeline.fluentapi.AbstractFluentPipelineHolder.buildPipelineWithContext(AbstractFluentPipelineHolder.java:583)\n\tat com.google.cloud.ai.platform.boq.shared.pipeline.fluentpipeline.LroFluentPipeline.buildPipeline(LroFluentPipeline.java:365)\n\tat com.google.cloud.ai.platform.boq.shared.pipeline.fluentpipeline.LroFluentPipeline.lambda$build$2(LroFluentPipeline.java:168)\n\tat com.google.async.promisegraph.ThenPromise.lambda$buildResultFromInputs$0(ThenPromise.java:91)\n\tat com.google.async.promisegraph.PromiseResult.fromSupplier(PromiseResult.java:29)\n\tat com.google.async.promisegraph.ThenPromise.buildResultFromInputs(ThenPromise.java:87)\n\tat com.google.async.promisegraph.AbstractPromise.complete(AbstractPromise.java:210)\n\tat com.google.async.promisegraph.GraphEnvironment.pumpLoop(GraphEnvironment.java:367)\n\tat com.google.async.promisegraph.GraphEnvironment.pumpWithoutBlocking(GraphEnvironment.java:305)\n\tat com.google.async.promisegraph.GraphEnvironment.executeWorkUntilBlockedOrCompleted(GraphEnvironment.java:553)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)\n\tat com.google.apps.framework.server.AbstractThreadPoolModule$InitializingThreadFactory.lambda$newThread$0(AbstractThreadPoolModule.java:424)\n\tat java.base/java.lang.Thread.run(Unknown Source)\n\tSuppressed: com.google.async.promisegraph.PromiseOriginException\n\t\tat com.google.cloud.ai.platform.boq.shared.pipeline.fluentpipeline.LroFluentPipeline.build(LroFluentPipeline.java:165)\n\tSuppressed: java.util.concurrent.ExecutionException: \n"
},
"authenticationInfo": {
"principalEmail": "XXXX@gmail.com"
},
"requestMetadata": {
"callerIp": "24.62.179.155",
"callerSuppliedUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36,gzip(gfe),gzip(gfe)",
"requestAttributes": {
"time": "2024-08-12T12:37:26.465940566Z",
"reason": "8uSywAYQGg5Db2xpc2V1bSBGbG93cw",
"auth": {}
},
"destinationAttributes": {}
},
"serviceName": "aiplatform.googleapis.com",
"methodName": "google.cloud.aiplatform.ui.EndpointService.CreateEndpoint",
"authorizationInfo": [
{
"resource": "projects/1044600842123/locations/us-east4",
"permission": "aiplatform.endpoints.create",
"granted": true,
"resourceAttributes": {},
"permissionType": "ADMIN_WRITE"
}
],
"resourceName": "projects/1044600842123/locations/us-east4",
"request": {
"@type": "type.googleapis.com/google.cloud.aiplatform.ui.CreateEndpointRequest",
"endpoint": {
"trafficSplit": {
"0": 100
}
},
"parent": "projects/1044600842123/locations/us-east4"
},
"resourceLocation": {
"currentLocations": [
"us-east4"
]
}
},
"insertId": "nm2n6re1kpjh",
"resource": {
"type": "audited_resource",
"labels": {
"service": "aiplatform.googleapis.com",
"project_id": "hale-aurora-429222-t4",
"method": "google.cloud.aiplatform.ui.EndpointService.CreateEndpoint"
}
},
"timestamp": "2024-08-12T12:37:26.459643373Z",
"severity": "ERROR",
"logName": "projects/hale-aurora-429222-t4/logs/cloudaudit.googleapis.com%2Factivity",
"receiveTimestamp": "2024-08-12T12:37:27.023187794Z"
}