When I try adding `--runtime nvidia --gpus all` then I get an error "docker: Error response from daemon: unknown or invalid runtime name: nvidia.".
If I try with just `--gpus all` then I get an error "invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead".
Here's a minimal Dockerfile and job config to reproduce the issue:
FROM python:3-slim-bullseye
RUN pip install nvidia-ml-py
ENTRYPOINT ["python", "-c", "import pynvml ; pynvml.nvmlInit()"]
{
"taskGroups": [
{
"taskCount": "1",
"parallelism": "1",
"taskSpec": {
"computeResource": {
"cpuMilli": "1000",
"memoryMib": "1024"
},
"runnables": [
{
"container": {
"imageUri": "<my-docker-image>",
"entrypoint": "",
"volumes": [],
"options": "--volume /var/lib/nvidia/lib64:/usr/local/nvidia/lib64 --volume /var/lib/nvidia/bin:/usr/local/nvidia/bin --device /dev/nvidia0:/dev/nvidia0 --device /dev/nvidia-uvm:/dev/nvidia-uvm --device /dev/nvidiactl:/dev/nvidiactl"
}
}
],
"volumes": []
}
}
],
"allocationPolicy": {
"instances": [
{
"installGpuDrivers": true,
"policy": {
"provisioningModel": "SPOT",
"machineType": "n1-standard-2",
"accelerators": [
{
"type": "nvidia-tesla-t4",
"count": "1"
}
],
"bootDisk": {
"sizeGb": "75"
}
}
}
],
"labels": {
"user": "avishai"
},
"serviceAccount": {
"email": "<service-account-email>"
}
},
"logsPolicy": {
"destination": "CLOUD_LOGGING"
}
}
I submit with `gcloud beta batch jobs submit --project <project> --location us-central1 --job-prefix=my-prefix --config /path/to/config.json`
LinkedIn
Twitter
