Docker & GPU

1sudo apt update
2
3sudo apt install -y nvidia-container-runtime
4# You may need to replace above line with
5sudo apt install nvidia-docker2
6sudo apt install nvidia-container-toolkit
7
8sudo apt install -y nvidia-cuda-toolkit
9# restard required

1# Verify that your computer has a graphic card
2lspci | grep -i nvidia

1# First, install drivers and check
2nvidia-smi
3# output: NVIDIA-SMI 450.80.02 Driver Version: 450.80.02    CUDA Version: 11.0
4# It's the maximum CUDA version that your driver supports

1# Check current version of cuda
2nvcc --version
3# If nvcc is not available, it may be in /usr/local/cuda/bin/
4# Add this location to PATH
5# modify ~/.zshrc or ~/.bashrc
6export PATH=/usr/local/cuda/bin:$PATH
7
8# You may need to install
9sudo apt install -y nvidia-cuda-toolkit

1# Install and check nvidia-docker
2dpkg -l | grep nvidia-docker
3# or
4nvidia-docker version

1# Verifying –gpus option under docker run
2docker run --help | grep -i gpus
3# output: --gpus gpu-request GPU devices to add to the container ('all' to pass all GPUs)

1# List all GPU devices
2docker run -it --rm --gpus all ubuntu nvidia-smi -L
3# output: GPU 0: GeForce GTX 1650 (...)

1# ERROR ?
2# docker: Error response from daemon: failed to create shim: OCI runtime create failed: container_linux.go:380: starting container process caused: process_linux.go:545: container init caused: Running hook #0:: error running hook: exit status 1, stdout: , stderr: nvidia-container-cli: initialization error: load library failed: libnvidia-ml.so.1: cannot open shared object file: no such file or directory: unknown.

1# ERROR ?
2# Error response from daemon: could not select device driver "" with capabilities: [[gpu]]
3
4# Solution: install nvidia-docker2

1# Verifying again with nvidia-smi
2docker run -it --rm --gpus all ubuntu nvidia-smi
3
4# Return something like
5+-----------------------------------------------------------------------------+
6| NVIDIA-SMI 510.54       Driver Version: 510.54       CUDA Version: 11.6     |
7|-------------------------------+----------------------+----------------------+
8| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
9| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
10|                               |                      |               MIG M. |
11|===============================+======================+======================|
12|   0  NVIDIA GeForce ...  Off  | 00000000:01:00.0 Off |                  N/A |
13| N/A   55C    P0    11W /  N/A |    369MiB /  4096MiB |      5%      Default |
14|                               |                      |                  N/A |
15+-------------------------------+----------------------+----------------------+
16# and another box like this

1whereis cudnn
2# cudnn: /usr/include/cudnn.h
3
4# Check cudnn version
5cat /usr/include/cudnn.h | grep CUDNN_MAJOR -A 2
6# or try this (it works for me, cudnn 8)
7cat /usr/include/cudnn_version.h | grep CUDNN_MAJOR -A 2

1# check version
2nvidia-docker version

1# If "version" in docker-compose.yml < 2.3
2# Modify: /etc/docker/daemon.json
3{
4    "default-runtime": "nvidia",
5    "runtimes": {
6        "nvidia": {
7            "path": "nvidia-container-runtime",
8            "runtimeArgs": []
9        }
10    }
11}

1# restart our docker daemon
2sudo pkill -SIGHUP dockerd

1# If "version" in docker-compose.yml >=2.3
2# docker-compose.yml => able to use "runtime"
3version: '2.3' # MUST BE >=2.3 AND <3
4services:
5  testing:
6    ports:
7      - "8000:8000"
8    runtime: nvidia
9    volumes:
10      - ./object_detection:/object_detection

1docker pull tensorflow/tensorflow:latest-gpu-jupyter
2mkdir ~/Downloads/test/notebooks

1docker run --name docker_thi_test -it --rm -v $(realpath ~/Downloads/test/notebooks):/tf/notebooks -p 8888:8888 tensorflow/tensorflow:latest-gpu-jupyter

1# ~/Download/test/Dockerfile
2FROM tensorflow/tensorflow:latest-gpu-jupyter

1docker-compose run --rm jupyter

1# Linux only
2nvidia-smi

1# All processes that use GPU
2sudo fuser -v /dev/nvidia*

1# Kill a single process
2sudo kill -9 3019

1# Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
2# Function call stack:
3# train_function