GPU & AI Dedicated Servers

# Check detected NVIDIA GPUs and driver version
nvidia-smi

# Expected output for a dual RTX 5090 configuration:
# +-------------------------+
# | NVIDIA-SMI  570.86.16   |
# | Driver: 570.86.16       |
# | CUDA: 12.8              |
# +-------------------------+
# | GPU 0: NVIDIA RTX 5090  |
# |   32GB GDDR7 | 575W    |
# | GPU 1: NVIDIA RTX 5090  |
# |   32GB GDDR7 | 575W    |
# +-------------------------+

# Monitor GPU utilization in real-time
watch -n 1 nvidia-smi​

# Install CUDA toolkit (Ubuntu 22.04+)
wget https://developer.download.nvidia.com/\
  compute/cuda/repos/ubuntu2204/x86_64/\
  cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb
sudo apt-get update
sudo apt-get install -y cuda-toolkit

# Install PyTorch with CUDA support
pip3 install torch torchvision torchaudio \
  --index-url https://download.pytorch.org/\
  whl/cu128

# Verify CUDA is available in PyTorch
python3 -c "import torch; \
  print(f'CUDA: {torch.cuda.is_available()}'); \
  print(f'GPUs: {torch.cuda.device_count()}')"​

# Install vLLM for high-throughput inference
pip install vllm

# Serve a 7B model on a single GPU
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --gpu-memory-utilization 0.9

# Serve a 70B model across multiple GPUs
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --tensor-parallel-size 2 \
  --gpu-memory-utilization 0.9

# Test the endpoint
curl http://localhost:8000/v1/completions \
  -H "Content-Type: application/json" \
  -d '{"model": "meta-llama/Llama-3.1-8B-Instruct",
       "prompt": "Explain GPU computing.",
       "max_tokens": 256}'​

import torch
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel

# Initialize distributed training
dist.init_process_group(backend="nccl")
local_rank = int(os.environ["LOCAL_RANK"])
torch.cuda.set_device(local_rank)

# Wrap model for multi-GPU
model = YourModel().cuda(local_rank)
model = DistributedDataParallel(
    model, device_ids=[local_rank]
)

# Launch with torchrun (2 GPUs)
# $ torchrun --nproc_per_node=2 train.py

# Check memory allocation per GPU
for i in range(torch.cuda.device_count()):
    alloc = torch.cuda.memory_allocated(i)
    total = torch.cuda.get_device_properties(i)
    print(f"GPU {i}: {alloc/1e9:.1f}GB"
          f" / {total.total_memory/1e9:.1f}GB")​