祝融说。

[gpu_nodes]
192.168.1.101
192.168.1.102

---
- hosts: gpu_nodes
  become: yes
  vars:
    nvidia_driver_version: "535"

  tasks:
    - name: 1. Update APT cache and install prerequisite
      apt:
        update_cache: yes
        name: ['apt-transport-https', 'ca-certificates', 'curl', 'gnupg-agent', 'software-properties-common']
        state: present

    - name: 2. Add Docker's official GPG key
      apt_key:
        url: https://download.docker.com/linux/ubuntu/gpg
        state: present

    - name: 3. Add Docker repository
      apt_repository:
        repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable
        state: present

    - name: 4. Install Docker Engine
      apt:
        name: ['docker-ce', 'docker-ce-cli', 'containerd.io']
        state: present

    - name: 5. Add NVIDIA driver repository
      apt_repository:
        repo: ppa:graphics-drivers/ppa
        state: present

    - name: 6. Install NVIDIA Driver
      apt:
        name: "nvidia-driver-{{ nvidia_driver_version }}"
        state: present
      register: driver_install
      notify: Reboot node

    - name: 7. Add NVIDIA Container Toolkit repository
      shell: |
        curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
        curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
        sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
        tee /etc/apt/sources.list.d/nvidia-container-toolkit.list

    - name: 8. Install NVIDIA Container Toolkit
      apt:
        update_cache: yes
        name: nvidia-container-toolkit
        state: present

    - name: 9. Configure Docker to use NVIDIA runtime and restart
      shell: |
        nvidia-ctk runtime configure --runtime=docker
        systemctl restart docker

    - name: 10. Install k3s (Lightweight Kubernetes)
      shell: |
        curl -sfL https://get.k3s.io | sh -
        mkdir -p $HOME/.kube
        cp /etc/rancher/k3s/k3s.yaml $HOME/.kube/config
        chown $(id -u):$(id -g) $HOME/.kube/config
      args:
        creates: /usr/local/bin/k3s

  handlers:
    - name: Reboot node
      reboot:
        msg: "Rebooting node after NVIDIA driver installation"
        connect_timeout: 5
        reboot_timeout: 300
        pre_reboot_delay: 0
        post_reboot_delay: 30
        test_command: whoami

lspci | grep -i ascend

# Grant execute permission
chmod +x AIA-Ascend-Driver-*.run

# Execute installation with root privileges
sudo ./AIA-Ascend-Driver-*.run --install

sudo reboot

# Grant execute permission
chmod +x AIA-Ascend-Toolkit-*.run

# Execute installation with root privileges
# --install-path specifies the installation path, --install-for-all installs for all users
sudo ./AIA-Ascend-Toolkit-*.run --install --install-path=/usr/local/ascend --install-for-all

# Edit .bashrc file
vim ~/.bashrc

# Add the following line at the end of the file (modify the path according to your actual installation path)
source /usr/local/ascend/ascend-toolkit/set_env.sh

# Make the configuration effective immediately
source ~/.bashrc

# Execute the npu-smi command
npu-smi info

+-------------------------------------------------------------------------------------------+
| npu-smi 21.0.2                  Version: 21.0.2                                           |
+-------------------------------+-----------------+-----------------------------------------+
| NPU     Name                  | Health          | Power(W)          Temp(C)               |
| Chip    Device-Chip-Id        | Bus-Id          | AICore(%)         HBM(MB)               |
+===============================+=================+=========================================+
| 0       Ascend 910B           | OK              | 110.0             45                    |
| 0       0-0                   | 0000:C1:00.0    | 0                 0 / 32768             |
+-------------------------------+-----------------+-----------------------------------------+
| 1       Ascend 910B           | OK              | 108.0             44                    |
| 0       1-0                   | 0000:C2:00.0    | 0                 0 / 32768             |
+-------------------------------+-----------------+-----------------------------------------+
... (other NPU cards)

helm repo add volcano-sh https://volcano-sh.github.io/charts
helm repo update
helm install volcano volcano-sh/volcano -n volcano-system --create-namespace

kubectl get pods -n volcano-system

import torch
import torch.distributed as dist
import torch.nn as nn
import torch.optim as optim
from torch.nn.parallel import DistributedDataParallel as DDP
from torchvision import datasets, transforms
import os

def setup(rank, world_size):
    os.environ['MASTER_ADDR'] = os.environ.get('MASTER_ADDR', 'localhost')
    os.environ['MASTER_PORT'] = os.environ.get('MASTER_PORT', '12355')
    dist.init_process_group("nccl", rank=rank, world_size=world_size)

def cleanup():
    dist.destroy_process_group()

class Net(nn.Module):
    # ... (a simple CNN model definition)
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 32, 3, 1)
        self.conv2 = nn.Conv2d(32, 64, 3, 1)
        self.fc1 = nn.Linear(9216, 128)
        self.fc2 = nn.Linear(128, 10)
    def forward(self, x):
        x = self.conv1(x)
        x = torch.relu(x)
        x = self.conv2(x)
        x = torch.relu(x)
        x = torch.max_pool2d(x, 2)
        x = torch.flatten(x, 1)
        x = self.fc1(x)
        x = torch.relu(x)
        x = self.fc2(x)
        return torch.log_softmax(x, dim=1)

def train(rank, world_size):
    setup(rank, world_size)
  
    device = torch.device(f"cuda:{rank % torch.cuda.device_count()}")
  
    transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.1307,), (0.3081,))
    ])
    dataset = datasets.MNIST('../data', train=True, download=True, transform=transform)
    train_sampler = torch.utils.data.distributed.DistributedSampler(dataset, num_replicas=world_size, rank=rank)
    train_loader = torch.utils.data.DataLoader(dataset, batch_size=64, sampler=train_sampler)

    model = Net().to(device)
    ddp_model = DDP(model, device_ids=[device])

    optimizer = optim.SGD(ddp_model.parameters(), lr=0.01)

    for epoch in range(3):
        train_sampler.set_epoch(epoch)
        for batch_idx, (data, target) in enumerate(train_loader):
            data, target = data.to(device), target.to(device)
            optimizer.zero_grad()
            output = ddp_model(data)
            loss = nn.functional.nll_loss(output, target)
            loss.backward()
            optimizer.step()
            if batch_idx % 10 == 0:
                print(f"Rank {rank}, Epoch {epoch}, Batch {batch_idx}, Loss: {loss.item()}")
  
    cleanup()

if __name__ == '__main__':
    # Volcano injects VC_TASK_INDEX and VC_WORKER_NUM environment variables
    rank = int(os.environ.get("VC_TASK_INDEX", "0"))
    world_size = int(os.environ.get("VC_WORKER_NUM", "1"))
    print(f"Starting training on Rank {rank} of {world_size}...")
    train(rank, world_size)

FROM pytorch/pytorch:2.1.0-cuda12.1-cudnn8-runtime

WORKDIR /app

COPY mnist_distributed.py .

# Download dataset to avoid downloading at runtime
RUN python -c "from torchvision import datasets; datasets.MNIST('../data', download=True)"

CMD ["python", "mnist_distributed.py"]

docker build -t your-registry/pytorch-dist-mnist:v1 .
docker push your-registry/pytorch-dist-mnist:v1

apiVersion: batch.volcano.sh/v1alpha1
kind: Job
metadata:
  name: pytorch-mnist-dist-job
spec:
  schedulerName: volcano
  minAvailable: 2 # Core: Gang Scheduling, must assemble 2 Pods before starting
  queue: default
  tasks:
    - name: worker
      replicas: 2
      template:
        spec:
          containers:
            - name: pytorch-worker
              image: your-registry/pytorch-dist-mnist:v1
              resources:
                limits:
                  nvidia.com/gpu: 1 # Each Pod requests 1 GPU
          restartPolicy: OnFailure

kubectl apply -f volcano_pytorch_job.yml

# View PodGroup status -- the core object of Volcano scheduling
kubectl get podgroup

# View Pod status -- you will see both Pods created and entering Running state almost simultaneously
kubectl get pods -l volcanosh.dev/job-name=pytorch-mnist-dist-job

# View logs of one Pod
kubectl logs pytorch-mnist-dist-job-worker-0

# Pull the vLLM image
docker pull vllm/vllm-openai:latest

# Start the service
# Note: This requires a good network connection to download the model, which will be cached at ~/.cache/huggingface
docker run --gpus all -d \
  -p 8000:8000 \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  --name vllm_server \
  vllm/vllm-openai:latest \
  --model meta-llama/Meta-Llama-3-8B-Instruct

curl http://localhost:8000/v1/chat/completions \
    -H "Content-Type: application/json" \
    -d '{
        "model": "meta-llama/Meta-Llama-3-8B-Instruct",
        "messages": [
            {"role": "user", "content": "Hello! What is your name?"}
        ]
    }'

from locust import task, HttpUser
import random

class LLMUser(HttpUser):
    @task
    def generate(self):
        prompts = ["What is the capital of France?", "Write a short poem about the sea.", "Explain black holes to a 5-year-old."]
        payload = {
            "model": "meta-llama/Meta-Llama-3-8B-Instruct",
            "messages": [{"role": "user", "content": random.choice(prompts)}],
            "max_tokens": 100
        }
        self.client.post("/v1/chat/completions", json=payload, name="/v1/chat/completions")

locust -f locustfile.py --host http://localhost:8000

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: gpu-alerts
  labels:
    # This label must match the ruleSelector of your Prometheus instance
    prometheus: kube-prometheus 
    role: alert-rules
spec:
  groups:
    - name: gpu.rules
      rules:
        - alert: GPUTemperatureHigh
          expr: DCGM_FI_DEV_GPU_TEMP > 80
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "GPU high temperature on {{ $labels.nodename }}"
            description: "GPU {{ $labels.gpu }} on node {{ $labels.nodename }} has been over 80 C for 5 minutes. Current value is {{ $value }} C."

        - alert: GPUMemoryHigh
          expr: (DCGM_FI_DEV_FB_USED / DCGM_FI_DEV_FB_TOTAL) * 100 > 95
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: "GPU high memory usage on {{ $labels.nodename }}"
            description: "GPU {{ $labels.gpu }} on node {{ $labels.nodename }} memory usage has been over 95% for 10 minutes. Current usage is {{ $value | printf `%.2f` }}%."

        - alert: GPUXidErrorDetected
          expr: rate(DCGM_FI_DEV_XID_ERRORS[5m]) > 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: "GPU XID Error Detected on {{ $labels.nodename }}"
            description: "GPU {{ $labels.gpu }} on node {{ $labels.nodename }} is reporting XID errors. This may indicate a software or hardware issue. Please investigate dmesg logs."

# Apply the rules to your K8s cluster (typically in the monitoring namespace)
kubectl apply -f gpu-alerts.yml -n monitoring

# Run on a GPU node
docker run --rm --gpus all -it wshuyi/gpu-burn -t 600 # Run for 10 minutes

import argparse
import paramiko
import pandas as pd
from kubernetes import client, config
from datetime import datetime

def ssh_run_command(hostname, command):
    """Execute command on a remote node via SSH and return output."""
    try:
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(hostname, username='root', timeout=5) # Modify username as needed
        stdin, stdout, stderr = ssh.exec_command(command)
        output = stdout.read().decode('utf-8')
        error = stderr.read().decode('utf-8')
        ssh.close()
        if error and "command not found" not in error:
            return f"ERROR: {error}"
        return output
    except Exception as e:
        return f"SSH_ERROR: {str(e)}"

def parse_nvidia_smi(output):
    """Parse nvidia-smi output and extract key health metrics."""
    if "NVIDIA-SMI has failed" in output or "ERROR" in output:
        return [{"gpu_id": "ALL", "health": "FAIL", "message": output}]
  
    devices = []
    lines = output.split('\n')
    # Very simplified parsing; in production, use nvidia-smi --query-gpu=... --format=csv
    for i, line in enumerate(lines):
        if "MiB" in line and "%" in line:
            parts = line.split()
            gpu_id = parts[1]
            temp = parts[3]
            power = parts[5]
            mem_used = parts[9]
            mem_total = parts[11]
            gpu_util = parts[13]
          
            health = "OK"
            message = []
            if int(temp.replace('C','')) > 85:
                health = "WARN"
                message.append(f"Temp>85C({temp})")
            if int(power.replace('W','')) > 350: # Assuming power threshold of 350W
                health = "WARN"
                message.append(f"Power>350W({power})")

            devices.append({
                "gpu_id": gpu_id,
                "health": health if message else "OK",
                "message": ", ".join(message) if message else "N/A"
            })
    return devices

def get_k8s_nodes(label_selector=""):
    """Get the list of nodes from the K8s cluster matching the label selector."""
    config.load_kube_config()
    v1 = client.CoreV1Api()
    nodes = v1.list_node(label_selector=label_selector)
    return [node.status.addresses[0].address for node in nodes.items]

def main():
    parser = argparse.ArgumentParser(description="Cluster Health Checker")
    parser.add_argument("--vendor", type=str, default="nvidia", choices=["nvidia", "ascend"], help="GPU/NPU vendor")
    parser.add_argument("--nodes", type=str, help="Comma-separated list of node IPs. Overrides k8s discovery.")
    parser.add_argument("--label", type=str, default="nvidia.com/gpu=true", help="K8s node label selector for discovery.")
    args = parser.parse_args()

    if args.nodes:
        nodes_to_check = args.nodes.split(',')
    else:
        print(f"Discovering nodes with label '{args.label}' from Kubernetes...")
        nodes_to_check = get_k8s_nodes(args.label)
        print(f"Found {len(nodes_to_check)} nodes: {nodes_to_check}")

    if args.vendor == "nvidia":
        command = "nvidia-smi"
        parser_func = parse_nvidia_smi
    elif args.vendor == "ascend":
        command = "npu-smi info"
        # You would need to write a similar parsing function for npu-smi
        # parser_func = parse_npu_smi 
        print("Ascend NPU parser not implemented in this example.")
        return

    results = []
    for node in nodes_to_check:
        print(f"Checking node: {node} ...")
        output = ssh_run_command(node, command)
        health_info = parser_func(output)
        for device in health_info:
            results.append({
                "node": node,
                "device_type": args.vendor.upper(),
                "device_id": device["gpu_id"],
                "health": device["health"],
                "message": device["message"],
            })
  
    report_df = pd.DataFrame(results)
  
    print("\n" + "="*50)
    print(f"  Cluster Health Report - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print("="*50)
    print(report_df.to_string())

    # Print problem summary
    failed_nodes = report_df[report_df['health'] != 'OK']
    if not failed_nodes.empty:
        print("\n" + "!"*20 + "  PROBLEM SUMMARY  " + "!"*20)
        print(failed_nodes.to_string())
        print("!"*59)
    else:
        print("\n" + "="*10 + "  All checks passed! Cluster is healthy. " + "="*10)

if __name__ == "__main__":
    main()

import argparse

def sizeof_fmt(num, suffix=""):
    for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
        if abs(num) < 1000.0:
            return f"{num:3.1f}{unit}{suffix}"
        num /= 1000.0
    return f"{num:.1f}Y{suffix}"

def main():
    parser = argparse.ArgumentParser(description="Estimate LLM Training FLOPs and Time")
  
    # Model and Data parameters
    parser.add_argument("-p", "--params", type=float, required=True, help="Model parameters in billions (e.g., 70 for 70B)")
    parser.add_argument("-d", "--data_tokens", type=float, required=True, help="Training data size in trillions of tokens (e.g., 2 for 2T)")
  
    # Hardware and Performance parameters
    parser.add_argument("-n", "--num_gpus", type=int, required=True, help="Number of GPUs in the cluster")
    parser.add_argument("--peak_tflops", type=float, required=True, help="Peak TFLOPS of a single GPU at target precision (e.g., 312 for A100 FP16)")
    parser.add_argument("--mfu", type=float, default=0.4, help="Model FLOPs Utilization (MFU), a value between 0 and 1 (default: 0.4)")
  
    args = parser.parse_args()

    # --- Calculations ---
  
    # 1. Total FLOPs
    total_flops = 6 * (args.params * 1e9) * (args.data_tokens * 1e12)
  
    # 2. Cluster effective throughput
    single_gpu_effective_tflops = args.peak_tflops * args.mfu
    cluster_effective_tflops = single_gpu_effective_tflops * args.num_gpus
  
    # 3. Estimated training time
    if cluster_effective_tflops == 0:
        estimated_seconds = float('inf')
    else:
        estimated_seconds = total_flops / (cluster_effective_tflops * 1e12)
  
    estimated_days = estimated_seconds / (24 * 3600)

    # --- Print Report ---
  
    print("\n" + "="*50)
    print("  LLM Training Estimation Report")
    print("="*50)
  
    print("\n[Input Parameters]")
    print(f"  - Model Size: {args.params}B parameters")
    print(f"  - Dataset Size: {args.data_tokens}T tokens")
    print(f"  - Cluster Size: {args.num_gpus} GPUs")
    print(f"  - Single GPU Peak Performance: {args.peak_tflops} TFLOPS")
    print(f"  - Assumed MFU: {args.mfu:.2%}")
  
    print("\n[Estimated Requirements]")
    print(f"  - Total Training FLOPs: {sizeof_fmt(total_flops, 'FLOPs')}")
    print(f"  - Cluster Effective Throughput: {cluster_effective_tflops:,.2f} TFLOPS")
  
    print("\n[Final Estimation]")
    print(f"  - Estimated Training Time: {estimated_days:.2f} days")
    print("="*50)
  
    print("\nNote: This is a theoretical estimation. Actual time may vary based on network, storage, and software stack efficiency.")

if __name__ == "__main__":
    main()

python calc_model_flops.py \
  --params 70 \
  --data_tokens 2 \
  --num_gpus 1024 \
  --peak_tflops 312 \
  --mfu 0.4

==================================================
  LLM Training Estimation Report
==================================================

[Input Parameters]
  - Model Size: 70.0B parameters
  - Dataset Size: 2.0T tokens
  - Cluster Size: 1024 GPUs
  - Single GPU Peak Performance: 312.0 TFLOPS
  - Assumed MFU: 40.00%

[Estimated Requirements]
  - Total Training FLOPs: 840.0EFLOPs
  - Cluster Effective Throughput: 127,795.20 TFLOPS

[Final Estimation]
  - Estimated Training Time: 76.08 days
==================================================

Note: This is a theoretical estimation. Actual time may vary based on network, storage, and software stack efficiency.