祝融说。

[gpu_nodes]
192.168.1.101
192.168.1.102

---
- hosts: gpu_nodes
  become: yes # 以root权限执行
  vars:
    nvidia_driver_version: "535" # 指定你想安装的驱动版本

  tasks:
    - name: 1. Update APT cache and install prerequisite
      apt:
        update_cache: yes
        name: ['apt-transport-https', 'ca-certificates', 'curl', 'gnupg-agent', 'software-properties-common']
        state: present

    - name: 2. Add Docker's official GPG key
      apt_key:
        url: https://download.docker.com/linux/ubuntu/gpg
        state: present

    - name: 3. Add Docker repository
      apt_repository:
        repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable
        state: present

    - name: 4. Install Docker Engine
      apt:
        name: ['docker-ce', 'docker-ce-cli', 'containerd.io']
        state: present

    - name: 5. Add NVIDIA driver repository
      apt_repository:
        repo: ppa:graphics-drivers/ppa
        state: present

    - name: 6. Install NVIDIA Driver
      apt:
        name: "nvidia-driver-{{ nvidia_driver_version }}"
        state: present
      register: driver_install
      notify: Reboot node # 驱动安装后通常需要重启

    - name: 7. Add NVIDIA Container Toolkit repository
      shell: |
        curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
        curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
        sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
        tee /etc/apt/sources.list.d/nvidia-container-toolkit.list

    - name: 8. Install NVIDIA Container Toolkit
      apt:
        update_cache: yes
        name: nvidia-container-toolkit
        state: present

    - name: 9. Configure Docker to use NVIDIA runtime and restart
      shell: |
        nvidia-ctk runtime configure --runtime=docker
        systemctl restart docker

    - name: 10. Install k3s (Lightweight Kubernetes)
      shell: |
        curl -sfL https://get.k3s.io | sh -
        mkdir -p $HOME/.kube
        cp /etc/rancher/k3s/k3s.yaml $HOME/.kube/config
        chown $(id -u):$(id -g) $HOME/.kube/config
      args:
        creates: /usr/local/bin/k3s # 避免重复安装

  handlers:
    - name: Reboot node
      reboot:
        msg: "Rebooting node after NVIDIA driver installation"
        connect_timeout: 5
        reboot_timeout: 300
        pre_reboot_delay: 0
        post_reboot_delay: 30
        test_command: whoami

lspci | grep -i ascend

# 赋予执行权限
chmod +x AIA-Ascend-Driver-*.run

# 以root权限执行安装
sudo ./AIA-Ascend-Driver-*.run --install

sudo reboot

# 赋予执行权限
chmod +x AIA-Ascend-Toolkit-*.run

# 以root权限执行安装
# --install-path指定安装路径，--install-for-all表示为所有用户安装
sudo ./AIA-Ascend-Toolkit-*.run --install --install-path=/usr/local/ascend --install-for-all

# 编辑.bashrc文件
vim ~/.bashrc

# 在文件末尾添加以下行 (路径请根据你的实际安装路径修改)
source /usr/local/ascend/ascend-toolkit/set_env.sh

# 使配置立即生效
source ~/.bashrc

# 执行npu-smi命令
npu-smi info

+-------------------------------------------------------------------------------------------+
| npu-smi 21.0.2                  Version: 21.0.2                                           |
+-------------------------------+-----------------+-----------------------------------------+
| NPU     Name                  | Health          | Power(W)          Temp(C)               |
| Chip    Device-Chip-Id        | Bus-Id          | AICore(%)         HBM(MB)               |
+===============================+=================+=========================================+
| 0       Ascend 910B           | OK              | 110.0             45                    |
| 0       0-0                   | 0000:C1:00.0    | 0                 0 / 32768             |
+-------------------------------+-----------------+-----------------------------------------+
| 1       Ascend 910B           | OK              | 108.0             44                    |
| 0       1-0                   | 0000:C2:00.0    | 0                 0 / 32768             |
+-------------------------------+-----------------+-----------------------------------------+
... (其他NPU卡)

helm repo add volcano-sh https://volcano-sh.github.io/charts
helm repo update
helm install volcano volcano-sh/volcano -n volcano-system --create-namespace

kubectl get pods -n volcano-system

import torch
import torch.distributed as dist
import torch.nn as nn
import torch.optim as optim
from torch.nn.parallel import DistributedDataParallel as DDP
from torchvision import datasets, transforms
import os

def setup(rank, world_size):
    os.environ['MASTER_ADDR'] = os.environ.get('MASTER_ADDR', 'localhost')
    os.environ['MASTER_PORT'] = os.environ.get('MASTER_PORT', '12355')
    dist.init_process_group("nccl", rank=rank, world_size=world_size)

def cleanup():
    dist.destroy_process_group()

class Net(nn.Module):
    # ... (一个简单的CNN模型定义)
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 32, 3, 1)
        self.conv2 = nn.Conv2d(32, 64, 3, 1)
        self.fc1 = nn.Linear(9216, 128)
        self.fc2 = nn.Linear(128, 10)
    def forward(self, x):
        x = self.conv1(x)
        x = torch.relu(x)
        x = self.conv2(x)
        x = torch.relu(x)
        x = torch.max_pool2d(x, 2)
        x = torch.flatten(x, 1)
        x = self.fc1(x)
        x = torch.relu(x)
        x = self.fc2(x)
        return torch.log_softmax(x, dim=1)

def train(rank, world_size):
    setup(rank, world_size)
  
    device = torch.device(f"cuda:{rank % torch.cuda.device_count()}")
  
    transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.1307,), (0.3081,))
    ])
    dataset = datasets.MNIST('../data', train=True, download=True, transform=transform)
    train_sampler = torch.utils.data.distributed.DistributedSampler(dataset, num_replicas=world_size, rank=rank)
    train_loader = torch.utils.data.DataLoader(dataset, batch_size=64, sampler=train_sampler)

    model = Net().to(device)
    ddp_model = DDP(model, device_ids=[device])

    optimizer = optim.SGD(ddp_model.parameters(), lr=0.01)

    for epoch in range(3):
        train_sampler.set_epoch(epoch)
        for batch_idx, (data, target) in enumerate(train_loader):
            data, target = data.to(device), target.to(device)
            optimizer.zero_grad()
            output = ddp_model(data)
            loss = nn.functional.nll_loss(output, target)
            loss.backward()
            optimizer.step()
            if batch_idx % 10 == 0:
                print(f"Rank {rank}, Epoch {epoch}, Batch {batch_idx}, Loss: {loss.item()}")
  
    cleanup()

if __name__ == '__main__':
    # Volcano会注入VC_TASK_INDEX和VC_WORKER_NUM环境变量
    rank = int(os.environ.get("VC_TASK_INDEX", "0"))
    world_size = int(os.environ.get("VC_WORKER_NUM", "1"))
    print(f"Starting training on Rank {rank} of {world_size}...")
    train(rank, world_size)

FROM pytorch/pytorch:2.1.0-cuda12.1-cudnn8-runtime

WORKDIR /app

COPY mnist_distributed.py .

# 下载数据集，避免在运行时下载
RUN python -c "from torchvision import datasets; datasets.MNIST('../data', download=True)"

CMD ["python", "mnist_distributed.py"]

docker build -t your-registry/pytorch-dist-mnist:v1 .
docker push your-registry/pytorch-dist-mnist:v1

apiVersion: batch.volcano.sh/v1alpha1
kind: Job
metadata:
  name: pytorch-mnist-dist-job
spec:
  schedulerName: volcano
  minAvailable: 2 # 核心：Gang Scheduling，必须凑齐2个Pod才能开始
  queue: default
  tasks:
    - name: worker
      replicas: 2
      template:
        spec:
          containers:
            - name: pytorch-worker
              image: your-registry/pytorch-dist-mnist:v1
              resources:
                limits:
                  nvidia.com/gpu: 1 # 每个Pod申请1张GPU
          restartPolicy: OnFailure

kubectl apply -f volcano_pytorch_job.yml

# 查看PodGroup状态，Volcano调度的核心对象
kubectl get podgroup

# 查看Pod状态，你会看到两个Pod几乎是同时被创建并进入Running状态
kubectl get pods -l volcanosh.dev/job-name=pytorch-mnist-dist-job

# 查看其中一个Pod的日志
kubectl logs pytorch-mnist-dist-job-worker-0

# 拉取vLLM镜像
docker pull vllm/vllm-openai:latest

# 启动服务
# 注意：这需要较好的网络来下载模型，模型会被缓存到~/.cache/huggingface
docker run --gpus all -d \
  -p 8000:8000 \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  --name vllm_server \
  vllm/vllm-openai:latest \
  --model meta-llama/Meta-Llama-3-8B-Instruct

curl http://localhost:8000/v1/chat/completions \
    -H "Content-Type: application/json" \
    -d '{
        "model": "meta-llama/Meta-Llama-3-8B-Instruct",
        "messages": [
            {"role": "user", "content": "Hello! What is your name?"}
        ]
    }'

from locust import task, HttpUser
import random

class LLMUser(HttpUser):
    @task
    def generate(self):
        prompts = ["What is the capital of France?", "Write a short poem about the sea.", "Explain black holes to a 5-year-old."]
        payload = {
            "model": "meta-llama/Meta-Llama-3-8B-Instruct",
            "messages": [{"role": "user", "content": random.choice(prompts)}],
            "max_tokens": 100
        }
        self.client.post("/v1/chat/completions", json=payload, name="/v1/chat/completions")

locust -f locustfile.py --host http://localhost:8000

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: gpu-alerts
  labels:
    # 这个label需要与你的Prometheus实例的ruleSelector匹配
    prometheus: kube-prometheus 
    role: alert-rules
spec:
  groups:
    - name: gpu.rules
      rules:
        - alert: GPUTemperatureHigh
          expr: DCGM_FI_DEV_GPU_TEMP > 80
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "GPU high temperature on {{ $labels.nodename }}"
            description: "GPU {{ $labels.gpu }} on node {{ $labels.nodename }} has been over 80°C for 5 minutes. Current value is {{ $value }}°C."

        - alert: GPUMemoryHigh
          expr: (DCGM_FI_DEV_FB_USED / DCGM_FI_DEV_FB_TOTAL) * 100 > 95
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: "GPU high memory usage on {{ $labels.nodename }}"
            description: "GPU {{ $labels.gpu }} on node {{ $labels.nodename }} memory usage has been over 95% for 10 minutes. Current usage is {{ $value | printf `%.2f` }}%."

        - alert: GPUXidErrorDetected
          expr: rate(DCGM_FI_DEV_XID_ERRORS[5m]) > 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: "GPU XID Error Detected on {{ $labels.nodename }}"
            description: "GPU {{ $labels.gpu }} on node {{ $labels.nodename }} is reporting XID errors. This may indicate a software or hardware issue. Please investigate dmesg logs."

# 应用规则到你的K8s集群 (通常在monitoring命名空间)
kubectl apply -f gpu-alerts.yml -n monitoring

# 在GPU节点上运行
docker run --rm --gpus all -it wshuyi/gpu-burn -t 600 # 运行10分钟

import argparse
import paramiko
import pandas as pd
from kubernetes import client, config
from datetime import datetime

def ssh_run_command(hostname, command):
    """通过SSH在远程节点上执行命令并返回输出。"""
    try:
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(hostname, username='root', timeout=5) # 请根据实际情况修改用户名
        stdin, stdout, stderr = ssh.exec_command(command)
        output = stdout.read().decode('utf-8')
        error = stderr.read().decode('utf-8')
        ssh.close()
        if error and "command not found" not in error:
            return f"ERROR: {error}"
        return output
    except Exception as e:
        return f"SSH_ERROR: {str(e)}"

def parse_nvidia_smi(output):
    """解析nvidia-smi的输出，提取关键健康指标。"""
    if "NVIDIA-SMI has failed" in output or "ERROR" in output:
        return [{"gpu_id": "ALL", "health": "FAIL", "message": output}]
  
    devices = []
    lines = output.split('\n')
    # 非常简化的解析，实际生产中建议使用nvidia-smi --query-gpu=... --format=csv
    for i, line in enumerate(lines):
        if "MiB" in line and "%" in line:
            parts = line.split()
            gpu_id = parts[1]
            temp = parts[3]
            power = parts[5]
            mem_used = parts[9]
            mem_total = parts[11]
            gpu_util = parts[13]
          
            health = "OK"
            message = []
            if int(temp.replace('C','')) > 85:
                health = "WARN"
                message.append(f"Temp>85C({temp})")
            if int(power.replace('W','')) > 350: # 假设功耗阈值为350W
                health = "WARN"
                message.append(f"Power>350W({power})")

            devices.append({
                "gpu_id": gpu_id,
                "health": health if message else "OK",
                "message": ", ".join(message) if message else "N/A"
            })
    return devices

def get_k8s_nodes(label_selector=""):
    """获取K8s集群中符合标签选择器的节点列表。"""
    config.load_kube_config()
    v1 = client.CoreV1Api()
    nodes = v1.list_node(label_selector=label_selector)
    return [node.status.addresses[0].address for node in nodes.items]

def main():
    parser = argparse.ArgumentParser(description="Cluster Health Checker")
    parser.add_argument("--vendor", type=str, default="nvidia", choices=["nvidia", "ascend"], help="GPU/NPU vendor")
    parser.add_argument("--nodes", type=str, help="Comma-separated list of node IPs. Overrides k8s discovery.")
    parser.add_argument("--label", type=str, default="nvidia.com/gpu=true", help="K8s node label selector for discovery.")
    args = parser.parse_args()

    if args.nodes:
        nodes_to_check = args.nodes.split(',')
    else:
        print(f"Discovering nodes with label '{args.label}' from Kubernetes...")
        nodes_to_check = get_k8s_nodes(args.label)
        print(f"Found {len(nodes_to_check)} nodes: {nodes_to_check}")

    if args.vendor == "nvidia":
        command = "nvidia-smi"
        parser_func = parse_nvidia_smi
    elif args.vendor == "ascend":
        command = "npu-smi info"
        # 你需要为npu-smi编写一个类似的解析函数
        # parser_func = parse_npu_smi 
        print("Ascend NPU parser not implemented in this example.")
        return

    results = []
    for node in nodes_to_check:
        print(f"Checking node: {node} ...")
        output = ssh_run_command(node, command)
        health_info = parser_func(output)
        for device in health_info:
            results.append({
                "node": node,
                "device_type": args.vendor.upper(),
                "device_id": device["gpu_id"],
                "health": device["health"],
                "message": device["message"],
            })
  
    report_df = pd.DataFrame(results)
  
    print("\n" + "="*50)
    print(f"  Cluster Health Report - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print("="*50)
    print(report_df.to_string())

    # 打印问题摘要
    failed_nodes = report_df[report_df['health'] != 'OK']
    if not failed_nodes.empty:
        print("\n" + "!"*20 + "  PROBLEM SUMMARY  " + "!"*20)
        print(failed_nodes.to_string())
        print("!"*59)
    else:
        print("\n" + "✅"*10 + "  All checks passed! Cluster is healthy. " + "✅"*10)

if __name__ == "__main__":
    main()

import argparse

def sizeof_fmt(num, suffix=""):
    for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
        if abs(num) < 1000.0:
            return f"{num:3.1f}{unit}{suffix}"
        num /= 1000.0
    return f"{num:.1f}Y{suffix}"

def main():
    parser = argparse.ArgumentParser(description="Estimate LLM Training FLOPs and Time")
  
    # Model and Data parameters
    parser.add_argument("-p", "--params", type=float, required=True, help="Model parameters in billions (e.g., 70 for 70B)")
    parser.add_argument("-d", "--data_tokens", type=float, required=True, help="Training data size in trillions of tokens (e.g., 2 for 2T)")
  
    # Hardware and Performance parameters
    parser.add_argument("-n", "--num_gpus", type=int, required=True, help="Number of GPUs in the cluster")
    parser.add_argument("--peak_tflops", type=float, required=True, help="Peak TFLOPS of a single GPU at target precision (e.g., 312 for A100 FP16)")
    parser.add_argument("--mfu", type=float, default=0.4, help="Model FLOPs Utilization (MFU), a value between 0 and 1 (default: 0.4)")
  
    args = parser.parse_args()

    # --- Calculations ---
  
    # 1. Total FLOPs
    total_flops = 6 * (args.params * 1e9) * (args.data_tokens * 1e12)
  
    # 2. Cluster effective throughput
    single_gpu_effective_tflops = args.peak_tflops * args.mfu
    cluster_effective_tflops = single_gpu_effective_tflops * args.num_gpus
  
    # 3. Estimated training time
    if cluster_effective_tflops == 0:
        estimated_seconds = float('inf')
    else:
        estimated_seconds = total_flops / (cluster_effective_tflops * 1e12)
  
    estimated_days = estimated_seconds / (24 * 3600)

    # --- Print Report ---
  
    print("\n" + "="*50)
    print("  LLM Training Estimation Report")
    print("="*50)
  
    print("\n[Input Parameters]")
    print(f"  - Model Size: {args.params}B parameters")
    print(f"  - Dataset Size: {args.data_tokens}T tokens")
    print(f"  - Cluster Size: {args.num_gpus} GPUs")
    print(f"  - Single GPU Peak Performance: {args.peak_tflops} TFLOPS")
    print(f"  - Assumed MFU: {args.mfu:.2%}")
  
    print("\n[Estimated Requirements]")
    print(f"  - Total Training FLOPs: {sizeof_fmt(total_flops, 'FLOPs')}")
    print(f"  - Cluster Effective Throughput: {cluster_effective_tflops:,.2f} TFLOPS")
  
    print("\n[Final Estimation]")
    print(f"  - Estimated Training Time: {estimated_days:.2f} days")
    print("="*50)
  
    print("\nNote: This is a theoretical estimation. Actual time may vary based on network, storage, and software stack efficiency.")

if __name__ == "__main__":
    main()

python calc_model_flops.py \
  --params 70 \
  --data_tokens 2 \
  --num_gpus 1024 \
  --peak_tflops 312 \
  --mfu 0.4

==================================================
  LLM Training Estimation Report
==================================================

[Input Parameters]
  - Model Size: 70.0B parameters
  - Dataset Size: 2.0T tokens
  - Cluster Size: 1024 GPUs
  - Single GPU Peak Performance: 312.0 TFLOPS
  - Assumed MFU: 40.00%

[Estimated Requirements]
  - Total Training FLOPs: 840.0EFLOPs
  - Cluster Effective Throughput: 127,795.20 TFLOPS

[Final Estimation]
  - Estimated Training Time: 76.08 days
==================================================

Note: This is a theoretical estimation. Actual time may vary based on network, storage, and software stack efficiency.