# 使用modelscope库下载模型
from modelscope import snapshot_download
model_dir = snapshot_download(
    'Qwen/Qwen3-235B-A22B-Instruct-2507',
    cache_dir='/workspace/models',
    revision='v1.0.0'
)

vllm serve Qwen/Qwen3-235B-A22B-Instruct-2507 \
    --tensor-parallel-size 8 \
    --gpu-memory-utilization 0.85 \
    --max-model-len 131072 \
    --served-model-name qwen3-235b \
    --port 9997 \
    --host 0.0.0.0 \
    --trust-remote-code \
    --dtype auto \
    --enable-prefix-caching \
    --enable-chunked-prefill \
    --api-key sk-xxxxxxxxx \
    --tool-call-parser hermes \
    --enable-auto-tool-choice \
    --swap-space 16 \
    --disable-log-requests

apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    app: vllm-qwen3-235b
    nvidia.com/app: vllm-qwen3-235b
    nvidia.com/framework: python
    nvidia.com/unit: application
  name: vllm-qwen3-235b
  namespace: ai-serving
spec:
  podManagementPolicy: OrderedReady
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: vllm-qwen3-235b
  serviceName: vllm-qwen3-235b
  template:
    metadata:
      annotations:
        nvidia.com/use-gputype: A100,A800,H100,H800
        nvidia.com/globalid: AIP_MDIS_QWEN3_235B
        nvidia.com/gpu: "8"
        nvidia.com/gpumem: "638976"  # 8 * 79872
        nvidia.com/model-uid: Qwen3-235B-A22B-Instruct-2507
      labels:
        app: vllm-qwen3-235b
        type: llm
        model-size: 235b
        nvidia.com/app: vllm-qwen3-235b
        nvidia.com/framework: python
        nvidia.com/unit: application
        nvidia.com/4pd-scheduler: "true"
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: accelerator
                operator: In
                values: ["nvidia-a100-80gb", "nvidia-a800-80gb", "nvidia-h100-80gb"]
              - key: topology.kubernetes.io/zone
                operator: In
                values: ["zone-gpu-high"]
      initContainers:
      - name: download-model
        image: download-model:latest
        imagePullPolicy: IfNotPresent
        command: ["sh", "-c"]
        args:
        - |
          echo "开始下载Qwen3-235B模型..."
          model_download /opt/app/models/ || {
            echo "模型下载失败，尝试从备用源下载..."
            exit 1
          }
          echo "模型下载完成，验证模型文件..."
          ls -la /opt/app/models/
          if [ ! -f "/opt/app/models/config.json" ]; then
            echo "模型文件验证失败"
            exit 1
          fi
          echo "模型文件验证成功"
        env:
        - name: S3_URL
          valueFrom:
            secretKeyRef:
              key: S3_URL
              name: vllm-qwen3-235b
        - name: S3_AK
          valueFrom:
            secretKeyRef:
              key: S3_AK
              name: vllm-qwen3-235b
        - name: S3_SK
          valueFrom:
            secretKeyRef:
              key: S3_SK
              name: vllm-qwen3-235b
        - name: S3_BUCKET
          valueFrom:
            secretKeyRef:
              key: S3_BUCKET
              name: vllm-qwen3-235b
        - name: USERNAME
          value: "ai-platform"
        - name: MODEL_NAME
          value: "qwen3-235b"
        - name: VERSION
          value: "Qwen3-235B-A22B-Instruct-2507"
        resources:
          requests:
            cpu: "2"
            memory: "4Gi"
          limits:
            cpu: "4"
            memory: "8Gi"
        volumeMounts:
        - mountPath: /opt/app/models
          name: share-volume
      containers:
      - name: vllm-qwen3-235b
        image: vllm/vllm-openai:v0.12.0
        imagePullPolicy: IfNotPresent
        command: ["/bin/bash"]
        args:
        - -c
        - |
          set -e
          
          # 环境变量设置
          export NCCL_SHM_DISABLE=1
          export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
          export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
          export NCCL_IB_DISABLE=0
          export NCCL_NET_GDR_LEVEL=2
          
          # 创建日志目录
          mkdir -p /log/logs
          
          # 等待模型文件就绪
          while [ ! -f "/opt/app/models/config.json" ]; do
            echo "等待模型文件就绪..."
            sleep 10
          done
          
          echo "启动vLLM服务..."
          exec python3 -m vllm.entrypoints.openai.api_server \
            --model=/opt/app/models \
            --served-model-name=Qwen3-235B-A22B-Instruct-2507 \
            --port=9997 \
            --host=0.0.0.0 \
            --trust-remote-code \
            --dtype=auto \
            --enable-prefix-caching \
            --enable-chunked-prefill \
            --tensor-parallel-size=8 \
            --gpu-memory-utilization=0.85 \
            --api-key=${API_KEY} \
            --max-model-len=131072 \
            --tool-call-parser=hermes \
            --enable-auto-tool-choice \
            --swap-space=16 \
            --disable-log-requests \
            --max-num-seqs=128 \
            --max-paddings=256 \
            > /log/logs/vllm-qwen3-235b.log 2>&1
        env:
        - name: ACTIVE_OOM_KILLER
          value: "0"
        - name: LIBCUDA_LOG_LEVEL
          value: "0"
        - name: GPU_CORE_UTILIZATION_POLICY
          value: "disable"
        - name: VLLM_WORKER_MULTIPROC_METHOD
          value: "spawn"
        - name: VLLM_LOGGING_LEVEL
          value: "INFO"
        - name: MODEL_UID
          value: "Qwen3-235B-A22B-Instruct-2507"
        - name: MODEL_TYPE
          value: "chat"
        - name: API_KEY
          valueFrom:
            secretKeyRef:
              key: api-key
              name: vllm-qwen3-235b
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        lifecycle:
          preStop:
            exec:
              command:
              - /bin/bash
              - -c
              - |
                echo "正在优雅关闭vLLM服务..."
                kill -TERM $(pgrep -f "vllm.entrypoints.openai.api_server") || true
                sleep 30
        ports:
        - containerPort: 9997
          name: api-port
          protocol: TCP
        livenessProbe:
          httpGet:
            path: /health
            port: 9997
            scheme: HTTP
          failureThreshold: 3
          initialDelaySeconds: 600
          periodSeconds: 60
          successThreshold: 1
          timeoutSeconds: 30
        readinessProbe:
          httpGet:
            path: /health
            port: 9997
            scheme: HTTP
          failureThreshold: 3
          initialDelaySeconds: 300
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 10
        startupProbe:
          httpGet:
            path: /health
            port: 9997
            scheme: HTTP
          failureThreshold: 30
          initialDelaySeconds: 120
          periodSeconds: 30
          successThreshold: 1
          timeoutSeconds: 10
        resources:
          requests:
            cpu: "16"
            memory: "64Gi"
            nvidia.com/gpu: "8"
            nvidia.com/gpumem: "638976"
          limits:
            cpu: "64"
            memory: "256Gi"
            nvidia.com/gpu: "8"
            nvidia.com/gpumem: "638976"
        volumeMounts:
        - mountPath: /opt/app/models
          name: share-volume
          readOnly: true
        - mountPath: /dev/shm
          name: cache-volume
        - mountPath: /etc/localtime
          name: timezone-volume
          readOnly: true
        - mountPath: /log
          name: log-volume
        - mountPath: /tmp
          name: temp-volume
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext:
        runAsNonRoot: false
        fsGroup: 0
      terminationGracePeriodSeconds: 180
      tolerations:
      - effect: NoSchedule
        key: role
        operator: Equal
        value: bigdata
      - effect: NoSchedule
        key: gpurole
        operator: Equal
        value: gpu
      - effect: NoSchedule
        key: model-size
        operator: Equal
        value: large
      volumes:
      - name: share-volume
        persistentVolumeClaim:
          claimName: vllm-qwen3-235b-model-pvc
      - name: cache-volume
        emptyDir:
          medium: Memory
          sizeLimit: 256Gi
      - name: log-volume
        hostPath:
          path: /data/logs/vllm-qwen3-235b
          type: DirectoryOrCreate
      - name: temp-volume
        emptyDir:
          sizeLimit: 32Gi
      - name: timezone-volume
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai
          type: FileOrCreate
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      partition: 0
  volumeClaimTemplates: []

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: vllm-qwen3-235b-model-pvc
  namespace: ai-serving
  labels:
    app: vllm-qwen3-235b
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 1Ti
  storageClassName: nfs-client

---
apiVersion: v1
kind: Service
metadata:
  name: vllm-qwen3-235b
  namespace: ai-serving
  labels:
    app: vllm-qwen3-235b
spec:
  type: ClusterIP
  ports:
  - port: 9997
    targetPort: 9997
    protocol: TCP
    name: api-port
  selector:
    app: vllm-qwen3-235b

---
apiVersion: v1
kind: Secret
metadata:
  name: vllm-qwen3-235b
  namespace: ai-serving
type: Opaque
data:
  api-key: c2stdmxsbS1xd2VuMy0yMzViLWFwaS1rZXk=  # base64编码的API key
  S3_URL: <base64-encoded-s3-url>
  S3_AK: <base64-encoded-access-key>
  S3_SK: <base64-encoded-secret-key>
  S3_BUCKET: <base64-encoded-bucket-name>
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: vllm-qwen3-235b-model-pvc
  namespace: ai-serving
  labels:
    app: vllm-qwen3-235b
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 1Ti
  storageClassName: nfs-client

kubectl label nodes {nodeid} gpu=on

helm repo add hami-charts https://project-hami.github.io/HAMi/
helm repo update

helm install hami hami-charts/hami -n kube-system

kubectl get pods -n kube-system | grep vgpu

# NVIDIA GPU 参数配置
resourceName: "nvidia.com/gpu"                    # GPU设备资源名
resourceMem: "nvidia.com/gpumem"                  # GPU内存资源名
resourceMemPercentage: "nvidia.com/gpumem-percentage"  # GPU内存百分比
resourceCores: "nvidia.com/gpucores"              # GPU核心数
resourcePriority: "nvidia.com/priority"           # GPU优先级

# 自定义为公司标识
resourceName: "company.com/gpu"
resourceMem: "company.com/gpumem"
resourceMemPercentage: "company.com/gpumem-percentage"
resourceCores: "company.com/gpucores"

apiVersion: v1
kind: Service
metadata:
  name: vllm-service
  namespace: ai-serving
spec:
  selector:
    app: vllm-qwen3-235b
  ports:
  - port: 9997
    targetPort: 9997
  type: ClusterIP

---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: vllm-ingress
  namespace: ai-serving
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: /
spec:
  rules:
  - host: vllm.ai-serving.company.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: vllm-service
            port:
              number: 9997

# 手动扩展副本数应对流量高峰
kubectl scale sts vllm-qwen3-235b --replicas=2 -n ai-serving