# 部署Jupyter Notebook
deploy_notebook() {
local name=$1
local namespace=$2
local gpu=$3

echo “=== 部署Jupyter Notebook: $name ===”
kubectl apply -f – << EOF apiVersion: kubeflow.org/v1 kind: Notebook metadata: name: $name namespace: $namespace spec: template: spec: containers: - name: notebook image: kubeflownotebookswg/jupyter-tensorflow-full:v1.2.0 resources: requests: cpu: "2" memory: "8Gi" nvidia.com/gpu: "$gpu" limits: cpu: "4" memory: "16Gi" nvidia.com/gpu: "$gpu" volumeMounts: - name: data mountPath: /data volumes: - name: data persistentVolumeClaim: claimName: ${name}-data --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: ${name}-data namespace: $namespace spec: accessModes: - ReadWriteOnce resources: requests: storage: 100Gi EOF } # 部署模型训练任务 deploy_training() { local name=$1 local namespace=$2 local replicas=$3 local gpu=$4 echo "=== 部署模型训练任务: $name ===" kubectl apply -f - << EOF apiVersion: kubeflow.org/v1 kind: TFJob metadata: name: $name namespace: $namespace spec: tfReplicaSpecs: Worker: replicas: $replicas restartPolicy: OnFailure template: spec: containers: - name: tensorflow image: tensorflow/tensorflow:2.8.0-gpu command: - python - /app/train.py resources: requests: cpu: "2" memory: "8Gi" nvidia.com/gpu: "$gpu" limits: cpu: "4" memory: "16Gi" nvidia.com/gpu: "$gpu" volumeMounts: - name: data mountPath: /data - name: code mountPath: /app volumes: - name: data persistentVolumeClaim: claimName: ${name}-data - name: code configMap: name: ${name}-code --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: ${name}-data namespace: $namespace spec: accessModes: - ReadWriteOnce resources: requests: storage: 50Gi --- apiVersion: v1 kind: ConfigMap metadata: name: ${name}-code namespace: $namespace data: train.py: | import tensorflow as tf from tensorflow.keras.datasets import mnist from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense, Dropout, Flatten from tensorflow.keras.layers import Conv2D, MaxPooling2D from tensorflow.keras import backend as K # 加载数据 (x_train, y_train), (x_test, y_test) = mnist.load_data() # 数据预处理 if K.image_data_format() == 'channels_first': x_train = x_train.reshape(x_train.shape[0], 1, 28, 28) x_test = x_test.reshape(x_test.shape[0], 1, 28, 28) input_shape = (1, 28, 28) else: x_train = x_train.reshape(x_train.shape[0], 28, 28, 1) x_test = x_test.reshape(x_test.shape[0], 28, 28, 1) input_shape = (28, 28, 1) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 # 转换标签 y_train = tf.keras.utils.to_categorical(y_train, 10) y_test = tf.keras.utils.to_categorical(y_test, 10) # 构建模型 model = Sequential() model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(10, activation='softmax')) # 编译模型 model.compile(loss=tf.keras.losses.categorical_crossentropy, optimizer=tf.keras.optimizers.Adadelta(), metrics=['accuracy']) # 训练模型 model.fit(x_train, y_train, batch_size=128, epochs=10, verbose=1, validation_data=(x_test, y_test)) # 评估模型 score = model.evaluate(x_test, y_test, verbose=0) print('Test loss:', score[0]) print('Test accuracy:', score[1]) # 保存模型 model.save('/data/model.h5') EOF } # 部署模型服务 deploy_model_service() { local name=$1 local namespace=$2 local model_path=$3 local gpu=$4 echo "=== 部署模型服务: $name ===" kubectl apply -f - << EOF apiVersion: serving.kubeflow.org/v1alpha2 kind: InferenceService metadata: name: $name namespace: $namespace spec: default: predictor: tensorflow: storageUri: $model_path resources: requests: cpu: "1" memory: "4Gi" nvidia.com/gpu: "$gpu" limits: cpu: "2" memory: "8Gi" nvidia.com/gpu: "$gpu" EOF } # 主菜单 main() { echo "AI平台管理脚本" echo "1. 部署Jupyter Notebook" echo "2. 部署模型训练任务" echo "3. 部署模型服务" echo "4. 查看资源状态" echo "5. 退出" read -p "请选择操作: " choice case $choice in 1) read -p "请输入Notebook名称: " name read -p "请输入命名空间: " namespace read -p "请输入GPU数量: " gpu deploy_notebook $name $namespace $gpu ;; 2) read -p "请输入训练任务名称: " name read -p "请输入命名空间: " namespace read -p "请输入Worker数量: " replicas read -p "请输入每个Worker的GPU数量: " gpu deploy_training $name $namespace $replicas $gpu ;; 3) read -p "请输入模型服务名称: " name read -p "请输入命名空间: " namespace read -p "请输入模型存储路径: " model_path read -p "请输入GPU数量: " gpu deploy_model_service $name $namespace $model_path $gpu ;; 4) read -p "请输入命名空间: " namespace echo "=== 查看Pod状态 ===" kubectl get pods -n $namespace echo "=== 查看服务状态 ===" kubectl get svc -n $namespace echo "=== 查看GPU资源 ===" kubectl get nodes -o json | jq '.items[].status.allocatable' | grep -E 'nvidia|cpu|memory' ;; 5) exit 0 ;; *) echo "无效选择" ;; esac main } main

from Linux:www.itpux.com。

Part05-风哥经验总结与分享

5.1 AI/机器学习平台最佳实践

• 硬件选择：根据模型类型和训练规模选择合适的GPU，如NVIDIA A100、H100等
• 软件栈选择：选择适合的AI框架和工具，如TensorFlow、PyTorch、Kubeflow等
• 资源管理：合理配置GPU资源，使用K8s的资源配额和限制
• 分布式训练：对于大规模模型，使用分布式训练提高训练速度
• 模型管理：建立模型版本管理和部署流程，确保模型的可追溯性

5.2 常见问题与解决方案

• GPU资源不足：使用GPU共享技术，合理调度GPU资源
• 训练速度慢：优化模型结构，使用混合精度训练，增加批量大小
• 模型部署延迟高：使用模型量化、模型压缩等技术，减少推理延迟
• 数据处理瓶颈：使用分布式数据处理框架，如Apache Spark
• 平台维护复杂：使用自动化工具，简化平台的维护和管理

5.3 性能优化建议

• GPU优化：使用CUDA和cuDNN的最新版本，优化GPU内存使用
• 数据优化：使用数据预加载和缓存，减少数据加载时间
• 模型优化：使用模型压缩、量化等技术，减少模型大小和推理时间
• 网络优化：使用高速网络，减少分布式训练的通信开销
• 存储优化：使用高速存储，如NVMe SSD，提高数据读写速度

5.4 未来发展趋势

• 大模型时代：支持更大规模的模型训练和部署
• 边缘AI：将AI模型部署到边缘设备，实现实时推理
• 自动化机器学习：使用AutoML技术，自动选择和优化模型
• 联邦学习：在保护数据隐私的前提下进行模型训练
• AI与云原生深度集成：利用云原生技术，提供更灵活、可扩展的AI平台

风哥提示：AI/机器学习平台的搭建需要综合考虑硬件、软件、网络等多个因素，需要根据具体的应用场景和需求进行优化和调整。